From d75d26c23d22670ead51817684c279ba7b9ae498 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 07:49:26 +0900 Subject: [PATCH] Stabilize paired reviewer recovery and owner follow-up --- README.md | 2 +- docs/architecture.md | 2 +- runners/agent-runner/src/index.ts | 54 +++- runners/agent-runner/src/reviewer-runtime.ts | 85 ++++++ .../test/reviewer-runtime.test.ts | 101 ++++++- setup/environment.ts | 27 +- setup/platform.test.ts | 215 ++++++++++++++ setup/platform.ts | 110 ++++++++ setup/service.ts | 23 +- src/agent-runner.ts | 3 +- src/config.ts | 2 +- src/message-agent-executor.test.ts | 223 ++++++++++++++- src/message-agent-executor.ts | 86 +++--- src/message-runtime.test.ts | 224 +++++++++++++++ src/message-runtime.ts | 40 +++ src/paired-execution-context.test.ts | 34 ++- src/paired-execution-context.ts | 3 + src/paired-workspace-manager.test.ts | 7 +- src/provider-retry.test.ts | 63 +++++ src/provider-retry.ts | 263 +++++++++++------- src/task-scheduler.test.ts | 9 + src/token-refresh.ts | 14 +- src/token-rotation.ts | 7 + src/verification.test.ts | 4 +- src/verification.ts | 6 +- src/workspace-package-manager.test.ts | 10 +- src/workspace-package-manager.ts | 23 +- 27 files changed, 1474 insertions(+), 166 deletions(-) diff --git a/README.md b/README.md index 0eba661..5bab52a 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ User message ├─ Arbiter enabled → Arbiter judges → PROCEED/REVISE/RESET/ESCALATE └─ Arbiter disabled → Escalate to user → @user ⚠️ → Owner BLOCKED/NEEDS_CONTEXT → Arbiter (same path as reviewer) - → Deadlock (3+ round trips without progress) + → Deadlock (2+ round trips without progress) → Arbiter summoned → binding verdict → loop resumes ``` diff --git a/docs/architecture.md b/docs/architecture.md index f23f857..542151c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -73,7 +73,7 @@ User message ├─ Arbiter enabled → Arbiter judges → PROCEED/REVISE/RESET/ESCALATE └─ Arbiter disabled → Escalate to user → @user ⚠️ → Owner BLOCKED/NEEDS_CONTEXT → Arbiter (same path as reviewer) - → Deadlock (3+ round trips without progress) + → Deadlock (2+ round trips without progress) → Arbiter summoned → binding verdict → loop resumes ``` diff --git a/runners/agent-runner/src/index.ts b/runners/agent-runner/src/index.ts index 1f7a38d..4478502 100644 --- a/runners/agent-runner/src/index.ts +++ b/runners/agent-runner/src/index.ts @@ -25,7 +25,10 @@ import { } from './room-role-context.js'; import { assertReadonlyWorkspaceRepoConnectivity, + buildClaudeReadonlySandboxSettings, buildReviewerGitGuardEnv, + getClaudeReadonlySandboxMode, + isClaudeReadonlyReviewerRuntime, isReviewerMutatingShellCommand, isReviewerRuntime, } from './reviewer-runtime.js'; @@ -545,6 +548,8 @@ async function runQuery( containerInput: ContainerInput, sdkEnv: Record, reviewerRuntime: boolean, + claudeReadonlyReviewerRuntime: boolean, + claudeReadonlySandboxMode: 'strict' | 'best-effort' | null, ): Promise<{ newSessionId?: string; closedDuringQuery: boolean; @@ -622,8 +627,20 @@ async function runQuery( if (thinking) log(`Thinking config: ${JSON.stringify(thinking)}`); if (effort) log(`Effort: ${effort}`); if (reviewerRuntime) log('Reviewer runtime restrictions enabled'); + if (claudeReadonlyReviewerRuntime) { + log( + `Claude host reviewer read-only sandbox enabled (${claudeReadonlySandboxMode || 'unknown'})`, + ); + } + if (claudeReadonlySandboxMode === 'best-effort') { + log( + 'Claude host reviewer sandbox capability unavailable on this host, using best-effort read-only mode', + ); + } - const allowedTools = reviewerRuntime + const readonlyReviewerRuntime = + reviewerRuntime || claudeReadonlyReviewerRuntime; + const allowedTools = readonlyReviewerRuntime ? [ 'Bash', 'Read', 'Glob', 'Grep', @@ -644,6 +661,18 @@ async function runQuery( 'mcp__ejclaw__*' ]; + const readonlyProtectedPaths = [effectiveCwd, ...extraDirs].filter( + (value): value is string => Boolean(value), + ); + const claudeReadonlySandboxSettings = + claudeReadonlyReviewerRuntime && claudeReadonlySandboxMode + ? buildClaudeReadonlySandboxSettings( + readonlyProtectedPaths, + undefined, + claudeReadonlySandboxMode, + ) + : undefined; + for await (const message of query({ prompt: stream, options: { @@ -657,6 +686,11 @@ async function runQuery( env: sdkEnv, permissionMode: 'bypassPermissions', allowDangerouslySkipPermissions: true, + ...(claudeReadonlySandboxSettings + ? { + sandbox: claudeReadonlySandboxSettings, + } + : {}), settingSources: ['project', 'user'], abortController: agentAbortController, mcpServers: { @@ -684,7 +718,7 @@ async function runQuery( PreToolUse: [ { matcher: 'Bash', - hooks: reviewerRuntime + hooks: readonlyReviewerRuntime ? [createReviewerBashGuardHook(), createSanitizeBashHook()] : [createSanitizeBashHook()], }, @@ -950,9 +984,19 @@ async function main(): Promise { const reviewerRuntime = process.env.EJCLAW_REVIEWER_RUNTIME === '1' || isReviewerRuntime(containerInput.roomRoleContext); + const claudeReadonlyReviewerRuntime = + isClaudeReadonlyReviewerRuntime(containerInput.roomRoleContext); + const claudeReadonlySandboxMode = claudeReadonlyReviewerRuntime + ? getClaudeReadonlySandboxMode() + : null; const readonlyRuntime = - reviewerRuntime || process.env.EJCLAW_ARBITER_RUNTIME === '1'; - const guardedSdkEnv = buildReviewerGitGuardEnv(sdkEnv, reviewerRuntime); + reviewerRuntime || + claudeReadonlyReviewerRuntime || + process.env.EJCLAW_ARBITER_RUNTIME === '1'; + const guardedSdkEnv = buildReviewerGitGuardEnv( + sdkEnv, + reviewerRuntime || claudeReadonlyReviewerRuntime, + ); assertReadonlyWorkspaceRepoConnectivity(guardedSdkEnv, readonlyRuntime); const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -1092,6 +1136,8 @@ async function main(): Promise { containerInput, guardedSdkEnv, reviewerRuntime, + claudeReadonlyReviewerRuntime, + claudeReadonlySandboxMode, ); if (queryResult.newSessionId) { sessionId = queryResult.newSessionId; diff --git a/runners/agent-runner/src/reviewer-runtime.ts b/runners/agent-runner/src/reviewer-runtime.ts index baf26da..9496995 100644 --- a/runners/agent-runner/src/reviewer-runtime.ts +++ b/runners/agent-runner/src/reviewer-runtime.ts @@ -5,6 +5,8 @@ import path from 'path'; import type { RoomRoleContext } from './room-role-context.js'; +export type ClaudeReadonlySandboxMode = 'strict' | 'best-effort'; + const BLOCKED_GIT_SUBCOMMANDS = new Set([ 'add', 'am', @@ -40,6 +42,89 @@ export function isReviewerRuntime( return roomRoleContext?.role === 'reviewer'; } +export function isClaudeReadonlyReviewerRuntime( + roomRoleContext?: RoomRoleContext, +): boolean { + return ( + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1' && + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY === '1' && + roomRoleContext?.role === 'reviewer' + ); +} + +let cachedLinuxBubblewrapReadonlyCapability: boolean | undefined; + +function commandExistsForRuntime(command: string): boolean { + try { + execFileSync('bash', ['-lc', `command -v ${command}`], { + stdio: ['ignore', 'ignore', 'ignore'], + }); + return true; + } catch { + return false; + } +} + +export function canUseLinuxBubblewrapReadonlySandbox(): boolean { + if (cachedLinuxBubblewrapReadonlyCapability != null) { + return cachedLinuxBubblewrapReadonlyCapability; + } + if (os.platform() !== 'linux') { + cachedLinuxBubblewrapReadonlyCapability = false; + return false; + } + if (!commandExistsForRuntime('bwrap')) { + cachedLinuxBubblewrapReadonlyCapability = false; + return false; + } + + try { + execFileSync('bwrap', ['--ro-bind', '/', '/', '/bin/true'], { + stdio: ['ignore', 'ignore', 'ignore'], + }); + cachedLinuxBubblewrapReadonlyCapability = true; + } catch { + cachedLinuxBubblewrapReadonlyCapability = false; + } + + return cachedLinuxBubblewrapReadonlyCapability; +} + +export function getClaudeReadonlySandboxMode( + platform: NodeJS.Platform = os.platform(), + linuxCapabilityProbe: () => boolean = canUseLinuxBubblewrapReadonlySandbox, +): ClaudeReadonlySandboxMode { + if (platform === 'linux') { + return linuxCapabilityProbe() ? 'strict' : 'best-effort'; + } + return 'best-effort'; +} + +export function buildClaudeReadonlySandboxSettings( + protectedPaths: string[], + platform: NodeJS.Platform = os.platform(), + sandboxMode: ClaudeReadonlySandboxMode = getClaudeReadonlySandboxMode( + platform, + ), +) { + const normalizedPaths = [...new Set( + protectedPaths + .filter((value): value is string => Boolean(value)) + .map((value) => path.resolve(value)), + )]; + + return { + enabled: true, + failIfUnavailable: sandboxMode === 'strict', + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: + normalizedPaths.length > 0 + ? { denyWrite: normalizedPaths } + : undefined, + }; +} + function resolveGitBinary(baseEnv: NodeJS.ProcessEnv): string { return execFileSync('bash', ['-lc', 'command -v git'], { encoding: 'utf-8', diff --git a/runners/agent-runner/test/reviewer-runtime.test.ts b/runners/agent-runner/test/reviewer-runtime.test.ts index 1dafd50..d5826e7 100644 --- a/runners/agent-runner/test/reviewer-runtime.test.ts +++ b/runners/agent-runner/test/reviewer-runtime.test.ts @@ -3,15 +3,44 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertReadonlyWorkspaceRepoConnectivity, + buildClaudeReadonlySandboxSettings, buildReviewerGitGuardEnv, + getClaudeReadonlySandboxMode, + isClaudeReadonlyReviewerRuntime, isReviewerMutatingShellCommand, isReviewerRuntime, } from '../src/reviewer-runtime.js'; +const ORIGINAL_CLAUDE_REVIEWER_READONLY = + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + +afterEach(() => { + if (ORIGINAL_CLAUDE_REVIEWER_READONLY == null) { + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + } else { + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY = + ORIGINAL_CLAUDE_REVIEWER_READONLY; + } + + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } +}); + +beforeEach(() => { + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; +}); + function createTempRepo(prefix: string): string { const cwd = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); execFileSync('git', ['init'], { @@ -56,6 +85,76 @@ describe('claude reviewer runtime guard', () => { ).toBe(true); }); + it('detects claude host reviewer read-only mode from env + role', () => { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY = '1'; + + expect( + isClaudeReadonlyReviewerRuntime({ + serviceId: 'claude', + role: 'reviewer', + ownerServiceId: 'codex-main', + reviewerServiceId: 'claude', + failoverOwner: false, + }), + ).toBe(true); + expect( + isClaudeReadonlyReviewerRuntime({ + serviceId: 'claude', + role: 'owner', + ownerServiceId: 'codex-main', + reviewerServiceId: 'claude', + failoverOwner: false, + }), + ).toBe(false); + }); + + it('builds a read-only sandbox with normalized protected paths', () => { + const sandbox = buildClaudeReadonlySandboxSettings([ + '/repo/work', + '/repo/work', + '/repo/owner/../owner', + ], 'linux', 'strict'); + + expect(sandbox).toMatchObject({ + enabled: true, + failIfUnavailable: true, + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: { + denyWrite: ['/repo/work', '/repo/owner'], + }, + }); + }); + + it('uses best-effort sandbox mode when linux capability probe fails', () => { + expect(getClaudeReadonlySandboxMode('linux', () => false)).toBe( + 'best-effort', + ); + + const sandbox = buildClaudeReadonlySandboxSettings( + ['/repo/work'], + 'linux', + 'best-effort', + ); + + expect(sandbox.failIfUnavailable).toBe(false); + }); + + it('keeps non-linux reviewers in best-effort sandbox mode', () => { + expect(getClaudeReadonlySandboxMode('darwin', () => true)).toBe( + 'best-effort', + ); + + const sandbox = buildClaudeReadonlySandboxSettings( + ['/repo/work'], + 'darwin', + 'best-effort', + ); + + expect(sandbox.failIfUnavailable).toBe(false); + }); + it('flags mutating shell commands', () => { expect(isReviewerMutatingShellCommand('git commit -m "x"')).toBe(false); expect(isReviewerMutatingShellCommand('git -c color.ui=false commit -m "x"')).toBe( diff --git a/setup/environment.ts b/setup/environment.ts index e6715b5..15a68fc 100644 --- a/setup/environment.ts +++ b/setup/environment.ts @@ -9,7 +9,14 @@ import { Database } from 'bun:sqlite'; import { STORE_DIR } from '../src/config.js'; import { logger } from '../src/logger.js'; -import { getPlatform, isHeadless, isWSL } from './platform.js'; +import { + canUseLinuxBubblewrapReadonlySandbox, + commandExists, + getAppArmorRestrictUnprivilegedUsernsValue, + getPlatform, + isHeadless, + isWSL, +} from './platform.js'; import { emitStatus } from './status.js'; export async function run(_args: string[]): Promise { @@ -20,6 +27,14 @@ export async function run(_args: string[]): Promise { const platform = getPlatform(); const wsl = isWSL(); const headless = isHeadless(); + const hasBubblewrap = platform === 'linux' && commandExists('bwrap'); + const hasSocat = platform === 'linux' && commandExists('socat'); + const apparmorRestrictUnprivilegedUserns = + platform === 'linux' + ? getAppArmorRestrictUnprivilegedUsernsValue() + : null; + const hasBubblewrapReadonlySandboxCapability = + platform === 'linux' && canUseLinuxBubblewrapReadonlySandbox(); // Check existing config const hasEnv = fs.existsSync(path.join(projectRoot, '.env')); @@ -55,6 +70,10 @@ export async function run(_args: string[]): Promise { hasEnv, hasAuth, hasRegisteredGroups, + hasBubblewrap, + hasSocat, + apparmorRestrictUnprivilegedUserns, + hasBubblewrapReadonlySandboxCapability, }, 'Environment check complete', ); @@ -66,6 +85,12 @@ export async function run(_args: string[]): Promise { HAS_ENV: hasEnv, HAS_AUTH: hasAuth, HAS_REGISTERED_GROUPS: hasRegisteredGroups, + HAS_BWRAP: hasBubblewrap, + HAS_SOCAT: hasSocat, + APPARMOR_RESTRICT_UNPRIVILEGED_USERNS: + apparmorRestrictUnprivilegedUserns ?? 'n/a', + HAS_BWRAP_READONLY_SANDBOX_CAPABILITY: + hasBubblewrapReadonlySandboxCapability, STATUS: 'success', LOG: 'logs/setup.log', }); diff --git a/setup/platform.test.ts b/setup/platform.test.ts index 1604d3e..7dba53c 100644 --- a/setup/platform.test.ts +++ b/setup/platform.test.ts @@ -1,6 +1,9 @@ import { describe, it, expect } from 'vitest'; import { + ensureLinuxReadonlySandboxAppArmorSupport, + getAppArmorRestrictUnprivilegedUsernsValue, + canUseLinuxBubblewrapReadonlySandbox, getPlatform, isWSL, isRoot, @@ -99,6 +102,218 @@ describe('commandExists', () => { }); }); +describe('canUseLinuxBubblewrapReadonlySandbox', () => { + it('returns false outside linux', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'macos', + () => { + throw new Error('should not probe commands on macOS'); + }, + ), + ).toBe(false); + }); + + it('returns false when bwrap is not installed', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'linux', + () => false, + ), + ).toBe(false); + }); + + it('returns true when linux bwrap readonly probe succeeds', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'linux', + () => true, + (() => Buffer.from('')) as typeof import('child_process').execSync, + ), + ).toBe(true); + }); + + it('returns false when linux bwrap readonly probe fails', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'linux', + () => true, + (() => { + throw new Error('permission denied'); + }) as typeof import('child_process').execSync, + ), + ).toBe(false); + }); +}); + +describe('getAppArmorRestrictUnprivilegedUsernsValue', () => { + it('returns null when the proc sysctl path is unavailable', () => { + expect( + getAppArmorRestrictUnprivilegedUsernsValue((() => { + throw new Error('missing'); + }) as typeof import('fs').readFileSync), + ).toBeNull(); + }); + + it('returns the trimmed proc sysctl value', () => { + expect( + getAppArmorRestrictUnprivilegedUsernsValue((() => + '1\n') as typeof import('fs').readFileSync), + ).toBe('1'); + }); +}); + +describe('ensureLinuxReadonlySandboxAppArmorSupport', () => { + it('does nothing outside linux', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'macos', + }), + ).toBe('not-linux'); + }); + + it('does nothing when apparmor does not block sandboxing', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '0', + sandboxCapable: false, + }), + ).toBe('not-needed'); + }); + + it('does nothing when sandbox is already capable', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: true, + }), + ).toBe('not-needed'); + }); + + it('requires root when linux apparmor blocks sandboxing', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: false, + }), + ).toBe('requires-root'); + }); + + it('writes and applies the sysctl when running as root', () => { + const writes: string[] = []; + const commands: string[] = []; + const fileContents = new Map([ + ['/proc/sys/kernel/apparmor_restrict_unprivileged_userns', '1\n'], + ]); + + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: true, + readFileSyncFn: ((target) => { + const value = fileContents.get(String(target)); + if (value == null) throw new Error('missing'); + return value; + }) as typeof import('fs').readFileSync, + mkdirSyncFn: (() => undefined) as typeof import('fs').mkdirSync, + writeFileSyncFn: ((target, contents) => { + const normalizedTarget = String(target); + const normalizedContents = String(contents); + writes.push(`${normalizedTarget}\n${normalizedContents}`); + fileContents.set(normalizedTarget, normalizedContents); + }) as typeof import('fs').writeFileSync, + execFn: ((command) => { + commands.push(String(command)); + fileContents.set( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + '0\n', + ); + return Buffer.from(''); + }) as typeof import('child_process').execSync, + }), + ).toBe('configured'); + + expect(writes[0]).toContain('/etc/sysctl.d/90-ejclaw-sandbox.conf'); + expect(writes[0]).toContain( + 'kernel.apparmor_restrict_unprivileged_userns=0', + ); + expect(commands).toContain( + 'sysctl -w kernel.apparmor_restrict_unprivileged_userns=0', + ); + }); + + it('skips rewriting an already-correct sysctl file but still applies and verifies', () => { + const writes: string[] = []; + const commands: string[] = []; + const fileContents = new Map([ + ['/etc/sysctl.d/90-ejclaw-sandbox.conf', + '# Managed by EJClaw setup to allow bubblewrap readonly sandboxing.\n' + + 'kernel.apparmor_restrict_unprivileged_userns=0\n'], + ['/proc/sys/kernel/apparmor_restrict_unprivileged_userns', '1\n'], + ]); + + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: true, + readFileSyncFn: ((target) => { + const value = fileContents.get(String(target)); + if (value == null) throw new Error('missing'); + return value; + }) as typeof import('fs').readFileSync, + mkdirSyncFn: (() => undefined) as typeof import('fs').mkdirSync, + writeFileSyncFn: ((target, contents) => { + writes.push(`${String(target)}\n${String(contents)}`); + }) as typeof import('fs').writeFileSync, + execFn: ((command) => { + commands.push(String(command)); + fileContents.set( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + '0\n', + ); + return Buffer.from(''); + }) as typeof import('child_process').execSync, + }), + ).toBe('configured'); + + expect(writes).toHaveLength(0); + expect(commands).toContain( + 'sysctl -w kernel.apparmor_restrict_unprivileged_userns=0', + ); + }); + + it('returns failed when the sysctl apply does not take effect', () => { + const fileContents = new Map([ + ['/proc/sys/kernel/apparmor_restrict_unprivileged_userns', '1\n'], + ]); + + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: true, + readFileSyncFn: ((target) => { + const value = fileContents.get(String(target)); + if (value == null) throw new Error('missing'); + return value; + }) as typeof import('fs').readFileSync, + mkdirSyncFn: (() => undefined) as typeof import('fs').mkdirSync, + writeFileSyncFn: (() => undefined) as typeof import('fs').writeFileSync, + execFn: (() => Buffer.from('')) as typeof import('child_process').execSync, + }), + ).toBe('failed'); + }); +}); + // --- getNodeVersion --- describe('getNodeVersion', () => { diff --git a/setup/platform.ts b/setup/platform.ts index a0fba9e..3ed57eb 100644 --- a/setup/platform.ts +++ b/setup/platform.ts @@ -7,6 +7,12 @@ import os from 'os'; export type Platform = 'macos' | 'linux' | 'unknown'; export type ServiceManager = 'launchd' | 'systemd' | 'none'; +export type LinuxReadonlySandboxAppArmorSetupResult = + | 'not-linux' + | 'not-needed' + | 'requires-root' + | 'failed' + | 'configured'; export function getPlatform(): Platform { const platform = os.platform(); @@ -115,6 +121,110 @@ export function commandExists(name: string): boolean { } } +export function canUseLinuxBubblewrapReadonlySandbox( + platform: Platform = getPlatform(), + commandExistsFn: (name: string) => boolean = commandExists, + execFn: typeof execSync = execSync, +): boolean { + if (platform !== 'linux') return false; + if (!commandExistsFn('bwrap')) return false; + + try { + execFn('bwrap --ro-bind / / /bin/true', { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function readProcSysValue( + procPath: string, + readFileSyncFn: typeof fs.readFileSync = fs.readFileSync, +): string | null { + try { + return readFileSyncFn(procPath, 'utf-8').trim(); + } catch { + return null; + } +} + +export function getAppArmorRestrictUnprivilegedUsernsValue( + readFileSyncFn: typeof fs.readFileSync = fs.readFileSync, +): string | null { + return readProcSysValue( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + readFileSyncFn, + ); +} + +export function ensureLinuxReadonlySandboxAppArmorSupport(options?: { + platform?: Platform; + isRootUser?: boolean; + apparmorRestrictUnprivilegedUserns?: string | null; + sandboxCapable?: boolean; + readFileSyncFn?: typeof fs.readFileSync; + mkdirSyncFn?: typeof fs.mkdirSync; + writeFileSyncFn?: typeof fs.writeFileSync; + execFn?: typeof execSync; +}): LinuxReadonlySandboxAppArmorSetupResult { + const platform = options?.platform ?? getPlatform(); + if (platform !== 'linux') return 'not-linux'; + const readFileSyncFn = options?.readFileSyncFn ?? fs.readFileSync; + + const apparmorRestrictUnprivilegedUserns = + options?.apparmorRestrictUnprivilegedUserns ?? + getAppArmorRestrictUnprivilegedUsernsValue(readFileSyncFn); + const sandboxCapable = + options?.sandboxCapable ?? canUseLinuxBubblewrapReadonlySandbox(platform); + + if ( + apparmorRestrictUnprivilegedUserns !== '1' || + sandboxCapable + ) { + return 'not-needed'; + } + + const isRootUser = options?.isRootUser ?? isRoot(); + if (!isRootUser) return 'requires-root'; + + const mkdirSyncFn = options?.mkdirSyncFn ?? fs.mkdirSync; + const writeFileSyncFn = options?.writeFileSyncFn ?? fs.writeFileSync; + const execFn = options?.execFn ?? execSync; + const sysctlPath = '/etc/sysctl.d/90-ejclaw-sandbox.conf'; + const sysctlContents = + '# Managed by EJClaw setup to allow bubblewrap readonly sandboxing.\n' + + 'kernel.apparmor_restrict_unprivileged_userns=0\n'; + let existingContents: string | null = null; + + try { + existingContents = readFileSyncFn(sysctlPath, 'utf-8'); + } catch { + existingContents = null; + } + + try { + mkdirSyncFn('/etc/sysctl.d', { recursive: true }); + if (existingContents !== sysctlContents) { + writeFileSyncFn(sysctlPath, sysctlContents); + } + execFn('sysctl -w kernel.apparmor_restrict_unprivileged_userns=0', { + stdio: 'ignore', + }); + + const appliedValue = getAppArmorRestrictUnprivilegedUsernsValue( + readFileSyncFn, + ); + if (appliedValue !== '0') { + throw new Error( + `kernel.apparmor_restrict_unprivileged_userns remained ${appliedValue ?? 'unavailable'} after sysctl apply`, + ); + } + return 'configured'; + } catch { + return 'failed'; + } +} + export function getNodeVersion(): string | null { try { const version = execSync('node --version', { encoding: 'utf-8' }).trim(); diff --git a/setup/service.ts b/setup/service.ts index 26e82d0..2c0f8d0 100644 --- a/setup/service.ts +++ b/setup/service.ts @@ -11,7 +11,12 @@ import os from 'os'; import path from 'path'; import { logger } from '../src/logger.js'; -import { getPlatform, getNodePath, getServiceManager } from './platform.js'; +import { + ensureLinuxReadonlySandboxAppArmorSupport, + getPlatform, + getNodePath, + getServiceManager, +} from './platform.js'; import { detectLegacyServiceIssues, formatLegacyServiceFailureMessage, @@ -33,6 +38,22 @@ export async function run(_args: string[]): Promise { logger.info({ platform, nodePath, projectRoot }, 'Setting up service'); + const readonlySandboxAppArmorSetup = + ensureLinuxReadonlySandboxAppArmorSupport(); + if (readonlySandboxAppArmorSetup === 'configured') { + logger.info( + 'Configured AppArmor user namespace sysctl for EJClaw readonly sandbox support', + ); + } else if (readonlySandboxAppArmorSetup === 'failed') { + logger.warn( + 'Failed to apply AppArmor user namespace sysctl for EJClaw readonly sandbox support', + ); + } else if (readonlySandboxAppArmorSetup === 'requires-root') { + logger.warn( + 'Readonly sandbox strict mode requires root setup to disable AppArmor unprivileged userns restriction', + ); + } + const legacyServiceIssues = detectLegacyServiceIssues( projectRoot, serviceManager, diff --git a/src/agent-runner.ts b/src/agent-runner.ts index eec92d4..8978e8c 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -108,8 +108,7 @@ export async function runAgentProcess( // Reviewers always run inside a Docker container with read-only source // mount for kernel-level write protection. Docker is required. if ( - !unsafeHostPairedMode && - envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1' || + (!unsafeHostPairedMode && envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1') || (!unsafeHostPairedMode && envOverrides?.EJCLAW_ARBITER_RUNTIME === '1') ) { const ownerWorkspaceDir = diff --git a/src/config.ts b/src/config.ts index 23af224..c208b14 100644 --- a/src/config.ts +++ b/src/config.ts @@ -120,7 +120,7 @@ export const AGENT_LANGUAGE = getEnv('AGENT_LANGUAGE') || ''; /** Number of consecutive owner↔reviewer round trips before arbiter is auto-requested. */ export const ARBITER_DEADLOCK_THRESHOLD = parseInt( - getEnv('ARBITER_DEADLOCK_THRESHOLD') || '3', + getEnv('ARBITER_DEADLOCK_THRESHOLD') || '2', 10, ); diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index fe92ff7..2a815b6 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('./agent-runner.js', () => ({ runAgentProcess: vi.fn(), @@ -156,9 +156,14 @@ vi.mock('./session-recovery.js', () => ({ vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), + getCurrentTokenIndex: vi.fn(() => 0), markTokenHealthy: vi.fn(), })); +vi.mock('./token-refresh.js', () => ({ + forceRefreshToken: vi.fn(async () => null), +})); + vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn((error?: string | null) => { const lower = (error || '').toLowerCase(); @@ -198,6 +203,7 @@ import { runAgentForGroup } from './message-agent-executor.js'; import * as pairedExecutionContext from './paired-execution-context.js'; import * as sessionRecovery from './session-recovery.js'; import * as serviceRouting from './service-routing.js'; +import * as tokenRefresh from './token-refresh.js'; import * as tokenRotation from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; @@ -226,9 +232,13 @@ function makeDeps() { }; } +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + describe('runAgentForGroup room memory', () => { beforeEach(() => { vi.resetAllMocks(); + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; vi.mocked(agentRunner.runAgentProcess).mockImplementation( async (_group, _input, _onProcess, onOutput) => { await onOutput?.({ @@ -245,6 +255,15 @@ describe('runAgentForGroup room memory', () => { ); }); + afterAll(() => { + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE === undefined) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } + }); + it('injects a room memory briefing when starting a fresh session', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); @@ -1031,6 +1050,108 @@ describe('runAgentForGroup room memory', () => { ); }); + it('starts reviewer Claude fresh in unsafe host mode instead of resuming a stored session', async () => { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + const group = { + ...makeGroup(), + folder: 'test-group', + agentType: 'codex' as const, + }; + const deps = { + ...makeDeps(), + getSessions: () => ({ 'test-group:reviewer': 'reviewer-session' }), + }; + + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_agent_type: 'codex', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + arbiter_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'paired-task-review', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-review', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: { + EJCLAW_PAIRED_TASK_ID: 'paired-task-review', + EJCLAW_PAIRED_ROLE: 'reviewer', + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + CLAUDE_CONFIG_DIR: '/tmp/test-group-reviewer', + }, + }); + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: 'review ok', + newSessionId: 'review-session-new', + }); + + await runAgentForGroup(deps, { + group, + prompt: 'please review', + chatJid: 'group@test', + runId: 'run-review-plan-unsafe-host', + }); + + expect(deps.clearSession).toHaveBeenCalledWith('test-group:reviewer'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledWith( + expect.objectContaining({ + folder: 'test-group', + agentType: 'claude-code', + }), + expect.objectContaining({ + sessionId: undefined, + }), + expect.any(Function), + undefined, + expect.objectContaining({ + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + }), + ); + expect(deps.persistSession).not.toHaveBeenCalled(); + }); + it('does not enqueue a second generic follow-up when reviewer approval already moved the task to merge_ready', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); @@ -1123,7 +1244,9 @@ describe('runAgentForGroup Claude rotation', () => { vi.resetAllMocks(); vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined); vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + vi.mocked(tokenRotation.getCurrentTokenIndex).mockReturnValue(0); vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); + vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null); }); it('rotates to another Claude account on usage exhaustion', async () => { @@ -1229,6 +1352,62 @@ describe('runAgentForGroup Claude rotation', () => { expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']); }); + it('force-refreshes the active Claude token before rotating on auth-expired', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValueOnce( + 'new-access-token', + ); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }); + await onOutput?.({ + status: 'success', + phase: 'final', + result: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'force refresh 뒤 Claude 응답입니다.', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-auth-expired-force-refresh', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRefresh.forceRefreshToken).toHaveBeenCalledWith(0); + expect(tokenRotation.rotateToken).not.toHaveBeenCalled(); + expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); + expect(outputs).toEqual(['force refresh 뒤 Claude 응답입니다.']); + }); + it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => { const outputs: string[] = []; @@ -1505,6 +1684,48 @@ describe('runAgentForGroup Claude rotation', () => { ); }); + it('drops a stale Claude session id before retrying a fresh session', async () => { + const deps = { + ...makeDeps(), + getSessions: () => ({ 'test-claude': 'stale-session-id' }), + }; + + vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, input) => { + expect(input.sessionId).toBe('stale-session-id'); + throw new Error('No conversation found with session ID: stale-session-id'); + }) + .mockImplementationOnce(async (_group, input, _onProcess, onOutput) => { + expect(input.sessionId).toBeUndefined(); + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'fresh retry success', + }); + return { + status: 'success', + result: 'fresh retry success', + newSessionId: 'fresh-session-id', + }; + }); + + const result = await runAgentForGroup(deps, { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-stale-session-id-retry', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(deps.clearSession).toHaveBeenCalledWith('test-claude'); + }); + it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => { const outputs: string[] = []; diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 44661f1..17bf68d 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -150,12 +150,22 @@ export async function runAgentForGroup( activeRole, group.agentType, ); + const unsafeHostPairedMode = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; + const forceFreshClaudeReviewerSession = + reviewerMode && isClaudeCodeAgent && unsafeHostPairedMode; + const storedSessionId = sessions[sessionFolder]; + if (forceFreshClaudeReviewerSession && storedSessionId) { + deps.clearSession(sessionFolder); + } // Arbiter always starts fresh — never resume a previous session - const sessionId = - activeRole === 'arbiter' || args.forcedAgentType + let currentSessionId = + activeRole === 'arbiter' || + args.forcedAgentType || + forceFreshClaudeReviewerSession ? undefined - : sessions[sessionFolder]; - const memoryBriefing = sessionId + : storedSessionId; + const memoryBriefing = currentSessionId ? undefined : await buildRoomMemoryBriefing({ groupFolder: group.folder, @@ -185,7 +195,9 @@ export async function runAgentForGroup( let resetSessionRequested = false; - const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1; + const claudeTokenCount = isClaudeCodeAgent ? getTokenCount() : 0; + const canRetryClaudeCredentials = isClaudeCodeAgent && claudeTokenCount > 0; + const canRotateToken = isClaudeCodeAgent && claudeTokenCount > 1; const roomRoleContext = buildRoomRoleContext( currentLease, effectiveServiceId, @@ -212,6 +224,7 @@ export async function runAgentForGroup( pairedExecutionContext.envOverrides[effortKey] = roleConfig.effort; } } + const log = createScopedLogger({ chatJid, groupName: group.name, @@ -243,11 +256,27 @@ export async function runAgentForGroup( reviewerMode, arbiterMode, sessionFolder, - resumedSession: sessionId ?? null, + resumedSession: currentSessionId ?? null, }, 'Resolved execution target for agent turn', ); + const clearRoleSdkSessions = (): void => { + const configDir = pairedExecutionContext?.envOverrides?.CLAUDE_CONFIG_DIR; + if (!configDir) return; + for (const sdkDir of ['.claude', '.codex']) { + const sessionsDir = path.join(configDir, sdkDir, 'sessions'); + if (fs.existsSync(sessionsDir)) { + fs.rmSync(sessionsDir, { recursive: true, force: true }); + log.info({ sessionsDir }, 'Cleared SDK sessions for fresh Claude turn'); + } + } + }; + + if (forceFreshClaudeReviewerSession) { + clearRoleSdkSessions(); + } + // ── MoA prompt enrichment ───────────────────────────────────── // When MoA is enabled and we're in arbiter mode, query external API // models (Kimi, GLM, etc.) in parallel for their opinions, then inject @@ -394,7 +423,7 @@ export async function runAgentForGroup( const agentInput = { prompt: effectivePrompt, - sessionId, + sessionId: currentSessionId, memoryBriefing, groupFolder: group.folder, chatJid, @@ -451,6 +480,7 @@ export async function runAgentForGroup( sawVisibleOutput: false, sawSuccessNullResultWithoutOutput: false, }; + const attemptSessionId = provider === 'claude' ? currentSessionId : undefined; const wrappedOnOutput = onOutput ? async (output: AgentOutput) => { @@ -476,7 +506,7 @@ export async function runAgentForGroup( effectiveServiceId, effectiveAgentType, sessionFolder, - resumedSession: sessionId ?? null, + resumedSession: attemptSessionId ?? null, streamedSessionId: output.newSessionId ?? null, roomRoleServiceId: roomRoleContext?.serviceId ?? null, roomRole: roomRoleContext?.role ?? null, @@ -499,9 +529,11 @@ export async function runAgentForGroup( if ( provider === 'claude' && output.newSessionId && - !resetSessionRequested + !resetSessionRequested && + !forceFreshClaudeReviewerSession ) { deps.persistSession(sessionFolder, output.newSessionId); + currentSessionId = output.newSessionId; } const evaluation = evaluateStreamedOutput(output, streamedState, { agentType: isClaudeCodeAgent ? 'claude-code' : 'codex', @@ -510,7 +542,7 @@ export async function runAgentForGroup( trackSuccessNullResult: true, shortCircuitTriggeredErrors: provider === 'claude' - ? canRotateToken + ? canRetryClaudeCredentials : getCodexAccountCount() > 1, }); streamedState = evaluation.state; @@ -601,7 +633,7 @@ export async function runAgentForGroup( effectiveGroup, { ...agentInput, - sessionId: provider === 'claude' ? sessionId : undefined, + sessionId: attemptSessionId, }, (proc, processName, ipcDir) => deps.queue.registerProcess(chatJid, proc, processName, ipcDir), @@ -609,8 +641,13 @@ export async function runAgentForGroup( pairedExecutionContext?.envOverrides, ); - if (provider === 'claude' && output.newSessionId) { + if ( + provider === 'claude' && + output.newSessionId && + !forceFreshClaudeReviewerSession + ) { deps.persistSession(sessionFolder, output.newSessionId); + currentSessionId = output.newSessionId; } providerLog.info( @@ -804,23 +841,9 @@ export async function runAgentForGroup( }))); if (isRetryableClaudeSessionFailure(primaryAttempt)) { + currentSessionId = undefined; deps.clearSession(sessionFolder); - // Also clear SDK session state inside the reviewer/arbiter config dir - // so the persistent container doesn't resume the same stale session. - const configDir = - pairedExecutionContext?.envOverrides?.CLAUDE_CONFIG_DIR; - if (configDir) { - for (const sdkDir of ['.claude', '.codex']) { - const sessionsDir = path.join(configDir, sdkDir, 'sessions'); - if (fs.existsSync(sessionsDir)) { - fs.rmSync(sessionsDir, { recursive: true, force: true }); - log.info( - { sessionsDir }, - 'Cleared container SDK sessions for stale session recovery', - ); - } - } - } + clearRoleSdkSessions(); log.warn( 'Cleared poisoned Claude session before visible output, retrying fresh session', ); @@ -828,6 +851,7 @@ export async function runAgentForGroup( primaryAttempt = await runAttempt('claude'); if (isRetryableClaudeSessionFailure(primaryAttempt)) { + currentSessionId = undefined; deps.clearSession(sessionFolder); log.warn('Fresh Claude retry also hit a retryable session failure'); @@ -840,7 +864,7 @@ export async function runAgentForGroup( if (primaryAttempt.error) { if ( - canRotateToken && + canRetryClaudeCredentials && provider === 'claude' && !primaryAttempt.sawOutput ) { @@ -913,7 +937,7 @@ export async function runAgentForGroup( } if ( - canRotateToken && + canRetryClaudeCredentials && provider === 'claude' && !primaryAttempt.sawOutput && primaryAttempt.streamedTriggerReason && @@ -945,7 +969,7 @@ export async function runAgentForGroup( if (output.status === 'error') { if ( - canRotateToken && + canRetryClaudeCredentials && provider === 'claude' && !primaryAttempt.sawOutput ) { diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 2a7b3e1..3f59290 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -1704,6 +1704,230 @@ describe('createMessageRuntime', () => { ); }); + it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + + vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'task-active-owner-follow-up', + chat_jid: chatJid, + group_folder: group.folder, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: '2026-03-30T00:00:00.000Z', + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + }); + vi.mocked(db.getMessagesSince).mockReturnValue([]); + vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ + { + id: 1, + task_id: 'task-active-owner-follow-up', + turn_number: 1, + role: 'owner', + output_text: 'owner 초안', + created_at: '2026-03-30T00:00:01.000Z', + }, + { + id: 2, + task_id: 'task-active-owner-follow-up', + turn_number: 2, + role: 'reviewer', + output_text: '리뷰어가 수정 요청을 남김', + created_at: '2026-03-30T00:00:02.000Z', + }, + ]); + vi.mocked(db.getRecentChatMessages).mockReturnValue([ + { + id: 'human-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: '눈쟁이', + content: '이 기능 마무리해줘', + timestamp: '2026-03-30T00:00:00.500Z', + seq: 1, + is_bot_message: false, + } as any, + ]); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, input, _onProcess, onOutput) => { + expect(input.prompt).toContain('이 기능 마무리해줘'); + expect(input.prompt).toContain('리뷰어가 수정 요청을 남김'); + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'owner가 reviewer 피드백을 반영했습니다.', + newSessionId: 'session-owner-follow-up', + }); + return { + status: 'success', + result: 'owner가 reviewer 피드백을 반영했습니다.', + newSessionId: 'session-owner-follow-up', + }; + }, + ); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => ({}), + saveState: vi.fn(), + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-active-owner-follow-up', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); + expect(channel.sendMessage).toHaveBeenCalledWith( + chatJid, + 'owner가 reviewer 피드백을 반영했습니다.', + ); + }); + + it('builds owner follow-up prompts from paired turn outputs instead of raw reviewer bot delivery text', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + + vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'task-active-bot-follow-up', + chat_jid: chatJid, + group_folder: group.folder, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: '2026-03-30T00:00:00.000Z', + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + }); + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'reviewer-bot-message-active', + chat_jid: chatJid, + sender: 'reviewer-bot@test', + sender_name: '리뷰어', + content: 'DONE_WITH_CONCERNS\n\n리뷰어 디스코드 출력 원문', + timestamp: '2026-03-30T00:00:04.000Z', + seq: 42, + is_bot_message: true, + } as any, + ]); + vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ + { + id: 1, + task_id: 'task-active-bot-follow-up', + turn_number: 1, + role: 'owner', + output_text: 'owner 초안', + created_at: '2026-03-30T00:00:01.000Z', + }, + { + id: 2, + task_id: 'task-active-bot-follow-up', + turn_number: 2, + role: 'reviewer', + output_text: 'paired_turn_outputs 에 저장된 reviewer 요약', + created_at: '2026-03-30T00:00:02.000Z', + }, + ]); + vi.mocked(db.getRecentChatMessages).mockReturnValue([ + { + id: 'human-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: '눈쟁이', + content: '이 기능 마무리해줘', + timestamp: '2026-03-30T00:00:00.500Z', + seq: 1, + is_bot_message: false, + } as any, + ]); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, input, _onProcess, onOutput) => { + expect(input.prompt).toContain('paired_turn_outputs 에 저장된 reviewer 요약'); + expect(input.prompt).not.toContain('리뷰어 디스코드 출력 원문'); + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'owner follow-up ok', + newSessionId: 'session-owner-bot-follow-up', + }); + return { + status: 'success', + result: 'owner follow-up ok', + newSessionId: 'session-owner-bot-follow-up', + }; + }, + ); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => ({}), + saveState: vi.fn(), + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-active-bot-follow-up', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); + expect(channel.sendMessage).toHaveBeenCalledWith(chatJid, 'owner follow-up ok'); + }); + it('reuses the shared arbiter prompt builder for pending arbiter turns', async () => { const chatJid = 'group@test'; const group = makeGroup('codex'); diff --git a/src/message-runtime.ts b/src/message-runtime.ts index 95de8d5..fbff013 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -270,6 +270,30 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { return `User request:\n---\n${userMessage}\n---\n\nReview the latest owner changes in the workspace.`; }; + const buildOwnerPendingPrompt = ( + task: PairedTask, + chatJid: string, + timezone: string, + ): string => { + const turnOutputs = getPairedTurnOutputs(task.id); + if (turnOutputs.length > 0) { + const humanMessages = getRecentChatMessages(chatJid, 20).filter( + (message) => !message.is_bot_message, + ); + return formatMessages( + mergeHumanAndTurnOutputMessages(chatJid, humanMessages, turnOutputs), + timezone, + ); + } + + const userMessage = getLastHumanMessageContent(chatJid); + if (!userMessage) { + return 'Continue the owner turn using the latest reviewer or arbiter feedback.'; + } + + return `User request:\n---\n${userMessage}\n---\n\nContinue the owner turn using the latest reviewer or arbiter feedback.`; + }; + const buildArbiterPromptForTask = ( task: PairedTask, chatJid: string, @@ -825,6 +849,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }; } + if (taskStatus === 'active') { + const turnOutputs = getPairedTurnOutputs(task.id); + const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; + if ( + lastTurnOutput && + (lastTurnOutput.role === 'reviewer' || + lastTurnOutput.role === 'arbiter') + ) { + return { + prompt: buildOwnerPendingPrompt(task, chatJid, deps.timezone), + channel: resolveChannel(taskStatus), + cursor, + }; + } + } + return null; }; diff --git a/src/paired-execution-context.test.ts b/src/paired-execution-context.test.ts index afb32d8..73e653b 100644 --- a/src/paired-execution-context.test.ts +++ b/src/paired-execution-context.test.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('./db.js', () => ({ createPairedTask: vi.fn(), @@ -78,6 +78,12 @@ const failoverOwnerContext: RoomRoleContext = { failoverOwner: true, }; +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; +const ORIGINAL_REVIEWER_RUNTIME = process.env.EJCLAW_REVIEWER_RUNTIME; +const ORIGINAL_CLAUDE_REVIEWER_READONLY = + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + function createCanonicalRepoWithCommit(commitMessage: string): string { const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-finalize-')); execFileSync('git', ['init'], { cwd: repoDir, stdio: 'ignore' }); @@ -150,6 +156,9 @@ function buildWorkspace( describe('paired execution context', () => { beforeEach(() => { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + delete process.env.EJCLAW_REVIEWER_RUNTIME; + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; vi.resetAllMocks(); vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue(undefined); vi.mocked(db.getPairedTaskById).mockReturnValue(undefined); @@ -176,6 +185,28 @@ describe('paired execution context', () => { }); }); + afterAll(() => { + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } + + if (ORIGINAL_REVIEWER_RUNTIME == null) { + delete process.env.EJCLAW_REVIEWER_RUNTIME; + } else { + process.env.EJCLAW_REVIEWER_RUNTIME = ORIGINAL_REVIEWER_RUNTIME; + } + + if (ORIGINAL_CLAUDE_REVIEWER_READONLY == null) { + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + } else { + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY = + ORIGINAL_CLAUDE_REVIEWER_READONLY; + } + }); + it('creates an owner execution with a worktree override', () => { const result = preparePairedExecutionContext({ group, @@ -413,6 +444,7 @@ describe('paired execution context', () => { EJCLAW_PAIRED_ROLE: 'reviewer', EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', }); + expect(result?.envOverrides.EJCLAW_CLAUDE_REVIEWER_READONLY).toBe('1'); expect(result?.envOverrides.EJCLAW_REVIEWER_RUNTIME).toBeUndefined(); expect(result?.envOverrides.CLAUDE_CONFIG_DIR).toContain( '/data/sessions/paired-room-reviewer', diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index 005b864..fde5eb6 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -387,6 +387,9 @@ export function preparePairedExecutionContext(args: { envOverrides.CLAUDE_CONFIG_DIR = reviewerSessionDir; if (unsafeHostPairedMode) { envOverrides.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + if (REVIEWER_AGENT_TYPE === 'claude-code') { + envOverrides.EJCLAW_CLAUDE_REVIEWER_READONLY = '1'; + } } else { envOverrides.EJCLAW_REVIEWER_RUNTIME = '1'; } diff --git a/src/paired-workspace-manager.test.ts b/src/paired-workspace-manager.test.ts index 29fde61..929547f 100644 --- a/src/paired-workspace-manager.test.ts +++ b/src/paired-workspace-manager.test.ts @@ -219,10 +219,9 @@ describe('paired workspace manager', () => { packageManager: 'pnpm' as const, })); vi.doMock('./workspace-package-manager.js', async () => { - const actual = - await vi.importActual( - './workspace-package-manager.js', - ); + const actual = await vi.importActual< + typeof import('./workspace-package-manager.js') + >('./workspace-package-manager.js'); return { ...actual, ensureWorkspaceDependenciesInstalled: diff --git a/src/provider-retry.test.ts b/src/provider-retry.test.ts index 0708f42..dc32d51 100644 --- a/src/provider-retry.test.ts +++ b/src/provider-retry.test.ts @@ -12,21 +12,30 @@ vi.mock('./logger.js', () => ({ vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), + getCurrentTokenIndex: vi.fn(() => 0), markTokenHealthy: vi.fn(), })); +vi.mock('./token-refresh.js', () => ({ + forceRefreshToken: vi.fn(async () => null), +})); + import { runClaudeRotationLoop } from './provider-retry.js'; import { + getCurrentTokenIndex, getTokenCount, markTokenHealthy, rotateToken, } from './token-rotation.js'; +import { forceRefreshToken } from './token-refresh.js'; describe('runClaudeRotationLoop', () => { beforeEach(() => { vi.clearAllMocks(); vi.mocked(getTokenCount).mockReturnValue(1); + vi.mocked(getCurrentTokenIndex).mockReturnValue(0); vi.mocked(rotateToken).mockReturnValue(false); + vi.mocked(forceRefreshToken).mockResolvedValue(null); }); it('rotates and succeeds after an org-access-denied trigger', async () => { @@ -85,6 +94,60 @@ describe('runClaudeRotationLoop', () => { }); }); + it('force-refreshes the active token before rotating on auth-expired', async () => { + vi.mocked(forceRefreshToken).mockResolvedValueOnce('new-access-token'); + + const outcome = await runClaudeRotationLoop( + { reason: 'auth-expired' }, + async () => ({ + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }), + { runId: 'force-refresh-auth-expired' }, + ); + + expect(outcome).toEqual({ type: 'success', sawOutput: true }); + expect(forceRefreshToken).toHaveBeenCalledWith(0); + expect(rotateToken).not.toHaveBeenCalled(); + expect(markTokenHealthy).toHaveBeenCalledTimes(1); + }); + + it('falls back to rotation when force refresh does not recover auth-expired', async () => { + vi.mocked(getTokenCount).mockReturnValue(2); + vi.mocked(forceRefreshToken).mockResolvedValueOnce('new-access-token'); + vi.mocked(rotateToken).mockReturnValueOnce(true); + + let attempts = 0; + const outcome = await runClaudeRotationLoop( + { reason: 'auth-expired' }, + async () => { + attempts += 1; + if (attempts === 1) { + return { + output: { + status: 'error', + result: null, + error: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }, + sawOutput: false, + }; + } + + return { + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }; + }, + { runId: 'force-refresh-then-rotate' }, + ); + + expect(outcome).toEqual({ type: 'success', sawOutput: true }); + expect(forceRefreshToken).toHaveBeenCalledWith(0); + expect(rotateToken).toHaveBeenCalledTimes(1); + expect(markTokenHealthy).toHaveBeenCalledTimes(1); + }); + it('uses structured public output text as the next rotation message', async () => { vi.mocked(getTokenCount).mockReturnValue(2); vi.mocked(rotateToken).mockReturnValueOnce(true).mockReturnValueOnce(false); diff --git a/src/provider-retry.ts b/src/provider-retry.ts index da6c783..149b5d1 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -15,10 +15,12 @@ import { import { logger } from './logger.js'; import { getErrorMessage } from './utils.js'; import { + getCurrentTokenIndex, rotateToken, getTokenCount, markTokenHealthy, } from './token-rotation.js'; +import { forceRefreshToken } from './token-refresh.js'; import { clearGlobalFailover } from './service-routing.js'; // ── Types ──────────────────────────────────────────────────────── @@ -40,6 +42,113 @@ export type RotationOutcome = | { type: 'success'; sawOutput: boolean } | { type: 'error'; trigger?: TriggerInfo }; +type AttemptOutcome = + | { type: 'success'; sawOutput: boolean } + | { type: 'error'; trigger?: TriggerInfo } + | { + type: 'continue'; + trigger: TriggerInfo; + rotationMessage?: string; + }; + +function evaluateClaudeAttempt( + attempt: RotationAttemptResult, + logContext: Record, +): AttemptOutcome { + if (attempt.thrownError) { + if (!attempt.sawOutput) { + const errMsg = getErrorMessage(attempt.thrownError); + const retryTrigger = attempt.streamedTriggerReason + ? { + shouldRetry: true, + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + } + : classifyRotationTrigger(errMsg); + if (retryTrigger.shouldRetry) { + return { + type: 'continue', + trigger: { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }, + rotationMessage: errMsg, + }; + } + } + + logger.error( + { ...logContext, provider: 'claude', err: attempt.thrownError }, + 'Rotated Claude account also threw', + ); + return { type: 'error' }; + } + + const output = attempt.output; + if (!output) { + logger.error( + { ...logContext, provider: 'claude' }, + 'Rotated Claude account produced no output object', + ); + return { type: 'error' }; + } + + if ( + !attempt.sawOutput && + attempt.streamedTriggerReason && + output.status !== 'error' + ) { + return { + type: 'continue', + trigger: { + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + }, + rotationMessage: getAgentOutputText(output) ?? undefined, + }; + } + + if (!attempt.sawOutput && attempt.sawSuccessNullResult) { + logger.warn( + { ...logContext, provider: 'claude' }, + 'All rotated tokens returned success with null result', + ); + return { type: 'error', trigger: { reason: 'success-null-result' } }; + } + + if (output.status === 'error') { + if (!attempt.sawOutput) { + const retryTrigger = attempt.streamedTriggerReason + ? { + shouldRetry: true, + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + } + : classifyRotationTrigger(output.error); + if (retryTrigger.shouldRetry) { + return { + type: 'continue', + trigger: { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }, + rotationMessage: output.error ?? undefined, + }; + } + } + + logger.error( + { ...logContext, provider: 'claude', error: output.error }, + 'Rotated Claude account failed', + ); + return { type: 'error' }; + } + + markTokenHealthy(); + clearGlobalFailover(); + return { type: 'success', sawOutput: attempt.sawOutput }; +} + // ── Shared rotation loop ───────────────────────────────────────── /** @@ -56,112 +165,70 @@ export async function runClaudeRotationLoop( ): Promise { let trigger = initialTrigger; let lastRotationMessage = rotationMessage; + const attemptedForcedRefreshIndexes = new Set(); + + while (shouldRotateClaudeToken(trigger.reason)) { + if (trigger.reason === 'auth-expired') { + const tokenIndex = getCurrentTokenIndex(); + if ( + tokenIndex != null && + !attemptedForcedRefreshIndexes.has(tokenIndex) + ) { + attemptedForcedRefreshIndexes.add(tokenIndex); + const refreshedToken = await forceRefreshToken(tokenIndex); + if (refreshedToken) { + logger.info( + { ...logContext, tokenIndex }, + 'Claude auth-expired recovered by force-refreshing current token', + ); + + const refreshAttemptOutcome = evaluateClaudeAttempt( + await runAttempt(), + logContext, + ); + if (refreshAttemptOutcome.type === 'success') { + return refreshAttemptOutcome; + } + if (refreshAttemptOutcome.type === 'error') { + return refreshAttemptOutcome; + } + + trigger = refreshAttemptOutcome.trigger; + lastRotationMessage = refreshAttemptOutcome.rotationMessage; + continue; + } + } + } + + if ( + !( + getTokenCount() > 1 && + // Respect per-token cooldowns so exhausted auth/quota failures can + // terminate instead of cycling forever. + rotateToken(lastRotationMessage) + ) + ) { + break; + } - while ( - shouldRotateClaudeToken(trigger.reason) && - getTokenCount() > 1 && - // Respect per-token cooldowns so exhausted auth/quota failures can - // terminate instead of cycling forever. - rotateToken(lastRotationMessage) - ) { logger.info( { ...logContext, reason: trigger.reason }, 'Claude account unavailable, retrying with rotated account', ); - const attempt = await runAttempt(); - - // ── Thrown error (exception from spawn/process) ── - if (attempt.thrownError) { - if (!attempt.sawOutput) { - const errMsg = getErrorMessage(attempt.thrownError); - const retryTrigger = attempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(errMsg); - if (retryTrigger.shouldRetry) { - trigger = { - reason: retryTrigger.reason, - retryAfterMs: retryTrigger.retryAfterMs, - }; - lastRotationMessage = errMsg; - continue; - } - } - - logger.error( - { ...logContext, provider: 'claude', err: attempt.thrownError }, - 'Rotated Claude account also threw', - ); - return { type: 'error' }; + const rotatedAttemptOutcome = evaluateClaudeAttempt( + await runAttempt(), + logContext, + ); + if (rotatedAttemptOutcome.type === 'success') { + return rotatedAttemptOutcome; + } + if (rotatedAttemptOutcome.type === 'error') { + return rotatedAttemptOutcome; } - const output = attempt.output; - if (!output) { - logger.error( - { ...logContext, provider: 'claude' }, - 'Rotated Claude account produced no output object', - ); - return { type: 'error' }; - } - - // ── Streamed trigger in non-error success ── - if ( - !attempt.sawOutput && - attempt.streamedTriggerReason && - output.status !== 'error' - ) { - trigger = { - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - }; - lastRotationMessage = getAgentOutputText(output) ?? undefined; - continue; - } - - // ── Success with null result ── - if (!attempt.sawOutput && attempt.sawSuccessNullResult) { - logger.warn( - { ...logContext, provider: 'claude' }, - 'All rotated tokens returned success with null result', - ); - return { type: 'error', trigger: { reason: 'success-null-result' } }; - } - - // ── Error status ── - if (output.status === 'error') { - if (!attempt.sawOutput) { - const retryTrigger = attempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(output.error); - if (retryTrigger.shouldRetry) { - trigger = { - reason: retryTrigger.reason, - retryAfterMs: retryTrigger.retryAfterMs, - }; - lastRotationMessage = output.error ?? undefined; - continue; - } - } - - logger.error( - { ...logContext, provider: 'claude', error: output.error }, - 'Rotated Claude account failed', - ); - return { type: 'error' }; - } - - // ── Success ── - markTokenHealthy(); - clearGlobalFailover(); - return { type: 'success', sawOutput: attempt.sawOutput }; + trigger = rotatedAttemptOutcome.trigger; + lastRotationMessage = rotatedAttemptOutcome.rotationMessage; } // ── All tokens exhausted ── diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index 6fdeda4..ecef805 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -54,9 +54,14 @@ vi.mock('./agent-error-detection.js', async (importOriginal) => { vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), + getCurrentTokenIndex: vi.fn(() => 0), markTokenHealthy: vi.fn(), })); +vi.mock('./token-refresh.js', () => ({ + forceRefreshToken: vi.fn(async () => null), +})); + vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn((error?: string | null) => { const lower = (error || '').toLowerCase(); @@ -142,6 +147,7 @@ import { _initTestDatabase, createTask, getTaskById } from './db.js'; import * as codexTokenRotation from './codex-token-rotation.js'; import { TIMEZONE } from './config.js'; import * as serviceRouting from './service-routing.js'; +import * as tokenRefresh from './token-refresh.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js'; import * as tokenRotation from './token-rotation.js'; @@ -183,9 +189,12 @@ describe('task scheduler', () => { }); // No fallback provider setup needed vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + vi.mocked(tokenRotation.getCurrentTokenIndex).mockReturnValue(0); vi.mocked(tokenRotation.markTokenHealthy).mockClear(); vi.mocked(tokenRotation.rotateToken).mockClear(); vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); + vi.mocked(tokenRefresh.forceRefreshToken).mockReset(); + vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null); vi.mocked(codexTokenRotation.rotateCodexToken).mockClear(); vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValue(false); vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(1); diff --git a/src/token-refresh.ts b/src/token-refresh.ts index 020fb41..63a0ab4 100644 --- a/src/token-refresh.ts +++ b/src/token-refresh.ts @@ -237,6 +237,7 @@ async function refreshToken( */ async function checkAndRefreshAccount( accountIndex: number, + opts?: { force?: boolean }, ): Promise { const creds = readCredentials(accountIndex); if (!creds?.claudeAiOauth) return null; @@ -250,7 +251,7 @@ async function checkAndRefreshAccount( const now = Date.now(); const remaining = expiresAt - now; - if (remaining > REFRESH_BEFORE_EXPIRY_MS) { + if (!opts?.force && remaining > REFRESH_BEFORE_EXPIRY_MS) { logger.debug( { accountIndex, remainingMin: Math.round(remaining / 60000) }, 'Token still valid, no refresh needed', @@ -260,7 +261,12 @@ async function checkAndRefreshAccount( const isExpired = remaining <= 0; logger.info( - { accountIndex, remainingMin: Math.round(remaining / 60000), isExpired }, + { + accountIndex, + remainingMin: Math.round(remaining / 60000), + isExpired, + force: opts?.force === true, + }, 'Refreshing Claude OAuth token', ); @@ -407,7 +413,9 @@ export async function forceRefreshToken( ): Promise { logger.info({ tokenIndex }, 'Force-refreshing token after 401'); - const newAccessToken = await checkAndRefreshAccount(tokenIndex); + const newAccessToken = await checkAndRefreshAccount(tokenIndex, { + force: true, + }); if (newAccessToken) { const allTokens = getAllTokens(); if (tokenIndex < allTokens.length) { diff --git a/src/token-rotation.ts b/src/token-rotation.ts index da88989..6f68598 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -155,6 +155,13 @@ export function getCurrentToken(): string | undefined { return tokens[currentIndex % tokens.length]?.token; } +/** Get the current active token index. */ +export function getCurrentTokenIndex(): number | null { + if (tokens.length === 0) return null; + refreshRuntimeTokenSelection(); + return currentIndex % tokens.length; +} + /** * Try to rotate to the next available (non-rate-limited) token. * Returns true if a fresh token was found, false if all are exhausted. diff --git a/src/verification.test.ts b/src/verification.test.ts index 1947dd4..54cda99 100644 --- a/src/verification.test.ts +++ b/src/verification.test.ts @@ -20,7 +20,9 @@ describe('verification helpers', () => { }); it('builds deterministic commands for each profile', () => { - const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-verification-')); + const repoDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'ejclaw-verification-'), + ); fs.writeFileSync(path.join(repoDir, 'package.json'), JSON.stringify({})); expect(buildVerificationCommand('test', repoDir)).toMatchObject({ diff --git a/src/verification.ts b/src/verification.ts index 23e66df..e6bd918 100644 --- a/src/verification.ts +++ b/src/verification.ts @@ -77,7 +77,11 @@ export function buildVerificationCommand( repoDir: string = process.cwd(), ): VerificationCommandSpec { const scriptName = - profile === 'test' ? 'test' : profile === 'typecheck' ? 'typecheck' : 'build'; + profile === 'test' + ? 'test' + : profile === 'typecheck' + ? 'typecheck' + : 'build'; const command = buildWorkspaceScriptCommand(repoDir, scriptName); return { file: command.file, diff --git a/src/workspace-package-manager.test.ts b/src/workspace-package-manager.test.ts index b942bd7..406b606 100644 --- a/src/workspace-package-manager.test.ts +++ b/src/workspace-package-manager.test.ts @@ -57,7 +57,10 @@ describe('workspace package manager helpers', () => { JSON.stringify({ packageManager: 'bun@1.3.11' }), ); fs.writeFileSync(path.join(mixedRepo, 'bun.lock'), ''); - fs.writeFileSync(path.join(mixedRepo, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n'); + fs.writeFileSync( + path.join(mixedRepo, 'pnpm-lock.yaml'), + 'lockfileVersion: 9.0\n', + ); expect(detectWorkspacePackageManager(pnpmRepo)).toBe('pnpm'); expect(detectWorkspacePackageManager(bunRepo)).toBe('bun'); @@ -118,7 +121,10 @@ describe('workspace package manager helpers', () => { scripts: { typecheck: 'tsc --noEmit' }, }), ); - fs.writeFileSync(path.join(repoDir, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n'); + fs.writeFileSync( + path.join(repoDir, 'pnpm-lock.yaml'), + 'lockfileVersion: 9.0\n', + ); execFileSyncMock.mockImplementation((_file, _args, options) => { const cwd = (options as { cwd: string }).cwd; diff --git a/src/workspace-package-manager.ts b/src/workspace-package-manager.ts index 7158d2c..208ae44 100644 --- a/src/workspace-package-manager.ts +++ b/src/workspace-package-manager.ts @@ -44,7 +44,9 @@ function readPackageJsonMetadata(repoDir: string): PackageJsonMetadata | null { return null; } - return JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) as PackageJsonMetadata; + return JSON.parse( + fs.readFileSync(packageJsonPath, 'utf-8'), + ) as PackageJsonMetadata; } function detectPackageManagerFromField( @@ -60,10 +62,7 @@ function detectPackageManagerFromField( return null; } -function hasLockfile( - repoDir: string, - ...names: readonly string[] -): boolean { +function hasLockfile(repoDir: string, ...names: readonly string[]): boolean { return names.some((name) => fs.existsSync(path.join(repoDir, name))); } @@ -91,7 +90,9 @@ function detectYarnBerry( packageManagerField: string | undefined, ): boolean { if (packageManagerField?.startsWith('yarn@')) { - const version = packageManagerField.slice('yarn@'.length).split(/[+-]/, 1)[0]; + const version = packageManagerField + .slice('yarn@'.length) + .split(/[+-]/, 1)[0]; const major = Number.parseInt(version.split('.', 1)[0] ?? '', 10); if (Number.isFinite(major) && major >= 2) { return true; @@ -142,11 +143,7 @@ function computeInstallFingerprint(repoDir: string): string | null { } function readInstallFingerprint(repoDir: string): string | null { - const statePath = path.join( - repoDir, - 'node_modules', - INSTALL_STATE_FILENAME, - ); + const statePath = path.join(repoDir, 'node_modules', INSTALL_STATE_FILENAME); if (!fs.existsSync(statePath)) { return null; } @@ -356,7 +353,9 @@ export function ensureWorkspaceDependenciesInstalled( ? (error as { stdout: string }).stdout.trim() : ''; const detail = - stderr || stdout || (error instanceof Error ? error.message : String(error)); + stderr || + stdout || + (error instanceof Error ? error.message : String(error)); throw new Error( `Failed to install workspace dependencies with "${command.commandText}" in ${repoDir}: ${detail}`, );