From 2983e0501c4d50ddf047ccf41325fbbec0e42500 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Sun, 5 Apr 2026 20:11:14 +0900 Subject: [PATCH 01/29] Serialize merge-ready owner finalize handoff --- src/message-agent-executor.test.ts | 86 ++++++++++++++++++++++++++++ src/message-agent-executor.ts | 14 ++++- src/paired-execution-context.test.ts | 21 +++++++ src/paired-execution-context.ts | 12 ++++ 4 files changed, 132 insertions(+), 1 deletion(-) diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 9972ea7..fe92ff7 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1030,6 +1030,92 @@ describe('runAgentForGroup room memory', () => { undefined, ); }); + + it('does not enqueue a second generic follow-up when reviewer approval already moved the task to merge_ready', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + const deps = makeDeps(); + + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_agent_type: 'claude-code', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + owner_service_id: 'claude', + reviewer_service_id: 'claude', + arbiter_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-merge-ready', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'claude', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'in_review', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: {}, + }); + vi.mocked(db.getPairedTaskById).mockReturnValue({ + id: 'paired-task-merge-ready', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'claude', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'merge_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + result: 'DONE\nreview complete', + output: { visibility: 'public', text: 'DONE\nreview complete' }, + phase: 'final', + }); + return { + status: 'success', + result: 'DONE\nreview complete', + }; + }, + ); + + const result = await runAgentForGroup(deps, { + group, + prompt: 'please review', + chatJid: 'group@test', + runId: 'run-review-merge-ready', + forcedRole: 'reviewer', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); + }); }); describe('runAgentForGroup Claude rotation', () => { diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 66220b6..44661f1 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -1124,8 +1124,20 @@ export async function runAgentForGroup( pairedExecutionStatus === 'succeeded' && pairedSawOutput ) { + const completedRole = roomRoleContext?.role ?? 'owner'; const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id); - if (finishedCheck?.status !== 'completed') { + const skipGenericFollowUpForInlineFinalize = + completedRole === 'reviewer' && finishedCheck?.status === 'merge_ready'; + if (skipGenericFollowUpForInlineFinalize) { + log.info( + { + taskId: pairedExecutionContext.task.id, + role: completedRole, + taskStatus: finishedCheck?.status ?? null, + }, + 'Skipping generic follow-up after reviewer approval because inline finalize will handle the owner handoff', + ); + } else if (finishedCheck?.status !== 'completed') { deps.queue.enqueueMessageCheck(chatJid); } } diff --git a/src/paired-execution-context.test.ts b/src/paired-execution-context.test.ts index c441299..af7b99c 100644 --- a/src/paired-execution-context.test.ts +++ b/src/paired-execution-context.test.ts @@ -391,6 +391,27 @@ describe('paired execution context', () => { // Should not throw; just logs. }); + it('ignores late completions after the paired task is already completed', () => { + vi.mocked(db.getPairedTaskById).mockReturnValue( + buildPairedTask({ + status: 'completed', + completion_reason: 'done', + }), + ); + + completePairedExecutionContext({ + taskId: 'task-1', + role: 'owner', + status: 'succeeded', + summary: 'DONE', + }); + + expect(db.updatePairedTask).not.toHaveBeenCalled(); + expect( + pairedWorkspaceManager.markPairedTaskReviewReady, + ).not.toHaveBeenCalled(); + }); + it('completes owner finalize when only the commit object changed after approval', () => { const repoDir = createCanonicalRepoWithCommit('reviewed'); const approvedSourceRef = resolveTreeRef(repoDir); diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index 86dd418..966702d 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -429,6 +429,18 @@ export function completePairedExecutionContext(args: { const task = getPairedTaskById(taskId); if (!task) return; + if (task.status === 'completed') { + logger.info( + { + taskId, + role, + status, + completionReason: task.completion_reason ?? null, + }, + 'Ignoring late paired execution completion for an already completed task', + ); + return; + } // On failure: for reviewers, still check verdict from summary — output may // have been delivered even though the executor classified it as failed From 446b194e2c6d2915e2990f11da526100dbaa8d38 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Sun, 5 Apr 2026 22:34:16 +0900 Subject: [PATCH 02/29] Automate worktree dependency installs --- package.json | 1 + runners/agent-runner/src/ipc-mcp-stdio.ts | 2 +- src/container-runner.ts | 32 +- src/paired-workspace-manager.test.ts | 46 +++ src/paired-workspace-manager.ts | 41 +++ src/verification.test.ts | 57 ++- src/verification.ts | 76 ++-- src/workspace-package-manager.test.ts | 178 +++++++++ src/workspace-package-manager.ts | 419 ++++++++++++++++++++++ 9 files changed, 762 insertions(+), 90 deletions(-) create mode 100644 src/workspace-package-manager.test.ts create mode 100644 src/workspace-package-manager.ts diff --git a/package.json b/package.json index 71a47f0..f7ec7d2 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "ejclaw", "version": "0.1.0", "description": "Personal Claude assistant. Lightweight, secure, customizable.", + "packageManager": "bun@1.3.11", "type": "module", "main": "dist/index.js", "scripts": { diff --git a/runners/agent-runner/src/ipc-mcp-stdio.ts b/runners/agent-runner/src/ipc-mcp-stdio.ts index 83aab69..2c77deb 100644 --- a/runners/agent-runner/src/ipc-mcp-stdio.ts +++ b/runners/agent-runner/src/ipc-mcp-stdio.ts @@ -427,7 +427,7 @@ server.tool( profile: z .enum(VERIFICATION_PROFILES) .describe( - 'Fixed verification profile. test=npm test, typecheck=npm run typecheck, build=npm run build.', + 'Fixed verification profile. Runs the workspace test/typecheck/build scripts with the repo-configured package manager.', ), }, async (args) => { diff --git a/src/container-runner.ts b/src/container-runner.ts index d3cd7dd..283fc38 100644 --- a/src/container-runner.ts +++ b/src/container-runner.ts @@ -37,6 +37,7 @@ import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js'; import { logger } from './logger.js'; import type { AgentOutput } from './agent-runner.js'; import type { RegisteredGroup } from './types.js'; +import { detectPnpmStorePath } from './workspace-package-manager.js'; // ── Config ──────────────────────────────────────────────────────── @@ -161,37 +162,6 @@ function pushWorktreeGitMetadataMounts( } } -// ── pnpm store detection ───────────────────────────────────────── - -function detectPnpmStorePath(workspaceDir: string): string | null { - if (!fs.existsSync(path.join(workspaceDir, 'pnpm-lock.yaml'))) { - return null; - } - if (process.env.PNPM_STORE_DIR && fs.existsSync(process.env.PNPM_STORE_DIR)) { - return process.env.PNPM_STORE_DIR; - } - try { - const storePath = execFileSync('pnpm', ['store', 'path'], { - cwd: workspaceDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 5000, - }).trim(); - if (storePath && fs.existsSync(storePath)) return storePath; - } catch { - /* pnpm not available */ - } - const defaultStore = path.join( - os.homedir(), - '.local', - 'share', - 'pnpm', - 'store', - ); - if (fs.existsSync(defaultStore)) return defaultStore; - return null; -} - // ── Pre-flight checks ──────────────────────────────────────────── let containerRuntimeChecked = false; diff --git a/src/paired-workspace-manager.test.ts b/src/paired-workspace-manager.test.ts index fd8e687..29fde61 100644 --- a/src/paired-workspace-manager.test.ts +++ b/src/paired-workspace-manager.test.ts @@ -213,6 +213,52 @@ describe('paired workspace manager', () => { ).toBe(null); }); + it('ensures owner workspace dependencies on provision and review handoff', async () => { + const ensureWorkspaceDependenciesInstalledMock = vi.fn(() => ({ + installed: false, + packageManager: 'pnpm' as const, + })); + vi.doMock('./workspace-package-manager.js', async () => { + const actual = + await vi.importActual( + './workspace-package-manager.js', + ); + return { + ...actual, + ensureWorkspaceDependenciesInstalled: + ensureWorkspaceDependenciesInstalledMock, + }; + }); + + const { db, manager } = await loadModules(); + db._initTestDatabase(); + + const canonicalDir = path.join(tempRoot, 'canonical'); + initCanonicalRepo(canonicalDir); + seedPairedTask(db, canonicalDir, { + taskId: 'paired-task-install-owner-deps', + groupFolder: 'install-room', + }); + + const ownerWorkspace = manager.provisionOwnerWorkspaceForPairedTask( + 'paired-task-install-owner-deps', + ); + + expect(ensureWorkspaceDependenciesInstalledMock).toHaveBeenCalledTimes(1); + expect(ensureWorkspaceDependenciesInstalledMock).toHaveBeenNthCalledWith( + 1, + ownerWorkspace.workspace_dir, + ); + + manager.markPairedTaskReviewReady('paired-task-install-owner-deps'); + + expect(ensureWorkspaceDependenciesInstalledMock).toHaveBeenCalledTimes(2); + expect(ensureWorkspaceDependenciesInstalledMock).toHaveBeenNthCalledWith( + 2, + ownerWorkspace.workspace_dir, + ); + }); + it('uses the shared DB owner workspace across service-local data dirs', async () => { const canonicalDir = path.join(tempRoot, 'canonical'); fs.mkdirSync(canonicalDir, { recursive: true }); diff --git a/src/paired-workspace-manager.ts b/src/paired-workspace-manager.ts index cb99dcf..3bfa486 100644 --- a/src/paired-workspace-manager.ts +++ b/src/paired-workspace-manager.ts @@ -14,6 +14,7 @@ import { import { resolvePairedTaskWorkspacePath } from './group-folder.js'; import { logger } from './logger.js'; import type { PairedTask, PairedWorkspace } from './types.js'; +import { ensureWorkspaceDependenciesInstalled } from './workspace-package-manager.js'; const REVIEWER_SNAPSHOT_STALE_BLOCK_MESSAGE = 'Review snapshot is stale after owner changes. Retry the review once to refresh against the latest owner workspace.'; @@ -539,6 +540,18 @@ export function registerOwnerCanonicalWorkspace( canonicalWorkDir: string, ): PairedWorkspace { ensureGitRepository(canonicalWorkDir); + const installResult = ensureWorkspaceDependenciesInstalled(canonicalWorkDir); + if (installResult.installed) { + logger.info( + { + taskId, + workspaceDir: canonicalWorkDir, + packageManager: installResult.packageManager, + command: installResult.commandText ?? null, + }, + 'Installed owner workspace dependencies', + ); + } const workspace = makeWorkspaceRecord({ taskId, role: 'owner', @@ -644,6 +657,19 @@ export function provisionOwnerWorkspaceForPairedTask( } } + const installResult = ensureWorkspaceDependenciesInstalled(workspaceDir); + if (installResult.installed) { + logger.info( + { + taskId, + workspaceDir, + packageManager: installResult.packageManager, + command: installResult.commandText ?? null, + }, + 'Installed owner workspace dependencies', + ); + } + const workspace = makeWorkspaceRecord({ taskId, role: 'owner', @@ -741,6 +767,21 @@ export function markPairedTaskReviewReady(taskId: string): { return null; } + const installResult = ensureWorkspaceDependenciesInstalled( + ownerWorkspace.workspace_dir, + ); + if (installResult.installed) { + logger.info( + { + taskId, + ownerDir: ownerWorkspace.workspace_dir, + packageManager: installResult.packageManager, + command: installResult.commandText ?? null, + }, + 'Installed owner workspace dependencies before review handoff', + ); + } + // Reviewer runs in a container with the owner workspace mounted read-only. // No file-level snapshot copy needed — just register the path. const reviewerWorkspace = makeWorkspaceRecord({ diff --git a/src/verification.test.ts b/src/verification.test.ts index 35b02f9..1947dd4 100644 --- a/src/verification.test.ts +++ b/src/verification.test.ts @@ -8,6 +8,7 @@ import { buildVerificationCommand, computeVerificationSnapshot, isVerificationProfile, + runVerificationRequest, } from './verification.js'; describe('verification helpers', () => { @@ -19,23 +20,43 @@ describe('verification helpers', () => { }); it('builds deterministic commands for each profile', () => { - expect(buildVerificationCommand('test')).toMatchObject({ + const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-verification-')); + fs.writeFileSync(path.join(repoDir, 'package.json'), JSON.stringify({})); + + expect(buildVerificationCommand('test', repoDir)).toMatchObject({ file: 'npm', args: ['test'], requiredScript: 'test', }); - expect(buildVerificationCommand('typecheck')).toMatchObject({ + expect(buildVerificationCommand('typecheck', repoDir)).toMatchObject({ file: 'npm', args: ['run', 'typecheck'], requiredScript: 'typecheck', }); - expect(buildVerificationCommand('build')).toMatchObject({ + expect(buildVerificationCommand('build', repoDir)).toMatchObject({ file: 'npm', args: ['run', 'build'], requiredScript: 'build', }); }); + it('selects the workspace package manager for verification commands', () => { + const repoDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'ejclaw-verification-pnpm-'), + ); + fs.writeFileSync( + path.join(repoDir, 'package.json'), + JSON.stringify({ packageManager: 'pnpm@10.11.0' }), + ); + + expect(buildVerificationCommand('typecheck', repoDir)).toMatchObject({ + file: 'corepack', + args: ['pnpm', 'run', 'typecheck'], + commandText: 'corepack pnpm run typecheck', + requiredScript: 'typecheck', + }); + }); + it('computes a stable snapshot over the readable workspace inputs', () => { const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-snapshot-')); fs.mkdirSync(path.join(repoDir, 'src'), { recursive: true }); @@ -58,4 +79,34 @@ describe('verification helpers', () => { expect(second).toBe(first); expect(third).not.toBe(first); }); + + it('fails verification when node_modules contains only cache noise', async () => { + const repoDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'ejclaw-verification-noise-'), + ); + fs.mkdirSync(path.join(repoDir, 'node_modules', '.vite'), { + recursive: true, + }); + fs.writeFileSync( + path.join(repoDir, 'package.json'), + JSON.stringify({ + scripts: { + test: 'vitest run', + typecheck: 'tsc --noEmit', + build: 'tsc', + }, + }), + ); + + const result = await runVerificationRequest( + { + requestId: 'req-noise', + profile: 'typecheck', + }, + { repoDir }, + ); + + expect(result.ok).toBe(false); + expect(result.error).toContain('installed node_modules tree'); + }); }); diff --git a/src/verification.ts b/src/verification.ts index 5f42ce0..23e66df 100644 --- a/src/verification.ts +++ b/src/verification.ts @@ -13,6 +13,11 @@ import { writableMountArgs, } from './container-runtime.js'; import { resolveGroupIpcPath } from './group-folder.js'; +import { + buildWorkspaceScriptCommand, + detectPnpmStorePath, + hasInstalledNodeModules, +} from './workspace-package-manager.js'; export const VERIFICATION_PROFILES = ['test', 'typecheck', 'build'] as const; @@ -69,30 +74,17 @@ export function isVerificationProfile( export function buildVerificationCommand( profile: VerificationProfile, + repoDir: string = process.cwd(), ): VerificationCommandSpec { - switch (profile) { - case 'test': - return { - file: 'npm', - args: ['test'], - commandText: 'npm test', - requiredScript: 'test', - }; - case 'typecheck': - return { - file: 'npm', - args: ['run', 'typecheck'], - commandText: 'npm run typecheck', - requiredScript: 'typecheck', - }; - case 'build': - return { - file: 'npm', - args: ['run', 'build'], - commandText: 'npm run build', - requiredScript: 'build', - }; - } + const scriptName = + profile === 'test' ? 'test' : profile === 'typecheck' ? 'typecheck' : 'build'; + const command = buildWorkspaceScriptCommand(repoDir, scriptName); + return { + file: command.file, + args: command.args, + commandText: command.commandText, + requiredScript: scriptName, + }; } function shouldExcludePath(name: string): boolean { @@ -146,35 +138,6 @@ function truncateOutput(value: string | undefined): string { return `${value.slice(0, MAX_OUTPUT_CHARS)}\n...[truncated]`; } -function detectPnpmStorePath(workspaceDir: string): string | null { - if (!fs.existsSync(path.join(workspaceDir, 'pnpm-lock.yaml'))) { - return null; - } - if (process.env.PNPM_STORE_DIR && fs.existsSync(process.env.PNPM_STORE_DIR)) { - return process.env.PNPM_STORE_DIR; - } - try { - const storePath = execFileSync('pnpm', ['store', 'path'], { - cwd: workspaceDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 5000, - }).trim(); - if (storePath && fs.existsSync(storePath)) return storePath; - } catch { - /* ignore */ - } - - const defaultStore = path.join( - os.homedir(), - '.local', - 'share', - 'pnpm', - 'store', - ); - return fs.existsSync(defaultStore) ? defaultStore : null; -} - function readPackageScripts(repoDir: string): Record { const packageJsonPath = path.join(repoDir, 'package.json'); if (!fs.existsSync(packageJsonPath)) { @@ -248,7 +211,10 @@ function buildDockerRunArgs( args.push(...writableMountArgs(scratchWorkspace, PRIMARY_PROJECT_MOUNT)); const sourceNodeModulesDir = path.join(sourceRepoDir, 'node_modules'); - if (fs.existsSync(sourceNodeModulesDir)) { + if ( + hasInstalledNodeModules(sourceRepoDir) && + fs.existsSync(sourceNodeModulesDir) + ) { args.push( ...readonlyMountArgs( sourceNodeModulesDir, @@ -314,7 +280,7 @@ export async function runVerificationRequest( ): Promise { const repoDir = options?.repoDir || process.cwd(); const runtimeVersion = detectRuntimeVersion(); - const command = buildVerificationCommand(request.profile); + const command = buildVerificationCommand(request.profile, repoDir); const scripts = readPackageScripts(repoDir); if (!scripts[command.requiredScript]) { @@ -331,7 +297,7 @@ export async function runVerificationRequest( }; } - if (!fs.existsSync(path.join(repoDir, 'node_modules'))) { + if (!hasInstalledNodeModules(repoDir)) { return { ok: false, profile: request.profile, diff --git a/src/workspace-package-manager.test.ts b/src/workspace-package-manager.test.ts new file mode 100644 index 0000000..b942bd7 --- /dev/null +++ b/src/workspace-package-manager.test.ts @@ -0,0 +1,178 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { execFileSyncMock } = vi.hoisted(() => ({ + execFileSyncMock: vi.fn(), +})); + +vi.mock('child_process', () => ({ + execFileSync: execFileSyncMock, +})); + +import { + buildWorkspaceScriptCommand, + detectWorkspacePackageManager, + ensureWorkspaceDependenciesInstalled, + hasInstalledNodeModules, + resolveWorkspaceInstallCommand, +} from './workspace-package-manager.js'; + +describe('workspace package manager helpers', () => { + let tempRoot: string; + + beforeEach(() => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-pkgmgr-')); + execFileSyncMock.mockReset(); + }); + + afterEach(() => { + fs.rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('detects package managers from packageManager field and lockfiles', () => { + const pnpmRepo = path.join(tempRoot, 'pnpm'); + fs.mkdirSync(pnpmRepo, { recursive: true }); + fs.writeFileSync( + path.join(pnpmRepo, 'package.json'), + JSON.stringify({ packageManager: 'pnpm@10.11.0' }), + ); + + const bunRepo = path.join(tempRoot, 'bun'); + fs.mkdirSync(bunRepo, { recursive: true }); + fs.writeFileSync(path.join(bunRepo, 'package.json'), JSON.stringify({})); + fs.writeFileSync(path.join(bunRepo, 'bun.lock'), ''); + + const yarnRepo = path.join(tempRoot, 'yarn'); + fs.mkdirSync(yarnRepo, { recursive: true }); + fs.writeFileSync(path.join(yarnRepo, 'package.json'), JSON.stringify({})); + fs.writeFileSync(path.join(yarnRepo, 'yarn.lock'), ''); + + const mixedRepo = path.join(tempRoot, 'mixed'); + fs.mkdirSync(mixedRepo, { recursive: true }); + fs.writeFileSync( + path.join(mixedRepo, 'package.json'), + JSON.stringify({ packageManager: 'bun@1.3.11' }), + ); + fs.writeFileSync(path.join(mixedRepo, 'bun.lock'), ''); + fs.writeFileSync(path.join(mixedRepo, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n'); + + expect(detectWorkspacePackageManager(pnpmRepo)).toBe('pnpm'); + expect(detectWorkspacePackageManager(bunRepo)).toBe('bun'); + expect(detectWorkspacePackageManager(yarnRepo)).toBe('yarn'); + expect(detectWorkspacePackageManager(mixedRepo)).toBe('bun'); + }); + + it('fails fast when multiple lockfiles exist without a packageManager field', () => { + const repoDir = path.join(tempRoot, 'ambiguous'); + fs.mkdirSync(repoDir, { recursive: true }); + fs.writeFileSync(path.join(repoDir, 'package.json'), JSON.stringify({})); + fs.writeFileSync(path.join(repoDir, 'bun.lock'), ''); + fs.writeFileSync(path.join(repoDir, 'package-lock.json'), '{}\n'); + + expect(() => detectWorkspacePackageManager(repoDir)).toThrow( + /Ambiguous package manager/i, + ); + }); + + it('builds script and install commands for non-npm workspaces', () => { + const pnpmRepo = path.join(tempRoot, 'pnpm'); + fs.mkdirSync(pnpmRepo, { recursive: true }); + fs.writeFileSync( + path.join(pnpmRepo, 'package.json'), + JSON.stringify({ packageManager: 'pnpm@10.11.0' }), + ); + + const bunRepo = path.join(tempRoot, 'bun'); + fs.mkdirSync(bunRepo, { recursive: true }); + fs.writeFileSync(path.join(bunRepo, 'package.json'), JSON.stringify({})); + fs.writeFileSync(path.join(bunRepo, 'bun.lock'), ''); + + expect(buildWorkspaceScriptCommand(pnpmRepo, 'typecheck')).toMatchObject({ + file: 'corepack', + args: ['pnpm', 'run', 'typecheck'], + commandText: 'corepack pnpm run typecheck', + }); + expect(resolveWorkspaceInstallCommand(pnpmRepo)).toMatchObject({ + file: 'corepack', + args: ['pnpm', 'install', '--frozen-lockfile'], + commandText: 'corepack pnpm install --frozen-lockfile', + }); + expect(buildWorkspaceScriptCommand(bunRepo, 'build')).toMatchObject({ + file: 'bun', + args: ['run', 'build'], + commandText: 'bun run build', + }); + }); + + it('installs dependencies once and tracks install fingerprint', () => { + const repoDir = path.join(tempRoot, 'repo'); + fs.mkdirSync(repoDir, { recursive: true }); + fs.writeFileSync( + path.join(repoDir, 'package.json'), + JSON.stringify({ + name: 'repo', + packageManager: 'pnpm@10.11.0', + scripts: { typecheck: 'tsc --noEmit' }, + }), + ); + fs.writeFileSync(path.join(repoDir, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n'); + + execFileSyncMock.mockImplementation((_file, _args, options) => { + const cwd = (options as { cwd: string }).cwd; + fs.mkdirSync(path.join(cwd, 'node_modules', '.bin'), { recursive: true }); + fs.writeFileSync(path.join(cwd, 'node_modules', '.bin', 'tsc'), ''); + return ''; + }); + + const first = ensureWorkspaceDependenciesInstalled(repoDir); + expect(first).toMatchObject({ + installed: true, + packageManager: 'pnpm', + commandText: 'corepack pnpm install --frozen-lockfile', + }); + expect(hasInstalledNodeModules(repoDir)).toBe(true); + + execFileSyncMock.mockClear(); + const second = ensureWorkspaceDependenciesInstalled(repoDir); + expect(second).toMatchObject({ + installed: false, + packageManager: 'pnpm', + }); + expect(execFileSyncMock).not.toHaveBeenCalled(); + + fs.writeFileSync( + path.join(repoDir, 'package.json'), + JSON.stringify({ + name: 'repo', + packageManager: 'pnpm@10.11.0', + scripts: { typecheck: 'tsc --noEmit', test: 'vitest run' }, + }), + ); + + expect(hasInstalledNodeModules(repoDir)).toBe(false); + }); + + it('rejects no-op installs that only leave the sentinel file behind', () => { + const repoDir = path.join(tempRoot, 'noop'); + fs.mkdirSync(repoDir, { recursive: true }); + fs.writeFileSync( + path.join(repoDir, 'package.json'), + JSON.stringify({ + name: 'noop', + packageManager: 'npm@10.0.0', + scripts: { typecheck: 'tsc --noEmit' }, + }), + ); + fs.writeFileSync(path.join(repoDir, 'package-lock.json'), '{}\n'); + + execFileSyncMock.mockImplementation(() => ''); + + expect(() => ensureWorkspaceDependenciesInstalled(repoDir)).toThrow( + /did not produce a usable node_modules tree/i, + ); + expect(hasInstalledNodeModules(repoDir)).toBe(false); + }); +}); diff --git a/src/workspace-package-manager.ts b/src/workspace-package-manager.ts new file mode 100644 index 0000000..7158d2c --- /dev/null +++ b/src/workspace-package-manager.ts @@ -0,0 +1,419 @@ +import { execFileSync } from 'child_process'; +import { createHash } from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +export type WorkspacePackageManager = 'bun' | 'pnpm' | 'npm' | 'yarn'; + +export interface WorkspaceCommandSpec { + packageManager: WorkspacePackageManager; + file: string; + args: string[]; + commandText: string; +} + +export interface WorkspaceDependencyInstallResult { + installed: boolean; + packageManager: WorkspacePackageManager | null; + commandText?: string; +} + +const INSTALL_STATE_FILENAME = '.ejclaw-install-state.json'; +const NODE_MODULES_NOISE_ENTRIES = new Set([ + '.cache', + '.vite', + '.vite-temp', + INSTALL_STATE_FILENAME, +]); +const LOCKFILE_NAMES = [ + 'bun.lock', + 'bun.lockb', + 'pnpm-lock.yaml', + 'package-lock.json', + 'npm-shrinkwrap.json', + 'yarn.lock', +] as const; + +type PackageJsonMetadata = { + packageManager?: string; +}; + +function readPackageJsonMetadata(repoDir: string): PackageJsonMetadata | null { + const packageJsonPath = path.join(repoDir, 'package.json'); + if (!fs.existsSync(packageJsonPath)) { + return null; + } + + return JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) as PackageJsonMetadata; +} + +function detectPackageManagerFromField( + packageManager: string | undefined, +): WorkspacePackageManager | null { + if (!packageManager) { + return null; + } + if (packageManager.startsWith('bun@')) return 'bun'; + if (packageManager.startsWith('pnpm@')) return 'pnpm'; + if (packageManager.startsWith('npm@')) return 'npm'; + if (packageManager.startsWith('yarn@')) return 'yarn'; + return null; +} + +function hasLockfile( + repoDir: string, + ...names: readonly string[] +): boolean { + return names.some((name) => fs.existsSync(path.join(repoDir, name))); +} + +function detectPackageManagersFromLockfiles( + repoDir: string, +): WorkspacePackageManager[] { + const detected = new Set(); + if (hasLockfile(repoDir, 'bun.lock', 'bun.lockb')) { + detected.add('bun'); + } + if (hasLockfile(repoDir, 'pnpm-lock.yaml')) { + detected.add('pnpm'); + } + if (hasLockfile(repoDir, 'package-lock.json', 'npm-shrinkwrap.json')) { + detected.add('npm'); + } + if (hasLockfile(repoDir, 'yarn.lock')) { + detected.add('yarn'); + } + return [...detected]; +} + +function detectYarnBerry( + repoDir: string, + packageManagerField: string | undefined, +): boolean { + if (packageManagerField?.startsWith('yarn@')) { + const version = packageManagerField.slice('yarn@'.length).split(/[+-]/, 1)[0]; + const major = Number.parseInt(version.split('.', 1)[0] ?? '', 10); + if (Number.isFinite(major) && major >= 2) { + return true; + } + } + return fs.existsSync(path.join(repoDir, '.yarnrc.yml')); +} + +function buildCommandText(file: string, args: string[]): string { + return [file, ...args].join(' '); +} + +function buildCorepackCommand( + packageManager: 'pnpm' | 'yarn', + args: string[], +): WorkspaceCommandSpec { + return { + packageManager, + file: 'corepack', + args: [packageManager, ...args], + commandText: buildCommandText('corepack', [packageManager, ...args]), + }; +} + +function computeInstallFingerprint(repoDir: string): string | null { + const packageJsonPath = path.join(repoDir, 'package.json'); + if (!fs.existsSync(packageJsonPath)) { + return null; + } + + const hash = createHash('sha256'); + const fingerprintFiles = [packageJsonPath]; + for (const lockfileName of LOCKFILE_NAMES) { + const lockfilePath = path.join(repoDir, lockfileName); + if (fs.existsSync(lockfilePath)) { + fingerprintFiles.push(lockfilePath); + } + } + + for (const filePath of fingerprintFiles.sort()) { + hash.update(path.basename(filePath)); + hash.update('\0'); + hash.update(fs.readFileSync(filePath)); + hash.update('\0'); + } + + return hash.digest('hex'); +} + +function readInstallFingerprint(repoDir: string): string | null { + const statePath = path.join( + repoDir, + 'node_modules', + INSTALL_STATE_FILENAME, + ); + if (!fs.existsSync(statePath)) { + return null; + } + + try { + const state = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as { + fingerprint?: string; + }; + return typeof state.fingerprint === 'string' ? state.fingerprint : null; + } catch { + return null; + } +} + +function writeInstallFingerprint( + repoDir: string, + packageManager: WorkspacePackageManager, + fingerprint: string, +): void { + const nodeModulesDir = path.join(repoDir, 'node_modules'); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync( + path.join(nodeModulesDir, INSTALL_STATE_FILENAME), + JSON.stringify( + { + packageManager, + fingerprint, + installedAt: new Date().toISOString(), + }, + null, + 2, + ), + ); +} + +function hasRunnableNodeModulesTree(repoDir: string): boolean { + const nodeModulesDir = path.join(repoDir, 'node_modules'); + if (!fs.existsSync(nodeModulesDir)) { + return false; + } + + try { + return fs + .readdirSync(nodeModulesDir) + .some((entry) => !NODE_MODULES_NOISE_ENTRIES.has(entry)); + } catch { + return false; + } +} + +export function detectWorkspacePackageManager( + repoDir: string, +): WorkspacePackageManager | null { + const packageJson = readPackageJsonMetadata(repoDir); + if (!packageJson) { + return null; + } + + const fromField = detectPackageManagerFromField(packageJson.packageManager); + if (fromField) { + return fromField; + } + const lockfilePackageManagers = detectPackageManagersFromLockfiles(repoDir); + if (lockfilePackageManagers.length > 1) { + throw new Error( + `Ambiguous package manager for ${repoDir}: multiple lockfiles detected (${lockfilePackageManagers.join( + ', ', + )}). Add packageManager to package.json to disambiguate.`, + ); + } + if (lockfilePackageManagers.length === 1) { + return lockfilePackageManagers[0]!; + } + return 'npm'; +} + +export function buildWorkspaceScriptCommand( + repoDir: string, + scriptName: string, +): WorkspaceCommandSpec { + const packageManager = detectWorkspacePackageManager(repoDir) ?? 'npm'; + + switch (packageManager) { + case 'bun': + return { + packageManager, + file: 'bun', + args: ['run', scriptName], + commandText: buildCommandText('bun', ['run', scriptName]), + }; + case 'pnpm': + return buildCorepackCommand(packageManager, ['run', scriptName]); + case 'yarn': + return buildCorepackCommand(packageManager, ['run', scriptName]); + case 'npm': + if (scriptName === 'test') { + return { + packageManager, + file: 'npm', + args: ['test'], + commandText: 'npm test', + }; + } + return { + packageManager, + file: 'npm', + args: ['run', scriptName], + commandText: buildCommandText('npm', ['run', scriptName]), + }; + } +} + +export function resolveWorkspaceInstallCommand( + repoDir: string, +): WorkspaceCommandSpec | null { + const packageJson = readPackageJsonMetadata(repoDir); + if (!packageJson) { + return null; + } + + const packageManager = detectWorkspacePackageManager(repoDir) ?? 'npm'; + switch (packageManager) { + case 'bun': + return { + packageManager, + file: 'bun', + args: ['install', '--frozen-lockfile'], + commandText: buildCommandText('bun', ['install', '--frozen-lockfile']), + }; + case 'pnpm': + return buildCorepackCommand(packageManager, [ + 'install', + '--frozen-lockfile', + ]); + case 'yarn': { + const immutableArg = detectYarnBerry(repoDir, packageJson.packageManager) + ? '--immutable' + : '--frozen-lockfile'; + return buildCorepackCommand(packageManager, ['install', immutableArg]); + } + case 'npm': + if (hasLockfile(repoDir, 'package-lock.json', 'npm-shrinkwrap.json')) { + return { + packageManager, + file: 'npm', + args: ['ci'], + commandText: 'npm ci', + }; + } + return { + packageManager, + file: 'npm', + args: ['install'], + commandText: 'npm install', + }; + } +} + +export function hasInstalledNodeModules(repoDir: string): boolean { + if (!hasRunnableNodeModulesTree(repoDir)) { + return false; + } + + const expectedFingerprint = computeInstallFingerprint(repoDir); + if (!expectedFingerprint) { + return false; + } + + return readInstallFingerprint(repoDir) === expectedFingerprint; +} + +export function ensureWorkspaceDependenciesInstalled( + repoDir: string, +): WorkspaceDependencyInstallResult { + const packageManager = detectWorkspacePackageManager(repoDir); + if (!packageManager) { + return { installed: false, packageManager: null }; + } + if (hasInstalledNodeModules(repoDir)) { + return { installed: false, packageManager }; + } + + const command = resolveWorkspaceInstallCommand(repoDir); + if (!command) { + return { installed: false, packageManager }; + } + + try { + execFileSync(command.file, command.args, { + cwd: repoDir, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + const stderr = + typeof error === 'object' && + error !== null && + 'stderr' in error && + typeof (error as { stderr?: unknown }).stderr === 'string' + ? (error as { stderr: string }).stderr.trim() + : ''; + const stdout = + typeof error === 'object' && + error !== null && + 'stdout' in error && + typeof (error as { stdout?: unknown }).stdout === 'string' + ? (error as { stdout: string }).stdout.trim() + : ''; + const detail = + stderr || stdout || (error instanceof Error ? error.message : String(error)); + throw new Error( + `Failed to install workspace dependencies with "${command.commandText}" in ${repoDir}: ${detail}`, + ); + } + + const fingerprint = computeInstallFingerprint(repoDir); + if (fingerprint) { + writeInstallFingerprint(repoDir, packageManager, fingerprint); + } + + if (!hasInstalledNodeModules(repoDir)) { + throw new Error( + `Workspace dependency install did not produce a usable node_modules tree in ${repoDir}.`, + ); + } + + return { + installed: true, + packageManager, + commandText: command.commandText, + }; +} + +export function detectPnpmStorePath(workspaceDir: string): string | null { + if (detectWorkspacePackageManager(workspaceDir) !== 'pnpm') { + return null; + } + if (process.env.PNPM_STORE_DIR && fs.existsSync(process.env.PNPM_STORE_DIR)) { + return process.env.PNPM_STORE_DIR; + } + + const candidates: Array<[string, string[]]> = [ + ['pnpm', ['store', 'path']], + ['corepack', ['pnpm', 'store', 'path']], + ]; + for (const [file, args] of candidates) { + try { + const storePath = execFileSync(file, args, { + cwd: workspaceDir, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5000, + }).trim(); + if (storePath && fs.existsSync(storePath)) { + return storePath; + } + } catch { + // Try the next candidate. + } + } + + const defaultStore = path.join( + process.env.HOME || '', + '.local', + 'share', + 'pnpm', + 'store', + ); + return fs.existsSync(defaultStore) ? defaultStore : null; +} From 994e957767a7d4be0c107e78b29ef866aa8c5b09 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 02:40:15 +0900 Subject: [PATCH 03/29] Enable unsafe host paired runtime --- runners/agent-runner/src/reviewer-runtime.ts | 3 + runners/codex-runner/src/reviewer-runtime.ts | 3 + setup/service-defs.ts | 8 +- setup/service.test.ts | 3 + src/agent-runner-environment.test.ts | 3 + src/agent-runner-environment.ts | 8 ++ src/agent-runner.test.ts | 125 ++++++++++++++++++- src/agent-runner.ts | 38 +++++- src/container-runner.test.ts | 14 +++ src/container-runner.ts | 7 ++ src/paired-execution-context.test.ts | 40 ++++++ src/paired-execution-context.ts | 14 ++- 12 files changed, 261 insertions(+), 5 deletions(-) diff --git a/runners/agent-runner/src/reviewer-runtime.ts b/runners/agent-runner/src/reviewer-runtime.ts index d217e6f..baf26da 100644 --- a/runners/agent-runner/src/reviewer-runtime.ts +++ b/runners/agent-runner/src/reviewer-runtime.ts @@ -34,6 +34,9 @@ const MUTATING_SHELL_PATTERNS = [ export function isReviewerRuntime( roomRoleContext?: RoomRoleContext, ): boolean { + if (process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1') { + return false; + } return roomRoleContext?.role === 'reviewer'; } diff --git a/runners/codex-runner/src/reviewer-runtime.ts b/runners/codex-runner/src/reviewer-runtime.ts index 012c0b1..8b1b0b3 100644 --- a/runners/codex-runner/src/reviewer-runtime.ts +++ b/runners/codex-runner/src/reviewer-runtime.ts @@ -31,6 +31,9 @@ const BLOCKED_GIT_SUBCOMMANDS = new Set([ export function isReviewerRuntime( roomRoleContext?: RoomRoleContext, ): boolean { + if (process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1') { + return false; + } return roomRoleContext?.role === 'reviewer'; } diff --git a/setup/service-defs.ts b/setup/service-defs.ts index cf3ecd6..2125a5a 100644 --- a/setup/service-defs.ts +++ b/setup/service-defs.ts @@ -54,6 +54,12 @@ const LEGACY_SERVICE_TEMPLATES: ServiceTemplate[] = [ function materializeServiceDef(template: ServiceTemplate): ServiceDef { const environmentFile = undefined; + const extraEnv = + template.kind === 'primary' + ? { + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + } + : undefined; return { kind: template.kind, @@ -62,7 +68,7 @@ function materializeServiceDef(template: ServiceTemplate): ServiceDef { description: template.description, logName: template.logName, environmentFile, - extraEnv: undefined, + extraEnv, }; } diff --git a/setup/service.test.ts b/setup/service.test.ts index cb7f44e..eff0c9e 100644 --- a/setup/service.test.ts +++ b/setup/service.test.ts @@ -184,6 +184,9 @@ describe('service definitions', () => { expect(defs.map((def) => def.name)).toEqual(['ejclaw']); expect(defs.map((def) => def.kind)).toEqual(['primary']); + expect(defs[0]?.extraEnv).toMatchObject({ + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + }); }); it('keeps legacy service identities available for migration guards', () => { diff --git a/src/agent-runner-environment.test.ts b/src/agent-runner-environment.test.ts index 7f91cb4..386cfdd 100644 --- a/src/agent-runner-environment.test.ts +++ b/src/agent-runner-environment.test.ts @@ -425,6 +425,9 @@ args = ["other.js"] expect( fs.readFileSync(path.join(sessionDir, '.codex', 'auth.json'), 'utf-8'), ).toContain('"auth_mode":"chatgpt"'); + expect( + fs.readFileSync(path.join(sessionDir, '.claude.json'), 'utf-8'), + ).toBe('{}\n'); const toml = fs.readFileSync( path.join(sessionDir, '.codex', 'config.toml'), 'utf-8', diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index 49e18ba..9b5c1f9 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -80,6 +80,13 @@ function ensureClaudeSessionSettings(groupSessionsDir: string): void { ); } +export function ensureClaudeGlobalSettingsFile(sessionDir: string): void { + const settingsFile = path.join(sessionDir, '.claude.json'); + if (fs.existsSync(settingsFile)) return; + + fs.writeFileSync(settingsFile, '{}\n'); +} + function syncHostCodexSessionFiles(sessionCodexDir: string): void { const hostCodexDir = path.join(os.homedir(), '.codex'); fs.mkdirSync(sessionCodexDir, { recursive: true }); @@ -586,6 +593,7 @@ export function prepareContainerSessionEnvironment(args: { fs.mkdirSync(sessionDir, { recursive: true }); ensureClaudeSessionSettings(sessionDir); + ensureClaudeGlobalSettingsFile(sessionDir); // Sync skills from host const skillSources = [ diff --git a/src/agent-runner.test.ts b/src/agent-runner.test.ts index dba711c..d2a27de 100644 --- a/src/agent-runner.test.ts +++ b/src/agent-runner.test.ts @@ -105,7 +105,13 @@ vi.mock('child_process', async () => { }; }); -import { runAgentProcess, AgentOutput } from './agent-runner.js'; +import { + runAgentProcess, + AgentOutput, + mirrorContainerCodexSessionFiles, +} from './agent-runner.js'; +import * as agentRunnerEnvironment from './agent-runner-environment.js'; +import * as containerRunner from './container-runner.js'; import type { RegisteredGroup } from './types.js'; const testGroup: RegisteredGroup = { @@ -133,6 +139,7 @@ function emitOutputMarker( describe('agent-runner timeout behavior', () => { beforeEach(() => { vi.useFakeTimers(); + vi.clearAllMocks(); fakeProc = createFakeProcess(); }); @@ -313,6 +320,103 @@ describe('agent-runner timeout behavior', () => { ); }); + it('uses host IPC and workdir in unsafe host paired mode reviewer sessions', async () => { + vi.useRealTimers(); + fakeProc = createFakeProcess(); + const prepareContainerSessionEnvironmentSpy = vi.spyOn( + agentRunnerEnvironment, + 'prepareContainerSessionEnvironment', + ); + const runReviewerContainerSpy = vi.spyOn( + containerRunner, + 'runReviewerContainer', + ); + + vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => { + const str = String(p); + return ( + str.includes('dist/index.js') || + str.includes('dist/ipc-mcp-stdio.js') || + str.endsWith('/.codex/config.toml') + ); + }); + + const resultPromise = runAgentProcess( + testGroup, + { + ...testInput, + roomRoleContext: { + serviceId: 'claude', + role: 'reviewer', + ownerServiceId: 'codex-main', + reviewerServiceId: 'claude', + failoverOwner: false, + }, + }, + () => {}, + async () => {}, + { + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + CLAUDE_CONFIG_DIR: '/tmp/host-reviewer-session', + EJCLAW_WORK_DIR: '/tmp/paired/task-1/reviewer', + }, + ); + + fakeProc.emit('close', 0); + const result = await resultPromise; + expect(result.status).toBe('success'); + expect(runReviewerContainerSpy).not.toHaveBeenCalled(); + expect(prepareContainerSessionEnvironmentSpy).toHaveBeenCalledWith( + expect.objectContaining({ + sessionDir: '/tmp/host-reviewer-session', + ipcDir: '/tmp/ejclaw-test-data/ipc/test-group', + hostIpcDir: '/tmp/ejclaw-test-data/ipc/test-group', + workDir: '/tmp/paired/task-1/reviewer', + role: 'reviewer', + }), + ); + }); + + it('uses role-scoped CODEX_HOME in unsafe host paired mode for codex reviewer', async () => { + vi.useRealTimers(); + fakeProc = createFakeProcess(); + + const codexGroup: RegisteredGroup = { + ...testGroup, + agentType: 'codex', + }; + + const resultPromise = runAgentProcess( + codexGroup, + { + ...testInput, + roomRoleContext: { + serviceId: 'codex-review', + role: 'reviewer', + ownerServiceId: 'codex-main', + reviewerServiceId: 'codex-review', + failoverOwner: false, + }, + }, + () => {}, + async () => {}, + { + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + CLAUDE_CONFIG_DIR: '/tmp/host-reviewer-session', + EJCLAW_WORK_DIR: '/tmp/paired/task-1/reviewer', + }, + ); + + fakeProc.emit('close', 0); + const result = await resultPromise; + expect(result.status).toBe('success'); + + const spawnEnv = vi.mocked(spawn).mock.calls.at(-1)?.[2]?.env as + | Record + | undefined; + expect(spawnEnv?.CODEX_HOME).toBe('/tmp/host-reviewer-session/.codex'); + }); + it('serializes roomRoleContext into the runner stdin payload', async () => { vi.useRealTimers(); fakeProc = createFakeProcess(); @@ -584,4 +688,23 @@ OUROBOROS_LLM_BACKEND = "codex" expect.objectContaining({ newSessionId: 'stale-session' }), ); }); + + it('mirrors .claude.json alongside codex session files for container reuse', () => { + vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => { + const str = String(p); + return ( + str === '/tmp/source/.codex' || + str === '/tmp/mounted' || + str === '/tmp/source/.codex/AGENTS.md' || + str === '/tmp/source/.claude.json' + ); + }); + + mirrorContainerCodexSessionFiles('/tmp/source', '/tmp/mounted'); + + expect(vi.mocked(fs.copyFileSync)).toHaveBeenCalledWith( + '/tmp/source/.claude.json', + '/tmp/mounted/.claude.json', + ); + }); }); diff --git a/src/agent-runner.ts b/src/agent-runner.ts index cf1d218..eec92d4 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -13,6 +13,7 @@ import { IDLE_TIMEOUT, } from './config.js'; import { + ensureClaudeGlobalSettingsFile, prepareContainerSessionEnvironment, prepareGroupEnvironment, } from './agent-runner-environment.js'; @@ -78,6 +79,15 @@ export function mirrorContainerCodexSessionFiles( fs.copyFileSync(sourcePath, path.join(mountedCodexDir, file)); } } + + const sourceClaudeJson = path.join(sourceSessionDir, '.claude.json'); + if (fs.existsSync(sourceClaudeJson)) { + ensureClaudeGlobalSettingsFile(mountedSessionDir); + fs.copyFileSync( + sourceClaudeJson, + path.join(mountedSessionDir, '.claude.json'), + ); + } } export async function runAgentProcess( @@ -91,12 +101,16 @@ export async function runAgentProcess( onOutput?: (output: AgentOutput) => Promise, envOverrides?: Record, ): Promise { + const unsafeHostPairedMode = + envOverrides?.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; + // ── Reviewer container mode ───────────────────────────────────── // Reviewers always run inside a Docker container with read-only source // mount for kernel-level write protection. Docker is required. if ( + !unsafeHostPairedMode && envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1' || - envOverrides?.EJCLAW_ARBITER_RUNTIME === '1' + (!unsafeHostPairedMode && envOverrides?.EJCLAW_ARBITER_RUNTIME === '1') ) { const ownerWorkspaceDir = envOverrides?.EJCLAW_WORK_DIR || group.workDir || process.cwd(); @@ -171,6 +185,28 @@ export async function runAgentProcess( if (value) env[key] = value; } } + if ( + unsafeHostPairedMode && + envOverrides?.CLAUDE_CONFIG_DIR && + (input.roomRoleContext?.role === 'reviewer' || + input.roomRoleContext?.role === 'arbiter') + ) { + prepareContainerSessionEnvironment({ + sessionDir: envOverrides.CLAUDE_CONFIG_DIR, + chatJid: input.chatJid, + isMain: input.isMain, + groupFolder: group.folder, + agentType: group.agentType || 'claude-code', + memoryBriefing: input.memoryBriefing, + role: input.roomRoleContext.role, + ipcDir: env.EJCLAW_IPC_DIR, + hostIpcDir: env.EJCLAW_HOST_IPC_DIR, + workDir: envOverrides.EJCLAW_WORK_DIR || env.EJCLAW_WORK_DIR, + }); + if ((group.agentType || 'claude-code') === 'codex') { + env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex'); + } + } if (input.runId) { env.EJCLAW_RUN_ID = input.runId; } diff --git a/src/container-runner.test.ts b/src/container-runner.test.ts index b587bf5..0f491da 100644 --- a/src/container-runner.test.ts +++ b/src/container-runner.test.ts @@ -40,6 +40,10 @@ vi.mock('./container-runtime.js', () => ({ tmpfsMountArgs: (containerPath: string) => ['--tmpfs', containerPath], })); +vi.mock('./agent-runner-environment.js', () => ({ + ensureClaudeGlobalSettingsFile: vi.fn(), +})); + vi.mock('./credential-proxy.js', () => ({ detectAuthMode: mockDetectAuthMode, })); @@ -149,6 +153,16 @@ describe('container-runner path compatibility', () => { containerPath: ownerWorkspaceDir, readonly: true, }), + expect.objectContaining({ + hostPath: path.join( + TEST_DATA_DIR, + 'sessions', + `${group.folder}-reviewer`, + '.claude.json', + ), + containerPath: '/home/node/.claude.json', + readonly: false, + }), expect.objectContaining({ hostPath: '/dev/null', containerPath: '/workspace/project/.env', diff --git a/src/container-runner.ts b/src/container-runner.ts index 283fc38..7fb4771 100644 --- a/src/container-runner.ts +++ b/src/container-runner.ts @@ -32,6 +32,7 @@ import { tmpfsMountArgs, writableMountArgs, } from './container-runtime.js'; +import { ensureClaudeGlobalSettingsFile } from './agent-runner-environment.js'; import { detectAuthMode } from './credential-proxy.js'; import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js'; import { logger } from './logger.js'; @@ -435,11 +436,17 @@ export function buildReviewerMounts( `${group.folder}-reviewer`, ); fs.mkdirSync(groupSessionsDir, { recursive: true }); + ensureClaudeGlobalSettingsFile(groupSessionsDir); pushMountOnce(mounts, { hostPath: groupSessionsDir, containerPath: '/home/node/.claude', readonly: false, }); + pushMountOnce(mounts, { + hostPath: path.join(groupSessionsDir, '.claude.json'), + containerPath: '/home/node/.claude.json', + readonly: false, + }); // Owner session directory: read-only so reviewer can verify runtime state // files (cron state, configs, etc.) that the owner references by absolute path. diff --git a/src/paired-execution-context.test.ts b/src/paired-execution-context.test.ts index af7b99c..afb32d8 100644 --- a/src/paired-execution-context.test.ts +++ b/src/paired-execution-context.test.ts @@ -380,6 +380,46 @@ describe('paired execution context', () => { expect(result?.envOverrides.EJCLAW_WORK_DIR).toBeUndefined(); }); + it('routes reviewer to host mode when unsafe host paired mode is enabled', () => { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue( + buildPairedTask({ + status: 'review_ready', + review_requested_at: '2026-03-28T00:00:00.000Z', + }), + ); + vi.mocked(db.getPairedTaskById).mockReturnValue( + buildPairedTask({ + status: 'review_ready', + review_requested_at: '2026-03-28T00:00:00.000Z', + }), + ); + vi.mocked( + pairedWorkspaceManager.prepareReviewerWorkspaceForExecution, + ).mockReturnValue({ + workspace: buildWorkspace('reviewer', '/tmp/paired/task-1/reviewer'), + autoRefreshed: false, + }); + + const result = preparePairedExecutionContext({ + group, + chatJid: 'dc:test', + runId: 'run-host-reviewer', + roomRoleContext: reviewerContext, + }); + + expect(result?.envOverrides).toMatchObject({ + EJCLAW_WORK_DIR: '/tmp/paired/task-1/reviewer', + EJCLAW_PAIRED_ROLE: 'reviewer', + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + }); + expect(result?.envOverrides.EJCLAW_REVIEWER_RUNTIME).toBeUndefined(); + expect(result?.envOverrides.CLAUDE_CONFIG_DIR).toContain( + '/data/sessions/paired-room-reviewer', + ); + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + }); + it('completePairedExecutionContext logs without error', () => { completePairedExecutionContext({ taskId: 'task-1', diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index 966702d..005b864 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -368,12 +368,13 @@ export function preparePairedExecutionContext(args: { EJCLAW_PAIRED_TASK_ID: task.id, EJCLAW_PAIRED_ROLE: roomRoleContext.role, }; + const unsafeHostPairedMode = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; if (workspace?.workspace_dir) { envOverrides.EJCLAW_WORK_DIR = workspace.workspace_dir; } if (roomRoleContext.role === 'reviewer') { - envOverrides.EJCLAW_REVIEWER_RUNTIME = '1'; // Use a separate Claude config dir so the reviewer's SDK session cache // doesn't collide with the owner's. Without this, the Claude SDK picks // up the owner's cached session from disk even when sessionId is undefined. @@ -384,8 +385,12 @@ export function preparePairedExecutionContext(args: { ); fs.mkdirSync(reviewerSessionDir, { recursive: true }); envOverrides.CLAUDE_CONFIG_DIR = reviewerSessionDir; + if (unsafeHostPairedMode) { + envOverrides.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + } else { + envOverrides.EJCLAW_REVIEWER_RUNTIME = '1'; + } } else if (roomRoleContext.role === 'arbiter') { - envOverrides.EJCLAW_ARBITER_RUNTIME = '1'; const arbiterSessionDir = path.join( DATA_DIR, 'sessions', @@ -396,6 +401,11 @@ export function preparePairedExecutionContext(args: { fs.rmSync(arbiterSessionDir, { recursive: true, force: true }); fs.mkdirSync(arbiterSessionDir, { recursive: true }); envOverrides.CLAUDE_CONFIG_DIR = arbiterSessionDir; + if (unsafeHostPairedMode) { + envOverrides.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + } else { + envOverrides.EJCLAW_ARBITER_RUNTIME = '1'; + } } return { From d75d26c23d22670ead51817684c279ba7b9ae498 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 07:49:26 +0900 Subject: [PATCH 04/29] Stabilize paired reviewer recovery and owner follow-up --- README.md | 2 +- docs/architecture.md | 2 +- runners/agent-runner/src/index.ts | 54 +++- runners/agent-runner/src/reviewer-runtime.ts | 85 ++++++ .../test/reviewer-runtime.test.ts | 101 ++++++- setup/environment.ts | 27 +- setup/platform.test.ts | 215 ++++++++++++++ setup/platform.ts | 110 ++++++++ setup/service.ts | 23 +- src/agent-runner.ts | 3 +- src/config.ts | 2 +- src/message-agent-executor.test.ts | 223 ++++++++++++++- src/message-agent-executor.ts | 86 +++--- src/message-runtime.test.ts | 224 +++++++++++++++ src/message-runtime.ts | 40 +++ src/paired-execution-context.test.ts | 34 ++- src/paired-execution-context.ts | 3 + src/paired-workspace-manager.test.ts | 7 +- src/provider-retry.test.ts | 63 +++++ src/provider-retry.ts | 263 +++++++++++------- src/task-scheduler.test.ts | 9 + src/token-refresh.ts | 14 +- src/token-rotation.ts | 7 + src/verification.test.ts | 4 +- src/verification.ts | 6 +- src/workspace-package-manager.test.ts | 10 +- src/workspace-package-manager.ts | 23 +- 27 files changed, 1474 insertions(+), 166 deletions(-) diff --git a/README.md b/README.md index 0eba661..5bab52a 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ User message ├─ Arbiter enabled → Arbiter judges → PROCEED/REVISE/RESET/ESCALATE └─ Arbiter disabled → Escalate to user → @user ⚠️ → Owner BLOCKED/NEEDS_CONTEXT → Arbiter (same path as reviewer) - → Deadlock (3+ round trips without progress) + → Deadlock (2+ round trips without progress) → Arbiter summoned → binding verdict → loop resumes ``` diff --git a/docs/architecture.md b/docs/architecture.md index f23f857..542151c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -73,7 +73,7 @@ User message ├─ Arbiter enabled → Arbiter judges → PROCEED/REVISE/RESET/ESCALATE └─ Arbiter disabled → Escalate to user → @user ⚠️ → Owner BLOCKED/NEEDS_CONTEXT → Arbiter (same path as reviewer) - → Deadlock (3+ round trips without progress) + → Deadlock (2+ round trips without progress) → Arbiter summoned → binding verdict → loop resumes ``` diff --git a/runners/agent-runner/src/index.ts b/runners/agent-runner/src/index.ts index 1f7a38d..4478502 100644 --- a/runners/agent-runner/src/index.ts +++ b/runners/agent-runner/src/index.ts @@ -25,7 +25,10 @@ import { } from './room-role-context.js'; import { assertReadonlyWorkspaceRepoConnectivity, + buildClaudeReadonlySandboxSettings, buildReviewerGitGuardEnv, + getClaudeReadonlySandboxMode, + isClaudeReadonlyReviewerRuntime, isReviewerMutatingShellCommand, isReviewerRuntime, } from './reviewer-runtime.js'; @@ -545,6 +548,8 @@ async function runQuery( containerInput: ContainerInput, sdkEnv: Record, reviewerRuntime: boolean, + claudeReadonlyReviewerRuntime: boolean, + claudeReadonlySandboxMode: 'strict' | 'best-effort' | null, ): Promise<{ newSessionId?: string; closedDuringQuery: boolean; @@ -622,8 +627,20 @@ async function runQuery( if (thinking) log(`Thinking config: ${JSON.stringify(thinking)}`); if (effort) log(`Effort: ${effort}`); if (reviewerRuntime) log('Reviewer runtime restrictions enabled'); + if (claudeReadonlyReviewerRuntime) { + log( + `Claude host reviewer read-only sandbox enabled (${claudeReadonlySandboxMode || 'unknown'})`, + ); + } + if (claudeReadonlySandboxMode === 'best-effort') { + log( + 'Claude host reviewer sandbox capability unavailable on this host, using best-effort read-only mode', + ); + } - const allowedTools = reviewerRuntime + const readonlyReviewerRuntime = + reviewerRuntime || claudeReadonlyReviewerRuntime; + const allowedTools = readonlyReviewerRuntime ? [ 'Bash', 'Read', 'Glob', 'Grep', @@ -644,6 +661,18 @@ async function runQuery( 'mcp__ejclaw__*' ]; + const readonlyProtectedPaths = [effectiveCwd, ...extraDirs].filter( + (value): value is string => Boolean(value), + ); + const claudeReadonlySandboxSettings = + claudeReadonlyReviewerRuntime && claudeReadonlySandboxMode + ? buildClaudeReadonlySandboxSettings( + readonlyProtectedPaths, + undefined, + claudeReadonlySandboxMode, + ) + : undefined; + for await (const message of query({ prompt: stream, options: { @@ -657,6 +686,11 @@ async function runQuery( env: sdkEnv, permissionMode: 'bypassPermissions', allowDangerouslySkipPermissions: true, + ...(claudeReadonlySandboxSettings + ? { + sandbox: claudeReadonlySandboxSettings, + } + : {}), settingSources: ['project', 'user'], abortController: agentAbortController, mcpServers: { @@ -684,7 +718,7 @@ async function runQuery( PreToolUse: [ { matcher: 'Bash', - hooks: reviewerRuntime + hooks: readonlyReviewerRuntime ? [createReviewerBashGuardHook(), createSanitizeBashHook()] : [createSanitizeBashHook()], }, @@ -950,9 +984,19 @@ async function main(): Promise { const reviewerRuntime = process.env.EJCLAW_REVIEWER_RUNTIME === '1' || isReviewerRuntime(containerInput.roomRoleContext); + const claudeReadonlyReviewerRuntime = + isClaudeReadonlyReviewerRuntime(containerInput.roomRoleContext); + const claudeReadonlySandboxMode = claudeReadonlyReviewerRuntime + ? getClaudeReadonlySandboxMode() + : null; const readonlyRuntime = - reviewerRuntime || process.env.EJCLAW_ARBITER_RUNTIME === '1'; - const guardedSdkEnv = buildReviewerGitGuardEnv(sdkEnv, reviewerRuntime); + reviewerRuntime || + claudeReadonlyReviewerRuntime || + process.env.EJCLAW_ARBITER_RUNTIME === '1'; + const guardedSdkEnv = buildReviewerGitGuardEnv( + sdkEnv, + reviewerRuntime || claudeReadonlyReviewerRuntime, + ); assertReadonlyWorkspaceRepoConnectivity(guardedSdkEnv, readonlyRuntime); const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -1092,6 +1136,8 @@ async function main(): Promise { containerInput, guardedSdkEnv, reviewerRuntime, + claudeReadonlyReviewerRuntime, + claudeReadonlySandboxMode, ); if (queryResult.newSessionId) { sessionId = queryResult.newSessionId; diff --git a/runners/agent-runner/src/reviewer-runtime.ts b/runners/agent-runner/src/reviewer-runtime.ts index baf26da..9496995 100644 --- a/runners/agent-runner/src/reviewer-runtime.ts +++ b/runners/agent-runner/src/reviewer-runtime.ts @@ -5,6 +5,8 @@ import path from 'path'; import type { RoomRoleContext } from './room-role-context.js'; +export type ClaudeReadonlySandboxMode = 'strict' | 'best-effort'; + const BLOCKED_GIT_SUBCOMMANDS = new Set([ 'add', 'am', @@ -40,6 +42,89 @@ export function isReviewerRuntime( return roomRoleContext?.role === 'reviewer'; } +export function isClaudeReadonlyReviewerRuntime( + roomRoleContext?: RoomRoleContext, +): boolean { + return ( + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1' && + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY === '1' && + roomRoleContext?.role === 'reviewer' + ); +} + +let cachedLinuxBubblewrapReadonlyCapability: boolean | undefined; + +function commandExistsForRuntime(command: string): boolean { + try { + execFileSync('bash', ['-lc', `command -v ${command}`], { + stdio: ['ignore', 'ignore', 'ignore'], + }); + return true; + } catch { + return false; + } +} + +export function canUseLinuxBubblewrapReadonlySandbox(): boolean { + if (cachedLinuxBubblewrapReadonlyCapability != null) { + return cachedLinuxBubblewrapReadonlyCapability; + } + if (os.platform() !== 'linux') { + cachedLinuxBubblewrapReadonlyCapability = false; + return false; + } + if (!commandExistsForRuntime('bwrap')) { + cachedLinuxBubblewrapReadonlyCapability = false; + return false; + } + + try { + execFileSync('bwrap', ['--ro-bind', '/', '/', '/bin/true'], { + stdio: ['ignore', 'ignore', 'ignore'], + }); + cachedLinuxBubblewrapReadonlyCapability = true; + } catch { + cachedLinuxBubblewrapReadonlyCapability = false; + } + + return cachedLinuxBubblewrapReadonlyCapability; +} + +export function getClaudeReadonlySandboxMode( + platform: NodeJS.Platform = os.platform(), + linuxCapabilityProbe: () => boolean = canUseLinuxBubblewrapReadonlySandbox, +): ClaudeReadonlySandboxMode { + if (platform === 'linux') { + return linuxCapabilityProbe() ? 'strict' : 'best-effort'; + } + return 'best-effort'; +} + +export function buildClaudeReadonlySandboxSettings( + protectedPaths: string[], + platform: NodeJS.Platform = os.platform(), + sandboxMode: ClaudeReadonlySandboxMode = getClaudeReadonlySandboxMode( + platform, + ), +) { + const normalizedPaths = [...new Set( + protectedPaths + .filter((value): value is string => Boolean(value)) + .map((value) => path.resolve(value)), + )]; + + return { + enabled: true, + failIfUnavailable: sandboxMode === 'strict', + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: + normalizedPaths.length > 0 + ? { denyWrite: normalizedPaths } + : undefined, + }; +} + function resolveGitBinary(baseEnv: NodeJS.ProcessEnv): string { return execFileSync('bash', ['-lc', 'command -v git'], { encoding: 'utf-8', diff --git a/runners/agent-runner/test/reviewer-runtime.test.ts b/runners/agent-runner/test/reviewer-runtime.test.ts index 1dafd50..d5826e7 100644 --- a/runners/agent-runner/test/reviewer-runtime.test.ts +++ b/runners/agent-runner/test/reviewer-runtime.test.ts @@ -3,15 +3,44 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertReadonlyWorkspaceRepoConnectivity, + buildClaudeReadonlySandboxSettings, buildReviewerGitGuardEnv, + getClaudeReadonlySandboxMode, + isClaudeReadonlyReviewerRuntime, isReviewerMutatingShellCommand, isReviewerRuntime, } from '../src/reviewer-runtime.js'; +const ORIGINAL_CLAUDE_REVIEWER_READONLY = + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + +afterEach(() => { + if (ORIGINAL_CLAUDE_REVIEWER_READONLY == null) { + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + } else { + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY = + ORIGINAL_CLAUDE_REVIEWER_READONLY; + } + + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } +}); + +beforeEach(() => { + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; +}); + function createTempRepo(prefix: string): string { const cwd = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); execFileSync('git', ['init'], { @@ -56,6 +85,76 @@ describe('claude reviewer runtime guard', () => { ).toBe(true); }); + it('detects claude host reviewer read-only mode from env + role', () => { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY = '1'; + + expect( + isClaudeReadonlyReviewerRuntime({ + serviceId: 'claude', + role: 'reviewer', + ownerServiceId: 'codex-main', + reviewerServiceId: 'claude', + failoverOwner: false, + }), + ).toBe(true); + expect( + isClaudeReadonlyReviewerRuntime({ + serviceId: 'claude', + role: 'owner', + ownerServiceId: 'codex-main', + reviewerServiceId: 'claude', + failoverOwner: false, + }), + ).toBe(false); + }); + + it('builds a read-only sandbox with normalized protected paths', () => { + const sandbox = buildClaudeReadonlySandboxSettings([ + '/repo/work', + '/repo/work', + '/repo/owner/../owner', + ], 'linux', 'strict'); + + expect(sandbox).toMatchObject({ + enabled: true, + failIfUnavailable: true, + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: { + denyWrite: ['/repo/work', '/repo/owner'], + }, + }); + }); + + it('uses best-effort sandbox mode when linux capability probe fails', () => { + expect(getClaudeReadonlySandboxMode('linux', () => false)).toBe( + 'best-effort', + ); + + const sandbox = buildClaudeReadonlySandboxSettings( + ['/repo/work'], + 'linux', + 'best-effort', + ); + + expect(sandbox.failIfUnavailable).toBe(false); + }); + + it('keeps non-linux reviewers in best-effort sandbox mode', () => { + expect(getClaudeReadonlySandboxMode('darwin', () => true)).toBe( + 'best-effort', + ); + + const sandbox = buildClaudeReadonlySandboxSettings( + ['/repo/work'], + 'darwin', + 'best-effort', + ); + + expect(sandbox.failIfUnavailable).toBe(false); + }); + it('flags mutating shell commands', () => { expect(isReviewerMutatingShellCommand('git commit -m "x"')).toBe(false); expect(isReviewerMutatingShellCommand('git -c color.ui=false commit -m "x"')).toBe( diff --git a/setup/environment.ts b/setup/environment.ts index e6715b5..15a68fc 100644 --- a/setup/environment.ts +++ b/setup/environment.ts @@ -9,7 +9,14 @@ import { Database } from 'bun:sqlite'; import { STORE_DIR } from '../src/config.js'; import { logger } from '../src/logger.js'; -import { getPlatform, isHeadless, isWSL } from './platform.js'; +import { + canUseLinuxBubblewrapReadonlySandbox, + commandExists, + getAppArmorRestrictUnprivilegedUsernsValue, + getPlatform, + isHeadless, + isWSL, +} from './platform.js'; import { emitStatus } from './status.js'; export async function run(_args: string[]): Promise { @@ -20,6 +27,14 @@ export async function run(_args: string[]): Promise { const platform = getPlatform(); const wsl = isWSL(); const headless = isHeadless(); + const hasBubblewrap = platform === 'linux' && commandExists('bwrap'); + const hasSocat = platform === 'linux' && commandExists('socat'); + const apparmorRestrictUnprivilegedUserns = + platform === 'linux' + ? getAppArmorRestrictUnprivilegedUsernsValue() + : null; + const hasBubblewrapReadonlySandboxCapability = + platform === 'linux' && canUseLinuxBubblewrapReadonlySandbox(); // Check existing config const hasEnv = fs.existsSync(path.join(projectRoot, '.env')); @@ -55,6 +70,10 @@ export async function run(_args: string[]): Promise { hasEnv, hasAuth, hasRegisteredGroups, + hasBubblewrap, + hasSocat, + apparmorRestrictUnprivilegedUserns, + hasBubblewrapReadonlySandboxCapability, }, 'Environment check complete', ); @@ -66,6 +85,12 @@ export async function run(_args: string[]): Promise { HAS_ENV: hasEnv, HAS_AUTH: hasAuth, HAS_REGISTERED_GROUPS: hasRegisteredGroups, + HAS_BWRAP: hasBubblewrap, + HAS_SOCAT: hasSocat, + APPARMOR_RESTRICT_UNPRIVILEGED_USERNS: + apparmorRestrictUnprivilegedUserns ?? 'n/a', + HAS_BWRAP_READONLY_SANDBOX_CAPABILITY: + hasBubblewrapReadonlySandboxCapability, STATUS: 'success', LOG: 'logs/setup.log', }); diff --git a/setup/platform.test.ts b/setup/platform.test.ts index 1604d3e..7dba53c 100644 --- a/setup/platform.test.ts +++ b/setup/platform.test.ts @@ -1,6 +1,9 @@ import { describe, it, expect } from 'vitest'; import { + ensureLinuxReadonlySandboxAppArmorSupport, + getAppArmorRestrictUnprivilegedUsernsValue, + canUseLinuxBubblewrapReadonlySandbox, getPlatform, isWSL, isRoot, @@ -99,6 +102,218 @@ describe('commandExists', () => { }); }); +describe('canUseLinuxBubblewrapReadonlySandbox', () => { + it('returns false outside linux', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'macos', + () => { + throw new Error('should not probe commands on macOS'); + }, + ), + ).toBe(false); + }); + + it('returns false when bwrap is not installed', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'linux', + () => false, + ), + ).toBe(false); + }); + + it('returns true when linux bwrap readonly probe succeeds', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'linux', + () => true, + (() => Buffer.from('')) as typeof import('child_process').execSync, + ), + ).toBe(true); + }); + + it('returns false when linux bwrap readonly probe fails', () => { + expect( + canUseLinuxBubblewrapReadonlySandbox( + 'linux', + () => true, + (() => { + throw new Error('permission denied'); + }) as typeof import('child_process').execSync, + ), + ).toBe(false); + }); +}); + +describe('getAppArmorRestrictUnprivilegedUsernsValue', () => { + it('returns null when the proc sysctl path is unavailable', () => { + expect( + getAppArmorRestrictUnprivilegedUsernsValue((() => { + throw new Error('missing'); + }) as typeof import('fs').readFileSync), + ).toBeNull(); + }); + + it('returns the trimmed proc sysctl value', () => { + expect( + getAppArmorRestrictUnprivilegedUsernsValue((() => + '1\n') as typeof import('fs').readFileSync), + ).toBe('1'); + }); +}); + +describe('ensureLinuxReadonlySandboxAppArmorSupport', () => { + it('does nothing outside linux', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'macos', + }), + ).toBe('not-linux'); + }); + + it('does nothing when apparmor does not block sandboxing', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '0', + sandboxCapable: false, + }), + ).toBe('not-needed'); + }); + + it('does nothing when sandbox is already capable', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: true, + }), + ).toBe('not-needed'); + }); + + it('requires root when linux apparmor blocks sandboxing', () => { + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: false, + }), + ).toBe('requires-root'); + }); + + it('writes and applies the sysctl when running as root', () => { + const writes: string[] = []; + const commands: string[] = []; + const fileContents = new Map([ + ['/proc/sys/kernel/apparmor_restrict_unprivileged_userns', '1\n'], + ]); + + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: true, + readFileSyncFn: ((target) => { + const value = fileContents.get(String(target)); + if (value == null) throw new Error('missing'); + return value; + }) as typeof import('fs').readFileSync, + mkdirSyncFn: (() => undefined) as typeof import('fs').mkdirSync, + writeFileSyncFn: ((target, contents) => { + const normalizedTarget = String(target); + const normalizedContents = String(contents); + writes.push(`${normalizedTarget}\n${normalizedContents}`); + fileContents.set(normalizedTarget, normalizedContents); + }) as typeof import('fs').writeFileSync, + execFn: ((command) => { + commands.push(String(command)); + fileContents.set( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + '0\n', + ); + return Buffer.from(''); + }) as typeof import('child_process').execSync, + }), + ).toBe('configured'); + + expect(writes[0]).toContain('/etc/sysctl.d/90-ejclaw-sandbox.conf'); + expect(writes[0]).toContain( + 'kernel.apparmor_restrict_unprivileged_userns=0', + ); + expect(commands).toContain( + 'sysctl -w kernel.apparmor_restrict_unprivileged_userns=0', + ); + }); + + it('skips rewriting an already-correct sysctl file but still applies and verifies', () => { + const writes: string[] = []; + const commands: string[] = []; + const fileContents = new Map([ + ['/etc/sysctl.d/90-ejclaw-sandbox.conf', + '# Managed by EJClaw setup to allow bubblewrap readonly sandboxing.\n' + + 'kernel.apparmor_restrict_unprivileged_userns=0\n'], + ['/proc/sys/kernel/apparmor_restrict_unprivileged_userns', '1\n'], + ]); + + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: true, + readFileSyncFn: ((target) => { + const value = fileContents.get(String(target)); + if (value == null) throw new Error('missing'); + return value; + }) as typeof import('fs').readFileSync, + mkdirSyncFn: (() => undefined) as typeof import('fs').mkdirSync, + writeFileSyncFn: ((target, contents) => { + writes.push(`${String(target)}\n${String(contents)}`); + }) as typeof import('fs').writeFileSync, + execFn: ((command) => { + commands.push(String(command)); + fileContents.set( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + '0\n', + ); + return Buffer.from(''); + }) as typeof import('child_process').execSync, + }), + ).toBe('configured'); + + expect(writes).toHaveLength(0); + expect(commands).toContain( + 'sysctl -w kernel.apparmor_restrict_unprivileged_userns=0', + ); + }); + + it('returns failed when the sysctl apply does not take effect', () => { + const fileContents = new Map([ + ['/proc/sys/kernel/apparmor_restrict_unprivileged_userns', '1\n'], + ]); + + expect( + ensureLinuxReadonlySandboxAppArmorSupport({ + platform: 'linux', + apparmorRestrictUnprivilegedUserns: '1', + sandboxCapable: false, + isRootUser: true, + readFileSyncFn: ((target) => { + const value = fileContents.get(String(target)); + if (value == null) throw new Error('missing'); + return value; + }) as typeof import('fs').readFileSync, + mkdirSyncFn: (() => undefined) as typeof import('fs').mkdirSync, + writeFileSyncFn: (() => undefined) as typeof import('fs').writeFileSync, + execFn: (() => Buffer.from('')) as typeof import('child_process').execSync, + }), + ).toBe('failed'); + }); +}); + // --- getNodeVersion --- describe('getNodeVersion', () => { diff --git a/setup/platform.ts b/setup/platform.ts index a0fba9e..3ed57eb 100644 --- a/setup/platform.ts +++ b/setup/platform.ts @@ -7,6 +7,12 @@ import os from 'os'; export type Platform = 'macos' | 'linux' | 'unknown'; export type ServiceManager = 'launchd' | 'systemd' | 'none'; +export type LinuxReadonlySandboxAppArmorSetupResult = + | 'not-linux' + | 'not-needed' + | 'requires-root' + | 'failed' + | 'configured'; export function getPlatform(): Platform { const platform = os.platform(); @@ -115,6 +121,110 @@ export function commandExists(name: string): boolean { } } +export function canUseLinuxBubblewrapReadonlySandbox( + platform: Platform = getPlatform(), + commandExistsFn: (name: string) => boolean = commandExists, + execFn: typeof execSync = execSync, +): boolean { + if (platform !== 'linux') return false; + if (!commandExistsFn('bwrap')) return false; + + try { + execFn('bwrap --ro-bind / / /bin/true', { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function readProcSysValue( + procPath: string, + readFileSyncFn: typeof fs.readFileSync = fs.readFileSync, +): string | null { + try { + return readFileSyncFn(procPath, 'utf-8').trim(); + } catch { + return null; + } +} + +export function getAppArmorRestrictUnprivilegedUsernsValue( + readFileSyncFn: typeof fs.readFileSync = fs.readFileSync, +): string | null { + return readProcSysValue( + '/proc/sys/kernel/apparmor_restrict_unprivileged_userns', + readFileSyncFn, + ); +} + +export function ensureLinuxReadonlySandboxAppArmorSupport(options?: { + platform?: Platform; + isRootUser?: boolean; + apparmorRestrictUnprivilegedUserns?: string | null; + sandboxCapable?: boolean; + readFileSyncFn?: typeof fs.readFileSync; + mkdirSyncFn?: typeof fs.mkdirSync; + writeFileSyncFn?: typeof fs.writeFileSync; + execFn?: typeof execSync; +}): LinuxReadonlySandboxAppArmorSetupResult { + const platform = options?.platform ?? getPlatform(); + if (platform !== 'linux') return 'not-linux'; + const readFileSyncFn = options?.readFileSyncFn ?? fs.readFileSync; + + const apparmorRestrictUnprivilegedUserns = + options?.apparmorRestrictUnprivilegedUserns ?? + getAppArmorRestrictUnprivilegedUsernsValue(readFileSyncFn); + const sandboxCapable = + options?.sandboxCapable ?? canUseLinuxBubblewrapReadonlySandbox(platform); + + if ( + apparmorRestrictUnprivilegedUserns !== '1' || + sandboxCapable + ) { + return 'not-needed'; + } + + const isRootUser = options?.isRootUser ?? isRoot(); + if (!isRootUser) return 'requires-root'; + + const mkdirSyncFn = options?.mkdirSyncFn ?? fs.mkdirSync; + const writeFileSyncFn = options?.writeFileSyncFn ?? fs.writeFileSync; + const execFn = options?.execFn ?? execSync; + const sysctlPath = '/etc/sysctl.d/90-ejclaw-sandbox.conf'; + const sysctlContents = + '# Managed by EJClaw setup to allow bubblewrap readonly sandboxing.\n' + + 'kernel.apparmor_restrict_unprivileged_userns=0\n'; + let existingContents: string | null = null; + + try { + existingContents = readFileSyncFn(sysctlPath, 'utf-8'); + } catch { + existingContents = null; + } + + try { + mkdirSyncFn('/etc/sysctl.d', { recursive: true }); + if (existingContents !== sysctlContents) { + writeFileSyncFn(sysctlPath, sysctlContents); + } + execFn('sysctl -w kernel.apparmor_restrict_unprivileged_userns=0', { + stdio: 'ignore', + }); + + const appliedValue = getAppArmorRestrictUnprivilegedUsernsValue( + readFileSyncFn, + ); + if (appliedValue !== '0') { + throw new Error( + `kernel.apparmor_restrict_unprivileged_userns remained ${appliedValue ?? 'unavailable'} after sysctl apply`, + ); + } + return 'configured'; + } catch { + return 'failed'; + } +} + export function getNodeVersion(): string | null { try { const version = execSync('node --version', { encoding: 'utf-8' }).trim(); diff --git a/setup/service.ts b/setup/service.ts index 26e82d0..2c0f8d0 100644 --- a/setup/service.ts +++ b/setup/service.ts @@ -11,7 +11,12 @@ import os from 'os'; import path from 'path'; import { logger } from '../src/logger.js'; -import { getPlatform, getNodePath, getServiceManager } from './platform.js'; +import { + ensureLinuxReadonlySandboxAppArmorSupport, + getPlatform, + getNodePath, + getServiceManager, +} from './platform.js'; import { detectLegacyServiceIssues, formatLegacyServiceFailureMessage, @@ -33,6 +38,22 @@ export async function run(_args: string[]): Promise { logger.info({ platform, nodePath, projectRoot }, 'Setting up service'); + const readonlySandboxAppArmorSetup = + ensureLinuxReadonlySandboxAppArmorSupport(); + if (readonlySandboxAppArmorSetup === 'configured') { + logger.info( + 'Configured AppArmor user namespace sysctl for EJClaw readonly sandbox support', + ); + } else if (readonlySandboxAppArmorSetup === 'failed') { + logger.warn( + 'Failed to apply AppArmor user namespace sysctl for EJClaw readonly sandbox support', + ); + } else if (readonlySandboxAppArmorSetup === 'requires-root') { + logger.warn( + 'Readonly sandbox strict mode requires root setup to disable AppArmor unprivileged userns restriction', + ); + } + const legacyServiceIssues = detectLegacyServiceIssues( projectRoot, serviceManager, diff --git a/src/agent-runner.ts b/src/agent-runner.ts index eec92d4..8978e8c 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -108,8 +108,7 @@ export async function runAgentProcess( // Reviewers always run inside a Docker container with read-only source // mount for kernel-level write protection. Docker is required. if ( - !unsafeHostPairedMode && - envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1' || + (!unsafeHostPairedMode && envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1') || (!unsafeHostPairedMode && envOverrides?.EJCLAW_ARBITER_RUNTIME === '1') ) { const ownerWorkspaceDir = diff --git a/src/config.ts b/src/config.ts index 23af224..c208b14 100644 --- a/src/config.ts +++ b/src/config.ts @@ -120,7 +120,7 @@ export const AGENT_LANGUAGE = getEnv('AGENT_LANGUAGE') || ''; /** Number of consecutive owner↔reviewer round trips before arbiter is auto-requested. */ export const ARBITER_DEADLOCK_THRESHOLD = parseInt( - getEnv('ARBITER_DEADLOCK_THRESHOLD') || '3', + getEnv('ARBITER_DEADLOCK_THRESHOLD') || '2', 10, ); diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index fe92ff7..2a815b6 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('./agent-runner.js', () => ({ runAgentProcess: vi.fn(), @@ -156,9 +156,14 @@ vi.mock('./session-recovery.js', () => ({ vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), + getCurrentTokenIndex: vi.fn(() => 0), markTokenHealthy: vi.fn(), })); +vi.mock('./token-refresh.js', () => ({ + forceRefreshToken: vi.fn(async () => null), +})); + vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn((error?: string | null) => { const lower = (error || '').toLowerCase(); @@ -198,6 +203,7 @@ import { runAgentForGroup } from './message-agent-executor.js'; import * as pairedExecutionContext from './paired-execution-context.js'; import * as sessionRecovery from './session-recovery.js'; import * as serviceRouting from './service-routing.js'; +import * as tokenRefresh from './token-refresh.js'; import * as tokenRotation from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; @@ -226,9 +232,13 @@ function makeDeps() { }; } +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + describe('runAgentForGroup room memory', () => { beforeEach(() => { vi.resetAllMocks(); + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; vi.mocked(agentRunner.runAgentProcess).mockImplementation( async (_group, _input, _onProcess, onOutput) => { await onOutput?.({ @@ -245,6 +255,15 @@ describe('runAgentForGroup room memory', () => { ); }); + afterAll(() => { + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE === undefined) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } + }); + it('injects a room memory briefing when starting a fresh session', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); @@ -1031,6 +1050,108 @@ describe('runAgentForGroup room memory', () => { ); }); + it('starts reviewer Claude fresh in unsafe host mode instead of resuming a stored session', async () => { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + const group = { + ...makeGroup(), + folder: 'test-group', + agentType: 'codex' as const, + }; + const deps = { + ...makeDeps(), + getSessions: () => ({ 'test-group:reviewer': 'reviewer-session' }), + }; + + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_agent_type: 'codex', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + arbiter_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'paired-task-review', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-review', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: { + EJCLAW_PAIRED_TASK_ID: 'paired-task-review', + EJCLAW_PAIRED_ROLE: 'reviewer', + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + CLAUDE_CONFIG_DIR: '/tmp/test-group-reviewer', + }, + }); + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: 'review ok', + newSessionId: 'review-session-new', + }); + + await runAgentForGroup(deps, { + group, + prompt: 'please review', + chatJid: 'group@test', + runId: 'run-review-plan-unsafe-host', + }); + + expect(deps.clearSession).toHaveBeenCalledWith('test-group:reviewer'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledWith( + expect.objectContaining({ + folder: 'test-group', + agentType: 'claude-code', + }), + expect.objectContaining({ + sessionId: undefined, + }), + expect.any(Function), + undefined, + expect.objectContaining({ + EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', + }), + ); + expect(deps.persistSession).not.toHaveBeenCalled(); + }); + it('does not enqueue a second generic follow-up when reviewer approval already moved the task to merge_ready', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); @@ -1123,7 +1244,9 @@ describe('runAgentForGroup Claude rotation', () => { vi.resetAllMocks(); vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined); vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + vi.mocked(tokenRotation.getCurrentTokenIndex).mockReturnValue(0); vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); + vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null); }); it('rotates to another Claude account on usage exhaustion', async () => { @@ -1229,6 +1352,62 @@ describe('runAgentForGroup Claude rotation', () => { expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']); }); + it('force-refreshes the active Claude token before rotating on auth-expired', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValueOnce( + 'new-access-token', + ); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }); + await onOutput?.({ + status: 'success', + phase: 'final', + result: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'force refresh 뒤 Claude 응답입니다.', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-auth-expired-force-refresh', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRefresh.forceRefreshToken).toHaveBeenCalledWith(0); + expect(tokenRotation.rotateToken).not.toHaveBeenCalled(); + expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); + expect(outputs).toEqual(['force refresh 뒤 Claude 응답입니다.']); + }); + it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => { const outputs: string[] = []; @@ -1505,6 +1684,48 @@ describe('runAgentForGroup Claude rotation', () => { ); }); + it('drops a stale Claude session id before retrying a fresh session', async () => { + const deps = { + ...makeDeps(), + getSessions: () => ({ 'test-claude': 'stale-session-id' }), + }; + + vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, input) => { + expect(input.sessionId).toBe('stale-session-id'); + throw new Error('No conversation found with session ID: stale-session-id'); + }) + .mockImplementationOnce(async (_group, input, _onProcess, onOutput) => { + expect(input.sessionId).toBeUndefined(); + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'fresh retry success', + }); + return { + status: 'success', + result: 'fresh retry success', + newSessionId: 'fresh-session-id', + }; + }); + + const result = await runAgentForGroup(deps, { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-stale-session-id-retry', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(deps.clearSession).toHaveBeenCalledWith('test-claude'); + }); + it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => { const outputs: string[] = []; diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 44661f1..17bf68d 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -150,12 +150,22 @@ export async function runAgentForGroup( activeRole, group.agentType, ); + const unsafeHostPairedMode = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; + const forceFreshClaudeReviewerSession = + reviewerMode && isClaudeCodeAgent && unsafeHostPairedMode; + const storedSessionId = sessions[sessionFolder]; + if (forceFreshClaudeReviewerSession && storedSessionId) { + deps.clearSession(sessionFolder); + } // Arbiter always starts fresh — never resume a previous session - const sessionId = - activeRole === 'arbiter' || args.forcedAgentType + let currentSessionId = + activeRole === 'arbiter' || + args.forcedAgentType || + forceFreshClaudeReviewerSession ? undefined - : sessions[sessionFolder]; - const memoryBriefing = sessionId + : storedSessionId; + const memoryBriefing = currentSessionId ? undefined : await buildRoomMemoryBriefing({ groupFolder: group.folder, @@ -185,7 +195,9 @@ export async function runAgentForGroup( let resetSessionRequested = false; - const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1; + const claudeTokenCount = isClaudeCodeAgent ? getTokenCount() : 0; + const canRetryClaudeCredentials = isClaudeCodeAgent && claudeTokenCount > 0; + const canRotateToken = isClaudeCodeAgent && claudeTokenCount > 1; const roomRoleContext = buildRoomRoleContext( currentLease, effectiveServiceId, @@ -212,6 +224,7 @@ export async function runAgentForGroup( pairedExecutionContext.envOverrides[effortKey] = roleConfig.effort; } } + const log = createScopedLogger({ chatJid, groupName: group.name, @@ -243,11 +256,27 @@ export async function runAgentForGroup( reviewerMode, arbiterMode, sessionFolder, - resumedSession: sessionId ?? null, + resumedSession: currentSessionId ?? null, }, 'Resolved execution target for agent turn', ); + const clearRoleSdkSessions = (): void => { + const configDir = pairedExecutionContext?.envOverrides?.CLAUDE_CONFIG_DIR; + if (!configDir) return; + for (const sdkDir of ['.claude', '.codex']) { + const sessionsDir = path.join(configDir, sdkDir, 'sessions'); + if (fs.existsSync(sessionsDir)) { + fs.rmSync(sessionsDir, { recursive: true, force: true }); + log.info({ sessionsDir }, 'Cleared SDK sessions for fresh Claude turn'); + } + } + }; + + if (forceFreshClaudeReviewerSession) { + clearRoleSdkSessions(); + } + // ── MoA prompt enrichment ───────────────────────────────────── // When MoA is enabled and we're in arbiter mode, query external API // models (Kimi, GLM, etc.) in parallel for their opinions, then inject @@ -394,7 +423,7 @@ export async function runAgentForGroup( const agentInput = { prompt: effectivePrompt, - sessionId, + sessionId: currentSessionId, memoryBriefing, groupFolder: group.folder, chatJid, @@ -451,6 +480,7 @@ export async function runAgentForGroup( sawVisibleOutput: false, sawSuccessNullResultWithoutOutput: false, }; + const attemptSessionId = provider === 'claude' ? currentSessionId : undefined; const wrappedOnOutput = onOutput ? async (output: AgentOutput) => { @@ -476,7 +506,7 @@ export async function runAgentForGroup( effectiveServiceId, effectiveAgentType, sessionFolder, - resumedSession: sessionId ?? null, + resumedSession: attemptSessionId ?? null, streamedSessionId: output.newSessionId ?? null, roomRoleServiceId: roomRoleContext?.serviceId ?? null, roomRole: roomRoleContext?.role ?? null, @@ -499,9 +529,11 @@ export async function runAgentForGroup( if ( provider === 'claude' && output.newSessionId && - !resetSessionRequested + !resetSessionRequested && + !forceFreshClaudeReviewerSession ) { deps.persistSession(sessionFolder, output.newSessionId); + currentSessionId = output.newSessionId; } const evaluation = evaluateStreamedOutput(output, streamedState, { agentType: isClaudeCodeAgent ? 'claude-code' : 'codex', @@ -510,7 +542,7 @@ export async function runAgentForGroup( trackSuccessNullResult: true, shortCircuitTriggeredErrors: provider === 'claude' - ? canRotateToken + ? canRetryClaudeCredentials : getCodexAccountCount() > 1, }); streamedState = evaluation.state; @@ -601,7 +633,7 @@ export async function runAgentForGroup( effectiveGroup, { ...agentInput, - sessionId: provider === 'claude' ? sessionId : undefined, + sessionId: attemptSessionId, }, (proc, processName, ipcDir) => deps.queue.registerProcess(chatJid, proc, processName, ipcDir), @@ -609,8 +641,13 @@ export async function runAgentForGroup( pairedExecutionContext?.envOverrides, ); - if (provider === 'claude' && output.newSessionId) { + if ( + provider === 'claude' && + output.newSessionId && + !forceFreshClaudeReviewerSession + ) { deps.persistSession(sessionFolder, output.newSessionId); + currentSessionId = output.newSessionId; } providerLog.info( @@ -804,23 +841,9 @@ export async function runAgentForGroup( }))); if (isRetryableClaudeSessionFailure(primaryAttempt)) { + currentSessionId = undefined; deps.clearSession(sessionFolder); - // Also clear SDK session state inside the reviewer/arbiter config dir - // so the persistent container doesn't resume the same stale session. - const configDir = - pairedExecutionContext?.envOverrides?.CLAUDE_CONFIG_DIR; - if (configDir) { - for (const sdkDir of ['.claude', '.codex']) { - const sessionsDir = path.join(configDir, sdkDir, 'sessions'); - if (fs.existsSync(sessionsDir)) { - fs.rmSync(sessionsDir, { recursive: true, force: true }); - log.info( - { sessionsDir }, - 'Cleared container SDK sessions for stale session recovery', - ); - } - } - } + clearRoleSdkSessions(); log.warn( 'Cleared poisoned Claude session before visible output, retrying fresh session', ); @@ -828,6 +851,7 @@ export async function runAgentForGroup( primaryAttempt = await runAttempt('claude'); if (isRetryableClaudeSessionFailure(primaryAttempt)) { + currentSessionId = undefined; deps.clearSession(sessionFolder); log.warn('Fresh Claude retry also hit a retryable session failure'); @@ -840,7 +864,7 @@ export async function runAgentForGroup( if (primaryAttempt.error) { if ( - canRotateToken && + canRetryClaudeCredentials && provider === 'claude' && !primaryAttempt.sawOutput ) { @@ -913,7 +937,7 @@ export async function runAgentForGroup( } if ( - canRotateToken && + canRetryClaudeCredentials && provider === 'claude' && !primaryAttempt.sawOutput && primaryAttempt.streamedTriggerReason && @@ -945,7 +969,7 @@ export async function runAgentForGroup( if (output.status === 'error') { if ( - canRotateToken && + canRetryClaudeCredentials && provider === 'claude' && !primaryAttempt.sawOutput ) { diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 2a7b3e1..3f59290 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -1704,6 +1704,230 @@ describe('createMessageRuntime', () => { ); }); + it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + + vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'task-active-owner-follow-up', + chat_jid: chatJid, + group_folder: group.folder, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: '2026-03-30T00:00:00.000Z', + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + }); + vi.mocked(db.getMessagesSince).mockReturnValue([]); + vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ + { + id: 1, + task_id: 'task-active-owner-follow-up', + turn_number: 1, + role: 'owner', + output_text: 'owner 초안', + created_at: '2026-03-30T00:00:01.000Z', + }, + { + id: 2, + task_id: 'task-active-owner-follow-up', + turn_number: 2, + role: 'reviewer', + output_text: '리뷰어가 수정 요청을 남김', + created_at: '2026-03-30T00:00:02.000Z', + }, + ]); + vi.mocked(db.getRecentChatMessages).mockReturnValue([ + { + id: 'human-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: '눈쟁이', + content: '이 기능 마무리해줘', + timestamp: '2026-03-30T00:00:00.500Z', + seq: 1, + is_bot_message: false, + } as any, + ]); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, input, _onProcess, onOutput) => { + expect(input.prompt).toContain('이 기능 마무리해줘'); + expect(input.prompt).toContain('리뷰어가 수정 요청을 남김'); + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'owner가 reviewer 피드백을 반영했습니다.', + newSessionId: 'session-owner-follow-up', + }); + return { + status: 'success', + result: 'owner가 reviewer 피드백을 반영했습니다.', + newSessionId: 'session-owner-follow-up', + }; + }, + ); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => ({}), + saveState: vi.fn(), + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-active-owner-follow-up', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); + expect(channel.sendMessage).toHaveBeenCalledWith( + chatJid, + 'owner가 reviewer 피드백을 반영했습니다.', + ); + }); + + it('builds owner follow-up prompts from paired turn outputs instead of raw reviewer bot delivery text', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + + vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'task-active-bot-follow-up', + chat_jid: chatJid, + group_folder: group.folder, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: '2026-03-30T00:00:00.000Z', + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + }); + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'reviewer-bot-message-active', + chat_jid: chatJid, + sender: 'reviewer-bot@test', + sender_name: '리뷰어', + content: 'DONE_WITH_CONCERNS\n\n리뷰어 디스코드 출력 원문', + timestamp: '2026-03-30T00:00:04.000Z', + seq: 42, + is_bot_message: true, + } as any, + ]); + vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ + { + id: 1, + task_id: 'task-active-bot-follow-up', + turn_number: 1, + role: 'owner', + output_text: 'owner 초안', + created_at: '2026-03-30T00:00:01.000Z', + }, + { + id: 2, + task_id: 'task-active-bot-follow-up', + turn_number: 2, + role: 'reviewer', + output_text: 'paired_turn_outputs 에 저장된 reviewer 요약', + created_at: '2026-03-30T00:00:02.000Z', + }, + ]); + vi.mocked(db.getRecentChatMessages).mockReturnValue([ + { + id: 'human-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: '눈쟁이', + content: '이 기능 마무리해줘', + timestamp: '2026-03-30T00:00:00.500Z', + seq: 1, + is_bot_message: false, + } as any, + ]); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, input, _onProcess, onOutput) => { + expect(input.prompt).toContain('paired_turn_outputs 에 저장된 reviewer 요약'); + expect(input.prompt).not.toContain('리뷰어 디스코드 출력 원문'); + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'owner follow-up ok', + newSessionId: 'session-owner-bot-follow-up', + }); + return { + status: 'success', + result: 'owner follow-up ok', + newSessionId: 'session-owner-bot-follow-up', + }; + }, + ); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => ({}), + saveState: vi.fn(), + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-active-bot-follow-up', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); + expect(channel.sendMessage).toHaveBeenCalledWith(chatJid, 'owner follow-up ok'); + }); + it('reuses the shared arbiter prompt builder for pending arbiter turns', async () => { const chatJid = 'group@test'; const group = makeGroup('codex'); diff --git a/src/message-runtime.ts b/src/message-runtime.ts index 95de8d5..fbff013 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -270,6 +270,30 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { return `User request:\n---\n${userMessage}\n---\n\nReview the latest owner changes in the workspace.`; }; + const buildOwnerPendingPrompt = ( + task: PairedTask, + chatJid: string, + timezone: string, + ): string => { + const turnOutputs = getPairedTurnOutputs(task.id); + if (turnOutputs.length > 0) { + const humanMessages = getRecentChatMessages(chatJid, 20).filter( + (message) => !message.is_bot_message, + ); + return formatMessages( + mergeHumanAndTurnOutputMessages(chatJid, humanMessages, turnOutputs), + timezone, + ); + } + + const userMessage = getLastHumanMessageContent(chatJid); + if (!userMessage) { + return 'Continue the owner turn using the latest reviewer or arbiter feedback.'; + } + + return `User request:\n---\n${userMessage}\n---\n\nContinue the owner turn using the latest reviewer or arbiter feedback.`; + }; + const buildArbiterPromptForTask = ( task: PairedTask, chatJid: string, @@ -825,6 +849,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }; } + if (taskStatus === 'active') { + const turnOutputs = getPairedTurnOutputs(task.id); + const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; + if ( + lastTurnOutput && + (lastTurnOutput.role === 'reviewer' || + lastTurnOutput.role === 'arbiter') + ) { + return { + prompt: buildOwnerPendingPrompt(task, chatJid, deps.timezone), + channel: resolveChannel(taskStatus), + cursor, + }; + } + } + return null; }; diff --git a/src/paired-execution-context.test.ts b/src/paired-execution-context.test.ts index afb32d8..73e653b 100644 --- a/src/paired-execution-context.test.ts +++ b/src/paired-execution-context.test.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('./db.js', () => ({ createPairedTask: vi.fn(), @@ -78,6 +78,12 @@ const failoverOwnerContext: RoomRoleContext = { failoverOwner: true, }; +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; +const ORIGINAL_REVIEWER_RUNTIME = process.env.EJCLAW_REVIEWER_RUNTIME; +const ORIGINAL_CLAUDE_REVIEWER_READONLY = + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + function createCanonicalRepoWithCommit(commitMessage: string): string { const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-finalize-')); execFileSync('git', ['init'], { cwd: repoDir, stdio: 'ignore' }); @@ -150,6 +156,9 @@ function buildWorkspace( describe('paired execution context', () => { beforeEach(() => { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + delete process.env.EJCLAW_REVIEWER_RUNTIME; + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; vi.resetAllMocks(); vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue(undefined); vi.mocked(db.getPairedTaskById).mockReturnValue(undefined); @@ -176,6 +185,28 @@ describe('paired execution context', () => { }); }); + afterAll(() => { + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } + + if (ORIGINAL_REVIEWER_RUNTIME == null) { + delete process.env.EJCLAW_REVIEWER_RUNTIME; + } else { + process.env.EJCLAW_REVIEWER_RUNTIME = ORIGINAL_REVIEWER_RUNTIME; + } + + if (ORIGINAL_CLAUDE_REVIEWER_READONLY == null) { + delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY; + } else { + process.env.EJCLAW_CLAUDE_REVIEWER_READONLY = + ORIGINAL_CLAUDE_REVIEWER_READONLY; + } + }); + it('creates an owner execution with a worktree override', () => { const result = preparePairedExecutionContext({ group, @@ -413,6 +444,7 @@ describe('paired execution context', () => { EJCLAW_PAIRED_ROLE: 'reviewer', EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', }); + expect(result?.envOverrides.EJCLAW_CLAUDE_REVIEWER_READONLY).toBe('1'); expect(result?.envOverrides.EJCLAW_REVIEWER_RUNTIME).toBeUndefined(); expect(result?.envOverrides.CLAUDE_CONFIG_DIR).toContain( '/data/sessions/paired-room-reviewer', diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index 005b864..fde5eb6 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -387,6 +387,9 @@ export function preparePairedExecutionContext(args: { envOverrides.CLAUDE_CONFIG_DIR = reviewerSessionDir; if (unsafeHostPairedMode) { envOverrides.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1'; + if (REVIEWER_AGENT_TYPE === 'claude-code') { + envOverrides.EJCLAW_CLAUDE_REVIEWER_READONLY = '1'; + } } else { envOverrides.EJCLAW_REVIEWER_RUNTIME = '1'; } diff --git a/src/paired-workspace-manager.test.ts b/src/paired-workspace-manager.test.ts index 29fde61..929547f 100644 --- a/src/paired-workspace-manager.test.ts +++ b/src/paired-workspace-manager.test.ts @@ -219,10 +219,9 @@ describe('paired workspace manager', () => { packageManager: 'pnpm' as const, })); vi.doMock('./workspace-package-manager.js', async () => { - const actual = - await vi.importActual( - './workspace-package-manager.js', - ); + const actual = await vi.importActual< + typeof import('./workspace-package-manager.js') + >('./workspace-package-manager.js'); return { ...actual, ensureWorkspaceDependenciesInstalled: diff --git a/src/provider-retry.test.ts b/src/provider-retry.test.ts index 0708f42..dc32d51 100644 --- a/src/provider-retry.test.ts +++ b/src/provider-retry.test.ts @@ -12,21 +12,30 @@ vi.mock('./logger.js', () => ({ vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), + getCurrentTokenIndex: vi.fn(() => 0), markTokenHealthy: vi.fn(), })); +vi.mock('./token-refresh.js', () => ({ + forceRefreshToken: vi.fn(async () => null), +})); + import { runClaudeRotationLoop } from './provider-retry.js'; import { + getCurrentTokenIndex, getTokenCount, markTokenHealthy, rotateToken, } from './token-rotation.js'; +import { forceRefreshToken } from './token-refresh.js'; describe('runClaudeRotationLoop', () => { beforeEach(() => { vi.clearAllMocks(); vi.mocked(getTokenCount).mockReturnValue(1); + vi.mocked(getCurrentTokenIndex).mockReturnValue(0); vi.mocked(rotateToken).mockReturnValue(false); + vi.mocked(forceRefreshToken).mockResolvedValue(null); }); it('rotates and succeeds after an org-access-denied trigger', async () => { @@ -85,6 +94,60 @@ describe('runClaudeRotationLoop', () => { }); }); + it('force-refreshes the active token before rotating on auth-expired', async () => { + vi.mocked(forceRefreshToken).mockResolvedValueOnce('new-access-token'); + + const outcome = await runClaudeRotationLoop( + { reason: 'auth-expired' }, + async () => ({ + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }), + { runId: 'force-refresh-auth-expired' }, + ); + + expect(outcome).toEqual({ type: 'success', sawOutput: true }); + expect(forceRefreshToken).toHaveBeenCalledWith(0); + expect(rotateToken).not.toHaveBeenCalled(); + expect(markTokenHealthy).toHaveBeenCalledTimes(1); + }); + + it('falls back to rotation when force refresh does not recover auth-expired', async () => { + vi.mocked(getTokenCount).mockReturnValue(2); + vi.mocked(forceRefreshToken).mockResolvedValueOnce('new-access-token'); + vi.mocked(rotateToken).mockReturnValueOnce(true); + + let attempts = 0; + const outcome = await runClaudeRotationLoop( + { reason: 'auth-expired' }, + async () => { + attempts += 1; + if (attempts === 1) { + return { + output: { + status: 'error', + result: null, + error: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }, + sawOutput: false, + }; + } + + return { + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }; + }, + { runId: 'force-refresh-then-rotate' }, + ); + + expect(outcome).toEqual({ type: 'success', sawOutput: true }); + expect(forceRefreshToken).toHaveBeenCalledWith(0); + expect(rotateToken).toHaveBeenCalledTimes(1); + expect(markTokenHealthy).toHaveBeenCalledTimes(1); + }); + it('uses structured public output text as the next rotation message', async () => { vi.mocked(getTokenCount).mockReturnValue(2); vi.mocked(rotateToken).mockReturnValueOnce(true).mockReturnValueOnce(false); diff --git a/src/provider-retry.ts b/src/provider-retry.ts index da6c783..149b5d1 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -15,10 +15,12 @@ import { import { logger } from './logger.js'; import { getErrorMessage } from './utils.js'; import { + getCurrentTokenIndex, rotateToken, getTokenCount, markTokenHealthy, } from './token-rotation.js'; +import { forceRefreshToken } from './token-refresh.js'; import { clearGlobalFailover } from './service-routing.js'; // ── Types ──────────────────────────────────────────────────────── @@ -40,6 +42,113 @@ export type RotationOutcome = | { type: 'success'; sawOutput: boolean } | { type: 'error'; trigger?: TriggerInfo }; +type AttemptOutcome = + | { type: 'success'; sawOutput: boolean } + | { type: 'error'; trigger?: TriggerInfo } + | { + type: 'continue'; + trigger: TriggerInfo; + rotationMessage?: string; + }; + +function evaluateClaudeAttempt( + attempt: RotationAttemptResult, + logContext: Record, +): AttemptOutcome { + if (attempt.thrownError) { + if (!attempt.sawOutput) { + const errMsg = getErrorMessage(attempt.thrownError); + const retryTrigger = attempt.streamedTriggerReason + ? { + shouldRetry: true, + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + } + : classifyRotationTrigger(errMsg); + if (retryTrigger.shouldRetry) { + return { + type: 'continue', + trigger: { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }, + rotationMessage: errMsg, + }; + } + } + + logger.error( + { ...logContext, provider: 'claude', err: attempt.thrownError }, + 'Rotated Claude account also threw', + ); + return { type: 'error' }; + } + + const output = attempt.output; + if (!output) { + logger.error( + { ...logContext, provider: 'claude' }, + 'Rotated Claude account produced no output object', + ); + return { type: 'error' }; + } + + if ( + !attempt.sawOutput && + attempt.streamedTriggerReason && + output.status !== 'error' + ) { + return { + type: 'continue', + trigger: { + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + }, + rotationMessage: getAgentOutputText(output) ?? undefined, + }; + } + + if (!attempt.sawOutput && attempt.sawSuccessNullResult) { + logger.warn( + { ...logContext, provider: 'claude' }, + 'All rotated tokens returned success with null result', + ); + return { type: 'error', trigger: { reason: 'success-null-result' } }; + } + + if (output.status === 'error') { + if (!attempt.sawOutput) { + const retryTrigger = attempt.streamedTriggerReason + ? { + shouldRetry: true, + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + } + : classifyRotationTrigger(output.error); + if (retryTrigger.shouldRetry) { + return { + type: 'continue', + trigger: { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }, + rotationMessage: output.error ?? undefined, + }; + } + } + + logger.error( + { ...logContext, provider: 'claude', error: output.error }, + 'Rotated Claude account failed', + ); + return { type: 'error' }; + } + + markTokenHealthy(); + clearGlobalFailover(); + return { type: 'success', sawOutput: attempt.sawOutput }; +} + // ── Shared rotation loop ───────────────────────────────────────── /** @@ -56,112 +165,70 @@ export async function runClaudeRotationLoop( ): Promise { let trigger = initialTrigger; let lastRotationMessage = rotationMessage; + const attemptedForcedRefreshIndexes = new Set(); + + while (shouldRotateClaudeToken(trigger.reason)) { + if (trigger.reason === 'auth-expired') { + const tokenIndex = getCurrentTokenIndex(); + if ( + tokenIndex != null && + !attemptedForcedRefreshIndexes.has(tokenIndex) + ) { + attemptedForcedRefreshIndexes.add(tokenIndex); + const refreshedToken = await forceRefreshToken(tokenIndex); + if (refreshedToken) { + logger.info( + { ...logContext, tokenIndex }, + 'Claude auth-expired recovered by force-refreshing current token', + ); + + const refreshAttemptOutcome = evaluateClaudeAttempt( + await runAttempt(), + logContext, + ); + if (refreshAttemptOutcome.type === 'success') { + return refreshAttemptOutcome; + } + if (refreshAttemptOutcome.type === 'error') { + return refreshAttemptOutcome; + } + + trigger = refreshAttemptOutcome.trigger; + lastRotationMessage = refreshAttemptOutcome.rotationMessage; + continue; + } + } + } + + if ( + !( + getTokenCount() > 1 && + // Respect per-token cooldowns so exhausted auth/quota failures can + // terminate instead of cycling forever. + rotateToken(lastRotationMessage) + ) + ) { + break; + } - while ( - shouldRotateClaudeToken(trigger.reason) && - getTokenCount() > 1 && - // Respect per-token cooldowns so exhausted auth/quota failures can - // terminate instead of cycling forever. - rotateToken(lastRotationMessage) - ) { logger.info( { ...logContext, reason: trigger.reason }, 'Claude account unavailable, retrying with rotated account', ); - const attempt = await runAttempt(); - - // ── Thrown error (exception from spawn/process) ── - if (attempt.thrownError) { - if (!attempt.sawOutput) { - const errMsg = getErrorMessage(attempt.thrownError); - const retryTrigger = attempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(errMsg); - if (retryTrigger.shouldRetry) { - trigger = { - reason: retryTrigger.reason, - retryAfterMs: retryTrigger.retryAfterMs, - }; - lastRotationMessage = errMsg; - continue; - } - } - - logger.error( - { ...logContext, provider: 'claude', err: attempt.thrownError }, - 'Rotated Claude account also threw', - ); - return { type: 'error' }; + const rotatedAttemptOutcome = evaluateClaudeAttempt( + await runAttempt(), + logContext, + ); + if (rotatedAttemptOutcome.type === 'success') { + return rotatedAttemptOutcome; + } + if (rotatedAttemptOutcome.type === 'error') { + return rotatedAttemptOutcome; } - const output = attempt.output; - if (!output) { - logger.error( - { ...logContext, provider: 'claude' }, - 'Rotated Claude account produced no output object', - ); - return { type: 'error' }; - } - - // ── Streamed trigger in non-error success ── - if ( - !attempt.sawOutput && - attempt.streamedTriggerReason && - output.status !== 'error' - ) { - trigger = { - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - }; - lastRotationMessage = getAgentOutputText(output) ?? undefined; - continue; - } - - // ── Success with null result ── - if (!attempt.sawOutput && attempt.sawSuccessNullResult) { - logger.warn( - { ...logContext, provider: 'claude' }, - 'All rotated tokens returned success with null result', - ); - return { type: 'error', trigger: { reason: 'success-null-result' } }; - } - - // ── Error status ── - if (output.status === 'error') { - if (!attempt.sawOutput) { - const retryTrigger = attempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(output.error); - if (retryTrigger.shouldRetry) { - trigger = { - reason: retryTrigger.reason, - retryAfterMs: retryTrigger.retryAfterMs, - }; - lastRotationMessage = output.error ?? undefined; - continue; - } - } - - logger.error( - { ...logContext, provider: 'claude', error: output.error }, - 'Rotated Claude account failed', - ); - return { type: 'error' }; - } - - // ── Success ── - markTokenHealthy(); - clearGlobalFailover(); - return { type: 'success', sawOutput: attempt.sawOutput }; + trigger = rotatedAttemptOutcome.trigger; + lastRotationMessage = rotatedAttemptOutcome.rotationMessage; } // ── All tokens exhausted ── diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index 6fdeda4..ecef805 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -54,9 +54,14 @@ vi.mock('./agent-error-detection.js', async (importOriginal) => { vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), + getCurrentTokenIndex: vi.fn(() => 0), markTokenHealthy: vi.fn(), })); +vi.mock('./token-refresh.js', () => ({ + forceRefreshToken: vi.fn(async () => null), +})); + vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn((error?: string | null) => { const lower = (error || '').toLowerCase(); @@ -142,6 +147,7 @@ import { _initTestDatabase, createTask, getTaskById } from './db.js'; import * as codexTokenRotation from './codex-token-rotation.js'; import { TIMEZONE } from './config.js'; import * as serviceRouting from './service-routing.js'; +import * as tokenRefresh from './token-refresh.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js'; import * as tokenRotation from './token-rotation.js'; @@ -183,9 +189,12 @@ describe('task scheduler', () => { }); // No fallback provider setup needed vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + vi.mocked(tokenRotation.getCurrentTokenIndex).mockReturnValue(0); vi.mocked(tokenRotation.markTokenHealthy).mockClear(); vi.mocked(tokenRotation.rotateToken).mockClear(); vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); + vi.mocked(tokenRefresh.forceRefreshToken).mockReset(); + vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null); vi.mocked(codexTokenRotation.rotateCodexToken).mockClear(); vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValue(false); vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(1); diff --git a/src/token-refresh.ts b/src/token-refresh.ts index 020fb41..63a0ab4 100644 --- a/src/token-refresh.ts +++ b/src/token-refresh.ts @@ -237,6 +237,7 @@ async function refreshToken( */ async function checkAndRefreshAccount( accountIndex: number, + opts?: { force?: boolean }, ): Promise { const creds = readCredentials(accountIndex); if (!creds?.claudeAiOauth) return null; @@ -250,7 +251,7 @@ async function checkAndRefreshAccount( const now = Date.now(); const remaining = expiresAt - now; - if (remaining > REFRESH_BEFORE_EXPIRY_MS) { + if (!opts?.force && remaining > REFRESH_BEFORE_EXPIRY_MS) { logger.debug( { accountIndex, remainingMin: Math.round(remaining / 60000) }, 'Token still valid, no refresh needed', @@ -260,7 +261,12 @@ async function checkAndRefreshAccount( const isExpired = remaining <= 0; logger.info( - { accountIndex, remainingMin: Math.round(remaining / 60000), isExpired }, + { + accountIndex, + remainingMin: Math.round(remaining / 60000), + isExpired, + force: opts?.force === true, + }, 'Refreshing Claude OAuth token', ); @@ -407,7 +413,9 @@ export async function forceRefreshToken( ): Promise { logger.info({ tokenIndex }, 'Force-refreshing token after 401'); - const newAccessToken = await checkAndRefreshAccount(tokenIndex); + const newAccessToken = await checkAndRefreshAccount(tokenIndex, { + force: true, + }); if (newAccessToken) { const allTokens = getAllTokens(); if (tokenIndex < allTokens.length) { diff --git a/src/token-rotation.ts b/src/token-rotation.ts index da88989..6f68598 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -155,6 +155,13 @@ export function getCurrentToken(): string | undefined { return tokens[currentIndex % tokens.length]?.token; } +/** Get the current active token index. */ +export function getCurrentTokenIndex(): number | null { + if (tokens.length === 0) return null; + refreshRuntimeTokenSelection(); + return currentIndex % tokens.length; +} + /** * Try to rotate to the next available (non-rate-limited) token. * Returns true if a fresh token was found, false if all are exhausted. diff --git a/src/verification.test.ts b/src/verification.test.ts index 1947dd4..54cda99 100644 --- a/src/verification.test.ts +++ b/src/verification.test.ts @@ -20,7 +20,9 @@ describe('verification helpers', () => { }); it('builds deterministic commands for each profile', () => { - const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-verification-')); + const repoDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'ejclaw-verification-'), + ); fs.writeFileSync(path.join(repoDir, 'package.json'), JSON.stringify({})); expect(buildVerificationCommand('test', repoDir)).toMatchObject({ diff --git a/src/verification.ts b/src/verification.ts index 23e66df..e6bd918 100644 --- a/src/verification.ts +++ b/src/verification.ts @@ -77,7 +77,11 @@ export function buildVerificationCommand( repoDir: string = process.cwd(), ): VerificationCommandSpec { const scriptName = - profile === 'test' ? 'test' : profile === 'typecheck' ? 'typecheck' : 'build'; + profile === 'test' + ? 'test' + : profile === 'typecheck' + ? 'typecheck' + : 'build'; const command = buildWorkspaceScriptCommand(repoDir, scriptName); return { file: command.file, diff --git a/src/workspace-package-manager.test.ts b/src/workspace-package-manager.test.ts index b942bd7..406b606 100644 --- a/src/workspace-package-manager.test.ts +++ b/src/workspace-package-manager.test.ts @@ -57,7 +57,10 @@ describe('workspace package manager helpers', () => { JSON.stringify({ packageManager: 'bun@1.3.11' }), ); fs.writeFileSync(path.join(mixedRepo, 'bun.lock'), ''); - fs.writeFileSync(path.join(mixedRepo, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n'); + fs.writeFileSync( + path.join(mixedRepo, 'pnpm-lock.yaml'), + 'lockfileVersion: 9.0\n', + ); expect(detectWorkspacePackageManager(pnpmRepo)).toBe('pnpm'); expect(detectWorkspacePackageManager(bunRepo)).toBe('bun'); @@ -118,7 +121,10 @@ describe('workspace package manager helpers', () => { scripts: { typecheck: 'tsc --noEmit' }, }), ); - fs.writeFileSync(path.join(repoDir, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n'); + fs.writeFileSync( + path.join(repoDir, 'pnpm-lock.yaml'), + 'lockfileVersion: 9.0\n', + ); execFileSyncMock.mockImplementation((_file, _args, options) => { const cwd = (options as { cwd: string }).cwd; diff --git a/src/workspace-package-manager.ts b/src/workspace-package-manager.ts index 7158d2c..208ae44 100644 --- a/src/workspace-package-manager.ts +++ b/src/workspace-package-manager.ts @@ -44,7 +44,9 @@ function readPackageJsonMetadata(repoDir: string): PackageJsonMetadata | null { return null; } - return JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) as PackageJsonMetadata; + return JSON.parse( + fs.readFileSync(packageJsonPath, 'utf-8'), + ) as PackageJsonMetadata; } function detectPackageManagerFromField( @@ -60,10 +62,7 @@ function detectPackageManagerFromField( return null; } -function hasLockfile( - repoDir: string, - ...names: readonly string[] -): boolean { +function hasLockfile(repoDir: string, ...names: readonly string[]): boolean { return names.some((name) => fs.existsSync(path.join(repoDir, name))); } @@ -91,7 +90,9 @@ function detectYarnBerry( packageManagerField: string | undefined, ): boolean { if (packageManagerField?.startsWith('yarn@')) { - const version = packageManagerField.slice('yarn@'.length).split(/[+-]/, 1)[0]; + const version = packageManagerField + .slice('yarn@'.length) + .split(/[+-]/, 1)[0]; const major = Number.parseInt(version.split('.', 1)[0] ?? '', 10); if (Number.isFinite(major) && major >= 2) { return true; @@ -142,11 +143,7 @@ function computeInstallFingerprint(repoDir: string): string | null { } function readInstallFingerprint(repoDir: string): string | null { - const statePath = path.join( - repoDir, - 'node_modules', - INSTALL_STATE_FILENAME, - ); + const statePath = path.join(repoDir, 'node_modules', INSTALL_STATE_FILENAME); if (!fs.existsSync(statePath)) { return null; } @@ -356,7 +353,9 @@ export function ensureWorkspaceDependenciesInstalled( ? (error as { stdout: string }).stdout.trim() : ''; const detail = - stderr || stdout || (error instanceof Error ? error.message : String(error)); + stderr || + stdout || + (error instanceof Error ? error.message : String(error)); throw new Error( `Failed to install workspace dependencies with "${command.commandText}" in ${repoDir}: ${detail}`, ); From aaa9eeb50a1b6bbf5175b01209de76a0a62588be Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 07:56:24 +0900 Subject: [PATCH 05/29] Remove paired reviewer container execution path --- README.md | 12 +- docs/architecture.md | 20 +- src/agent-runner.test.ts | 25 -- src/agent-runner.ts | 91 ---- src/config.ts | 4 - src/container-runner.test.ts | 264 ------------ src/container-runner.ts | 795 ----------------------------------- src/container-runtime.ts | 6 +- src/credential-proxy.ts | 170 -------- src/index.ts | 27 +- src/token-refresh.ts | 34 +- src/token-rotation.ts | 19 - 12 files changed, 21 insertions(+), 1446 deletions(-) delete mode 100644 src/container-runner.test.ts delete mode 100644 src/container-runner.ts delete mode 100644 src/credential-proxy.ts diff --git a/README.md b/README.md index 5bab52a..60e564f 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ No extra SDK processes. External references use lightweight API calls (Anthropic - **Discord-independent communication** — Agent-to-agent data flows directly via DB, Discord is display-only - **Mixture of Agents** — External model opinions (Kimi, GLM) enrich arbiter verdicts - **Per-role model selection** — `OWNER_MODEL`, `REVIEWER_MODEL`, `ARBITER_MODEL` + effort + fallback toggle -- **Container-isolated reviewer** — Persistent Docker container with read-only source mount +- **Host reviewer with read-only guards** — Reviewer runs on host with role-scoped sandbox and guard policy - **Global failover** — Account-level Claude failure → all channels switch to codex, auto-recovers - **Post-approval change detection** — Re-triggers review if owner modifies code after approval - **Auto user notification** — @mention on task completion (✅ done, ⚠️ escalated) @@ -109,7 +109,7 @@ No extra SDK processes. External references use lightweight API calls (Anthropic Discord ──► SQLite (WAL) ──► GroupQueue ──┬──► Owner (host process) │ │ │ ▼ (auto-trigger) - ├──► Reviewer (Docker container, :ro mount) + ├──► Reviewer (host process, read-only guarded) │ │ │ Verdict routing │ ├─ DONE → change detection → finalize or re-review @@ -140,7 +140,7 @@ Discord ──► SQLite (WAL) ──► GroupQueue ──┬──► Owner (ho - Linux (Ubuntu 22.04+) or macOS - [Bun](https://bun.sh/) 1.3+ -- Docker (required for reviewer container isolation) +- Docker (currently used for isolated verification runs) - [Claude Code CLI](https://docs.anthropic.com/en/docs/claude-code) - [Codex CLI](https://github.com/openai/codex) (`npm install -g @openai/codex`) - Discord bot tokens (3: owner, reviewer, arbiter) @@ -153,12 +153,12 @@ cd EJClaw bun install bun run build:runners bun run build -bun run build:container # Build reviewer Docker image +bun run build:container # Build verification container image ``` ## Documentation -- [Architecture](docs/architecture.md) — Data flow, room model, container isolation, key files +- [Architecture](docs/architecture.md) — Data flow, room model, verification isolation, key files - [Configuration](docs/configuration.md) — Full `.env` reference, debugging paths ### Environment @@ -213,7 +213,7 @@ bun run deploy ```bash bun run build # Build main project bun run build:runners # Install + build both runners -bun run build:container # Rebuild reviewer Docker image +bun run build:container # Rebuild verification container image bun run dev # Dev mode with hot reload bun test # Run tests ``` diff --git a/docs/architecture.md b/docs/architecture.md index 542151c..d259d09 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ Single unified service (`ejclaw`) manages all three Discord bots in one process: Discord ──► SQLite (WAL) ──► GroupQueue ──┬──► Owner (host process) │ │ │ ▼ (auto-trigger) - ├──► Reviewer (Docker container, :ro mount) + ├──► Reviewer (host process, read-only guarded) │ │ │ Verdict routing │ ├─ DONE → change detection → finalize or re-review @@ -77,16 +77,15 @@ User message → Arbiter summoned → binding verdict → loop resumes ``` -## Container Isolation +## Verification Isolation -Reviewer and arbiter run inside a Docker container with: +Reviewer and arbiter now run as host processes with role-scoped read-only +guards and sandbox settings. Docker remains in use for verification profiles: -- Read-only source mount (kernel-level write protection) -- Both Claude (agent-runner) and Codex (codex-runner) supported -- Runner selected at `docker exec` time based on agent type -- tmpfs overlays for test runner caches -- IPC via filesystem for follow-up messages -- Credentials injected per-exec (never baked into container) +- Scratch workspace copied before execution +- Verification command run inside the container image +- Isolated filesystem view for test/typecheck/build checks +- Runtime image shared with host tooling and evidence inspection ## Key Files @@ -94,7 +93,8 @@ Reviewer and arbiter run inside a Docker container with: |------|---------| | `src/index.ts` | Orchestrator: state, message loop, agent invocation | | `src/agent-runner.ts` | Spawns agent processes, manages env/sessions/skills | -| `src/container-runner.ts` | Docker container execution for reviewer/arbiter | +| `src/verification.ts` | Verification execution using the container image | +| `src/container-runtime.ts` | Shared Docker runtime helpers for verification | | `src/channels/discord.ts` | Discord channel (8s typing refresh, Whisper transcription) | | `src/ipc.ts` | IPC watcher and task processing | | `src/router.ts` | Message formatting and outbound routing | diff --git a/src/agent-runner.test.ts b/src/agent-runner.test.ts index d2a27de..b7744cf 100644 --- a/src/agent-runner.test.ts +++ b/src/agent-runner.test.ts @@ -108,10 +108,8 @@ vi.mock('child_process', async () => { import { runAgentProcess, AgentOutput, - mirrorContainerCodexSessionFiles, } from './agent-runner.js'; import * as agentRunnerEnvironment from './agent-runner-environment.js'; -import * as containerRunner from './container-runner.js'; import type { RegisteredGroup } from './types.js'; const testGroup: RegisteredGroup = { @@ -327,10 +325,6 @@ describe('agent-runner timeout behavior', () => { agentRunnerEnvironment, 'prepareContainerSessionEnvironment', ); - const runReviewerContainerSpy = vi.spyOn( - containerRunner, - 'runReviewerContainer', - ); vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => { const str = String(p); @@ -365,7 +359,6 @@ describe('agent-runner timeout behavior', () => { fakeProc.emit('close', 0); const result = await resultPromise; expect(result.status).toBe('success'); - expect(runReviewerContainerSpy).not.toHaveBeenCalled(); expect(prepareContainerSessionEnvironmentSpy).toHaveBeenCalledWith( expect.objectContaining({ sessionDir: '/tmp/host-reviewer-session', @@ -689,22 +682,4 @@ OUROBOROS_LLM_BACKEND = "codex" ); }); - it('mirrors .claude.json alongside codex session files for container reuse', () => { - vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => { - const str = String(p); - return ( - str === '/tmp/source/.codex' || - str === '/tmp/mounted' || - str === '/tmp/source/.codex/AGENTS.md' || - str === '/tmp/source/.claude.json' - ); - }); - - mirrorContainerCodexSessionFiles('/tmp/source', '/tmp/mounted'); - - expect(vi.mocked(fs.copyFileSync)).toHaveBeenCalledWith( - '/tmp/source/.claude.json', - '/tmp/mounted/.claude.json', - ); - }); }); diff --git a/src/agent-runner.ts b/src/agent-runner.ts index 8978e8c..e275282 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -13,7 +13,6 @@ import { IDLE_TIMEOUT, } from './config.js'; import { - ensureClaudeGlobalSettingsFile, prepareContainerSessionEnvironment, prepareGroupEnvironment, } from './agent-runner-environment.js'; @@ -25,7 +24,6 @@ export { } from './agent-runner-snapshot.js'; import { logger } from './logger.js'; import { OUTPUT_END_MARKER, OUTPUT_START_MARKER } from './agent-protocol.js'; -import { runReviewerContainer } from './container-runner.js'; import { AgentOutputPhase, AgentType, @@ -62,34 +60,6 @@ export interface AgentOutput { error?: string; } -export function mirrorContainerCodexSessionFiles( - sourceSessionDir: string, - mountedSessionDir: string, -): void { - const sourceCodexDir = path.join(sourceSessionDir, '.codex'); - const mountedCodexDir = path.join(mountedSessionDir, '.codex'); - if (!fs.existsSync(sourceCodexDir) || !fs.existsSync(mountedSessionDir)) { - return; - } - - fs.mkdirSync(mountedCodexDir, { recursive: true }); - for (const file of ['AGENTS.md', 'config.toml', 'config.json', 'auth.json']) { - const sourcePath = path.join(sourceCodexDir, file); - if (fs.existsSync(sourcePath)) { - fs.copyFileSync(sourcePath, path.join(mountedCodexDir, file)); - } - } - - const sourceClaudeJson = path.join(sourceSessionDir, '.claude.json'); - if (fs.existsSync(sourceClaudeJson)) { - ensureClaudeGlobalSettingsFile(mountedSessionDir); - fs.copyFileSync( - sourceClaudeJson, - path.join(mountedSessionDir, '.claude.json'), - ); - } -} - export async function runAgentProcess( group: RegisteredGroup, input: AgentInput, @@ -104,67 +74,6 @@ export async function runAgentProcess( const unsafeHostPairedMode = envOverrides?.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; - // ── Reviewer container mode ───────────────────────────────────── - // Reviewers always run inside a Docker container with read-only source - // mount for kernel-level write protection. Docker is required. - if ( - (!unsafeHostPairedMode && envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1') || - (!unsafeHostPairedMode && envOverrides?.EJCLAW_ARBITER_RUNTIME === '1') - ) { - const ownerWorkspaceDir = - envOverrides?.EJCLAW_WORK_DIR || group.workDir || process.cwd(); - - // Prepare session directory for the container (CLAUDE.md, skills, settings) - // so the Claude SDK inside the container has platform & paired room prompts. - const sessionDir = envOverrides?.CLAUDE_CONFIG_DIR; - if (sessionDir) { - const containerRole = - envOverrides?.EJCLAW_ARBITER_RUNTIME === '1' - ? ('arbiter' as const) - : ('reviewer' as const); - prepareContainerSessionEnvironment({ - sessionDir, - chatJid: input.chatJid, - isMain: input.isMain, - groupFolder: group.folder, - agentType: group.agentType || 'claude-code', - memoryBriefing: input.memoryBriefing, - role: containerRole, - }); - // The persistent container always mounts the reviewer session path at - // /home/node/.claude. When the arbiter stages a separate session dir, - // mirror the Codex session files into the mounted reviewer dir so the - // container sees the arbiter prompt/config, including MCP servers. - if (containerRole === 'arbiter') { - const reviewerSessionDir = path.join( - path.dirname(sessionDir), - `${group.folder}-reviewer`, - ); - mirrorContainerCodexSessionFiles(sessionDir, reviewerSessionDir); - } - } - - return runReviewerContainer({ - group, - input: { - prompt: input.prompt, - sessionId: input.sessionId, - groupFolder: input.groupFolder, - chatJid: input.chatJid, - runId: input.runId || `${Date.now()}`, - isMain: input.isMain, - assistantName: input.assistantName, - roomRoleContext: input.roomRoleContext, - }, - ownerWorkspaceDir, - envOverrides, - onOutput, - onProcess: (proc, containerName) => { - onProcess(proc, containerName, ''); - }, - }); - } - // ── Host process mode (owner) ─────────────────────────────────── const startTime = Date.now(); const { env, groupDir, runnerDir } = prepareGroupEnvironment( diff --git a/src/config.ts b/src/config.ts index c208b14..eacb69d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -210,10 +210,6 @@ export const PAIRED_MAX_ROUND_TRIPS = // ── Container isolation ─────────────────────────────────────────── export const REVIEWER_CONTAINER_IMAGE = getEnv('REVIEWER_CONTAINER_IMAGE') || 'ejclaw-reviewer:latest'; -export const CREDENTIAL_PROXY_PORT = parseInt( - getEnv('CREDENTIAL_PROXY_PORT') || '3001', - 10, -); export const RECOVERY_STAGGER_MS = parseInt( getEnv('RECOVERY_STAGGER_MS') || '2000', 10, diff --git a/src/container-runner.test.ts b/src/container-runner.test.ts deleted file mode 100644 index 0f491da..0000000 --- a/src/container-runner.test.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { execFileSync } from 'child_process'; -import fs from 'fs'; -import path from 'path'; - -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -const { TEST_ROOT, TEST_DATA_DIR, TEST_GROUPS_DIR, mockDetectAuthMode } = - vi.hoisted(() => { - const testRoot = '/tmp/ejclaw-container-runner-test'; - return { - TEST_ROOT: testRoot, - TEST_DATA_DIR: `${testRoot}/data`, - TEST_GROUPS_DIR: `${testRoot}/groups`, - mockDetectAuthMode: vi.fn<() => 'oauth' | 'api-key'>(), - }; - }); - -vi.mock('./config.js', () => ({ - AGENT_MAX_OUTPUT_SIZE: 1024 * 1024, - AGENT_TIMEOUT: 60_000, - DATA_DIR: TEST_DATA_DIR, - GROUPS_DIR: TEST_GROUPS_DIR, - IDLE_TIMEOUT: 1_000, - REVIEWER_CONTAINER_IMAGE: 'ejclaw-reviewer:latest', - TIMEZONE: 'Asia/Seoul', -})); - -vi.mock('./container-runtime.js', () => ({ - CONTAINER_RUNTIME_BIN: 'docker', - ensureContainerRuntimeRunning: vi.fn(), - hostGatewayArgs: () => ['--add-host=host.docker.internal:host-gateway'], - readonlyMountArgs: (hostPath: string, containerPath: string) => [ - '-v', - `${hostPath}:${containerPath}:ro`, - ], - writableMountArgs: (hostPath: string, containerPath: string) => [ - '-v', - `${hostPath}:${containerPath}`, - ], - tmpfsMountArgs: (containerPath: string) => ['--tmpfs', containerPath], -})); - -vi.mock('./agent-runner-environment.js', () => ({ - ensureClaudeGlobalSettingsFile: vi.fn(), -})); - -vi.mock('./credential-proxy.js', () => ({ - detectAuthMode: mockDetectAuthMode, -})); - -vi.mock('./group-folder.js', () => ({ - resolveGroupFolderPath: (folder: string) => - path.join(TEST_GROUPS_DIR, folder), - resolveGroupIpcPath: (folder: string) => - path.join(TEST_DATA_DIR, 'ipc', folder), -})); - -vi.mock('./logger.js', () => ({ - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }, -})); - -import { - appendExecEnvArgs, - buildCreateArgs, - buildReviewerMounts, -} from './container-runner.js'; -import type { RegisteredGroup } from './types.js'; - -function makeGroup(): RegisteredGroup { - return { - name: 'Test Group', - folder: 'brain', - trigger: '@Codex', - added_at: new Date().toISOString(), - agentType: 'codex', - }; -} - -function initGitRepo(repoDir: string): void { - fs.mkdirSync(repoDir, { recursive: true }); - execFileSync('git', ['init'], { - cwd: repoDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - }); - execFileSync('git', ['config', 'user.email', 'test@example.com'], { - cwd: repoDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - }); - execFileSync('git', ['config', 'user.name', 'EJClaw Test'], { - cwd: repoDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - }); -} - -describe('container-runner path compatibility', () => { - const previousCodeXHome = process.env.CODEX_HOME; - const previousOauthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN; - - beforeEach(() => { - fs.rmSync(TEST_ROOT, { recursive: true, force: true }); - fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); - fs.mkdirSync(TEST_GROUPS_DIR, { recursive: true }); - process.env.CODEX_HOME = path.join(TEST_ROOT, 'codex-home'); - fs.mkdirSync(process.env.CODEX_HOME, { recursive: true }); - process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-token'; - mockDetectAuthMode.mockReset(); - mockDetectAuthMode.mockReturnValue('oauth'); - }); - - afterEach(() => { - fs.rmSync(TEST_ROOT, { recursive: true, force: true }); - if (previousCodeXHome === undefined) { - delete process.env.CODEX_HOME; - } else { - process.env.CODEX_HOME = previousCodeXHome; - } - if (previousOauthToken === undefined) { - delete process.env.CLAUDE_CODE_OAUTH_TOKEN; - } else { - process.env.CLAUDE_CODE_OAUTH_TOKEN = previousOauthToken; - } - }); - - it('mounts the owner workspace at both canonical and host absolute paths and shadows both .env paths', () => { - const group = makeGroup(); - const ownerWorkspaceDir = path.join(TEST_ROOT, 'workspace', 'owner'); - fs.mkdirSync(ownerWorkspaceDir, { recursive: true }); - fs.writeFileSync(path.join(ownerWorkspaceDir, '.env'), 'SECRET=1\n'); - fs.mkdirSync(path.join(TEST_GROUPS_DIR, group.folder), { recursive: true }); - fs.mkdirSync(path.join(TEST_DATA_DIR, 'ipc', group.folder), { - recursive: true, - }); - - const mounts = buildReviewerMounts(group, ownerWorkspaceDir); - - expect(mounts).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - hostPath: ownerWorkspaceDir, - containerPath: '/workspace/project', - readonly: true, - }), - expect.objectContaining({ - hostPath: ownerWorkspaceDir, - containerPath: ownerWorkspaceDir, - readonly: true, - }), - expect.objectContaining({ - hostPath: path.join( - TEST_DATA_DIR, - 'sessions', - `${group.folder}-reviewer`, - '.claude.json', - ), - containerPath: '/home/node/.claude.json', - readonly: false, - }), - expect.objectContaining({ - hostPath: '/dev/null', - containerPath: '/workspace/project/.env', - readonly: true, - }), - expect.objectContaining({ - hostPath: '/dev/null', - containerPath: path.join(ownerWorkspaceDir, '.env'), - readonly: true, - }), - ]), - ); - }); - - it('mounts a local absolute origin target and shadows its .env', () => { - const group = makeGroup(); - const canonicalRepoDir = path.join(TEST_ROOT, 'canonical'); - const ownerWorkspaceDir = path.join(TEST_ROOT, 'workspace', 'owner'); - initGitRepo(canonicalRepoDir); - initGitRepo(ownerWorkspaceDir); - execFileSync('git', ['remote', 'add', 'origin', canonicalRepoDir], { - cwd: ownerWorkspaceDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - }); - fs.writeFileSync(path.join(canonicalRepoDir, '.env'), 'CANONICAL=1\n'); - fs.mkdirSync(path.join(TEST_GROUPS_DIR, group.folder), { recursive: true }); - fs.mkdirSync(path.join(TEST_DATA_DIR, 'ipc', group.folder), { - recursive: true, - }); - - const mounts = buildReviewerMounts(group, ownerWorkspaceDir); - - expect(mounts).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - hostPath: canonicalRepoDir, - containerPath: canonicalRepoDir, - readonly: true, - }), - expect.objectContaining({ - hostPath: '/dev/null', - containerPath: path.join(canonicalRepoDir, '.env'), - readonly: true, - }), - ]), - ); - }); - - it('keeps dynamic workdir and role env on docker exec instead of baking them into container creation', () => { - const createArgs = buildCreateArgs([], 'ejclaw-reviewer-brain'); - const createText = createArgs.join(' '); - - expect(createText).not.toContain('EJCLAW_WORK_DIR='); - expect(createText).not.toContain('EJCLAW_PAIRED_ROLE='); - expect(createText).not.toContain('CLAUDE_CONFIG_DIR='); - expect(createText).not.toContain('EJCLAW_REVIEWER_RUNTIME='); - - const reviewerExecArgs = ['exec', '-i']; - appendExecEnvArgs( - reviewerExecArgs, - { - EJCLAW_WORK_DIR: '/home/clone-ej/project', - EJCLAW_PAIRED_TASK_ID: 'task-1', - EJCLAW_PAIRED_ROLE: 'reviewer', - EJCLAW_REVIEWER_RUNTIME: '1', - CLAUDE_CONFIG_DIR: '/host/reviewer-session', - CLAUDE_MODEL: 'claude-sonnet', - CODEX_MODEL: 'gpt-5-codex', - }, - false, - ); - - const reviewerExecText = reviewerExecArgs.join(' '); - expect(reviewerExecText).toContain( - 'EJCLAW_WORK_DIR=/home/clone-ej/project', - ); - expect(reviewerExecText).toContain('EJCLAW_PAIRED_TASK_ID=task-1'); - expect(reviewerExecText).toContain('EJCLAW_PAIRED_ROLE=reviewer'); - expect(reviewerExecText).toContain('EJCLAW_REVIEWER_RUNTIME=1'); - expect(reviewerExecText).toContain('CLAUDE_CONFIG_DIR=/home/node/.claude'); - expect(reviewerExecText).toContain('CLAUDE_MODEL=claude-sonnet'); - expect(reviewerExecText).not.toContain('CODEX_MODEL=gpt-5-codex'); - - const codexExecArgs = ['exec', '-i']; - appendExecEnvArgs( - codexExecArgs, - { - CODEX_MODEL: 'gpt-5-codex', - CLAUDE_MODEL: 'claude-sonnet', - }, - true, - ); - - const codexExecText = codexExecArgs.join(' '); - expect(codexExecText).toContain('CODEX_MODEL=gpt-5-codex'); - expect(codexExecText).not.toContain('CLAUDE_MODEL=claude-sonnet'); - }); -}); diff --git a/src/container-runner.ts b/src/container-runner.ts deleted file mode 100644 index 7fb4771..0000000 --- a/src/container-runner.ts +++ /dev/null @@ -1,795 +0,0 @@ -/** - * Container runner for EJClaw reviewer agents. - * - * Uses persistent Docker containers per channel. The container is created - * once on first reviewer call and reused across turns. Each turn runs via - * `docker exec` inside the warm container — no startup overhead. - * - * - Source code mounted read-only (kernel-level write protection) - * - tmpfs overlays for test runner caches (vitest, coverage) - * - IPC via filesystem (input/ directory for follow-up messages) - * - Credentials injected by the credential proxy (never exposed to container) - */ -import { ChildProcess, execFileSync, spawn } from 'child_process'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; - -import { OUTPUT_END_MARKER, OUTPUT_START_MARKER } from './agent-protocol.js'; -import { - AGENT_MAX_OUTPUT_SIZE, - AGENT_TIMEOUT, - DATA_DIR, - GROUPS_DIR, - IDLE_TIMEOUT, - TIMEZONE, -} from './config.js'; -import { - CONTAINER_RUNTIME_BIN, - ensureContainerRuntimeRunning, - hostGatewayArgs, - readonlyMountArgs, - tmpfsMountArgs, - writableMountArgs, -} from './container-runtime.js'; -import { ensureClaudeGlobalSettingsFile } from './agent-runner-environment.js'; -import { detectAuthMode } from './credential-proxy.js'; -import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js'; -import { logger } from './logger.js'; -import type { AgentOutput } from './agent-runner.js'; -import type { RegisteredGroup } from './types.js'; -import { detectPnpmStorePath } from './workspace-package-manager.js'; - -// ── Config ──────────────────────────────────────────────────────── - -import { REVIEWER_CONTAINER_IMAGE as CONTAINER_IMAGE } from './config.js'; - -// ── Types ───────────────────────────────────────────────────────── - -export interface ReviewerContainerInput { - prompt: string; - sessionId?: string; - groupFolder: string; - chatJid: string; - runId: string; - isMain: boolean; - assistantName?: string; - roomRoleContext?: import('./types.js').RoomRoleContext; -} - -interface VolumeMount { - hostPath: string; - containerPath: string; - readonly: boolean; -} - -interface InspectedContainerMount { - Source?: string; - Destination?: string; - RW?: boolean; -} - -interface InspectedContainer { - Mounts?: InspectedContainerMount[]; -} - -const PRIMARY_PROJECT_MOUNT = '/workspace/project'; - -function pushMountOnce(mounts: VolumeMount[], mount: VolumeMount): void { - const exists = mounts.some( - (entry) => - entry.hostPath === mount.hostPath && - entry.containerPath === mount.containerPath && - entry.readonly === mount.readonly, - ); - if (!exists) { - mounts.push(mount); - } -} - -function normalizeLocalGitPath(remoteUrl: string): string | null { - if (!remoteUrl) { - return null; - } - if (path.isAbsolute(remoteUrl)) { - return path.resolve(remoteUrl); - } - if (remoteUrl.startsWith('file://')) { - try { - const parsed = new URL(remoteUrl); - if (parsed.protocol === 'file:') { - return path.resolve(parsed.pathname); - } - } catch { - return null; - } - } - return null; -} - -function resolveLocalOriginTarget(workspaceDir: string): string | null { - try { - const remoteUrl = execFileSync( - 'git', - ['config', '--get', 'remote.origin.url'], - { - cwd: workspaceDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 5000, - }, - ).trim(); - const localPath = normalizeLocalGitPath(remoteUrl); - if (!localPath || !fs.existsSync(localPath)) { - return null; - } - return localPath; - } catch { - return null; - } -} - -function pushWorktreeGitMetadataMounts( - mounts: VolumeMount[], - repoDir: string, -): void { - const dotGitPath = path.join(repoDir, '.git'); - try { - const stat = fs.statSync(dotGitPath); - if (!stat.isFile()) { - return; - } - const content = fs.readFileSync(dotGitPath, 'utf-8').trim(); - const match = content.match(/^gitdir:\s*(.+)$/); - if (!match) { - return; - } - const worktreeGitDir = path.resolve(repoDir, match[1]); - const parentGitDir = path.resolve(worktreeGitDir, '..', '..'); - if (!fs.existsSync(parentGitDir)) { - return; - } - pushMountOnce(mounts, { - hostPath: parentGitDir, - containerPath: parentGitDir, - readonly: true, - }); - logger.debug( - { parentGitDir, worktreeGitDir, repoDir }, - 'Mounting parent .git for worktree resolution', - ); - } catch { - // Not a git repo or .git missing — skip - } -} - -// ── Pre-flight checks ──────────────────────────────────────────── - -let containerRuntimeChecked = false; - -function ensureContainerReady(): void { - if (containerRuntimeChecked) return; - ensureContainerRuntimeRunning(); - try { - execFileSync(CONTAINER_RUNTIME_BIN, ['image', 'inspect', CONTAINER_IMAGE], { - stdio: 'pipe', - timeout: 10000, - }); - } catch { - throw new Error( - `Container image '${CONTAINER_IMAGE}' not found. Build it with: ./container/build.sh`, - ); - } - containerRuntimeChecked = true; -} - -// ── Persistent container management ────────────────────────────── - -function getContainerName(groupFolder: string): string { - return `ejclaw-reviewer-${groupFolder.replace(/[^a-zA-Z0-9-]/g, '-')}`; -} - -function isContainerRunning(name: string): boolean { - try { - const state = execFileSync( - CONTAINER_RUNTIME_BIN, - ['inspect', '--format', '{{.State.Running}}', name], - { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 5000 }, - ).trim(); - return state === 'true'; - } catch { - return false; - } -} - -function inspectContainer(name: string): InspectedContainer | null { - try { - const output = execFileSync(CONTAINER_RUNTIME_BIN, ['inspect', name], { - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - timeout: 5000, - }); - const parsed = JSON.parse(output) as InspectedContainer[]; - return parsed[0] ?? null; - } catch { - return null; - } -} - -function containerHasExpectedMounts( - name: string, - expectedMounts: VolumeMount[], -): boolean { - const inspection = inspectContainer(name); - const actualMounts = inspection?.Mounts ?? []; - - return expectedMounts.every((expected) => - actualMounts.some( - (actual) => - actual.Source === expected.hostPath && - actual.Destination === expected.containerPath && - Boolean(actual.RW) === !expected.readonly, - ), - ); -} - -function ensurePersistentContainer( - group: RegisteredGroup, - ownerWorkspaceDir: string, - envOverrides?: Record, -): string { - const containerName = getContainerName(group.folder); - const mounts = buildReviewerMounts(group, ownerWorkspaceDir); - - if (isContainerRunning(containerName)) { - if (containerHasExpectedMounts(containerName, mounts)) { - return containerName; - } - logger.info( - { - containerName, - group: group.name, - groupFolder: group.folder, - }, - 'Recreating persistent reviewer container because mount layout changed', - ); - } - - // Remove stale stopped container with same name - try { - execFileSync(CONTAINER_RUNTIME_BIN, ['rm', '-f', containerName], { - stdio: 'pipe', - timeout: 10000, - }); - } catch { - /* doesn't exist */ - } - - const args = buildCreateArgs(mounts, containerName); - - logger.info( - { - containerName, - group: group.name, - groupFolder: group.folder, - image: CONTAINER_IMAGE, - mountCount: mounts.length, - }, - 'Creating persistent reviewer container', - ); - - execFileSync(CONTAINER_RUNTIME_BIN, args, { - stdio: 'pipe', - timeout: 30000, - }); - - return containerName; -} - -/** Stop and remove a persistent reviewer container for a channel. */ -export function stopReviewerContainer(groupFolder: string): void { - const name = getContainerName(groupFolder); - try { - execFileSync(CONTAINER_RUNTIME_BIN, ['rm', '-f', name], { - stdio: 'pipe', - timeout: 10000, - }); - logger.info( - { containerName: name }, - 'Stopped persistent reviewer container', - ); - } catch { - /* already gone */ - } -} - -/** - * No-op — container recreation is no longer needed after token refresh. - * Tokens are now synced to process.env, and `docker exec -e` injects - * the latest token at each turn. Kept as a no-op to avoid breaking callers. - */ -export function recreateAllReviewerContainers(): void { - // Intentionally empty — docker exec -e picks up refreshed tokens - // from process.env without needing to recreate the container. -} - -// ── Mount builder ───────────────────────────────────────────────── - -export function buildReviewerMounts( - group: RegisteredGroup, - ownerWorkspaceDir: string, -): VolumeMount[] { - const mounts: VolumeMount[] = []; - const groupDir = resolveGroupFolderPath(group.folder); - const groupIpcDir = resolveGroupIpcPath(group.folder); - - // Source code: READ-ONLY (kernel-level protection) - pushMountOnce(mounts, { - hostPath: ownerWorkspaceDir, - containerPath: PRIMARY_PROJECT_MOUNT, - readonly: true, - }); - const canonicalRepoDir = resolveLocalOriginTarget(ownerWorkspaceDir); - if ( - canonicalRepoDir && - canonicalRepoDir !== ownerWorkspaceDir && - canonicalRepoDir !== PRIMARY_PROJECT_MOUNT - ) { - pushMountOnce(mounts, { - hostPath: canonicalRepoDir, - containerPath: canonicalRepoDir, - readonly: true, - }); - } - // Compatibility mount: expose the owner workspace at the same absolute path - // inside the container so owner-authored absolute paths still resolve. - if (ownerWorkspaceDir !== PRIMARY_PROJECT_MOUNT) { - pushMountOnce(mounts, { - hostPath: ownerWorkspaceDir, - containerPath: ownerWorkspaceDir, - readonly: true, - }); - } - - // Git worktree support: worktree's .git file references the parent repo's - // .git directory via absolute path. Mount the parent .git at the same host - // path so git commands resolve inside the container. - pushWorktreeGitMetadataMounts(mounts, ownerWorkspaceDir); - if (canonicalRepoDir) { - pushWorktreeGitMetadataMounts(mounts, canonicalRepoDir); - } - - // pnpm global store: mount at the same host path so hardlinks resolve. - const pnpmStore = detectPnpmStorePath(ownerWorkspaceDir); - if (pnpmStore) { - pushMountOnce(mounts, { - hostPath: pnpmStore, - containerPath: pnpmStore, - readonly: true, - }); - logger.debug({ pnpmStore }, 'Mounting pnpm store for container'); - } - - // Shadow .env so reviewer cannot read secrets from mounted project - const envFile = path.join(ownerWorkspaceDir, '.env'); - const shadowPaths = new Set(); - if (fs.existsSync(envFile)) { - shadowPaths.add(path.join(PRIMARY_PROJECT_MOUNT, '.env')); - shadowPaths.add(path.join(ownerWorkspaceDir, '.env')); - } - const canonicalEnvFile = canonicalRepoDir - ? path.join(canonicalRepoDir, '.env') - : null; - if (canonicalEnvFile && fs.existsSync(canonicalEnvFile)) { - shadowPaths.add(canonicalEnvFile); - } - for (const shadowPath of shadowPaths) { - pushMountOnce(mounts, { - hostPath: '/dev/null', - containerPath: shadowPath, - readonly: true, - }); - } - - // Attachments directory: read-only (Discord file uploads downloaded here) - const attachmentsDir = path.join(DATA_DIR, 'attachments'); - if (fs.existsSync(attachmentsDir)) { - pushMountOnce(mounts, { - hostPath: attachmentsDir, - containerPath: attachmentsDir, - readonly: true, - }); - } - - // Group folder: writable (logs, session data) - pushMountOnce(mounts, { - hostPath: groupDir, - containerPath: '/workspace/group', - readonly: false, - }); - - // IPC directory: writable (output messages, task results) - fs.mkdirSync(path.join(groupIpcDir, 'messages'), { recursive: true }); - fs.mkdirSync(path.join(groupIpcDir, 'input'), { recursive: true }); - pushMountOnce(mounts, { - hostPath: groupIpcDir, - containerPath: '/workspace/ipc', - readonly: false, - }); - - // Global memory: read-only - const globalDir = path.join(GROUPS_DIR, 'global'); - if (fs.existsSync(globalDir)) { - pushMountOnce(mounts, { - hostPath: globalDir, - containerPath: '/workspace/global', - readonly: true, - }); - } - - // Session directory for Claude/Codex state. - // Use the reviewer-suffixed path to match paired-execution-context.ts - // which writes CLAUDE.md (prompts) to {folder}-reviewer/.claude/. - const groupSessionsDir = path.join( - DATA_DIR, - 'sessions', - `${group.folder}-reviewer`, - ); - fs.mkdirSync(groupSessionsDir, { recursive: true }); - ensureClaudeGlobalSettingsFile(groupSessionsDir); - pushMountOnce(mounts, { - hostPath: groupSessionsDir, - containerPath: '/home/node/.claude', - readonly: false, - }); - pushMountOnce(mounts, { - hostPath: path.join(groupSessionsDir, '.claude.json'), - containerPath: '/home/node/.claude.json', - readonly: false, - }); - - // Owner session directory: read-only so reviewer can verify runtime state - // files (cron state, configs, etc.) that the owner references by absolute path. - const ownerSessionDir = path.join(DATA_DIR, 'sessions', group.folder); - if (fs.existsSync(ownerSessionDir)) { - pushMountOnce(mounts, { - hostPath: ownerSessionDir, - containerPath: ownerSessionDir, - readonly: true, - }); - } - - // Codex OAuth: mount host's ~/.codex (read-only) so codex-runner can authenticate - const hostCodexHome = - process.env.CODEX_HOME || path.join(os.homedir(), '.codex'); - if (fs.existsSync(hostCodexHome)) { - pushMountOnce(mounts, { - hostPath: hostCodexHome, - containerPath: '/home/node/.codex', - readonly: true, - }); - } - - return mounts; -} - -// ── Container args builders ────────────────────────────────────── - -/** Build args for `docker run -d` (create persistent container). */ -export function buildCreateArgs( - mounts: VolumeMount[], - containerName: string, -): string[] { - // Start detached with sleep infinity — turns run via docker exec - const args: string[] = [ - 'run', - '-d', - '--name', - containerName, - '--entrypoint', - 'sleep', - ]; - - // Timezone - args.push('-e', `TZ=${TIMEZONE}`); - - // Pass real credentials — Claude Code SDK handles OAuth internally, - // raw Bearer tokens on api.anthropic.com are not supported. - const authMode = detectAuthMode(); - if (authMode === 'api-key') { - const apiKey = process.env.ANTHROPIC_API_KEY || ''; - args.push('-e', `ANTHROPIC_API_KEY=${apiKey}`); - } else { - const oauthToken = - process.env.CLAUDE_CODE_OAUTH_TOKEN || - process.env.ANTHROPIC_AUTH_TOKEN || - ''; - args.push('-e', `CLAUDE_CODE_OAUTH_TOKEN=${oauthToken}`); - } - - // Sentry read-only token for reviewer to verify error fixes - if (process.env.SENTRY_AUTH_TOKEN) { - args.push('-e', `SENTRY_AUTH_TOKEN=${process.env.SENTRY_AUTH_TOKEN}`); - } - - // Host gateway - args.push(...hostGatewayArgs()); - - // Run as host user for bind-mount compatibility - const hostUid = process.getuid?.(); - const hostGid = process.getgid?.(); - if (hostUid != null && hostUid !== 0 && hostUid !== 1000) { - args.push('--user', `${hostUid}:${hostGid}`); - args.push('-e', 'HOME=/home/node'); - } - - // Volume mounts - for (const mount of mounts) { - if (mount.readonly) { - args.push(...readonlyMountArgs(mount.hostPath, mount.containerPath)); - } else { - args.push(...writableMountArgs(mount.hostPath, mount.containerPath)); - } - } - - // Writable tmpfs for test runners and temp files - args.push(...tmpfsMountArgs('/tmp')); - args.push('-e', 'VITEST_CACHE_DIR=/tmp/.vitest'); - args.push('-e', 'JEST_CACHE_DIR=/tmp/.jest'); - args.push('-e', 'npm_config_cache=/tmp/.npm'); - - args.push(CONTAINER_IMAGE); - args.push('infinity'); // argument to sleep - - return args; -} - -export function appendExecEnvArgs( - execArgs: string[], - envOverrides: Record | undefined, - isCodexAgent: boolean, -): void { - if (!envOverrides) { - return; - } - - const ignoredKeys = new Set(['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN']); - const incompatibleModelKeys = isCodexAgent - ? new Set(['CLAUDE_MODEL', 'CLAUDE_EFFORT']) - : new Set(['CODEX_MODEL', 'CODEX_EFFORT']); - - for (const [key, value] of Object.entries(envOverrides)) { - if (!value || ignoredKeys.has(key) || incompatibleModelKeys.has(key)) { - continue; - } - if (key === 'CLAUDE_CONFIG_DIR') { - execArgs.push('-e', 'CLAUDE_CONFIG_DIR=/home/node/.claude'); - continue; - } - execArgs.push('-e', `${key}=${value}`); - } -} - -// ── Main runner ─────────────────────────────────────────────────── - -export async function runReviewerContainer(args: { - group: RegisteredGroup; - input: ReviewerContainerInput; - ownerWorkspaceDir: string; - envOverrides?: Record; - onOutput?: (output: AgentOutput) => Promise; - onProcess?: (proc: ChildProcess, containerName: string) => void; -}): Promise { - const { group, input, ownerWorkspaceDir, envOverrides, onOutput, onProcess } = - args; - const startTime = Date.now(); - - // Pre-flight: Docker running + image exists (cached after first check) - ensureContainerReady(); - - // Ensure persistent container is running for this channel - const containerName = ensurePersistentContainer( - group, - ownerWorkspaceDir, - envOverrides, - ); - - logger.info( - { - containerName, - group: group.name, - groupFolder: group.folder, - chatJid: input.chatJid, - runId: input.runId, - }, - 'Executing reviewer turn in persistent container', - ); - - // Run a turn inside the persistent container via docker exec. - // Inject credentials and model config at exec-time so token rotation - // and per-role model overrides are picked up without recreating the container. - const execArgs = ['exec', '-i']; - const authMode = detectAuthMode(); - if (authMode === 'api-key') { - execArgs.push( - '-e', - `ANTHROPIC_API_KEY=${process.env.ANTHROPIC_API_KEY || ''}`, - ); - } else { - const oauthToken = - process.env.CLAUDE_CODE_OAUTH_TOKEN || - process.env.ANTHROPIC_AUTH_TOKEN || - ''; - execArgs.push('-e', `CLAUDE_CODE_OAUTH_TOKEN=${oauthToken}`); - } - const isCodexAgent = (group.agentType || 'claude-code') === 'codex'; - logger.info( - { - containerName, - groupAgentType: group.agentType, - isCodexAgent, - runnerPath: isCodexAgent - ? '/app/codex/dist/index.js' - : '/app/agent/dist/index.js', - }, - 'Container exec runner selection', - ); - appendExecEnvArgs(execArgs, envOverrides, isCodexAgent); - if (isCodexAgent) { - // Use session-local .codex dir (contains AGENTS.md with role prompts) - // instead of the host-mounted ~/.codex (which has owner-only config). - execArgs.push('-e', 'CODEX_HOME=/home/node/.claude/.codex'); - } - const runnerPath = isCodexAgent - ? '/app/codex/dist/index.js' - : '/app/agent/dist/index.js'; - execArgs.push(containerName, 'bun', runnerPath); - - return new Promise((resolve) => { - const proc = spawn(CONTAINER_RUNTIME_BIN, execArgs, { - stdio: ['pipe', 'pipe', 'pipe'], - }); - - onProcess?.(proc, containerName); - - // Send input via stdin - proc.stdin.write(JSON.stringify(input)); - proc.stdin.end(); - - let stdout = ''; - let stderr = ''; - let lastOutput: AgentOutput | null = null; - let totalOutputSize = 0; - let parseBuffer = ''; - let outputChain = Promise.resolve(); - - // Streaming output: parse OUTPUT_START/END marker pairs - proc.stdout.on('data', (chunk: Buffer) => { - const text = chunk.toString(); - totalOutputSize += text.length; - - if (totalOutputSize > AGENT_MAX_OUTPUT_SIZE) { - logger.warn( - { containerName, size: totalOutputSize }, - 'Container output exceeds max size, killing exec', - ); - try { - proc.kill('SIGKILL'); - } catch { - /* ignore */ - } - return; - } - - stdout += text; - parseBuffer += text; - resetTimeout(); - - let startIdx: number; - while ((startIdx = parseBuffer.indexOf(OUTPUT_START_MARKER)) !== -1) { - const endIdx = parseBuffer.indexOf(OUTPUT_END_MARKER, startIdx); - if (endIdx === -1) break; - - const json = parseBuffer - .slice(startIdx + OUTPUT_START_MARKER.length, endIdx) - .trim(); - parseBuffer = parseBuffer.slice(endIdx + OUTPUT_END_MARKER.length); - - try { - const parsed = JSON.parse(json) as AgentOutput; - lastOutput = parsed; - if (onOutput) { - outputChain = outputChain.then(() => onOutput(parsed)); - } - } catch (err) { - logger.warn( - { containerName, err }, - 'Failed to parse container output JSON', - ); - } - } - }); - - proc.stderr.on('data', (chunk: Buffer) => { - stderr += chunk.toString(); - resetTimeout(); - }); - - // Activity-based timeout - const timeoutMs = Math.max(AGENT_TIMEOUT, IDLE_TIMEOUT + 30_000); - const killOnTimeout = () => { - logger.warn( - { containerName, timeoutMs }, - 'Reviewer exec timed out, killing', - ); - try { - proc.kill('SIGTERM'); - } catch { - /* ignore */ - } - }; - let timeout = setTimeout(killOnTimeout, timeoutMs); - const resetTimeout = () => { - clearTimeout(timeout); - timeout = setTimeout(killOnTimeout, timeoutMs); - }; - - proc.on('close', (code, signal) => { - clearTimeout(timeout); - const durationMs = Date.now() - startTime; - - logger.info( - { - containerName, - group: group.name, - runId: input.runId, - exitCode: code, - signal, - durationMs, - stdoutLen: stdout.length, - stderrLen: stderr.length, - }, - 'Reviewer exec completed', - ); - - if (lastOutput) { - outputChain.then(() => resolve(lastOutput!)); - return; - } - - // Fallback: try to parse from full stdout - const sIdx = stdout.indexOf(OUTPUT_START_MARKER); - const eIdx = stdout.indexOf(OUTPUT_END_MARKER); - if (sIdx !== -1 && eIdx !== -1) { - try { - const json = stdout - .slice(sIdx + OUTPUT_START_MARKER.length, eIdx) - .trim(); - resolve(JSON.parse(json) as AgentOutput); - return; - } catch { - /* fall through */ - } - } - - const errorMsg = stderr.trim() || `Exec exited with code ${code}`; - resolve({ - status: code === 0 ? 'success' : 'error', - result: null, - error: errorMsg, - phase: 'final', - }); - }); - - proc.on('error', (err) => { - clearTimeout(timeout); - logger.error({ containerName, err }, 'Failed to exec in container'); - resolve({ - status: 'error', - result: null, - error: `Container exec error: ${err.message}`, - phase: 'final', - }); - }); - }); -} diff --git a/src/container-runtime.ts b/src/container-runtime.ts index ade0ddf..e488e9e 100644 --- a/src/container-runtime.ts +++ b/src/container-runtime.ts @@ -1,7 +1,7 @@ /** - * Container runtime abstraction for EJClaw. - * Reviewer agents run inside Docker containers for read-only isolation. - * All runtime-specific logic lives here so swapping runtimes means changing one file. + * Container runtime abstraction for EJClaw verification. + * Verification profiles and related host evidence still use Docker. + * Runtime-specific logic lives here so swapping runtimes means changing one file. */ import { execSync } from 'child_process'; import fs from 'fs'; diff --git a/src/credential-proxy.ts b/src/credential-proxy.ts deleted file mode 100644 index 3d87052..0000000 --- a/src/credential-proxy.ts +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Credential proxy for reviewer container isolation. - * Containers connect here instead of directly to AI APIs. - * The proxy injects real credentials so containers never see them. - * - * Supports both Claude (Anthropic) and Codex (OpenAI) APIs: - * Claude API key: Proxy injects x-api-key on every request. - * Claude OAuth: Proxy replaces placeholder Bearer token with real one. - * Codex: Proxy injects Authorization: Bearer {OPENAI_API_KEY}. - */ -import { createServer, Server } from 'http'; -import { request as httpsRequest } from 'https'; -import { request as httpRequest, RequestOptions } from 'http'; - -import { getEnv } from './env.js'; -import { logger } from './logger.js'; - -export type AuthMode = 'api-key' | 'oauth'; - -export function startCredentialProxy( - port: number, - host = '127.0.0.1', -): Promise { - const anthropicApiKey = getEnv('ANTHROPIC_API_KEY') || ''; - const oauthToken = - getEnv('CLAUDE_CODE_OAUTH_TOKEN') || getEnv('ANTHROPIC_AUTH_TOKEN') || ''; - const anthropicBaseUrl = - getEnv('ANTHROPIC_BASE_URL') || 'https://api.anthropic.com'; - const openaiApiKey = getEnv('OPENAI_API_KEY') || ''; - - const authMode: AuthMode = anthropicApiKey ? 'api-key' : 'oauth'; - - const anthropicUrl = new URL(anthropicBaseUrl); - const anthropicIsHttps = anthropicUrl.protocol === 'https:'; - const makeAnthropicRequest = anthropicIsHttps ? httpsRequest : httpRequest; - - return new Promise((resolve, reject) => { - const server = createServer((req, res) => { - const chunks: Buffer[] = []; - req.on('data', (c) => chunks.push(c)); - req.on('end', () => { - const body = Buffer.concat(chunks); - - // Route: Codex/OpenAI requests go to OpenAI API - const isOpenAI = - req.headers['x-ejclaw-provider'] === 'openai' || - (req.url || '').startsWith('/v1/chat/completions'); - - if (isOpenAI) { - proxyToOpenAI(req, res, body, openaiApiKey); - return; - } - - // Default: Anthropic/Claude API - const headers: Record = - { - ...(req.headers as Record), - host: anthropicUrl.host, - 'content-length': body.length, - }; - - delete headers['connection']; - delete headers['keep-alive']; - delete headers['transfer-encoding']; - delete headers['x-ejclaw-provider']; - - if (authMode === 'api-key') { - delete headers['x-api-key']; - headers['x-api-key'] = anthropicApiKey; - } else if (headers['authorization']) { - delete headers['authorization']; - if (oauthToken) { - headers['authorization'] = `Bearer ${oauthToken}`; - } - } - - const upstream = makeAnthropicRequest( - { - hostname: anthropicUrl.hostname, - port: anthropicUrl.port || (anthropicIsHttps ? 443 : 80), - path: req.url, - method: req.method, - headers, - } as RequestOptions, - (upRes) => { - res.writeHead(upRes.statusCode!, upRes.headers); - upRes.pipe(res); - }, - ); - - upstream.on('error', (err) => { - logger.error( - { err, url: req.url }, - 'Credential proxy upstream error (Anthropic)', - ); - if (!res.headersSent) { - res.writeHead(502); - res.end('Bad Gateway'); - } - }); - - upstream.write(body); - upstream.end(); - }); - }); - - server.listen(port, host, () => { - logger.info({ port, host, authMode }, 'Credential proxy started'); - resolve(server); - }); - - server.on('error', reject); - }); -} - -function proxyToOpenAI( - req: import('http').IncomingMessage, - res: import('http').ServerResponse, - body: Buffer, - apiKey: string, -): void { - const headers: Record = { - ...(req.headers as Record), - host: 'api.openai.com', - 'content-length': body.length, - }; - - delete headers['connection']; - delete headers['keep-alive']; - delete headers['transfer-encoding']; - delete headers['x-ejclaw-provider']; - - // Inject real OpenAI API key - delete headers['authorization']; - if (apiKey) { - headers['authorization'] = `Bearer ${apiKey}`; - } - - const upstream = httpsRequest( - { - hostname: 'api.openai.com', - port: 443, - path: req.url, - method: req.method, - headers, - } as RequestOptions, - (upRes) => { - res.writeHead(upRes.statusCode!, upRes.headers); - upRes.pipe(res); - }, - ); - - upstream.on('error', (err) => { - logger.error( - { err, url: req.url }, - 'Credential proxy upstream error (OpenAI)', - ); - if (!res.headersSent) { - res.writeHead(502); - res.end('Bad Gateway'); - } - }); - - upstream.write(body); - upstream.end(); -} - -export function detectAuthMode(): AuthMode { - return getEnv('ANTHROPIC_API_KEY') ? 'api-key' : 'oauth'; -} diff --git a/src/index.ts b/src/index.ts index 5fddaf8..9479238 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,6 @@ import path from 'path'; import { ASSISTANT_NAME, - CREDENTIAL_PROXY_PORT, DATA_DIR, IDLE_TIMEOUT, POLL_INTERVAL, @@ -72,11 +71,7 @@ import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { normalizeStoredSeqCursor } from './message-cursor.js'; import { initCodexTokenRotation } from './codex-token-rotation.js'; -import { - hasAvailableClaudeToken, - initTokenRotation, - onTokenRotated, -} from './token-rotation.js'; +import { hasAvailableClaudeToken, initTokenRotation } from './token-rotation.js'; function isTerminalStatusMessage(text: string): boolean { const trimmed = text.trimStart(); @@ -85,7 +80,6 @@ function isTerminalStatusMessage(text: string): boolean { ); } import { - onTokenRefreshed, startTokenRefreshLoop, stopTokenRefreshLoop, } from './token-refresh.js'; @@ -94,9 +88,6 @@ import { getGlobalFailoverInfo, } from './service-routing.js'; import { FAILOVER_MIN_DURATION_MS } from './config.js'; -import { startCredentialProxy } from './credential-proxy.js'; -import { recreateAllReviewerContainers } from './container-runner.js'; -import { cleanupOrphans, PROXY_BIND_HOST } from './container-runtime.js'; // Token rotation is initialized lazily on first use or at startup below @@ -331,21 +322,6 @@ async function main(): Promise { logger.info('Database initialized'); initTokenRotation(); initCodexTokenRotation(); - - // Start credential proxy for container isolation and clean up orphaned containers - startCredentialProxy(CREDENTIAL_PROXY_PORT, PROXY_BIND_HOST).catch((err) => - logger.warn( - { err }, - 'Failed to start credential proxy (may already be running)', - ), - ); - cleanupOrphans(); - - // Recreate reviewer containers when OAuth tokens are rotated or refreshed. - // Persistent containers hold the old token in their running SDK process; - // removing them forces re-creation with fresh credentials on next turn. - onTokenRotated(recreateAllReviewerContainers); - onTokenRefreshed(recreateAllReviewerContainers); startTokenRefreshLoop(); loadState(); @@ -392,7 +368,6 @@ async function main(): Promise { ); } await queue.shutdown(10000); - cleanupOrphans(); for (const ch of channels) await ch.disconnect(); process.exit(0); }; diff --git a/src/token-refresh.ts b/src/token-refresh.ts index 63a0ab4..a913785 100644 --- a/src/token-refresh.ts +++ b/src/token-refresh.ts @@ -322,7 +322,7 @@ async function checkAndRefreshAll(): Promise { } if (anyRefreshed) { - // Sync refreshed tokens to process.env so docker exec picks them up + // Sync refreshed tokens to process.env so subsequent turns pick them up const refreshedTokens = getAllTokens(); if (refreshedTokens.length > 0) { process.env.CLAUDE_CODE_OAUTH_TOKEN = refreshedTokens[0].token; @@ -333,31 +333,9 @@ async function checkAndRefreshAll(): Promise { } } updateEnvTokens(); - if (onTokenRefreshedCallback) { - try { - onTokenRefreshedCallback(); - } catch (err) { - logger.warn( - { err: getErrorMessage(err) }, - 'Post-token-refresh callback failed', - ); - } - } } } -// ── Post-refresh callback ─────────────────────────────────────── -// Allows callers (e.g. index.ts) to register a hook that runs after -// any token is refreshed — used to recreate reviewer containers whose -// persistent Claude Code SDK process still holds the old token. -type TokenRefreshCallback = () => void; -let onTokenRefreshedCallback: TokenRefreshCallback | null = null; - -/** Register a callback to be invoked after a successful token refresh. */ -export function onTokenRefreshed(cb: TokenRefreshCallback): void { - onTokenRefreshedCallback = cb; -} - let refreshInterval: ReturnType | null = null; export function startTokenRefreshLoop(): void { @@ -421,16 +399,6 @@ export async function forceRefreshToken( if (tokenIndex < allTokens.length) { updateTokenValue(tokenIndex, newAccessToken); updateEnvTokens(); - if (onTokenRefreshedCallback) { - try { - onTokenRefreshedCallback(); - } catch (err) { - logger.warn( - { err: getErrorMessage(err) }, - 'Post-token-refresh callback failed (force refresh)', - ); - } - } } } return newAccessToken; diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 6f68598..4bc88e0 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -33,18 +33,6 @@ const tokens: TokenState[] = []; let currentIndex = 0; let initialized = false; -// ── Post-rotation callback ────────────────────────────────────── -// Invoked after a successful token rotation so that callers (e.g. -// index.ts) can tear down reviewer containers whose SDK process -// still holds the old token. -type TokenRotationCallback = () => void; -let onTokenRotatedCallback: TokenRotationCallback | null = null; - -/** Register a callback to be invoked after a successful token rotation. */ -export function onTokenRotated(cb: TokenRotationCallback): void { - onTokenRotatedCallback = cb; -} - export function initTokenRotation(): void { if (initialized) return; initialized = true; @@ -194,13 +182,6 @@ export function rotateToken( `Rotated to token #${currentIndex + 1}/${tokens.length}`, ); saveState(); - if (onTokenRotatedCallback) { - try { - onTokenRotatedCallback(); - } catch { - /* best effort — don't break rotation flow */ - } - } return true; } From bf55ad65fdad8c0eb106aaafd94a0c892f7892ea Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 07:56:59 +0900 Subject: [PATCH 06/29] Format files touched by reviewer runtime cleanup --- src/agent-runner.test.ts | 6 +----- src/index.ts | 5 ++++- src/message-agent-executor.test.ts | 4 +++- src/message-agent-executor.ts | 3 ++- src/message-runtime.test.ts | 9 +++++++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/agent-runner.test.ts b/src/agent-runner.test.ts index b7744cf..7b8ad05 100644 --- a/src/agent-runner.test.ts +++ b/src/agent-runner.test.ts @@ -105,10 +105,7 @@ vi.mock('child_process', async () => { }; }); -import { - runAgentProcess, - AgentOutput, -} from './agent-runner.js'; +import { runAgentProcess, AgentOutput } from './agent-runner.js'; import * as agentRunnerEnvironment from './agent-runner-environment.js'; import type { RegisteredGroup } from './types.js'; @@ -681,5 +678,4 @@ OUROBOROS_LLM_BACKEND = "codex" expect.objectContaining({ newSessionId: 'stale-session' }), ); }); - }); diff --git a/src/index.ts b/src/index.ts index 9479238..7f1c169 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,7 +71,10 @@ import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { normalizeStoredSeqCursor } from './message-cursor.js'; import { initCodexTokenRotation } from './codex-token-rotation.js'; -import { hasAvailableClaudeToken, initTokenRotation } from './token-rotation.js'; +import { + hasAvailableClaudeToken, + initTokenRotation, +} from './token-rotation.js'; function isTerminalStatusMessage(text: string): boolean { const trimmed = text.trimStart(); diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 2a815b6..240e139 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1697,7 +1697,9 @@ describe('runAgentForGroup Claude rotation', () => { vi.mocked(agentRunner.runAgentProcess) .mockImplementationOnce(async (_group, input) => { expect(input.sessionId).toBe('stale-session-id'); - throw new Error('No conversation found with session ID: stale-session-id'); + throw new Error( + 'No conversation found with session ID: stale-session-id', + ); }) .mockImplementationOnce(async (_group, input, _onProcess, onOutput) => { expect(input.sessionId).toBeUndefined(); diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 17bf68d..37ebe3b 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -480,7 +480,8 @@ export async function runAgentForGroup( sawVisibleOutput: false, sawSuccessNullResultWithoutOutput: false, }; - const attemptSessionId = provider === 'claude' ? currentSessionId : undefined; + const attemptSessionId = + provider === 'claude' ? currentSessionId : undefined; const wrappedOnOutput = onOutput ? async (output: AgentOutput) => { diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 3f59290..048a022 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -1880,7 +1880,9 @@ describe('createMessageRuntime', () => { ]); vi.mocked(agentRunner.runAgentProcess).mockImplementation( async (_group, input, _onProcess, onOutput) => { - expect(input.prompt).toContain('paired_turn_outputs 에 저장된 reviewer 요약'); + expect(input.prompt).toContain( + 'paired_turn_outputs 에 저장된 reviewer 요약', + ); expect(input.prompt).not.toContain('리뷰어 디스코드 출력 원문'); await onOutput?.({ status: 'success', @@ -1925,7 +1927,10 @@ describe('createMessageRuntime', () => { expect(result).toBe(true); expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); - expect(channel.sendMessage).toHaveBeenCalledWith(chatJid, 'owner follow-up ok'); + expect(channel.sendMessage).toHaveBeenCalledWith( + chatJid, + 'owner follow-up ok', + ); }); it('reuses the shared arbiter prompt builder for pending arbiter turns', async () => { From 4650abbd104f54b77a04d812fbed9af6afbcdddb Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 17:27:16 +0900 Subject: [PATCH 07/29] Rename readonly reviewer runtime remnants --- .gitignore | 1 + runners/agent-runner/package.json | 2 +- runners/agent-runner/src/index.ts | 48 ++++++++--------- runners/codex-runner/src/index.ts | 34 ++++++------ src/agent-runner-environment.test.ts | 12 ++--- src/agent-runner-environment.ts | 13 ++--- src/agent-runner.test.ts | 6 +-- src/agent-runner.ts | 4 +- src/message-agent-executor.test.ts | 77 ++++++++++++++++++++++++++++ src/message-agent-executor.ts | 22 ++++++++ 10 files changed, 160 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 5d89d6f..60ecf65 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ store-*/ data/ data-*/ logs/ +.ejclaw-reviewer-runtime/ # Groups - only track base structure and specific CLAUDE.md files groups-*/ diff --git a/runners/agent-runner/package.json b/runners/agent-runner/package.json index e183357..6715457 100644 --- a/runners/agent-runner/package.json +++ b/runners/agent-runner/package.json @@ -2,7 +2,7 @@ "name": "ejclaw-agent-runner", "version": "1.0.0", "type": "module", - "description": "Container-side agent runner for EJClaw", + "description": "Agent runner subprocess for EJClaw", "main": "dist/index.js", "scripts": { "build": "tsc", diff --git a/runners/agent-runner/src/index.ts b/runners/agent-runner/src/index.ts index 4478502..949762e 100644 --- a/runners/agent-runner/src/index.ts +++ b/runners/agent-runner/src/index.ts @@ -34,7 +34,7 @@ import { } from './reviewer-runtime.js'; import { selectCompactMemoriesFromSummary } from './memory-selection.js'; -interface ContainerInput { +interface RunnerInput { prompt: string; sessionId?: string; groupFolder: string; @@ -47,7 +47,7 @@ interface ContainerInput { } /** Mirrors AgentOutput in src/agent-runner.ts (separate package, can't import directly). */ -interface ContainerOutput { +interface RunnerOutput { status: 'success' | 'error'; phase?: 'progress' | 'final' | 'tool-activity' | 'intermediate'; agentId?: string; @@ -199,7 +199,7 @@ async function readStdin(): Promise { const OUTPUT_START_MARKER = '---EJCLAW_OUTPUT_START---'; const OUTPUT_END_MARKER = '---EJCLAW_OUTPUT_END---'; -function writeOutput(output: ContainerOutput): void { +function writeOutput(output: RunnerOutput): void { console.log(OUTPUT_START_MARKER); console.log(JSON.stringify(output)); console.log(OUTPUT_END_MARKER); @@ -207,7 +207,7 @@ function writeOutput(output: ContainerOutput): void { function normalizeStructuredOutput(result: string | null): { result: string | null; - output?: ContainerOutput['output']; + output?: RunnerOutput['output']; } { if (typeof result !== 'string' || result.length === 0) { return { result }; @@ -545,7 +545,7 @@ async function runQuery( prompt: string, sessionId: string | undefined, mcpServerPath: string, - containerInput: ContainerInput, + runnerInput: RunnerInput, sdkEnv: Record, reviewerRuntime: boolean, claudeReadonlyReviewerRuntime: boolean, @@ -698,12 +698,12 @@ async function runQuery( command: 'node', args: [mcpServerPath], env: { - EJCLAW_CHAT_JID: containerInput.chatJid, - EJCLAW_GROUP_FOLDER: containerInput.groupFolder, - EJCLAW_IS_MAIN: containerInput.isMain ? '1' : '0', + EJCLAW_CHAT_JID: runnerInput.chatJid, + EJCLAW_GROUP_FOLDER: runnerInput.groupFolder, + EJCLAW_IS_MAIN: runnerInput.isMain ? '1' : '0', EJCLAW_AGENT_TYPE: process.env.EJCLAW_AGENT_TYPE || 'claude-code', - EJCLAW_ROOM_ROLE: containerInput.roomRoleContext?.role || '', + EJCLAW_ROOM_ROLE: runnerInput.roomRoleContext?.role || '', ...(process.env.EJCLAW_IPC_DIR && { EJCLAW_IPC_DIR: process.env.EJCLAW_IPC_DIR, }), @@ -714,7 +714,7 @@ async function runQuery( }, }, hooks: { - PreCompact: [{ hooks: [createPreCompactHook(containerInput.assistantName)] }], + PreCompact: [{ hooks: [createPreCompactHook(runnerInput.assistantName)] }], PreToolUse: [ { matcher: 'Bash', @@ -958,14 +958,14 @@ async function runQuery( } async function main(): Promise { - let containerInput: ContainerInput; + let runnerInput: RunnerInput; try { const stdinData = await readStdin(); - containerInput = JSON.parse(stdinData); + runnerInput = JSON.parse(stdinData); // Delete the temp file the entrypoint wrote — it contains secrets try { fs.unlinkSync('/tmp/input.json'); } catch { /* may not exist */ } - log(`Received input for group: ${containerInput.groupFolder}`); + log(`Received input for group: ${runnerInput.groupFolder}`); } catch (err) { writeOutput({ status: 'error', @@ -978,14 +978,14 @@ async function main(): Promise { // Build SDK env: merge secrets into process.env for the SDK only. // Secrets never touch process.env itself, so Bash subprocesses can't see them. const sdkEnv: Record = { ...process.env }; - for (const [key, value] of Object.entries(containerInput.secrets || {})) { + for (const [key, value] of Object.entries(runnerInput.secrets || {})) { sdkEnv[key] = value; } const reviewerRuntime = process.env.EJCLAW_REVIEWER_RUNTIME === '1' || - isReviewerRuntime(containerInput.roomRoleContext); + isReviewerRuntime(runnerInput.roomRoleContext); const claudeReadonlyReviewerRuntime = - isClaudeReadonlyReviewerRuntime(containerInput.roomRoleContext); + isClaudeReadonlyReviewerRuntime(runnerInput.roomRoleContext); const claudeReadonlySandboxMode = claudeReadonlyReviewerRuntime ? getClaudeReadonlySandboxMode() : null; @@ -1002,18 +1002,18 @@ async function main(): Promise { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const mcpServerPath = path.join(__dirname, 'ipc-mcp-stdio.js'); - let sessionId = containerInput.sessionId; + let sessionId = runnerInput.sessionId; fs.mkdirSync(IPC_INPUT_DIR, { recursive: true }); - // Clean up stale _close sentinel from previous container runs + // Clean up stale _close sentinel from previous runner sessions try { fs.unlinkSync(IPC_INPUT_CLOSE_SENTINEL); } catch { /* ignore */ } // Effective working directory (WORK_DIR overrides GROUP_DIR) const mainEffectiveCwd = WORK_DIR || GROUP_DIR; // Build initial prompt (drain any pending IPC messages too) - let prompt = containerInput.prompt; - if (containerInput.isScheduledTask) { + let prompt = runnerInput.prompt; + if (runnerInput.isScheduledTask) { prompt = `[SCHEDULED TASK - The following message was sent automatically and is not coming directly from the user or group.]\n\n${prompt}`; } const pending = drainIpcInput(); @@ -1024,10 +1024,10 @@ async function main(): Promise { // --- Slash command handling --- // Check BEFORE prepending room role header so /compact isn't masked. const KNOWN_SESSION_COMMANDS = new Set(['/compact']); - const isSessionSlashCommand = KNOWN_SESSION_COMMANDS.has(containerInput.prompt.trim()); + const isSessionSlashCommand = KNOWN_SESSION_COMMANDS.has(runnerInput.prompt.trim()); if (!isSessionSlashCommand) { - prompt = prependRoomRoleHeader(prompt, containerInput.roomRoleContext); + prompt = prependRoomRoleHeader(prompt, runnerInput.roomRoleContext); } const trimmedPrompt = prompt.trim(); @@ -1052,7 +1052,7 @@ async function main(): Promise { settingSources: ['project', 'user'] as const, abortController: agentAbortController, hooks: { - PreCompact: [{ hooks: [createPreCompactHook(containerInput.assistantName)] }], + PreCompact: [{ hooks: [createPreCompactHook(runnerInput.assistantName)] }], }, }, })) { @@ -1133,7 +1133,7 @@ async function main(): Promise { prompt, sessionId, mcpServerPath, - containerInput, + runnerInput, guardedSdkEnv, reviewerRuntime, claudeReadonlyReviewerRuntime, diff --git a/runners/codex-runner/src/index.ts b/runners/codex-runner/src/index.ts index 7bb37ec..bfbdbf0 100644 --- a/runners/codex-runner/src/index.ts +++ b/runners/codex-runner/src/index.ts @@ -4,7 +4,7 @@ * App-server only runtime. * * Input protocol: - * Stdin: Full ContainerInput JSON (read until EOF) + * Stdin: Full RunnerInput JSON (read until EOF) * IPC: Follow-up messages as JSON files in $EJCLAW_IPC_DIR/input/ * Sentinel: _close — signals session end * @@ -31,7 +31,7 @@ import { // ── Types ────────────────────────────────────────────────────────── -interface ContainerInput { +interface RunnerInput { prompt: string; sessionId?: string; groupFolder: string; @@ -43,7 +43,7 @@ interface ContainerInput { roomRoleContext?: RoomRoleContext; } -interface ContainerOutput { +interface RunnerOutput { status: 'success' | 'error'; result: string | null; output?: { @@ -77,7 +77,7 @@ let closeRequested = false; // ── Helpers ──────────────────────────────────────────────────────── -function writeOutput(output: ContainerOutput): void { +function writeOutput(output: RunnerOutput): void { console.log(OUTPUT_START_MARKER); console.log(JSON.stringify(output)); console.log(OUTPUT_END_MARKER); @@ -85,7 +85,7 @@ function writeOutput(output: ContainerOutput): void { function normalizeStructuredOutput(result: string | null): { result: string | null; - output?: ContainerOutput['output']; + output?: RunnerOutput['output']; } { if (typeof result !== 'string' || result.length === 0) { return { result }; @@ -400,12 +400,12 @@ async function runAppServerCompact( } async function runAppServerSession( - containerInput: ContainerInput, + runnerInput: RunnerInput, prompt: string, ): Promise { const reviewerRuntime = process.env.EJCLAW_REVIEWER_RUNTIME === '1' || - isReviewerRuntime(containerInput.roomRoleContext); + isReviewerRuntime(runnerInput.roomRoleContext); const readonlyRuntime = reviewerRuntime || process.env.EJCLAW_ARBITER_RUNTIME === '1'; const clientEnv = buildReviewerGitGuardEnv(process.env, reviewerRuntime); @@ -421,17 +421,17 @@ async function runAppServerSession( let threadId: string | undefined; try { try { - threadId = await client.startOrResumeThread(containerInput.sessionId, { + threadId = await client.startOrResumeThread(runnerInput.sessionId, { cwd: EFFECTIVE_CWD, model: CODEX_MODEL || undefined, }); log( - containerInput.sessionId + runnerInput.sessionId ? `App-server thread resumed (${threadId})` : `App-server thread started (${threadId})`, ); } catch (err) { - if (!containerInput.sessionId) throw err; + if (!runnerInput.sessionId) throw err; log( `App-server resume failed, retrying with new thread: ${ err instanceof Error ? err.message : String(err) @@ -483,17 +483,17 @@ async function runAppServerSession( // ── Main ────────────────────────────────────────────────────────── async function main(): Promise { - let containerInput: ContainerInput; + let runnerInput: RunnerInput; try { const stdinData = await readStdin(); - containerInput = JSON.parse(stdinData); + runnerInput = JSON.parse(stdinData); try { fs.unlinkSync('/tmp/input.json'); } catch { /* may not exist */ } - log(`Received input for group: ${containerInput.groupFolder}`); + log(`Received input for group: ${runnerInput.groupFolder}`); } catch (err) { writeOutput({ status: 'error', @@ -513,9 +513,9 @@ async function main(): Promise { /* ignore */ } - const rawPrompt = containerInput.prompt; + const rawPrompt = runnerInput.prompt; let prompt = rawPrompt; - if (containerInput.isScheduledTask) { + if (runnerInput.isScheduledTask) { prompt = `[SCHEDULED TASK]\n\n${prompt}`; } const pending = drainIpcInput(); @@ -526,12 +526,12 @@ async function main(): Promise { // so /compact is not masked by the header prefix. const isSessionCommand = rawPrompt.trim() === '/compact'; if (!isSessionCommand) { - prompt = prependRoomRoleHeader(prompt, containerInput.roomRoleContext); + prompt = prependRoomRoleHeader(prompt, runnerInput.roomRoleContext); } try { log('Runtime selected: app-server'); - await runAppServerSession(containerInput, prompt); + await runAppServerSession(runnerInput, prompt); } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); log(`Runner error: ${errorMessage}`); diff --git a/src/agent-runner-environment.test.ts b/src/agent-runner-environment.test.ts index 386cfdd..ee8af84 100644 --- a/src/agent-runner-environment.test.ts +++ b/src/agent-runner-environment.test.ts @@ -79,7 +79,7 @@ vi.mock('os', async () => { }); import { - prepareContainerSessionEnvironment, + prepareReadonlySessionEnvironment, prepareGroupEnvironment, } from './agent-runner-environment.js'; import * as config from './config.js'; @@ -341,12 +341,12 @@ describe('prepareGroupEnvironment codex auth handling', () => { }); }); -describe('prepareContainerSessionEnvironment codex compatibility', () => { +describe('prepareReadonlySessionEnvironment codex compatibility', () => { let tempRoot: string; let previousCwd: string; beforeEach(() => { - tempRoot = fs.mkdtempSync(path.join('/tmp', 'ejclaw-container-env-')); + tempRoot = fs.mkdtempSync(path.join('/tmp', 'ejclaw-readonly-env-')); previousCwd = process.cwd(); process.chdir(tempRoot); @@ -369,7 +369,7 @@ describe('prepareContainerSessionEnvironment codex compatibility', () => { fs.rmSync(tempRoot, { recursive: true, force: true }); }); - it('writes matching AGENTS.md and copies host codex auth/config into the container session', () => { + it('writes matching AGENTS.md and copies host codex auth/config into the role-scoped session', () => { vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); const promptsDir = path.join(tempRoot, 'prompts'); @@ -404,8 +404,8 @@ args = ["other.js"] `, ); - const sessionDir = path.join(tempRoot, 'container-reviewer-session'); - prepareContainerSessionEnvironment({ + const sessionDir = path.join(tempRoot, 'readonly-reviewer-session'); + prepareReadonlySessionEnvironment({ sessionDir, chatJid: 'dc:test', isMain: false, diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index 9b5c1f9..4137ed0 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -487,7 +487,7 @@ export function prepareGroupEnvironment( : undefined; // Owner CLAUDE.md: platform rules + owner paired room rules. // Reviewer paired room rules are NOT included — those belong to the - // container reviewer only (via prepareContainerSessionEnvironment). + // Read-only reviewer/arbiter session only (via prepareReadonlySessionEnvironment). const sessionClaudeMd = [ ownerCommonPlatformPrompt, claudePlatformPrompt, @@ -558,14 +558,15 @@ export function prepareGroupEnvironment( } /** - * Prepare the Claude session directory for a container-based reviewer. + * Prepare a role-scoped session directory for a host-based read-only reviewer + * or arbiter run. * * Writes CLAUDE.md (platform + paired room prompts + global memory + briefing), * syncs skills, and ensures settings.json exist — the same steps that - * `prepareGroupEnvironment` does for host-mode agents, but targeted at an - * externally provided session directory (the one mounted into the container). + * `prepareGroupEnvironment` does for regular host-mode agents, but targeted at + * an externally provided session directory. */ -export function prepareContainerSessionEnvironment(args: { +export function prepareReadonlySessionEnvironment(args: { sessionDir: string; chatJid: string; isMain: boolean; @@ -665,7 +666,7 @@ export function prepareContainerSessionEnvironment(args: { hasGlobalMemory: !!globalClaudeMemory, hasMemoryBriefing: !!memoryBriefing, }, - 'Container session CLAUDE.md written', + 'Readonly session CLAUDE.md written', ); } else if (fs.existsSync(sessionClaudeMdPath)) { fs.unlinkSync(sessionClaudeMdPath); diff --git a/src/agent-runner.test.ts b/src/agent-runner.test.ts index 7b8ad05..c2ba3ad 100644 --- a/src/agent-runner.test.ts +++ b/src/agent-runner.test.ts @@ -318,9 +318,9 @@ describe('agent-runner timeout behavior', () => { it('uses host IPC and workdir in unsafe host paired mode reviewer sessions', async () => { vi.useRealTimers(); fakeProc = createFakeProcess(); - const prepareContainerSessionEnvironmentSpy = vi.spyOn( + const prepareReadonlySessionEnvironmentSpy = vi.spyOn( agentRunnerEnvironment, - 'prepareContainerSessionEnvironment', + 'prepareReadonlySessionEnvironment', ); vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => { @@ -356,7 +356,7 @@ describe('agent-runner timeout behavior', () => { fakeProc.emit('close', 0); const result = await resultPromise; expect(result.status).toBe('success'); - expect(prepareContainerSessionEnvironmentSpy).toHaveBeenCalledWith( + expect(prepareReadonlySessionEnvironmentSpy).toHaveBeenCalledWith( expect.objectContaining({ sessionDir: '/tmp/host-reviewer-session', ipcDir: '/tmp/ejclaw-test-data/ipc/test-group', diff --git a/src/agent-runner.ts b/src/agent-runner.ts index e275282..51aa9bb 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -13,7 +13,7 @@ import { IDLE_TIMEOUT, } from './config.js'; import { - prepareContainerSessionEnvironment, + prepareReadonlySessionEnvironment, prepareGroupEnvironment, } from './agent-runner-environment.js'; import { getEnv } from './env.js'; @@ -99,7 +99,7 @@ export async function runAgentProcess( (input.roomRoleContext?.role === 'reviewer' || input.roomRoleContext?.role === 'arbiter') ) { - prepareContainerSessionEnvironment({ + prepareReadonlySessionEnvironment({ sessionDir: envOverrides.CLAUDE_CONFIG_DIR, chatJid: input.chatJid, isMain: input.isMain, diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 240e139..4522e53 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1237,6 +1237,83 @@ describe('runAgentForGroup room memory', () => { expect(result).toBe('success'); expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); }); + + it('re-enqueues the group when a reviewer fails and the task remains review_ready', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + const deps = makeDeps(); + + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_agent_type: 'claude-code', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + owner_service_id: 'claude', + reviewer_service_id: 'claude', + arbiter_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-review-ready', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'claude', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'in_review', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: {}, + }); + vi.mocked(db.getPairedTaskById).mockReturnValue({ + id: 'paired-task-review-ready', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'claude', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'error', + result: null, + error: 'SDK crashed with exit code 1', + }); + + const result = await runAgentForGroup(deps, { + group, + prompt: 'please review', + chatJid: 'group@test', + runId: 'run-review-ready-requeue', + forcedRole: 'reviewer', + onOutput: async () => {}, + }); + + expect(result).toBe('error'); + expect(deps.queue.enqueueMessageCheck).toHaveBeenCalledWith('group@test'); + }); }); describe('runAgentForGroup Claude rotation', () => { diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 37ebe3b..3d207f1 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -1165,6 +1165,28 @@ export async function runAgentForGroup( } else if (finishedCheck?.status !== 'completed') { deps.queue.enqueueMessageCheck(chatJid); } + } else if (pairedExecutionContext) { + const completedRole = roomRoleContext?.role ?? 'owner'; + const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id); + const shouldRequeuePendingPairedTurn = + (completedRole === 'reviewer' || completedRole === 'arbiter') && + (finishedCheck?.status === 'review_ready' || + finishedCheck?.status === 'in_review' || + finishedCheck?.status === 'arbiter_requested' || + finishedCheck?.status === 'in_arbitration' || + finishedCheck?.status === 'merge_ready'); + if (shouldRequeuePendingPairedTurn) { + deps.queue.enqueueMessageCheck(chatJid); + log.info( + { + taskId: pairedExecutionContext.task.id, + role: completedRole, + pairedExecutionStatus, + taskStatus: finishedCheck?.status ?? null, + }, + 'Queued paired follow-up after failed reviewer/arbiter execution left a pending task state', + ); + } } } } From 24ffa3d2f1c44c514a8421115249c7b8249a7b88 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 19:32:36 +0900 Subject: [PATCH 08/29] Refactor paired turn transitions and scheduling --- src/message-runtime.ts | 339 ++++++----- src/paired-execution-context.test.ts | 32 + src/paired-execution-context.ts | 857 ++++++++++++++------------- 3 files changed, 665 insertions(+), 563 deletions(-) diff --git a/src/message-runtime.ts b/src/message-runtime.ts index fbff013..d263124 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -340,6 +340,153 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { (message) => message.is_from_me === true || !!message.is_bot_message, ); + type BotOnlyPairedFollowUpAction = + | { kind: 'none' } + | { + kind: 'inline-finalize'; + task: PairedTask; + cursor: string | number | null; + } + | { + kind: 'requeue-pending-turn'; + task: PairedTask; + cursor: string | number | null; + cursorKey: string; + nextRole: 'reviewer' | 'arbiter'; + }; + + const resolveBotOnlyPairedFollowUpAction = (args: { + chatJid: string; + task: PairedTask | null | undefined; + isBotOnlyPairedFollowUp: boolean; + pendingCursorSource: NewMessage | undefined; + }): BotOnlyPairedFollowUpAction => { + const { chatJid, task, isBotOnlyPairedFollowUp, pendingCursorSource } = + args; + if (!task || !isBotOnlyPairedFollowUp) { + return { kind: 'none' }; + } + + const cursor = + pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; + + if (task.status === 'merge_ready') { + return { + kind: 'inline-finalize', + task, + cursor, + }; + } + + if ( + task.status === 'review_ready' || + task.status === 'in_review' || + task.status === 'arbiter_requested' || + task.status === 'in_arbitration' + ) { + const nextRole = resolveActiveRole(task.status); + return { + kind: 'requeue-pending-turn', + task, + cursor, + cursorKey: resolveCursorKey(chatJid, task.status), + nextRole: nextRole === 'arbiter' ? 'arbiter' : 'reviewer', + }; + } + + return { kind: 'none' }; + }; + + const executeBotOnlyPairedFollowUpAction = async (args: { + action: BotOnlyPairedFollowUpAction; + chatJid: string; + group: RegisteredGroup; + runId: string; + channel: Channel; + log: typeof logger; + enqueueGroupMessageCheck: () => void; + }): Promise => { + const { + action, + chatJid, + group, + runId, + channel, + log, + enqueueGroupMessageCheck, + } = args; + + if (action.kind === 'none') { + return false; + } + + if (action.cursor != null) { + advanceLastAgentCursor( + deps.getLastAgentTimestamps(), + deps.saveState, + chatJid, + action.cursor, + action.kind === 'requeue-pending-turn' ? action.cursorKey : undefined, + ); + } + + if (action.kind === 'inline-finalize') { + log.info( + { + chatJid, + group: group.name, + groupFolder: group.folder, + taskId: action.task.id, + taskStatus: action.task.status, + handoffMode: 'inline-finalize', + nextRole: 'owner', + cursor: action.cursor, + }, + 'Executing merge_ready finalize turn inline after bot-only reviewer follow-up', + ); + const { deliverySucceeded } = await executeTurn({ + group, + prompt: buildFinalizePendingPrompt(action.task), + chatJid, + runId, + channel, + startSeq: null, + endSeq: null, + }); + if (!deliverySucceeded) { + enqueueGroupMessageCheck(); + } + return true; + } + + deps.queue.closeStdin(chatJid, { + reason: 'paired-pending-turn-follow-up', + }); + enqueueGroupMessageCheck(); + log.info( + { + chatJid, + group: group.name, + groupFolder: group.folder, + taskId: action.task.id, + taskStatus: action.task.status, + handoffMode: 'requeue', + nextRole: action.nextRole, + cursor: action.cursor, + cursorKey: action.cursorKey, + }, + 'Queued fresh paired pending turn instead of piping bot-only follow-up into the active agent', + ); + return true; + }; + + const enqueueScopedGroupMessageCheck = ( + chatJid: string, + groupFolder: string, + ): void => { + deps.queue.enqueueMessageCheck(chatJid, resolveGroupIpcPath(groupFolder)); + }; + /** * Check if a message is a duplicate of the last bot final message in a paired room. * Returns true if duplicate (should be suppressed). @@ -911,6 +1058,38 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { return deliverySucceeded; }; + const enqueueGroupMessageCheck = (): void => { + enqueueScopedGroupMessageCheck(chatJid, group.folder); + }; + + const shouldSkipGenericFollowUpAfterDeliveryRetry = (args: { + deliveryRole: PairedRoomRole; + pendingTask: PairedTask | null | undefined; + }): boolean => + hasReviewerLease(chatJid) && + args.deliveryRole === 'reviewer' && + args.pendingTask?.status === 'merge_ready'; + + const enqueueGenericFollowUpAfterDeliveryRetry = (args: { + deliveryRole: PairedRoomRole; + pendingTask: PairedTask | null | undefined; + workItemId: string | number; + }): void => { + if (shouldSkipGenericFollowUpAfterDeliveryRetry(args)) { + log.info( + { + workItemId: args.workItemId, + chatJid, + deliveryRole: args.deliveryRole, + pendingTaskStatus: args.pendingTask?.status ?? null, + }, + 'Skipping queued follow-up after reviewer merge_ready delivery because inline finalize will handle the handoff', + ); + return; + } + deps.queue.enqueueMessageCheck(chatJid); + }; + if (hasReviewerLease(chatJid)) { log.info( { @@ -965,30 +1144,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { openWorkItem, ); if (!delivered) return false; - const skipQueuedFollowUpForInlineFinalize = - hasReviewerLease(chatJid) && - deliveryRole === 'reviewer' && - pendingTask?.status === 'merge_ready'; - // Keep stale final delivery isolated from the next conversational turn. - // A follow-up run will pick up any queued user messages or paired-room - // auto-review/finalize transitions after this retry succeeds. - // - // merge_ready is special: the delivered reviewer bot message itself - // triggers the inline finalize path. Queueing a generic follow-up here - // can race with that path and produce a second owner finalize turn. - if (skipQueuedFollowUpForInlineFinalize) { - log.info( - { - workItemId: openWorkItem.id, - chatJid, - deliveryRole, - pendingTaskStatus: pendingTask?.status ?? null, - }, - 'Skipping queued follow-up after reviewer merge_ready delivery because inline finalize will handle the handoff', - ); - } else { - deps.queue.enqueueMessageCheck(chatJid); - } + enqueueGenericFollowUpAfterDeliveryRetry({ + deliveryRole, + pendingTask, + workItemId: openWorkItem.id, + }); return true; } @@ -1396,110 +1556,24 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { rawPendingMessages.length > 0 ? rawPendingMessages[rawPendingMessages.length - 1] : messagesToSend[messagesToSend.length - 1]; + const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({ + chatJid, + task: loopPendingTask, + isBotOnlyPairedFollowUp, + pendingCursorSource, + }); if ( - loopPendingTask && - isBotOnlyPairedFollowUp && - loopPendingTask.status === 'merge_ready' - ) { - const inlineRunId = `loop-merge-ready-${Date.now().toString(36)}`; - const mergeReadyCursor = - pendingCursorSource?.seq ?? - pendingCursorSource?.timestamp ?? - null; - if (mergeReadyCursor != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - mergeReadyCursor, - ); - } - logger.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - taskId: loopPendingTask?.id ?? null, - taskStatus: loopPendingTask?.status ?? null, - handoffMode: 'inline-finalize', - nextRole: 'owner', - cursor: mergeReadyCursor, - }, - 'Executing merge_ready finalize turn inline after bot-only reviewer follow-up', - ); - const { deliverySucceeded } = await executeTurn({ + await executeBotOnlyPairedFollowUpAction({ + action: botOnlyFollowUpAction, + chatJid, group, - prompt: buildFinalizePendingPrompt(loopPendingTask), - chatJid, - runId: inlineRunId, + runId: `loop-merge-ready-${Date.now().toString(36)}`, channel, - startSeq: null, - endSeq: null, - }); - if (!deliverySucceeded) { - deps.queue.enqueueMessageCheck( - chatJid, - resolveGroupIpcPath(group.folder), - ); - } - continue; - } - const botOnlyPendingTurn = - loopPendingTask && - isBotOnlyPairedFollowUp && - (loopPendingTask.status === 'review_ready' || - loopPendingTask.status === 'in_review' || - loopPendingTask.status === 'arbiter_requested' || - loopPendingTask.status === 'in_arbitration') - ? { - cursor: - pendingCursorSource?.seq ?? - pendingCursorSource?.timestamp ?? - null, - cursorKey: resolveCursorKey( - chatJid, - loopPendingTask.status, - ), - } - : null; - - if (botOnlyPendingTurn) { - if (botOnlyPendingTurn.cursor != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - botOnlyPendingTurn.cursor, - botOnlyPendingTurn.cursorKey, - ); - } - deps.queue.closeStdin(chatJid, { - reason: 'paired-pending-turn-follow-up', - }); - deps.queue.enqueueMessageCheck( - chatJid, - resolveGroupIpcPath(group.folder), - ); - logger.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - taskId: loopPendingTask?.id ?? null, - taskStatus: loopPendingTask?.status ?? null, - handoffMode: 'requeue', - nextRole: - loopPendingTask?.status === 'review_ready' - ? 'reviewer' - : loopPendingTask?.status === 'arbiter_requested' || - loopPendingTask?.status === 'in_arbitration' - ? 'arbiter' - : 'reviewer', - cursor: botOnlyPendingTurn.cursor, - cursorKey: botOnlyPendingTurn.cursorKey, - }, - 'Queued fresh paired pending turn instead of piping bot-only follow-up into the active agent', - ); + log: logger, + enqueueGroupMessageCheck: () => + enqueueScopedGroupMessageCheck(chatJid, group.folder), + }) + ) { continue; } @@ -1539,10 +1613,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { continue; } - deps.queue.enqueueMessageCheck( - chatJid, - resolveGroupIpcPath(group.folder), - ); + enqueueScopedGroupMessageCheck(chatJid, group.folder); } } } catch (err) { @@ -1561,10 +1632,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { { chatJid, group: group.name, workItemId: openWorkItem.id }, 'Recovery: found open work item awaiting delivery', ); - deps.queue.enqueueMessageCheck( - chatJid, - resolveGroupIpcPath(group.folder), - ); + enqueueScopedGroupMessageCheck(chatJid, group.folder); continue; } @@ -1585,10 +1653,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { { group: group.name, pendingCount: pending.length }, 'Recovery: found unprocessed messages', ); - deps.queue.enqueueMessageCheck( - chatJid, - resolveGroupIpcPath(group.folder), - ); + enqueueScopedGroupMessageCheck(chatJid, group.folder); continue; } else if (rawPending.length > 0) { const endSeq = rawPending[rawPending.length - 1].seq; diff --git a/src/paired-execution-context.test.ts b/src/paired-execution-context.test.ts index 73e653b..74cfc96 100644 --- a/src/paired-execution-context.test.ts +++ b/src/paired-execution-context.test.ts @@ -647,6 +647,38 @@ describe('paired execution context', () => { }, ); + it.each(['BLOCKED', 'NEEDS_CONTEXT'])( + 'escalates immediately when owner reports %s during a normal turn without arbiter', + (summary) => { + vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(false); + + vi.mocked(db.getPairedTaskById).mockReturnValue( + buildPairedTask({ + status: 'active', + round_trip_count: 0, + }), + ); + + completePairedExecutionContext({ + taskId: 'task-1', + role: 'owner', + status: 'succeeded', + summary, + }); + + expect(db.updatePairedTask).toHaveBeenCalledWith( + 'task-1', + expect.objectContaining({ + status: 'completed', + completion_reason: 'escalated', + }), + ); + expect( + pairedWorkspaceManager.markPairedTaskReviewReady, + ).not.toHaveBeenCalled(); + }, + ); + it('records source_ref when reviewer verdict DONE arrives via failed fallback', () => { const repoDir = createCanonicalRepoWithCommit('reviewed'); const approvedSourceRef = resolveTreeRef(repoDir); diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index fde5eb6..fbb0216 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -44,16 +44,14 @@ import { resolveRoleAgentPlan } from './role-agent-plan.js'; // Reviewer verdict detection // --------------------------------------------------------------------------- -type ReviewerVerdict = +type Verdict = | 'done' | 'done_with_concerns' | 'blocked' | 'needs_context' | 'continue'; -function classifyReviewerVerdict( - summary: string | null | undefined, -): ReviewerVerdict { +function classifyVerdict(summary: string | null | undefined): Verdict { if (!summary) return 'continue'; // Strip ... tags — these are internal reasoning, not the verdict const cleaned = summary.replace(/[\s\S]*?<\/internal>/g, '').trim(); @@ -146,6 +144,33 @@ function hasCodeChangesSinceRef( } } +function requestArbiterOrEscalate(args: { + taskId: string; + now: string; + arbiterLogMessage: string; + escalateLogMessage: string; + logContext?: Record; +}): void { + const { taskId, now, arbiterLogMessage, escalateLogMessage, logContext } = + args; + if (isArbiterEnabled()) { + updatePairedTask(taskId, { + status: 'arbiter_requested', + arbiter_requested_at: now, + updated_at: now, + }); + logger.info(logContext ?? { taskId }, arbiterLogMessage); + return; + } + + updatePairedTask(taskId, { + status: 'completed', + completion_reason: 'escalated', + updated_at: now, + }); + logger.info(logContext ?? { taskId }, escalateLogMessage); +} + function ensurePairedProject( group: RegisteredGroup, chatJid: string, @@ -423,6 +448,396 @@ export function preparePairedExecutionContext(args: { // completePairedExecutionContext // --------------------------------------------------------------------------- +function handleFailedReviewerExecution(args: { + task: PairedTask; + taskId: string; + summary?: string | null; +}): void { + const { task, taskId, summary } = args; + const now = new Date().toISOString(); + + if (summary) { + const verdict = classifyVerdict(summary); + if ( + verdict === 'done' || + verdict === 'blocked' || + verdict === 'needs_context' + ) { + const ownerWs = + verdict === 'done' ? getPairedWorkspace(taskId, 'owner') : null; + const approvedSourceRef = + verdict === 'done' && ownerWs?.workspace_dir + ? resolveCanonicalSourceRef(ownerWs.workspace_dir) + : task.source_ref; + updatePairedTask(taskId, { + status: verdict === 'done' ? 'merge_ready' : 'completed', + ...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), + ...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), + updated_at: now, + }); + logger.info( + { + taskId, + verdict, + approvedSourceRef, + summary: summary.slice(0, 100), + }, + 'Reviewer verdict detected from failed execution — stopping ping-pong', + ); + return; + } + } + + const fallbackStatus = + task.status === 'in_review' || task.status === 'review_ready' + ? 'review_ready' + : task.status; + if (fallbackStatus !== task.status) { + updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); + logger.warn( + { + taskId, + role: 'reviewer', + previousStatus: task.status, + nextStatus: fallbackStatus, + }, + 'Preserved reviewer task in review-ready state after failed execution', + ); + } +} + +function handleFailedArbiterExecution(args: { + task: PairedTask; + taskId: string; +}): void { + const { task, taskId } = args; + const now = new Date().toISOString(); + const fallbackStatus = + task.status === 'in_arbitration' || task.status === 'arbiter_requested' + ? 'arbiter_requested' + : task.status; + if (fallbackStatus !== task.status) { + updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); + logger.warn( + { + taskId, + role: 'arbiter', + previousStatus: task.status, + nextStatus: fallbackStatus, + }, + 'Preserved arbiter task in arbitration-requested state after failed execution', + ); + } +} + +function handleFailedOwnerExecution(args: { + task: PairedTask; + taskId: string; +}): void { + const { task, taskId } = args; + if (task.status !== 'active') { + const now = new Date().toISOString(); + updatePairedTask(taskId, { status: 'active', updated_at: now }); + logger.info( + { taskId, role: 'owner', previousStatus: task.status }, + 'Reset task to active after failed execution', + ); + } +} + +function handleOwnerFinalizeCompletion(args: { + task: PairedTask; + taskId: string; + summary?: string | null; + now: string; +}): boolean { + const { task, taskId, summary, now } = args; + const ownerVerdict = classifyVerdict(summary); + + if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter', + escalateLogMessage: 'Owner blocked during finalize — escalating to user', + logContext: { + taskId, + ownerVerdict, + summary: summary?.slice(0, 100), + }, + }); + return true; + } + + if (ownerVerdict === 'done_with_concerns') { + if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter', + escalateLogMessage: + 'Owner finalize loop detected — escalating to user', + logContext: { + taskId, + ownerVerdict, + roundTrips: task.round_trip_count, + }, + }); + return true; + } + updatePairedTask(taskId, { status: 'active', updated_at: now }); + logger.info( + { + taskId, + ownerVerdict, + summary: summary?.slice(0, 100), + }, + 'Owner raised concerns during finalize — task set back to active', + ); + return false; + } + + const workspace = getPairedWorkspace(task.id, 'owner'); + const hasNewChanges = workspace?.workspace_dir + ? hasCodeChangesSinceRef(workspace.workspace_dir, task.source_ref) + : null; + + if (hasNewChanges === true) { + if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: + 'Owner finalize DONE loop detected — requesting arbiter', + escalateLogMessage: + 'Owner finalize DONE loop detected — escalating to user', + logContext: { + taskId, + roundTrips: task.round_trip_count, + hasNewChanges, + }, + }); + return true; + } + logger.info( + { + taskId, + sourceRef: task.source_ref, + hasNewChanges, + }, + 'Owner made changes after reviewer approval — re-triggering review', + ); + return false; + } + + updatePairedTask(taskId, { + status: 'completed', + completion_reason: 'done', + updated_at: now, + }); + logger.info( + { taskId, hasNewChanges, summary: summary?.slice(0, 100) }, + 'Owner finalized after reviewer approval — task completed', + ); + return true; +} + +function handleOwnerCompletion(args: { + task: PairedTask; + taskId: string; + summary?: string | null; +}): void { + const { task, taskId, summary } = args; + const now = new Date().toISOString(); + + if (task.status === 'merge_ready') { + const shouldStop = handleOwnerFinalizeCompletion({ + task, + taskId, + summary, + now, + }); + if (shouldStop) { + return; + } + } + + if (hasActiveCiWatcherForChat(task.chat_jid)) { + logger.info( + { taskId, chatJid: task.chat_jid }, + 'Active CI watcher found, deferring auto-review until watcher completes', + ); + return; + } + + if (task.status !== 'merge_ready') { + const ownerVerdict = classifyVerdict(summary); + if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter', + escalateLogMessage: 'Owner blocked/needs_context — escalating to user', + logContext: { + taskId, + ownerVerdict, + summary: summary?.slice(0, 100), + }, + }); + return; + } + } + + if (task.round_trip_count >= PAIRED_MAX_ROUND_TRIPS) { + logger.info( + { + taskId, + roundTrips: task.round_trip_count, + max: PAIRED_MAX_ROUND_TRIPS, + }, + 'Round trip limit reached, skipping auto-review', + ); + return; + } + + const result = markPairedTaskReviewReady(taskId); + if (result) { + updatePairedTask(taskId, { + round_trip_count: task.round_trip_count + 1, + review_requested_at: now, + updated_at: now, + }); + logger.info( + { taskId, roundTrip: task.round_trip_count + 1 }, + 'Auto-triggered reviewer after owner completion', + ); + } +} + +function handleReviewerCompletion(args: { + task: PairedTask; + taskId: string; + summary?: string | null; +}): void { + const { task, taskId, summary } = args; + const now = new Date().toISOString(); + const verdict = classifyVerdict(summary); + + switch (verdict) { + case 'done': { + const ownerWs = getPairedWorkspace(taskId, 'owner'); + const approvedSourceRef = ownerWs?.workspace_dir + ? resolveCanonicalSourceRef(ownerWs.workspace_dir) + : task.source_ref; + updatePairedTask(taskId, { + status: 'merge_ready', + source_ref: approvedSourceRef, + updated_at: now, + }); + logger.info( + { + taskId, + verdict, + approvedSourceRef, + summary: summary?.slice(0, 100), + }, + 'Reviewer approved — owner gets final turn to finalize', + ); + return; + } + + case 'blocked': + case 'needs_context': + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: + 'Reviewer blocked/needs_context — requesting arbiter before escalating', + escalateLogMessage: 'Reviewer escalated to user — ping-pong stopped', + logContext: { taskId, verdict, summary: summary?.slice(0, 100) }, + }); + return; + + case 'done_with_concerns': + case 'continue': + default: + if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: + 'Deadlock detected — requesting arbiter intervention', + escalateLogMessage: + 'Stopped ping-pong — escalating to user (arbiter not configured)', + logContext: { + taskId, + verdict, + roundTrips: task.round_trip_count, + }, + }); + return; + } + updatePairedTask(taskId, { status: 'active', updated_at: now }); + logger.info( + { taskId, verdict }, + 'Reviewer has feedback, task set back to active for owner', + ); + return; + } +} + +function handleArbiterCompletion(args: { + taskId: string; + summary?: string | null; +}): void { + const { taskId, summary } = args; + const now = new Date().toISOString(); + const arbiterVerdict = classifyArbiterVerdict(summary); + + logger.info( + { taskId, arbiterVerdict, summary: summary?.slice(0, 200) }, + 'Arbiter verdict rendered', + ); + + switch (arbiterVerdict) { + case 'proceed': + case 'revise': + case 'reset': + updatePairedTask(taskId, { + status: 'active', + round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), + arbiter_verdict: arbiterVerdict, + updated_at: now, + }); + logger.info( + { taskId, arbiterVerdict }, + 'Arbiter resolved deadlock — resuming ping-pong', + ); + return; + case 'escalate': + updatePairedTask(taskId, { + status: 'completed', + arbiter_verdict: 'escalate', + completion_reason: 'arbiter_escalated', + updated_at: now, + }); + logger.info({ taskId }, 'Arbiter escalated to user — task completed'); + return; + default: + updatePairedTask(taskId, { + status: 'active', + round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), + arbiter_verdict: 'unknown', + updated_at: now, + }); + logger.warn( + { taskId, summary: summary?.slice(0, 200) }, + 'Arbiter verdict unrecognized — falling back to proceed', + ); + return; + } +} + export function completePairedExecutionContext(args: { taskId: string; role: PairedRoomRole; @@ -455,444 +870,34 @@ export function completePairedExecutionContext(args: { return; } - // On failure: for reviewers, still check verdict from summary — output may - // have been delivered even though the executor classified it as failed - // (e.g. intermediate buffer → null result). This prevents infinite loops. if (status !== 'succeeded') { - if (role === 'reviewer' && args.summary) { - const verdict = classifyReviewerVerdict(args.summary); - if ( - verdict === 'done' || - verdict === 'blocked' || - verdict === 'needs_context' - ) { - const now = new Date().toISOString(); - const ownerWs = - verdict === 'done' ? getPairedWorkspace(taskId, 'owner') : null; - const approvedSourceRef = - verdict === 'done' && ownerWs?.workspace_dir - ? resolveCanonicalSourceRef(ownerWs.workspace_dir) - : task.source_ref; - updatePairedTask(taskId, { - status: verdict === 'done' ? 'merge_ready' : 'completed', - ...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), - ...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), - updated_at: now, - }); - logger.info( - { - taskId, - verdict, - approvedSourceRef, - summary: args.summary?.slice(0, 100), - }, - 'Reviewer verdict detected from failed execution — stopping ping-pong', - ); - return; - } - } - const now = new Date().toISOString(); if (role === 'reviewer') { - const fallbackStatus = - task.status === 'in_review' || task.status === 'review_ready' - ? 'review_ready' - : task.status; - if (fallbackStatus !== task.status) { - updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); - logger.warn( - { - taskId, - role, - previousStatus: task.status, - nextStatus: fallbackStatus, - }, - 'Preserved reviewer task in review-ready state after failed execution', - ); - } + handleFailedReviewerExecution({ + task, + taskId, + summary: args.summary, + }); return; } if (role === 'arbiter') { - const fallbackStatus = - task.status === 'in_arbitration' || task.status === 'arbiter_requested' - ? 'arbiter_requested' - : task.status; - if (fallbackStatus !== task.status) { - updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); - logger.warn( - { - taskId, - role, - previousStatus: task.status, - nextStatus: fallbackStatus, - }, - 'Preserved arbiter task in arbitration-requested state after failed execution', - ); - } + handleFailedArbiterExecution({ task, taskId }); return; } - if (task.status !== 'active') { - updatePairedTask(taskId, { status: 'active', updated_at: now }); - logger.info( - { taskId, role, previousStatus: task.status }, - 'Reset task to active after failed execution', - ); - } + handleFailedOwnerExecution({ task, taskId }); return; } - // Owner finished if (role === 'owner') { - const now = new Date().toISOString(); - - // merge_ready → reviewer already approved. Check owner verdict and - // whether owner made additional changes that need re-review. - if (task.status === 'merge_ready') { - // Owner can raise concerns even during finalize (e.g. push failed, - // detached HEAD, discovered issue). Respect the owner's verdict. - const ownerVerdict = classifyReviewerVerdict(args.summary); - if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { - if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info( - { - taskId, - ownerVerdict, - summary: args.summary?.slice(0, 100), - }, - 'Owner blocked during finalize — requesting arbiter', - ); - } else { - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, - }); - logger.info( - { - taskId, - ownerVerdict, - summary: args.summary?.slice(0, 100), - }, - 'Owner blocked during finalize — escalating to user', - ); - } - return; - } - - if (ownerVerdict === 'done_with_concerns') { - // Check deadlock threshold before looping back — prevents - // merge_ready ↔ active infinite oscillation. - if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { - if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info( - { taskId, ownerVerdict, roundTrips: task.round_trip_count }, - 'Owner finalize loop detected — requesting arbiter', - ); - } else { - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, - }); - logger.info( - { taskId, ownerVerdict, roundTrips: task.round_trip_count }, - 'Owner finalize loop detected — escalating to user', - ); - } - return; - } - updatePairedTask(taskId, { status: 'active', updated_at: now }); - logger.info( - { - taskId, - ownerVerdict, - summary: args.summary?.slice(0, 100), - }, - 'Owner raised concerns during finalize — task set back to active', - ); - // Fall through to auto-trigger reviewer for the new concern - } else { - const workspace = getPairedWorkspace(task.id, 'owner'); - const hasNewChanges = workspace?.workspace_dir - ? hasCodeChangesSinceRef(workspace.workspace_dir, task.source_ref) - : null; - - if (hasNewChanges === true) { - if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { - if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info( - { taskId, roundTrips: task.round_trip_count, hasNewChanges }, - 'Owner finalize DONE loop detected — requesting arbiter', - ); - } else { - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, - }); - logger.info( - { taskId, roundTrips: task.round_trip_count, hasNewChanges }, - 'Owner finalize DONE loop detected — escalating to user', - ); - } - return; - } - // Owner made changes after approval → needs re-review - logger.info( - { - taskId, - sourceRef: task.source_ref, - hasNewChanges, - }, - 'Owner made changes after reviewer approval — re-triggering review', - ); - } else { - // No code changes (false) or unable to determine (null) → - // finalize complete. Treating null as "no changes" prevents - // infinite DONE↔DONE loops when source_ref is from a different - // repo or the workspace has no matching ref. - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'done', - updated_at: now, - }); - logger.info( - { taskId, hasNewChanges, summary: args.summary?.slice(0, 100) }, - 'Owner finalized after reviewer approval — task completed', - ); - return; - } - } - } - - // Active CI watcher → skip auto-review until watcher completes. - // The watcher result will be posted via reviewer bot, triggering - // the owner to act on it, which then resumes the review loop. - if (hasActiveCiWatcherForChat(task.chat_jid)) { - logger.info( - { taskId, chatJid: task.chat_jid }, - 'Active CI watcher found, deferring auto-review until watcher completes', - ); - return; - } - - // Owner blocked/needs_context → request arbiter (same as reviewer path). - // Without this, only reviewer verdicts can summon the arbiter, leaving - // owner-side deadlocks (e.g. owner can't implement reviewer's request) - // without a resolution mechanism. - if (task.status !== 'merge_ready') { - const normalOwnerVerdict = classifyReviewerVerdict(args.summary); - if ( - (normalOwnerVerdict === 'blocked' || - normalOwnerVerdict === 'needs_context') && - isArbiterEnabled() - ) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info( - { - taskId, - ownerVerdict: normalOwnerVerdict, - summary: args.summary?.slice(0, 100), - }, - 'Owner blocked/needs_context — requesting arbiter', - ); - return; - } - } - - // Normal turn → auto-trigger reviewer (if within round trip limit) - if (task.round_trip_count >= PAIRED_MAX_ROUND_TRIPS) { - logger.info( - { - taskId, - roundTrips: task.round_trip_count, - max: PAIRED_MAX_ROUND_TRIPS, - }, - 'Round trip limit reached, skipping auto-review', - ); - return; - } - - const result = markPairedTaskReviewReady(taskId); - if (result) { - updatePairedTask(taskId, { - round_trip_count: task.round_trip_count + 1, - review_requested_at: now, - updated_at: now, - }); - logger.info( - { taskId, roundTrip: task.round_trip_count + 1 }, - 'Auto-triggered reviewer after owner completion', - ); - } + handleOwnerCompletion({ task, taskId, summary: args.summary }); + return; } - // Reviewer finished → classify verdict and route accordingly if (role === 'reviewer') { - const now = new Date().toISOString(); - const verdict = classifyReviewerVerdict(args.summary); - - switch (verdict) { - case 'done': { - // Approved → owner gets final turn to commit/push. - // Record the current HEAD as source_ref so the finalize turn - // can detect if the owner made additional code changes. - const ownerWs = getPairedWorkspace(taskId, 'owner'); - const approvedSourceRef = ownerWs?.workspace_dir - ? resolveCanonicalSourceRef(ownerWs.workspace_dir) - : task.source_ref; - updatePairedTask(taskId, { - status: 'merge_ready', - source_ref: approvedSourceRef, - updated_at: now, - }); - logger.info( - { - taskId, - verdict, - approvedSourceRef, - summary: args.summary?.slice(0, 100), - }, - 'Reviewer approved — owner gets final turn to finalize', - ); - break; - } - - case 'blocked': - case 'needs_context': - // If arbiter is enabled, let arbiter judge before escalating to user. - // Arbiter may resolve the block (e.g. owner can fix it, info exists in context). - if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info( - { taskId, verdict, summary: args.summary?.slice(0, 100) }, - 'Reviewer blocked/needs_context — requesting arbiter before escalating', - ); - } else { - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, - }); - logger.info( - { taskId, verdict, summary: args.summary?.slice(0, 100) }, - 'Reviewer escalated to user — ping-pong stopped', - ); - } - break; - - case 'done_with_concerns': - case 'continue': - default: - // If both sides keep echoing DONE_WITH_CONCERNS without progress, - // request arbiter intervention (or escalate to user if arbiter not configured). - if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { - if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info( - { taskId, verdict, roundTrips: task.round_trip_count }, - 'Deadlock detected — requesting arbiter intervention', - ); - } else { - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, - }); - logger.info( - { taskId, verdict, roundTrips: task.round_trip_count }, - 'Stopped ping-pong — escalating to user (arbiter not configured)', - ); - } - break; - } - // Owner needs to address feedback — ping-pong continues - updatePairedTask(taskId, { status: 'active', updated_at: now }); - logger.info( - { taskId, verdict }, - 'Reviewer has feedback, task set back to active for owner', - ); - break; - } + handleReviewerCompletion({ task, taskId, summary: args.summary }); + return; } - // Arbiter finished → classify verdict and route accordingly if (role === 'arbiter') { - const now = new Date().toISOString(); - const arbiterVerdict = classifyArbiterVerdict(args.summary); - - logger.info( - { taskId, arbiterVerdict, summary: args.summary?.slice(0, 200) }, - 'Arbiter verdict rendered', - ); - - switch (arbiterVerdict) { - case 'proceed': - case 'revise': - case 'reset': - // Non-escalate: resume ping-pong with reduced headroom. - // Set to threshold-1 so agents get one more round before - // re-triggering the arbiter — prevents infinite arbiter loops. - updatePairedTask(taskId, { - status: 'active', - round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), - arbiter_verdict: arbiterVerdict, - updated_at: now, - }); - logger.info( - { taskId, arbiterVerdict }, - 'Arbiter resolved deadlock — resuming ping-pong', - ); - break; - case 'escalate': - updatePairedTask(taskId, { - status: 'completed', - arbiter_verdict: 'escalate', - completion_reason: 'arbiter_escalated', - updated_at: now, - }); - logger.info({ taskId }, 'Arbiter escalated to user — task completed'); - break; - default: - // Unknown verdict — fallback to proceed so the loop continues. - // Stopping the task on a parse failure is worse than letting it run. - updatePairedTask(taskId, { - status: 'active', - round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), - arbiter_verdict: 'unknown', - updated_at: now, - }); - logger.warn( - { taskId, summary: args.summary?.slice(0, 200) }, - 'Arbiter verdict unrecognized — falling back to proceed', - ); - break; - } + handleArbiterCompletion({ taskId, summary: args.summary }); } } From 8daa3d47798a1dc85c128ea293d57eda4e03cb46 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 19:39:56 +0900 Subject: [PATCH 09/29] Refactor agent executor retry and follow-up flow --- src/message-agent-executor.test.ts | 77 ++++++++ src/message-agent-executor.ts | 300 ++++++++++++++++------------- 2 files changed, 239 insertions(+), 138 deletions(-) diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 4522e53..5af174a 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1485,6 +1485,83 @@ describe('runAgentForGroup Claude rotation', () => { expect(outputs).toEqual(['force refresh 뒤 Claude 응답입니다.']); }); + it('marks paired execution as succeeded when Claude rotation recovers after a streamed auth-expired trigger', async () => { + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-claude-rotation-success', + chat_jid: 'group@test', + group_folder: 'test-claude', + owner_service_id: 'claude', + reviewer_service_id: 'codex-review', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: null, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-04-06T00:00:00.000Z', + updated_at: '2026-04-06T00:00:00.000Z', + }, + workspace: null, + envOverrides: {}, + }); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }); + await onOutput?.({ + status: 'success', + phase: 'final', + result: + 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: '회전 복구 후 paired success', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-paired-auth-expired-success', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(pairedExecutionContext.completePairedExecutionContext).toHaveBeenCalledWith( + expect.objectContaining({ + taskId: 'paired-task-claude-rotation-success', + role: 'owner', + status: 'succeeded', + }), + ); + }); + it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => { const outputs: string[] = []; diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 3d207f1..bfd7bc4 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -774,6 +774,8 @@ export async function runAgentForGroup( return 'error'; }; + type AgentAttempt = Awaited>; + const retryClaudeWithRotation = async ( initialTrigger: { reason: AgentTriggerReason; @@ -813,9 +815,90 @@ export async function runAgentForGroup( } }; + const resolveClaudeRetryTrigger = ( + attempt: AgentAttempt, + fallbackMessage?: string | null, + ): { reason: AgentTriggerReason; retryAfterMs?: number } | null => { + if ( + !( + canRetryClaudeCredentials && + provider === 'claude' && + !attempt.sawOutput + ) + ) { + return null; + } + + if (attempt.streamedTriggerReason) { + return { + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + }; + } + + const trigger = classifyRotationTrigger(fallbackMessage); + if (!trigger.shouldRetry) { + return null; + } + + return { + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }; + }; + + const retryClaudeAttemptIfNeeded = async ( + attempt: AgentAttempt, + rotationMessage?: string | null, + ): Promise<'success' | 'error' | null> => { + const trigger = resolveClaudeRetryTrigger(attempt, rotationMessage); + if (!trigger) { + return null; + } + + const result = await retryClaudeWithRotation( + trigger, + rotationMessage ?? undefined, + ); + if (result === 'error') { + return maybeHandoffAfterError(trigger.reason, attempt); + } + + pairedExecutionStatus = 'succeeded'; + return result; + }; + + const retryCodexAttemptIfNeeded = async ( + attempt: AgentAttempt, + rotationMessage?: string | null, + ): Promise<'success' | 'error' | null> => { + if (isClaudeCodeAgent || getCodexAccountCount() <= 1) { + return null; + } + + const trigger = attempt.streamedTriggerReason + ? { + shouldRotate: true, + reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + } + : detectCodexRotationTrigger(rotationMessage); + if (!trigger.shouldRotate) { + return null; + } + + const result = await retryCodexWithRotation( + { reason: trigger.reason }, + rotationMessage ?? undefined, + ); + if (result === 'success') { + pairedExecutionStatus = 'succeeded'; + } + return result; + }; + const maybeHandoffAfterError = ( reason: AgentTriggerReason, - attempt: Awaited>, + attempt: AgentAttempt, ): 'success' | 'error' => { if (maybeHandoffToCodex(reason, attempt.sawVisibleOutput)) { return 'success'; @@ -823,6 +906,38 @@ export async function runAgentForGroup( return 'error'; }; + type PairedFollowUpQueueAction = + | 'generic' + | 'pending' + | 'skip-inline-finalize' + | 'none'; + type PairedTaskStatus = + NonNullable>['status']; + + const resolvePairedFollowUpQueueAction = ( + completedRole: PairedRoomRole, + executionStatus: 'succeeded' | 'failed', + sawOutput: boolean, + taskStatus: PairedTaskStatus | null, + ): PairedFollowUpQueueAction => { + if (executionStatus === 'succeeded' && sawOutput) { + return completedRole === 'reviewer' && taskStatus === 'merge_ready' + ? 'skip-inline-finalize' + : taskStatus !== 'completed' + ? 'generic' + : 'none'; + } + + const shouldRequeuePendingPairedTurn = + (completedRole === 'reviewer' || completedRole === 'arbiter') && + (taskStatus === 'review_ready' || + taskStatus === 'in_review' || + taskStatus === 'arbiter_requested' || + taskStatus === 'in_arbitration' || + taskStatus === 'merge_ready'); + return shouldRequeuePendingPairedTurn ? 'pending' : 'none'; + }; + const provider = 'claude'; try { @@ -864,50 +979,21 @@ export async function runAgentForGroup( } if (primaryAttempt.error) { - if ( - canRetryClaudeCredentials && - provider === 'claude' && - !primaryAttempt.sawOutput - ) { - const errMsg = getErrorMessage(primaryAttempt.error); - const trigger = primaryAttempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: primaryAttempt.streamedTriggerReason.reason, - retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(errMsg); - if (trigger.shouldRetry) { - const result = await retryClaudeWithRotation( - { - reason: trigger.reason, - retryAfterMs: trigger.retryAfterMs, - }, - errMsg, - ); - if (result === 'error') { - return maybeHandoffAfterError(trigger.reason, primaryAttempt); - } - if (result === 'success') { - pairedExecutionStatus = 'succeeded'; - } - return result; - } + const errMsg = getErrorMessage(primaryAttempt.error); + const claudeRetryResult = await retryClaudeAttemptIfNeeded( + primaryAttempt, + errMsg, + ); + if (claudeRetryResult) { + return claudeRetryResult; } - if (!isClaudeCodeAgent) { - const errMsg = getErrorMessage(primaryAttempt.error); - const trigger = detectCodexRotationTrigger(errMsg); - if (trigger.shouldRotate && getCodexAccountCount() > 1) { - const result = await retryCodexWithRotation( - { reason: trigger.reason }, - errMsg, - ); - if (result === 'success') { - pairedExecutionStatus = 'succeeded'; - } - return result; - } + const codexRetryResult = await retryCodexAttemptIfNeeded( + primaryAttempt, + errMsg, + ); + if (codexRetryResult) { + return codexRetryResult; } log.error( @@ -937,24 +1023,11 @@ export async function runAgentForGroup( : null); } - if ( - canRetryClaudeCredentials && - provider === 'claude' && - !primaryAttempt.sawOutput && - primaryAttempt.streamedTriggerReason && - output.status !== 'error' - ) { - const result = await retryClaudeWithRotation({ - reason: primaryAttempt.streamedTriggerReason.reason, - retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, - }); - if (result === 'error') { - return maybeHandoffAfterError( - primaryAttempt.streamedTriggerReason.reason, - primaryAttempt, - ); + if (!primaryAttempt.sawOutput && output.status !== 'error') { + const claudeRetryResult = await retryClaudeAttemptIfNeeded(primaryAttempt); + if (claudeRetryResult) { + return claudeRetryResult; } - return result; } if ( @@ -969,48 +1042,20 @@ export async function runAgentForGroup( } if (output.status === 'error') { - if ( - canRetryClaudeCredentials && - provider === 'claude' && - !primaryAttempt.sawOutput - ) { - const trigger = primaryAttempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: primaryAttempt.streamedTriggerReason.reason, - retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(output.error); - if (trigger.shouldRetry) { - const result = await retryClaudeWithRotation( - { - reason: trigger.reason, - retryAfterMs: trigger.retryAfterMs, - }, - output.error ?? undefined, - ); - if (result === 'error') { - return maybeHandoffAfterError(trigger.reason, primaryAttempt); - } - if (result === 'success') { - pairedExecutionStatus = 'succeeded'; - } - return result; - } + const claudeRetryResult = await retryClaudeAttemptIfNeeded( + primaryAttempt, + output.error, + ); + if (claudeRetryResult) { + return claudeRetryResult; } - if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { - const trigger = detectCodexRotationTrigger(output.error); - if (trigger.shouldRotate) { - const result = await retryCodexWithRotation( - { reason: trigger.reason }, - output.error ?? undefined, - ); - if (result === 'success') { - pairedExecutionStatus = 'succeeded'; - } - return result; - } + const codexRetryResult = await retryCodexAttemptIfNeeded( + primaryAttempt, + output.error, + ); + if (codexRetryResult) { + return codexRetryResult; } log.error( @@ -1023,22 +1068,12 @@ export async function runAgentForGroup( return 'error'; } - if ( - !isClaudeCodeAgent && - primaryAttempt.streamedTriggerReason && - getCodexAccountCount() > 1 - ) { - const result = await retryCodexWithRotation( - { - reason: primaryAttempt.streamedTriggerReason - .reason as CodexRotationReason, - }, - output.error ?? output.result ?? undefined, - ); - if (result === 'success') { - pairedExecutionStatus = 'succeeded'; - } - return result; + const codexRetryResult = await retryCodexAttemptIfNeeded( + primaryAttempt, + output.error ?? output.result, + ); + if (codexRetryResult) { + return codexRetryResult; } // Unresolved streamed trigger — rotation was unavailable or output was @@ -1144,16 +1179,18 @@ export async function runAgentForGroup( // After owner/reviewer completes, enqueue the next turn so // the message loop picks it up without waiting for a new message. // Skip if: no output (interrupted), or task already completed (ESCALATE, done, etc.) - if ( - pairedExecutionContext && - pairedExecutionStatus === 'succeeded' && - pairedSawOutput - ) { + if (pairedExecutionContext) { const completedRole = roomRoleContext?.role ?? 'owner'; const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id); - const skipGenericFollowUpForInlineFinalize = - completedRole === 'reviewer' && finishedCheck?.status === 'merge_ready'; - if (skipGenericFollowUpForInlineFinalize) { + const queueAction = resolvePairedFollowUpQueueAction( + completedRole, + pairedExecutionStatus, + pairedSawOutput, + finishedCheck?.status ?? null, + ); + if (queueAction === 'generic') { + deps.queue.enqueueMessageCheck(chatJid); + } else if (queueAction === 'skip-inline-finalize') { log.info( { taskId: pairedExecutionContext.task.id, @@ -1162,20 +1199,7 @@ export async function runAgentForGroup( }, 'Skipping generic follow-up after reviewer approval because inline finalize will handle the owner handoff', ); - } else if (finishedCheck?.status !== 'completed') { - deps.queue.enqueueMessageCheck(chatJid); - } - } else if (pairedExecutionContext) { - const completedRole = roomRoleContext?.role ?? 'owner'; - const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id); - const shouldRequeuePendingPairedTurn = - (completedRole === 'reviewer' || completedRole === 'arbiter') && - (finishedCheck?.status === 'review_ready' || - finishedCheck?.status === 'in_review' || - finishedCheck?.status === 'arbiter_requested' || - finishedCheck?.status === 'in_arbitration' || - finishedCheck?.status === 'merge_ready'); - if (shouldRequeuePendingPairedTurn) { + } else if (queueAction === 'pending') { deps.queue.enqueueMessageCheck(chatJid); log.info( { From 5b832042daf356e7f855fc2220d40df1ac2ece1c Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 19:40:36 +0900 Subject: [PATCH 10/29] Format files touched by executor refactor --- src/message-agent-executor.test.ts | 4 +++- src/message-agent-executor.ts | 8 +++++--- src/paired-execution-context.ts | 3 +-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 5af174a..e617f0f 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1553,7 +1553,9 @@ describe('runAgentForGroup Claude rotation', () => { }); expect(result).toBe('success'); - expect(pairedExecutionContext.completePairedExecutionContext).toHaveBeenCalledWith( + expect( + pairedExecutionContext.completePairedExecutionContext, + ).toHaveBeenCalledWith( expect.objectContaining({ taskId: 'paired-task-claude-rotation-success', role: 'owner', diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index bfd7bc4..1ad0dc4 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -911,8 +911,9 @@ export async function runAgentForGroup( | 'pending' | 'skip-inline-finalize' | 'none'; - type PairedTaskStatus = - NonNullable>['status']; + type PairedTaskStatus = NonNullable< + ReturnType + >['status']; const resolvePairedFollowUpQueueAction = ( completedRole: PairedRoomRole, @@ -1024,7 +1025,8 @@ export async function runAgentForGroup( } if (!primaryAttempt.sawOutput && output.status !== 'error') { - const claudeRetryResult = await retryClaudeAttemptIfNeeded(primaryAttempt); + const claudeRetryResult = + await retryClaudeAttemptIfNeeded(primaryAttempt); if (claudeRetryResult) { return claudeRetryResult; } diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index fbb0216..639b499 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -575,8 +575,7 @@ function handleOwnerFinalizeCompletion(args: { taskId, now, arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter', - escalateLogMessage: - 'Owner finalize loop detected — escalating to user', + escalateLogMessage: 'Owner finalize loop detected — escalating to user', logContext: { taskId, ownerVerdict, From a3dac2b983cf401bf87edd5f61076a0bea014a9a Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 19:53:26 +0900 Subject: [PATCH 11/29] Extract paired turn fallback planning --- src/message-agent-executor.ts | 127 ++++++++----------------------- src/paired-turn-fallback.test.ts | 118 ++++++++++++++++++++++++++++ src/paired-turn-fallback.ts | 116 ++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 94 deletions(-) create mode 100644 src/paired-turn-fallback.test.ts create mode 100644 src/paired-turn-fallback.ts diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 1ad0dc4..ecdcf9f 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -30,6 +30,7 @@ import { completePairedExecutionContext, preparePairedExecutionContext, } from './paired-execution-context.js'; +import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; import { resolveActiveRole, resolveConfiguredRoleAgentPlan, @@ -50,7 +51,6 @@ import { ARBITER_AGENT_TYPE, CODEX_REVIEW_SERVICE_ID, REVIEWER_AGENT_TYPE, - SERVICE_SESSION_SCOPE, TIMEZONE, isClaudeService, getRoleModelConfig, @@ -154,6 +154,10 @@ export async function runAgentForGroup( process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; const forceFreshClaudeReviewerSession = reviewerMode && isClaudeCodeAgent && unsafeHostPairedMode; + const shouldPersistSession = + activeRole !== 'arbiter' && + !args.forcedAgentType && + !forceFreshClaudeReviewerSession; const storedSessionId = sessions[sessionFolder]; if (forceFreshClaudeReviewerSession && storedSessionId) { deps.clearSession(sessionFolder); @@ -321,103 +325,44 @@ export async function runAgentForGroup( let pairedExecutionCompleted = false; let pairedSawOutput = false; - const shouldHandoffToCodex = ( - reason: AgentTriggerReason, - sawVisibleOutput: boolean, - ): boolean => { - if (sawVisibleOutput) { - return false; - } - return ( - reason === '429' || - reason === 'usage-exhausted' || - reason === 'auth-expired' || - reason === 'org-access-denied' || - reason === 'session-failure' - ); - }; - const maybeHandoffToCodex = ( reason: AgentTriggerReason, sawVisibleOutput: boolean, ): boolean => { if (!isClaudeCodeAgent) return false; - if (!shouldHandoffToCodex(reason, sawVisibleOutput)) { - return false; - } - if (currentLease.reviewer_agent_type === null) { - return false; - } - // Per-role fallback toggle - const roleConfig = getRoleModelConfig(activeRole); - if (!roleConfig.fallbackEnabled) { - log.info({ reason }, 'Fallback disabled for role, skipping handoff'); + const handoffResolution = resolveCodexFallbackHandoff({ + activeRole, + effectiveAgentType, + hasReviewer: currentLease.reviewer_agent_type !== null, + fallbackEnabled: getRoleModelConfig(activeRole).fallbackEnabled, + reason, + sawVisibleOutput, + prompt, + startSeq, + endSeq, + }); + + if (handoffResolution.type === 'none') { return false; } - if (arbiterMode) { - // Arbiter failed (e.g. Claude 401/429) — re-trigger arbitration with codex - createServiceHandoff({ - chat_jid: chatJid, - group_folder: group.folder, - source_role: activeRole, - target_role: 'arbiter', - source_agent_type: effectiveAgentType, - target_agent_type: 'codex', - prompt, - start_seq: startSeq ?? null, - end_seq: endSeq ?? null, - reason: `arbiter-claude-${reason}`, - intended_role: 'arbiter', - }); - log.warn( - { reason }, - 'Claude arbiter unavailable, handed off arbiter turn to codex', + if (handoffResolution.type === 'skip') { + log.info({ reason }, handoffResolution.logMessage); + return false; + } + + if (handoffResolution.plan.activateOwnerFailoverReason) { + activateCodexFailover( + chatJid, + handoffResolution.plan.activateOwnerFailoverReason, ); - return true; } - - if (reviewerMode) { - // Reviewer failed (e.g. Claude 401/429) — re-trigger review with codex - // instead of swapping owner/reviewer roles. - createServiceHandoff({ - chat_jid: chatJid, - group_folder: group.folder, - source_role: activeRole, - target_role: 'reviewer', - source_agent_type: effectiveAgentType, - target_agent_type: 'codex', - prompt, - start_seq: startSeq ?? null, - end_seq: endSeq ?? null, - reason: `reviewer-claude-${reason}`, - intended_role: 'reviewer', - }); - log.warn( - { reason }, - 'Claude reviewer unavailable, handed off review turn to codex-review', - ); - return true; - } - - activateCodexFailover(chatJid, `claude-${reason}`); createServiceHandoff({ chat_jid: chatJid, group_folder: group.folder, - source_role: activeRole, - target_role: activeRole, - source_agent_type: effectiveAgentType, - target_agent_type: 'codex', - prompt, - start_seq: startSeq ?? null, - end_seq: endSeq ?? null, - reason: `claude-${reason}`, - intended_role: activeRole, + ...handoffResolution.plan.handoff, }); - log.warn( - { reason }, - 'Claude unavailable, handed off current owner turn to codex fallback', - ); + log.warn({ reason }, handoffResolution.plan.logMessage); return true; }; @@ -480,8 +425,7 @@ export async function runAgentForGroup( sawVisibleOutput: false, sawSuccessNullResultWithoutOutput: false, }; - const attemptSessionId = - provider === 'claude' ? currentSessionId : undefined; + const attemptSessionId = currentSessionId; const wrappedOnOutput = onOutput ? async (output: AgentOutput) => { @@ -528,10 +472,9 @@ export async function runAgentForGroup( resetSessionRequested = true; } if ( - provider === 'claude' && output.newSessionId && !resetSessionRequested && - !forceFreshClaudeReviewerSession + shouldPersistSession ) { deps.persistSession(sessionFolder, output.newSessionId); currentSessionId = output.newSessionId; @@ -642,11 +585,7 @@ export async function runAgentForGroup( pairedExecutionContext?.envOverrides, ); - if ( - provider === 'claude' && - output.newSessionId && - !forceFreshClaudeReviewerSession - ) { + if (output.newSessionId && shouldPersistSession) { deps.persistSession(sessionFolder, output.newSessionId); currentSessionId = output.newSessionId; } @@ -939,7 +878,7 @@ export async function runAgentForGroup( return shouldRequeuePendingPairedTurn ? 'pending' : 'none'; }; - const provider = 'claude'; + const provider = isClaudeCodeAgent ? 'claude' : 'codex'; try { let primaryAttempt = await runAttempt(provider); diff --git a/src/paired-turn-fallback.test.ts b/src/paired-turn-fallback.test.ts new file mode 100644 index 0000000..4917b86 --- /dev/null +++ b/src/paired-turn-fallback.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; + +describe('resolveCodexFallbackHandoff', () => { + it('returns a reviewer codex handoff plan for reviewer auth failures', () => { + const result = resolveCodexFallbackHandoff({ + activeRole: 'reviewer', + effectiveAgentType: 'claude-code', + hasReviewer: true, + fallbackEnabled: true, + reason: 'auth-expired', + sawVisibleOutput: false, + prompt: 'please review', + startSeq: 10, + endSeq: 12, + }); + + expect(result).toEqual({ + type: 'handoff', + plan: { + handoff: { + source_role: 'reviewer', + target_role: 'reviewer', + source_agent_type: 'claude-code', + target_agent_type: 'codex', + prompt: 'please review', + start_seq: 10, + end_seq: 12, + reason: 'reviewer-claude-auth-expired', + intended_role: 'reviewer', + }, + logMessage: + 'Claude reviewer unavailable, handed off review turn to codex-review', + }, + }); + }); + + it('returns an arbiter codex handoff plan for arbiter auth failures', () => { + const result = resolveCodexFallbackHandoff({ + activeRole: 'arbiter', + effectiveAgentType: 'claude-code', + hasReviewer: true, + fallbackEnabled: true, + reason: '429', + sawVisibleOutput: false, + prompt: 'please arbitrate', + }); + + expect(result).toEqual({ + type: 'handoff', + plan: { + handoff: { + source_role: 'arbiter', + target_role: 'arbiter', + source_agent_type: 'claude-code', + target_agent_type: 'codex', + prompt: 'please arbitrate', + start_seq: null, + end_seq: null, + reason: 'arbiter-claude-429', + intended_role: 'arbiter', + }, + logMessage: + 'Claude arbiter unavailable, handed off arbiter turn to codex', + }, + }); + }); + + it('returns an owner codex handoff plan and failover activation for owner failures', () => { + const result = resolveCodexFallbackHandoff({ + activeRole: 'owner', + effectiveAgentType: 'claude-code', + hasReviewer: true, + fallbackEnabled: true, + reason: 'session-failure', + sawVisibleOutput: false, + prompt: 'please continue', + }); + + expect(result).toEqual({ + type: 'handoff', + plan: { + handoff: { + source_role: 'owner', + target_role: 'owner', + source_agent_type: 'claude-code', + target_agent_type: 'codex', + prompt: 'please continue', + start_seq: null, + end_seq: null, + reason: 'claude-session-failure', + intended_role: 'owner', + }, + activateOwnerFailoverReason: 'claude-session-failure', + logMessage: + 'Claude unavailable, handed off current owner turn to codex fallback', + }, + }); + }); + + it('skips handoff when fallback is disabled for the role', () => { + const result = resolveCodexFallbackHandoff({ + activeRole: 'owner', + effectiveAgentType: 'claude-code', + hasReviewer: true, + fallbackEnabled: false, + reason: 'usage-exhausted', + sawVisibleOutput: false, + prompt: 'please continue', + }); + + expect(result).toEqual({ + type: 'skip', + logMessage: 'Fallback disabled for role, skipping handoff', + }); + }); +}); diff --git a/src/paired-turn-fallback.ts b/src/paired-turn-fallback.ts new file mode 100644 index 0000000..5edabbe --- /dev/null +++ b/src/paired-turn-fallback.ts @@ -0,0 +1,116 @@ +import type { AgentTriggerReason } from './agent-error-detection.js'; +import type { AgentType, PairedRoomRole } from './types.js'; + +const CODEX_HANDOFF_REASONS = new Set([ + '429', + 'usage-exhausted', + 'auth-expired', + 'org-access-denied', + 'session-failure', +]); + +export interface CodexFallbackHandoffRecord { + source_role: PairedRoomRole; + target_role: PairedRoomRole; + source_agent_type: AgentType; + target_agent_type: 'codex'; + prompt: string; + start_seq: number | null; + end_seq: number | null; + reason: string; + intended_role: PairedRoomRole; +} + +export interface CodexFallbackHandoffPlan { + handoff: CodexFallbackHandoffRecord; + activateOwnerFailoverReason?: string; + logMessage: string; +} + +export type CodexFallbackResolution = + | { type: 'none' } + | { type: 'skip'; logMessage: string } + | { type: 'handoff'; plan: CodexFallbackHandoffPlan }; + +export function resolveCodexFallbackHandoff(args: { + activeRole: PairedRoomRole; + effectiveAgentType: AgentType; + hasReviewer: boolean; + fallbackEnabled: boolean; + reason: AgentTriggerReason; + sawVisibleOutput: boolean; + prompt: string; + startSeq?: number | null; + endSeq?: number | null; +}): CodexFallbackResolution { + if (args.sawVisibleOutput || !CODEX_HANDOFF_REASONS.has(args.reason)) { + return { type: 'none' }; + } + + if (!args.hasReviewer) { + return { type: 'none' }; + } + + if (!args.fallbackEnabled) { + return { + type: 'skip', + logMessage: 'Fallback disabled for role, skipping handoff', + }; + } + + const baseHandoff = { + source_role: args.activeRole, + source_agent_type: args.effectiveAgentType, + target_agent_type: 'codex' as const, + prompt: args.prompt, + start_seq: args.startSeq ?? null, + end_seq: args.endSeq ?? null, + }; + + if (args.activeRole === 'arbiter') { + return { + type: 'handoff', + plan: { + handoff: { + ...baseHandoff, + target_role: 'arbiter', + intended_role: 'arbiter', + reason: `arbiter-claude-${args.reason}`, + }, + logMessage: + 'Claude arbiter unavailable, handed off arbiter turn to codex', + }, + }; + } + + if (args.activeRole === 'reviewer') { + return { + type: 'handoff', + plan: { + handoff: { + ...baseHandoff, + target_role: 'reviewer', + intended_role: 'reviewer', + reason: `reviewer-claude-${args.reason}`, + }, + logMessage: + 'Claude reviewer unavailable, handed off review turn to codex-review', + }, + }; + } + + return { + type: 'handoff', + plan: { + handoff: { + ...baseHandoff, + target_role: args.activeRole, + intended_role: args.activeRole, + reason: `claude-${args.reason}`, + }, + activateOwnerFailoverReason: `claude-${args.reason}`, + logMessage: + 'Claude unavailable, handed off current owner turn to codex fallback', + }, + }; +} From 83c45c3ce394a0f0e7d066b9e8871e769f672707 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 20:38:10 +0900 Subject: [PATCH 12/29] Extract executor retry decision rules --- src/message-agent-executor-rules.test.ts | 115 +++++++ src/message-agent-executor-rules.ts | 139 ++++++++ src/message-agent-executor.ts | 416 +++++++++-------------- src/paired-turn-fallback.test.ts | 28 ++ src/provider-retry.test.ts | 95 ++++++ src/provider-retry.ts | 132 +++++++ 6 files changed, 662 insertions(+), 263 deletions(-) create mode 100644 src/message-agent-executor-rules.test.ts create mode 100644 src/message-agent-executor-rules.ts diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts new file mode 100644 index 0000000..57471f3 --- /dev/null +++ b/src/message-agent-executor-rules.test.ts @@ -0,0 +1,115 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./agent-error-detection.js', () => ({ + classifyRotationTrigger: vi.fn(() => ({ + shouldRetry: false, + reason: '', + })), +})); + +vi.mock('./codex-token-rotation.js', () => ({ + detectCodexRotationTrigger: vi.fn(() => ({ + shouldRotate: false, + reason: '', + })), +})); + +import { classifyRotationTrigger } from './agent-error-detection.js'; +import { detectCodexRotationTrigger } from './codex-token-rotation.js'; +import { + isRetryableClaudeSessionFailureAttempt, + resolveClaudeRetryTrigger, + resolveCodexRetryTrigger, + resolvePairedFollowUpQueueAction, +} from './message-agent-executor-rules.js'; + +describe('message-agent-executor-rules', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('resolves a Claude retry trigger from streamed trigger state', () => { + expect( + resolveClaudeRetryTrigger({ + canRetryClaudeCredentials: true, + provider: 'claude', + attempt: { + sawOutput: false, + streamedTriggerReason: { reason: 'auth-expired', retryAfterMs: 5000 }, + }, + }), + ).toEqual({ + reason: 'auth-expired', + retryAfterMs: 5000, + }); + }); + + it('returns null for Claude retry when output was already visible', () => { + expect( + resolveClaudeRetryTrigger({ + canRetryClaudeCredentials: true, + provider: 'claude', + attempt: { sawOutput: true }, + fallbackMessage: '429 rate limit', + }), + ).toBeNull(); + expect(classifyRotationTrigger).not.toHaveBeenCalled(); + }); + + it('resolves a Codex retry trigger from the codex detector', () => { + vi.mocked(detectCodexRotationTrigger).mockReturnValue({ + shouldRotate: true, + reason: 'auth-expired', + }); + + expect( + resolveCodexRetryTrigger({ + canRetryCodex: true, + attempt: {}, + rotationMessage: '401 oauth token has expired', + }), + ).toEqual({ reason: 'auth-expired' }); + }); + + it('detects retryable Claude session failures from either flag or classifier', () => { + expect( + isRetryableClaudeSessionFailureAttempt({ + attempt: { + sawOutput: false, + retryableSessionFailureDetected: true, + }, + isClaudeCodeAgent: true, + provider: 'claude', + shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false), + }), + ).toBe(true); + + const shouldRetryFreshSessionOnAgentFailure = vi.fn(() => true); + expect( + isRetryableClaudeSessionFailureAttempt({ + attempt: { + sawOutput: false, + error: new Error('stale session'), + }, + isClaudeCodeAgent: true, + provider: 'claude', + shouldRetryFreshSessionOnAgentFailure, + }), + ).toBe(true); + expect(shouldRetryFreshSessionOnAgentFailure).toHaveBeenCalledWith({ + result: null, + error: 'stale session', + }); + }); + + it('resolves pending reviewer follow-up requeue from task state', () => { + expect( + resolvePairedFollowUpQueueAction({ + completedRole: 'reviewer', + executionStatus: 'failed', + sawOutput: false, + taskStatus: 'review_ready', + }), + ).toBe('pending'); + }); +}); diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts new file mode 100644 index 0000000..9215300 --- /dev/null +++ b/src/message-agent-executor-rules.ts @@ -0,0 +1,139 @@ +import { + classifyRotationTrigger, + type AgentTriggerReason, + type CodexRotationReason, +} from './agent-error-detection.js'; +import { detectCodexRotationTrigger } from './codex-token-rotation.js'; +import type { PairedRoomRole, PairedTaskStatus } from './types.js'; +import { getErrorMessage } from './utils.js'; + +export interface ExecutorStreamedTrigger { + reason: AgentTriggerReason; + retryAfterMs?: number; +} + +export interface ExecutorAttemptState { + sawOutput: boolean; + retryableSessionFailureDetected?: boolean; + streamedTriggerReason?: ExecutorStreamedTrigger; + error?: unknown; +} + +export function isRetryableClaudeSessionFailureAttempt(args: { + attempt: ExecutorAttemptState; + isClaudeCodeAgent: boolean; + provider: 'claude' | 'codex'; + shouldRetryFreshSessionOnAgentFailure: (args: { + result: null; + error: string; + }) => boolean; +}): boolean { + if ( + !args.isClaudeCodeAgent || + args.provider !== 'claude' || + args.attempt.sawOutput + ) { + return false; + } + + if (args.attempt.retryableSessionFailureDetected === true) { + return true; + } + + if (args.attempt.error == null) { + return false; + } + + return args.shouldRetryFreshSessionOnAgentFailure({ + result: null, + error: getErrorMessage(args.attempt.error), + }); +} + +export function resolveClaudeRetryTrigger(args: { + canRetryClaudeCredentials: boolean; + provider: 'claude' | 'codex'; + attempt: Pick; + fallbackMessage?: string | null; +}): { reason: AgentTriggerReason; retryAfterMs?: number } | null { + if ( + !( + args.canRetryClaudeCredentials && + args.provider === 'claude' && + !args.attempt.sawOutput + ) + ) { + return null; + } + + if (args.attempt.streamedTriggerReason) { + return { + reason: args.attempt.streamedTriggerReason.reason, + retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs, + }; + } + + const trigger = classifyRotationTrigger(args.fallbackMessage); + if (!trigger.shouldRetry) { + return null; + } + + return { + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }; +} + +export function resolveCodexRetryTrigger(args: { + canRetryCodex: boolean; + attempt: Pick; + rotationMessage?: string | null; +}): { reason: CodexRotationReason } | null { + if (!args.canRetryCodex) { + return null; + } + + if (args.attempt.streamedTriggerReason) { + return { + reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason, + }; + } + + const trigger = detectCodexRotationTrigger(args.rotationMessage); + if (!trigger.shouldRotate) { + return null; + } + + return { reason: trigger.reason }; +} + +export type PairedFollowUpQueueAction = + | 'generic' + | 'pending' + | 'skip-inline-finalize' + | 'none'; + +export function resolvePairedFollowUpQueueAction(args: { + completedRole: PairedRoomRole; + executionStatus: 'succeeded' | 'failed'; + sawOutput: boolean; + taskStatus: PairedTaskStatus | null; +}): PairedFollowUpQueueAction { + if (args.executionStatus === 'succeeded' && args.sawOutput) { + return args.completedRole === 'reviewer' && + args.taskStatus === 'merge_ready' + ? 'skip-inline-finalize' + : args.taskStatus !== 'completed' + ? 'generic' + : 'none'; + } + + const shouldRequeuePendingPairedTurn = + (args.completedRole === 'reviewer' || args.completedRole === 'arbiter') && + (args.taskStatus === 'review_ready' || + args.taskStatus === 'in_review' || + args.taskStatus === 'arbiter_requested' || + args.taskStatus === 'in_arbitration' || + args.taskStatus === 'merge_ready'); + return shouldRequeuePendingPairedTurn ? 'pending' : 'none'; +} diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index ecdcf9f..48ab6d5 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -37,12 +37,20 @@ import { resolveEffectiveAgentType, resolveSessionFolder, } from './message-runtime-rules.js'; +import { + isRetryableClaudeSessionFailureAttempt, + resolveClaudeRetryTrigger, + resolveCodexRetryTrigger, + resolvePairedFollowUpQueueAction, +} from './message-agent-executor-rules.js'; import { buildRoomRoleContext } from './room-role-context.js'; import { - classifyRotationTrigger, type AgentTriggerReason, } from './agent-error-detection.js'; -import { runClaudeRotationLoop } from './provider-retry.js'; +import { + runClaudeRotationLoop, + runCodexRotationLoop, +} from './provider-retry.js'; import { shouldResetSessionOnAgentFailure, shouldRetryFreshSessionOnAgentFailure, @@ -69,9 +77,7 @@ import { } from './streamed-output-evaluator.js'; import { detectCodexRotationTrigger, - rotateCodexToken, getCodexAccountCount, - markCodexTokenHealthy, } from './codex-token-rotation.js'; import type { CodexRotationReason } from './agent-error-detection.js'; import { getTokenCount } from './token-rotation.js'; @@ -169,6 +175,7 @@ export async function runAgentForGroup( forceFreshClaudeReviewerSession ? undefined : storedSessionId; + const provider = isClaudeCodeAgent ? 'claude' : 'codex'; const memoryBriefing = currentSessionId ? undefined : await buildRoomMemoryBriefing({ @@ -626,91 +633,27 @@ export async function runAgentForGroup( initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, ): Promise<'success' | 'error'> => { - let trigger = initialTrigger; - let lastRotationMessage = rotationMessage; - - while ( - getCodexAccountCount() > 1 && - rotateCodexToken(lastRotationMessage) - ) { - log.info( - { reason: trigger.reason }, - 'Codex account unhealthy, retrying with rotated account', - ); - - const retryAttempt = await runAttempt('codex'); - - if (retryAttempt.error) { - const errMsg = getErrorMessage(retryAttempt.error); - const retryTrigger = detectCodexRotationTrigger(errMsg); - if (retryTrigger.shouldRotate) { - trigger = { reason: retryTrigger.reason }; - lastRotationMessage = errMsg; - continue; - } - - log.error( - { provider: 'codex', err: retryAttempt.error }, - 'Rotated Codex account also threw', - ); - return 'error'; - } - - const retryOutput = retryAttempt.output; - if (!retryOutput) { - log.error( - { provider: 'codex' }, - 'Rotated Codex account produced no output object', - ); - return 'error'; - } - - if ( - !retryAttempt.sawOutput && - retryAttempt.streamedTriggerReason && - retryOutput.status !== 'error' - ) { - trigger = { - reason: retryAttempt.streamedTriggerReason - .reason as CodexRotationReason, + const outcome = await runCodexRotationLoop( + initialTrigger, + async () => { + const attempt = await runAttempt('codex'); + return { + output: attempt.output, + thrownError: attempt.error, + sawOutput: attempt.sawOutput, + streamedTriggerReason: attempt.streamedTriggerReason, }; - lastRotationMessage = - typeof retryOutput.result === 'string' - ? retryOutput.result - : undefined; - continue; - } + }, + { + chatJid, + group: group.name, + groupFolder: group.folder, + runId, + }, + rotationMessage, + ); - if (retryOutput.status === 'error') { - const retryTrigger = retryAttempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: retryAttempt.streamedTriggerReason - .reason as CodexRotationReason, - } - : detectCodexRotationTrigger(retryOutput.error); - - if (retryTrigger.shouldRotate) { - trigger = { reason: retryTrigger.reason }; - lastRotationMessage = retryOutput.error ?? undefined; - continue; - } - - log.error( - { - provider: 'codex', - error: retryOutput.error, - }, - 'Rotated Codex account failed', - ); - return 'error'; - } - - markCodexTokenHealthy(); - return 'success'; - } - - return 'error'; + return outcome.type === 'success' ? 'success' : 'error'; }; type AgentAttempt = Awaited>; @@ -754,43 +697,16 @@ export async function runAgentForGroup( } }; - const resolveClaudeRetryTrigger = ( - attempt: AgentAttempt, - fallbackMessage?: string | null, - ): { reason: AgentTriggerReason; retryAfterMs?: number } | null => { - if ( - !( - canRetryClaudeCredentials && - provider === 'claude' && - !attempt.sawOutput - ) - ) { - return null; - } - - if (attempt.streamedTriggerReason) { - return { - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - }; - } - - const trigger = classifyRotationTrigger(fallbackMessage); - if (!trigger.shouldRetry) { - return null; - } - - return { - reason: trigger.reason, - retryAfterMs: trigger.retryAfterMs, - }; - }; - const retryClaudeAttemptIfNeeded = async ( attempt: AgentAttempt, rotationMessage?: string | null, ): Promise<'success' | 'error' | null> => { - const trigger = resolveClaudeRetryTrigger(attempt, rotationMessage); + const trigger = resolveClaudeRetryTrigger({ + canRetryClaudeCredentials, + provider, + attempt, + fallbackMessage: rotationMessage, + }); if (!trigger) { return null; } @@ -811,17 +727,12 @@ export async function runAgentForGroup( attempt: AgentAttempt, rotationMessage?: string | null, ): Promise<'success' | 'error' | null> => { - if (isClaudeCodeAgent || getCodexAccountCount() <= 1) { - return null; - } - - const trigger = attempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, - } - : detectCodexRotationTrigger(rotationMessage); - if (!trigger.shouldRotate) { + const trigger = resolveCodexRetryTrigger({ + canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1, + attempt, + rotationMessage, + }); + if (!trigger) { return null; } @@ -845,108 +756,93 @@ export async function runAgentForGroup( return 'error'; }; - type PairedFollowUpQueueAction = - | 'generic' - | 'pending' - | 'skip-inline-finalize' - | 'none'; - type PairedTaskStatus = NonNullable< - ReturnType - >['status']; + const getPairedExecutionStatus = (): 'succeeded' | 'failed' => + pairedExecutionStatus; - const resolvePairedFollowUpQueueAction = ( - completedRole: PairedRoomRole, - executionStatus: 'succeeded' | 'failed', - sawOutput: boolean, - taskStatus: PairedTaskStatus | null, - ): PairedFollowUpQueueAction => { - if (executionStatus === 'succeeded' && sawOutput) { - return completedRole === 'reviewer' && taskStatus === 'merge_ready' - ? 'skip-inline-finalize' - : taskStatus !== 'completed' - ? 'generic' - : 'none'; + const isRetryableClaudeSessionFailure = ( + attempt: AgentAttempt, + ): boolean => + isRetryableClaudeSessionFailureAttempt({ + attempt, + isClaudeCodeAgent, + provider, + shouldRetryFreshSessionOnAgentFailure, + }); + + const recoverRetryableClaudeSessionFailure = async ( + attempt: AgentAttempt, + ): Promise<{ attempt: AgentAttempt; resolved: 'success' | 'error' | null }> => { + if (!isRetryableClaudeSessionFailure(attempt)) { + return { attempt, resolved: null }; } - const shouldRequeuePendingPairedTurn = - (completedRole === 'reviewer' || completedRole === 'arbiter') && - (taskStatus === 'review_ready' || - taskStatus === 'in_review' || - taskStatus === 'arbiter_requested' || - taskStatus === 'in_arbitration' || - taskStatus === 'merge_ready'); - return shouldRequeuePendingPairedTurn ? 'pending' : 'none'; + currentSessionId = undefined; + deps.clearSession(sessionFolder); + clearRoleSdkSessions(); + log.warn( + 'Cleared poisoned Claude session before visible output, retrying fresh session', + ); + + const freshAttempt = await runAttempt('claude'); + if (!isRetryableClaudeSessionFailure(freshAttempt)) { + return { attempt: freshAttempt, resolved: null }; + } + + currentSessionId = undefined; + deps.clearSession(sessionFolder); + log.warn('Fresh Claude retry also hit a retryable session failure'); + log.error('Retryable Claude session failure persisted after fresh retry'); + return { + attempt: freshAttempt, + resolved: maybeHandoffAfterError('session-failure', freshAttempt), + }; }; - const provider = isClaudeCodeAgent ? 'claude' : 'codex'; - - try { - let primaryAttempt = await runAttempt(provider); - - const isRetryableClaudeSessionFailure = ( - attempt: Awaited>, - ): boolean => - isClaudeCodeAgent && - provider === 'claude' && - !attempt.sawOutput && - (attempt.retryableSessionFailureDetected === true || - (attempt.error != null && - shouldRetryFreshSessionOnAgentFailure({ - result: null, - error: getErrorMessage(attempt.error), - }))); - - if (isRetryableClaudeSessionFailure(primaryAttempt)) { - currentSessionId = undefined; - deps.clearSession(sessionFolder); - clearRoleSdkSessions(); - log.warn( - 'Cleared poisoned Claude session before visible output, retrying fresh session', - ); - - primaryAttempt = await runAttempt('claude'); - - if (isRetryableClaudeSessionFailure(primaryAttempt)) { - currentSessionId = undefined; - deps.clearSession(sessionFolder); - log.warn('Fresh Claude retry also hit a retryable session failure'); - - log.error( - 'Retryable Claude session failure persisted after fresh retry', - ); - return maybeHandoffAfterError('session-failure', primaryAttempt); - } + const handlePrimaryAttemptFailure = async ( + attempt: AgentAttempt, + rotationMessage: string, + ): Promise<'success' | 'error'> => { + const claudeRetryResult = await retryClaudeAttemptIfNeeded( + attempt, + rotationMessage, + ); + if (claudeRetryResult) { + return claudeRetryResult; } - if (primaryAttempt.error) { - const errMsg = getErrorMessage(primaryAttempt.error); - const claudeRetryResult = await retryClaudeAttemptIfNeeded( - primaryAttempt, - errMsg, - ); - if (claudeRetryResult) { - return claudeRetryResult; - } - - const codexRetryResult = await retryCodexAttemptIfNeeded( - primaryAttempt, - errMsg, - ); - if (codexRetryResult) { - return codexRetryResult; - } + const codexRetryResult = await retryCodexAttemptIfNeeded( + attempt, + rotationMessage, + ); + if (codexRetryResult) { + return codexRetryResult; + } + if (attempt.error) { log.error( { provider, - err: primaryAttempt.error, + err: attempt.error, }, 'Agent error', ); return 'error'; } - const output = primaryAttempt.output; + log.error( + { + provider, + error: attempt.output?.error, + }, + 'Agent process error', + ); + return 'error'; + }; + + const finalizePrimaryAttempt = async ( + attempt: AgentAttempt, + ): Promise<'success' | 'error'> => { + const output = attempt.output; if (!output) { log.error({ provider }, 'Agent produced no output object'); return 'error'; @@ -963,9 +859,8 @@ export async function runAgentForGroup( : null); } - if (!primaryAttempt.sawOutput && output.status !== 'error') { - const claudeRetryResult = - await retryClaudeAttemptIfNeeded(primaryAttempt); + if (!attempt.sawOutput && output.status !== 'error') { + const claudeRetryResult = await retryClaudeAttemptIfNeeded(attempt); if (claudeRetryResult) { return claudeRetryResult; } @@ -983,34 +878,14 @@ export async function runAgentForGroup( } if (output.status === 'error') { - const claudeRetryResult = await retryClaudeAttemptIfNeeded( - primaryAttempt, - output.error, + return handlePrimaryAttemptFailure( + attempt, + output.error ?? 'Agent process error', ); - if (claudeRetryResult) { - return claudeRetryResult; - } - - const codexRetryResult = await retryCodexAttemptIfNeeded( - primaryAttempt, - output.error, - ); - if (codexRetryResult) { - return codexRetryResult; - } - - log.error( - { - provider, - error: output.error, - }, - 'Agent process error', - ); - return 'error'; } const codexRetryResult = await retryCodexAttemptIfNeeded( - primaryAttempt, + attempt, output.error ?? output.result, ); if (codexRetryResult) { @@ -1018,20 +893,20 @@ export async function runAgentForGroup( } // Unresolved streamed trigger — rotation was unavailable or output was - // already forwarded. Surfaces as an error since there is no alternative provider. - if (primaryAttempt.streamedTriggerReason) { + // already forwarded. Surfaces as an error since there is no alternative provider. + if (attempt.streamedTriggerReason) { if ( isClaudeCodeAgent && maybeHandoffToCodex( - primaryAttempt.streamedTriggerReason.reason, - primaryAttempt.sawVisibleOutput, + attempt.streamedTriggerReason.reason, + attempt.sawVisibleOutput, ) ) { return 'success'; } log.error( { - reason: primaryAttempt.streamedTriggerReason.reason, + reason: attempt.streamedTriggerReason.reason, }, 'Agent trigger detected but could not be resolved', ); @@ -1040,30 +915,45 @@ export async function runAgentForGroup( // success-null-result with no visible output — agent returned nothing useful. // But if output was already delivered to Discord (sawOutput), treat as success. - if ( - primaryAttempt.sawSuccessNullResultWithoutOutput && - !primaryAttempt.sawOutput - ) { - log.error( - 'Agent returned success with null result and no visible output', - ); + if (attempt.sawSuccessNullResultWithoutOutput && !attempt.sawOutput) { + log.error('Agent returned success with null result and no visible output'); return 'error'; } pairedExecutionStatus = 'succeeded'; - pairedSawOutput = primaryAttempt.sawOutput; + pairedSawOutput = attempt.sawOutput; return 'success'; + }; + + try { + let primaryAttempt = await runAttempt(provider); + const recoveredSessionAttempt = + await recoverRetryableClaudeSessionFailure(primaryAttempt); + if (recoveredSessionAttempt.resolved) { + return recoveredSessionAttempt.resolved; + } + primaryAttempt = recoveredSessionAttempt.attempt; + + if (primaryAttempt.error) { + return await handlePrimaryAttemptFailure( + primaryAttempt, + getErrorMessage(primaryAttempt.error), + ); + } + + return await finalizePrimaryAttempt(primaryAttempt); } finally { if (pairedExecutionContext && !pairedExecutionCompleted) { const completedRole = roomRoleContext?.role ?? 'owner'; + const completionStatus = getPairedExecutionStatus(); // Owner was interrupted without producing output (e.g. /stop) — // treat as failed so reviewer is not auto-triggered. const effectiveStatus = completedRole === 'owner' && - pairedExecutionStatus === 'succeeded' && + completionStatus === 'succeeded' && !pairedSawOutput ? 'failed' - : pairedExecutionStatus; + : completionStatus; completePairedExecutionContext({ taskId: pairedExecutionContext.task.id, role: completedRole, @@ -1123,12 +1013,12 @@ export async function runAgentForGroup( if (pairedExecutionContext) { const completedRole = roomRoleContext?.role ?? 'owner'; const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id); - const queueAction = resolvePairedFollowUpQueueAction( + const queueAction = resolvePairedFollowUpQueueAction({ completedRole, - pairedExecutionStatus, - pairedSawOutput, - finishedCheck?.status ?? null, - ); + executionStatus: pairedExecutionStatus, + sawOutput: pairedSawOutput, + taskStatus: finishedCheck?.status ?? null, + }); if (queueAction === 'generic') { deps.queue.enqueueMessageCheck(chatJid); } else if (queueAction === 'skip-inline-finalize') { diff --git a/src/paired-turn-fallback.test.ts b/src/paired-turn-fallback.test.ts index 4917b86..7a9143a 100644 --- a/src/paired-turn-fallback.test.ts +++ b/src/paired-turn-fallback.test.ts @@ -115,4 +115,32 @@ describe('resolveCodexFallbackHandoff', () => { logMessage: 'Fallback disabled for role, skipping handoff', }); }); + + it('returns none when visible output was already emitted', () => { + const result = resolveCodexFallbackHandoff({ + activeRole: 'reviewer', + effectiveAgentType: 'claude-code', + hasReviewer: true, + fallbackEnabled: true, + reason: 'auth-expired', + sawVisibleOutput: true, + prompt: 'please review', + }); + + expect(result).toEqual({ type: 'none' }); + }); + + it('returns none when no reviewer is configured for the room', () => { + const result = resolveCodexFallbackHandoff({ + activeRole: 'owner', + effectiveAgentType: 'claude-code', + hasReviewer: false, + fallbackEnabled: true, + reason: '429', + sawVisibleOutput: false, + prompt: 'please continue', + }); + + expect(result).toEqual({ type: 'none' }); + }); }); diff --git a/src/provider-retry.test.ts b/src/provider-retry.test.ts index dc32d51..4809b16 100644 --- a/src/provider-retry.test.ts +++ b/src/provider-retry.test.ts @@ -9,6 +9,16 @@ vi.mock('./logger.js', () => ({ }, })); +vi.mock('./codex-token-rotation.js', () => ({ + detectCodexRotationTrigger: vi.fn(() => ({ + shouldRotate: false, + reason: '', + })), + getCodexAccountCount: vi.fn(() => 1), + markCodexTokenHealthy: vi.fn(), + rotateCodexToken: vi.fn(() => false), +})); + vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), @@ -21,6 +31,13 @@ vi.mock('./token-refresh.js', () => ({ })); import { runClaudeRotationLoop } from './provider-retry.js'; +import { runCodexRotationLoop } from './provider-retry.js'; +import { + detectCodexRotationTrigger, + getCodexAccountCount, + markCodexTokenHealthy, + rotateCodexToken, +} from './codex-token-rotation.js'; import { getCurrentTokenIndex, getTokenCount, @@ -184,3 +201,81 @@ describe('runClaudeRotationLoop', () => { expect(rotateToken).toHaveBeenNthCalledWith(2, 'retry with next token'); }); }); + +describe('runCodexRotationLoop', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getCodexAccountCount).mockReturnValue(1); + vi.mocked(rotateCodexToken).mockReturnValue(false); + }); + + it('rotates and succeeds after a Codex auth trigger', async () => { + vi.mocked(getCodexAccountCount).mockReturnValue(2); + vi.mocked(rotateCodexToken).mockReturnValueOnce(true); + + const outcome = await runCodexRotationLoop( + { reason: 'auth-expired' }, + async () => ({ + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }), + { runId: 'rotate-codex-auth' }, + ); + + expect(outcome).toEqual({ type: 'success' }); + expect(rotateCodexToken).toHaveBeenCalledTimes(1); + expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1); + }); + + it('continues rotation when streamed trigger arrives before visible output', async () => { + vi.mocked(getCodexAccountCount).mockReturnValue(2); + vi.mocked(rotateCodexToken).mockReturnValueOnce(true).mockReturnValueOnce(false); + + let attempts = 0; + const outcome = await runCodexRotationLoop( + { reason: 'auth-expired' }, + async () => { + attempts += 1; + if (attempts === 1) { + return { + output: { status: 'success', result: 'rotate next' }, + sawOutput: false, + streamedTriggerReason: { reason: 'auth-expired' }, + }; + } + + return { + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }; + }, + { runId: 'rotate-codex-streamed' }, + ); + + expect(outcome).toEqual({ type: 'error' }); + expect(rotateCodexToken).toHaveBeenNthCalledWith(2, 'rotate next'); + }); + + it('uses the Codex detector for thrown errors', async () => { + vi.mocked(getCodexAccountCount).mockReturnValue(2); + vi.mocked(rotateCodexToken).mockReturnValueOnce(true); + vi.mocked(detectCodexRotationTrigger).mockReturnValue({ + shouldRotate: true, + reason: 'auth-expired', + }); + + const outcome = await runCodexRotationLoop( + { reason: 'auth-expired' }, + async () => ({ + thrownError: new Error('OAuth token expired'), + sawOutput: false, + }), + { runId: 'rotate-codex-error' }, + ); + + expect(outcome).toEqual({ type: 'error' }); + expect(detectCodexRotationTrigger).toHaveBeenCalledWith( + 'OAuth token expired', + ); + }); +}); diff --git a/src/provider-retry.ts b/src/provider-retry.ts index 149b5d1..f19081b 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -9,9 +9,16 @@ import type { AgentOutput } from './agent-runner.js'; import { getAgentOutputText } from './agent-output.js'; import { classifyRotationTrigger, + type CodexRotationReason, shouldRotateClaudeToken, type AgentTriggerReason, } from './agent-error-detection.js'; +import { + detectCodexRotationTrigger, + getCodexAccountCount, + markCodexTokenHealthy, + rotateCodexToken, +} from './codex-token-rotation.js'; import { logger } from './logger.js'; import { getErrorMessage } from './utils.js'; import { @@ -42,6 +49,15 @@ export type RotationOutcome = | { type: 'success'; sawOutput: boolean } | { type: 'error'; trigger?: TriggerInfo }; +export interface CodexRotationAttemptResult { + output?: Pick; + thrownError?: unknown; + sawOutput: boolean; + streamedTriggerReason?: TriggerInfo; +} + +export type CodexRotationOutcome = { type: 'success' } | { type: 'error' }; + type AttemptOutcome = | { type: 'success'; sawOutput: boolean } | { type: 'error'; trigger?: TriggerInfo } @@ -51,6 +67,15 @@ type AttemptOutcome = rotationMessage?: string; }; +type CodexAttemptOutcome = + | { type: 'success' } + | { type: 'error' } + | { + type: 'continue'; + trigger: { reason: CodexRotationReason }; + rotationMessage?: string; + }; + function evaluateClaudeAttempt( attempt: RotationAttemptResult, logContext: Record, @@ -149,6 +174,77 @@ function evaluateClaudeAttempt( return { type: 'success', sawOutput: attempt.sawOutput }; } +function evaluateCodexAttempt( + attempt: CodexRotationAttemptResult, + logContext: Record, +): CodexAttemptOutcome { + if (attempt.thrownError) { + const errMsg = getErrorMessage(attempt.thrownError); + const retryTrigger = detectCodexRotationTrigger(errMsg); + if (retryTrigger.shouldRotate) { + return { + type: 'continue', + trigger: { reason: retryTrigger.reason }, + rotationMessage: errMsg, + }; + } + + logger.error( + { ...logContext, provider: 'codex', err: attempt.thrownError }, + 'Rotated Codex account also threw', + ); + return { type: 'error' }; + } + + const output = attempt.output; + if (!output) { + logger.error( + { ...logContext, provider: 'codex' }, + 'Rotated Codex account produced no output object', + ); + return { type: 'error' }; + } + + if ( + !attempt.sawOutput && + attempt.streamedTriggerReason && + output.status !== 'error' + ) { + return { + type: 'continue', + trigger: { + reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + }, + rotationMessage: getAgentOutputText(output) ?? undefined, + }; + } + + if (output.status === 'error') { + const retryTrigger = attempt.streamedTriggerReason + ? { + shouldRotate: true, + reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + } + : detectCodexRotationTrigger(output.error); + if (retryTrigger.shouldRotate) { + return { + type: 'continue', + trigger: { reason: retryTrigger.reason }, + rotationMessage: output.error ?? undefined, + }; + } + + logger.error( + { ...logContext, provider: 'codex', error: output.error }, + 'Rotated Codex account failed', + ); + return { type: 'error' }; + } + + markCodexTokenHealthy(); + return { type: 'success' }; +} + // ── Shared rotation loop ───────────────────────────────────────── /** @@ -238,3 +334,39 @@ export async function runClaudeRotationLoop( ); return { type: 'error', trigger }; } + +export async function runCodexRotationLoop( + initialTrigger: { reason: CodexRotationReason }, + runAttempt: () => Promise, + logContext: Record, + rotationMessage?: string, +): Promise { + let trigger = initialTrigger; + let lastRotationMessage = rotationMessage; + + while ( + getCodexAccountCount() > 1 && + rotateCodexToken(lastRotationMessage) + ) { + logger.info( + { ...logContext, reason: trigger.reason }, + 'Codex account unhealthy, retrying with rotated account', + ); + + const rotatedAttemptOutcome = evaluateCodexAttempt( + await runAttempt(), + logContext, + ); + if (rotatedAttemptOutcome.type === 'success') { + return rotatedAttemptOutcome; + } + if (rotatedAttemptOutcome.type === 'error') { + return rotatedAttemptOutcome; + } + + trigger = rotatedAttemptOutcome.trigger; + lastRotationMessage = rotatedAttemptOutcome.rotationMessage; + } + + return { type: 'error' }; +} From 7612b2c4ccc82220e623bd689fd0b2fca8ac494f Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 20:38:38 +0900 Subject: [PATCH 13/29] Format retry rule refactor files --- src/message-agent-executor.ts | 17 +++++++++-------- src/provider-retry.test.ts | 4 +++- src/provider-retry.ts | 5 +---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 48ab6d5..01378f3 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -44,9 +44,7 @@ import { resolvePairedFollowUpQueueAction, } from './message-agent-executor-rules.js'; import { buildRoomRoleContext } from './room-role-context.js'; -import { - type AgentTriggerReason, -} from './agent-error-detection.js'; +import { type AgentTriggerReason } from './agent-error-detection.js'; import { runClaudeRotationLoop, runCodexRotationLoop, @@ -759,9 +757,7 @@ export async function runAgentForGroup( const getPairedExecutionStatus = (): 'succeeded' | 'failed' => pairedExecutionStatus; - const isRetryableClaudeSessionFailure = ( - attempt: AgentAttempt, - ): boolean => + const isRetryableClaudeSessionFailure = (attempt: AgentAttempt): boolean => isRetryableClaudeSessionFailureAttempt({ attempt, isClaudeCodeAgent, @@ -771,7 +767,10 @@ export async function runAgentForGroup( const recoverRetryableClaudeSessionFailure = async ( attempt: AgentAttempt, - ): Promise<{ attempt: AgentAttempt; resolved: 'success' | 'error' | null }> => { + ): Promise<{ + attempt: AgentAttempt; + resolved: 'success' | 'error' | null; + }> => { if (!isRetryableClaudeSessionFailure(attempt)) { return { attempt, resolved: null }; } @@ -916,7 +915,9 @@ export async function runAgentForGroup( // success-null-result with no visible output — agent returned nothing useful. // But if output was already delivered to Discord (sawOutput), treat as success. if (attempt.sawSuccessNullResultWithoutOutput && !attempt.sawOutput) { - log.error('Agent returned success with null result and no visible output'); + log.error( + 'Agent returned success with null result and no visible output', + ); return 'error'; } diff --git a/src/provider-retry.test.ts b/src/provider-retry.test.ts index 4809b16..7835af8 100644 --- a/src/provider-retry.test.ts +++ b/src/provider-retry.test.ts @@ -229,7 +229,9 @@ describe('runCodexRotationLoop', () => { it('continues rotation when streamed trigger arrives before visible output', async () => { vi.mocked(getCodexAccountCount).mockReturnValue(2); - vi.mocked(rotateCodexToken).mockReturnValueOnce(true).mockReturnValueOnce(false); + vi.mocked(rotateCodexToken) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); let attempts = 0; const outcome = await runCodexRotationLoop( diff --git a/src/provider-retry.ts b/src/provider-retry.ts index f19081b..52dfeea 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -344,10 +344,7 @@ export async function runCodexRotationLoop( let trigger = initialTrigger; let lastRotationMessage = rotationMessage; - while ( - getCodexAccountCount() > 1 && - rotateCodexToken(lastRotationMessage) - ) { + while (getCodexAccountCount() > 1 && rotateCodexToken(lastRotationMessage)) { logger.info( { ...logContext, reason: trigger.reason }, 'Codex account unhealthy, retrying with rotated account', From fdc27836a323aea582444f1d92d94e0722acfc37 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 20:53:48 +0900 Subject: [PATCH 14/29] Unify codex rotation retry guards --- src/agent-error-detection.ts | 12 +++ src/message-agent-executor-rules.test.ts | 29 +++++-- src/message-agent-executor-rules.ts | 6 +- src/provider-retry.ts | 28 +++++-- src/task-scheduler.ts | 99 ++++++++---------------- 5 files changed, 94 insertions(+), 80 deletions(-) diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts index e4e2af9..159a100 100644 --- a/src/agent-error-detection.ts +++ b/src/agent-error-detection.ts @@ -126,6 +126,18 @@ export function shouldRotateClaudeToken( ); } +export function isCodexRotationReason( + reason: AgentTriggerReason, +): reason is CodexRotationReason { + return ( + reason === '429' || + reason === 'auth-expired' || + reason === 'org-access-denied' || + reason === 'overloaded' || + reason === 'network-error' + ); +} + // ── Rotation trigger classification ───────────────────────────── export type RotationTriggerResult = diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts index 57471f3..e10a6fd 100644 --- a/src/message-agent-executor-rules.test.ts +++ b/src/message-agent-executor-rules.test.ts @@ -1,11 +1,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -vi.mock('./agent-error-detection.js', () => ({ - classifyRotationTrigger: vi.fn(() => ({ - shouldRetry: false, - reason: '', - })), -})); +vi.mock('./agent-error-detection.js', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + classifyRotationTrigger: vi.fn(() => ({ + shouldRetry: false, + reason: '', + })), + }; +}); vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn(() => ({ @@ -71,6 +76,18 @@ describe('message-agent-executor-rules', () => { ).toEqual({ reason: 'auth-expired' }); }); + it('ignores streamed reasons that are not valid Codex rotation triggers', () => { + expect( + resolveCodexRetryTrigger({ + canRetryCodex: true, + attempt: { + streamedTriggerReason: { reason: 'usage-exhausted' }, + }, + }), + ).toBeNull(); + expect(detectCodexRotationTrigger).not.toHaveBeenCalled(); + }); + it('detects retryable Claude session failures from either flag or classifier', () => { expect( isRetryableClaudeSessionFailureAttempt({ diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index 9215300..c03c33a 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -2,6 +2,7 @@ import { classifyRotationTrigger, type AgentTriggerReason, type CodexRotationReason, + isCodexRotationReason, } from './agent-error-detection.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js'; @@ -94,8 +95,11 @@ export function resolveCodexRetryTrigger(args: { } if (args.attempt.streamedTriggerReason) { + if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) { + return null; + } return { - reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason, + reason: args.attempt.streamedTriggerReason.reason, }; } diff --git a/src/provider-retry.ts b/src/provider-retry.ts index 52dfeea..a5ac85c 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -10,6 +10,7 @@ import { getAgentOutputText } from './agent-output.js'; import { classifyRotationTrigger, type CodexRotationReason, + isCodexRotationReason, shouldRotateClaudeToken, type AgentTriggerReason, } from './agent-error-detection.js'; @@ -210,22 +211,35 @@ function evaluateCodexAttempt( attempt.streamedTriggerReason && output.status !== 'error' ) { + if (!isCodexRotationReason(attempt.streamedTriggerReason.reason)) { + logger.error( + { + ...logContext, + provider: 'codex', + reason: attempt.streamedTriggerReason.reason, + }, + 'Unexpected non-Codex streamed trigger while rotating Codex account', + ); + return { type: 'error' }; + } return { type: 'continue', trigger: { - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + reason: attempt.streamedTriggerReason.reason, }, rotationMessage: getAgentOutputText(output) ?? undefined, }; } if (output.status === 'error') { - const retryTrigger = attempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, - } - : detectCodexRotationTrigger(output.error); + const retryTrigger = + attempt.streamedTriggerReason && + isCodexRotationReason(attempt.streamedTriggerReason.reason) + ? { + shouldRotate: true as const, + reason: attempt.streamedTriggerReason.reason, + } + : detectCodexRotationTrigger(output.error); if (retryTrigger.shouldRotate) { return { type: 'continue', diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index ce13440..99aaec4 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -27,7 +27,7 @@ import { } from './group-folder.js'; import { createScopedLogger, logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; -import { runClaudeRotationLoop } from './provider-retry.js'; +import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js'; import { detectCodexRotationTrigger, rotateCodexToken, @@ -38,6 +38,7 @@ import { classifyRotationTrigger, type AgentTriggerReason, type CodexRotationReason, + isCodexRotationReason, } from './agent-error-detection.js'; import { getTokenCount, @@ -489,65 +490,29 @@ async function runTask( initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, ): Promise => { - let trigger = initialTrigger; - let lastRotationMessage = rotationMessage; - - while ( - getCodexAccountCount() > 1 && - rotateCodexToken(lastRotationMessage) - ) { - log.info( - { - reason: trigger.reason, - }, - 'Codex account unhealthy, retrying scheduled task with rotated account', - ); - - const retryAttempt = await runTaskAttempt('codex'); - result = retryAttempt.attemptResult; - error = retryAttempt.attemptError; - - if ( - !retryAttempt.sawOutput && - retryAttempt.streamedTriggerReason && - retryAttempt.output.status !== 'error' - ) { - trigger = { - reason: retryAttempt.streamedTriggerReason - .reason as CodexRotationReason, + const outcome = await runCodexRotationLoop( + initialTrigger, + async () => { + const retryAttempt = await runTaskAttempt('codex'); + result = retryAttempt.attemptResult; + error = retryAttempt.attemptError; + return { + output: retryAttempt.output, + thrownError: null, + sawOutput: retryAttempt.sawOutput, + streamedTriggerReason: retryAttempt.streamedTriggerReason, }; - lastRotationMessage = - typeof retryAttempt.output.result === 'string' - ? retryAttempt.output.result - : undefined; - continue; - } + }, + { + taskId: task.id, + group: context.group.name, + groupFolder: task.group_folder, + }, + rotationMessage, + ); - if (retryAttempt.output.status === 'error') { - const retryTrigger = retryAttempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: retryAttempt.streamedTriggerReason - .reason as CodexRotationReason, - } - : detectCodexRotationTrigger( - retryAttempt.attemptError || retryAttempt.output.error, - ); - - if (retryTrigger.shouldRotate) { - trigger = { reason: retryTrigger.reason }; - lastRotationMessage = - retryAttempt.attemptError || - retryAttempt.output.error || - undefined; - continue; - } - return; - } - - markCodexTokenHealthy(); + if (outcome.type === 'success') { error = null; - return; } }; @@ -567,11 +532,12 @@ async function runTask( } else if ( provider === 'codex' && attempt.streamedTriggerReason && - !attempt.sawOutput + !attempt.sawOutput && + isCodexRotationReason(attempt.streamedTriggerReason.reason) ) { await retryCodexTaskWithRotation( { - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + reason: attempt.streamedTriggerReason.reason, }, typeof attempt.output.error === 'string' ? attempt.output.error @@ -592,13 +558,14 @@ async function runTask( }); } } else if (attempt.output.status === 'error' && provider === 'codex') { - const trigger = attempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: attempt.streamedTriggerReason - .reason as CodexRotationReason, - } - : detectCodexRotationTrigger(error); + const trigger = + attempt.streamedTriggerReason && + isCodexRotationReason(attempt.streamedTriggerReason.reason) + ? { + shouldRotate: true as const, + reason: attempt.streamedTriggerReason.reason, + } + : detectCodexRotationTrigger(error); if (trigger.shouldRotate) { await retryCodexTaskWithRotation( { reason: trigger.reason }, From 75bbeac70391bf80cde8f09346650d9b08459286 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 20:54:16 +0900 Subject: [PATCH 15/29] Format task scheduler retry imports --- src/task-scheduler.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 99aaec4..0154c7f 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -27,7 +27,10 @@ import { } from './group-folder.js'; import { createScopedLogger, logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; -import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js'; +import { + runClaudeRotationLoop, + runCodexRotationLoop, +} from './provider-retry.js'; import { detectCodexRotationTrigger, rotateCodexToken, From 1a6ffa55af500b1d804d6edebb17cf582fcff847 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 21:09:46 +0900 Subject: [PATCH 16/29] Avoid duplicate paired follow-up scheduling --- src/message-agent-executor-rules.test.ts | 22 +++ src/message-agent-executor-rules.ts | 4 +- src/message-agent-executor.test.ts | 165 +++++++++++++++++++++++ 3 files changed, 188 insertions(+), 3 deletions(-) diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts index e10a6fd..7ad449a 100644 --- a/src/message-agent-executor-rules.test.ts +++ b/src/message-agent-executor-rules.test.ts @@ -129,4 +129,26 @@ describe('message-agent-executor-rules', () => { }), ).toBe('pending'); }); + + it('does not request a generic follow-up after successful owner output moved the task to review_ready', () => { + expect( + resolvePairedFollowUpQueueAction({ + completedRole: 'owner', + executionStatus: 'succeeded', + sawOutput: true, + taskStatus: 'review_ready', + }), + ).toBe('none'); + }); + + it('does not request a generic follow-up after successful reviewer output moved the task back to active', () => { + expect( + resolvePairedFollowUpQueueAction({ + completedRole: 'reviewer', + executionStatus: 'succeeded', + sawOutput: true, + taskStatus: 'active', + }), + ).toBe('none'); + }); }); diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index c03c33a..2671e10 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -127,9 +127,7 @@ export function resolvePairedFollowUpQueueAction(args: { return args.completedRole === 'reviewer' && args.taskStatus === 'merge_ready' ? 'skip-inline-finalize' - : args.taskStatus !== 'completed' - ? 'generic' - : 'none'; + : 'none'; } const shouldRequeuePendingPairedTurn = diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index e617f0f..602078d 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1238,6 +1238,171 @@ describe('runAgentForGroup room memory', () => { expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); }); + it('does not enqueue a generic follow-up when owner output already moved the task to review_ready', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + const deps = makeDeps(); + + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-owner-review-ready', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'claude', + reviewer_service_id: 'codex-review', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: null, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: {}, + }); + vi.mocked(db.getPairedTaskById).mockReturnValue({ + id: 'paired-task-owner-review-ready', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'claude', + reviewer_service_id: 'codex-review', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 0, + review_requested_at: null, + status: 'review_ready', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + result: 'DONE_WITH_CONCERNS\nowner complete', + output: { + visibility: 'public', + text: 'DONE_WITH_CONCERNS\nowner complete', + }, + phase: 'final', + }); + return { + status: 'success', + result: 'DONE_WITH_CONCERNS\nowner complete', + }; + }, + ); + + const result = await runAgentForGroup(deps, { + group, + prompt: 'please implement', + chatJid: 'group@test', + runId: 'run-owner-review-ready', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); + }); + + it('does not enqueue a generic follow-up when reviewer output already returned the task to active', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + const deps = makeDeps(); + + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_agent_type: 'codex', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + arbiter_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-reviewer-active', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'in_review', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: {}, + }); + vi.mocked(db.getPairedTaskById).mockReturnValue({ + id: 'paired-task-reviewer-active', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + result: 'DONE_WITH_CONCERNS\nreview complete', + output: { + visibility: 'public', + text: 'DONE_WITH_CONCERNS\nreview complete', + }, + phase: 'final', + }); + return { + status: 'success', + result: 'DONE_WITH_CONCERNS\nreview complete', + }; + }, + ); + + const result = await runAgentForGroup(deps, { + group, + prompt: 'please review', + chatJid: 'group@test', + runId: 'run-reviewer-active', + forcedRole: 'reviewer', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); + }); + it('re-enqueues the group when a reviewer fails and the task remains review_ready', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); From f22947ba6114ea17a5b98291b7960c6d411a967e Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 21:58:22 +0900 Subject: [PATCH 17/29] Fix paired turn follow-up scheduling --- src/message-agent-executor-rules.test.ts | 4 ++-- src/message-agent-executor-rules.ts | 7 +++++++ src/message-agent-executor.test.ts | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts index 7ad449a..a7eb7d5 100644 --- a/src/message-agent-executor-rules.test.ts +++ b/src/message-agent-executor-rules.test.ts @@ -130,7 +130,7 @@ describe('message-agent-executor-rules', () => { ).toBe('pending'); }); - it('does not request a generic follow-up after successful owner output moved the task to review_ready', () => { + it('requests a generic follow-up after successful owner output moved the task to review_ready', () => { expect( resolvePairedFollowUpQueueAction({ completedRole: 'owner', @@ -138,7 +138,7 @@ describe('message-agent-executor-rules', () => { sawOutput: true, taskStatus: 'review_ready', }), - ).toBe('none'); + ).toBe('generic'); }); it('does not request a generic follow-up after successful reviewer output moved the task back to active', () => { diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index 2671e10..0a250af 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -124,6 +124,13 @@ export function resolvePairedFollowUpQueueAction(args: { taskStatus: PairedTaskStatus | null; }): PairedFollowUpQueueAction { if (args.executionStatus === 'succeeded' && args.sawOutput) { + if ( + args.completedRole === 'owner' && + args.taskStatus === 'review_ready' + ) { + return 'generic'; + } + return args.completedRole === 'reviewer' && args.taskStatus === 'merge_ready' ? 'skip-inline-finalize' diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 602078d..0cdf23a 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1238,7 +1238,7 @@ describe('runAgentForGroup room memory', () => { expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); }); - it('does not enqueue a generic follow-up when owner output already moved the task to review_ready', async () => { + it('enqueues a generic follow-up when owner output moved the task to review_ready', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); @@ -1311,7 +1311,8 @@ describe('runAgentForGroup room memory', () => { }); expect(result).toBe('success'); - expect(deps.queue.enqueueMessageCheck).not.toHaveBeenCalled(); + expect(deps.queue.enqueueMessageCheck).toHaveBeenCalledTimes(1); + expect(deps.queue.enqueueMessageCheck).toHaveBeenCalledWith('group@test'); }); it('does not enqueue a generic follow-up when reviewer output already returned the task to active', async () => { From b62eebc25b9a6e08b2f675ecc5ed2e5a41122a5b Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 22:44:38 +0900 Subject: [PATCH 18/29] Simplify paired turn routing and prompt builders --- src/message-agent-executor-rules.test.ts | 22 ++ src/message-agent-executor-rules.ts | 24 +- src/message-agent-executor.ts | 54 ++--- src/message-runtime-prompts.ts | 141 ++++++++++++ src/message-runtime-rules.test.ts | 132 +++++++++++ src/message-runtime-rules.ts | 101 ++++++++- src/message-runtime.ts | 272 ++++++++--------------- 7 files changed, 519 insertions(+), 227 deletions(-) create mode 100644 src/message-runtime-prompts.ts create mode 100644 src/message-runtime-rules.test.ts diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts index a7eb7d5..315461e 100644 --- a/src/message-agent-executor-rules.test.ts +++ b/src/message-agent-executor-rules.test.ts @@ -141,6 +141,17 @@ describe('message-agent-executor-rules', () => { ).toBe('generic'); }); + it('skips generic requeue when reviewer success moved the task to merge_ready for inline finalize', () => { + expect( + resolvePairedFollowUpQueueAction({ + completedRole: 'reviewer', + executionStatus: 'succeeded', + sawOutput: true, + taskStatus: 'merge_ready', + }), + ).toBe('skip-inline-finalize'); + }); + it('does not request a generic follow-up after successful reviewer output moved the task back to active', () => { expect( resolvePairedFollowUpQueueAction({ @@ -151,4 +162,15 @@ describe('message-agent-executor-rules', () => { }), ).toBe('none'); }); + + it('returns none after successful output when no next-turn action is needed', () => { + expect( + resolvePairedFollowUpQueueAction({ + completedRole: 'owner', + executionStatus: 'succeeded', + sawOutput: true, + taskStatus: 'active', + }), + ).toBe('none'); + }); }); diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index 0a250af..de2ca9b 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -5,6 +5,7 @@ import { isCodexRotationReason, } from './agent-error-detection.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; +import { resolveNextTurnAction } from './message-runtime-rules.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js'; import { getErrorMessage } from './utils.js'; @@ -123,26 +124,29 @@ export function resolvePairedFollowUpQueueAction(args: { sawOutput: boolean; taskStatus: PairedTaskStatus | null; }): PairedFollowUpQueueAction { + const nextTurnAction = resolveNextTurnAction({ + taskStatus: args.taskStatus, + lastTurnOutputRole: + args.executionStatus === 'succeeded' && args.sawOutput + ? args.completedRole + : null, + }); + if (args.executionStatus === 'succeeded' && args.sawOutput) { - if ( - args.completedRole === 'owner' && - args.taskStatus === 'review_ready' - ) { + if (nextTurnAction.kind === 'reviewer-turn') { return 'generic'; } return args.completedRole === 'reviewer' && - args.taskStatus === 'merge_ready' + nextTurnAction.kind === 'finalize-owner-turn' ? 'skip-inline-finalize' : 'none'; } const shouldRequeuePendingPairedTurn = (args.completedRole === 'reviewer' || args.completedRole === 'arbiter') && - (args.taskStatus === 'review_ready' || - args.taskStatus === 'in_review' || - args.taskStatus === 'arbiter_requested' || - args.taskStatus === 'in_arbitration' || - args.taskStatus === 'merge_ready'); + (nextTurnAction.kind === 'reviewer-turn' || + nextTurnAction.kind === 'arbiter-turn' || + nextTurnAction.kind === 'finalize-owner-turn'); return shouldRequeuePendingPairedTurn ? 'pending' : 'none'; } diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 01378f3..6bca327 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -32,10 +32,7 @@ import { } from './paired-execution-context.js'; import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; import { - resolveActiveRole, - resolveConfiguredRoleAgentPlan, - resolveEffectiveAgentType, - resolveSessionFolder, + resolveExecutionTarget, } from './message-runtime-rules.js'; import { isRetryableClaudeSessionFailureAttempt, @@ -67,7 +64,6 @@ import { readArbiterPrompt } from './platform-prompts.js'; import { activateCodexFailover, getEffectiveChannelLease, - resolveLeaseServiceId, } from './service-routing.js'; import { evaluateStreamedOutput, @@ -119,41 +115,33 @@ export async function runAgentForGroup( const pairedTask = currentLease.reviewer_agent_type ? getLatestOpenPairedTaskForChat(chatJid) : null; - const inferredRole = resolveActiveRole(pairedTask?.status); - const canHonorForcedRole = Boolean( - args.forcedRole === 'owner' || - (args.forcedRole === 'reviewer' && currentLease.reviewer_agent_type) || - (args.forcedRole === 'arbiter' && currentLease.arbiter_agent_type), - ); - const activeRole = canHonorForcedRole ? args.forcedRole! : inferredRole; - const effectiveServiceId = resolveLeaseServiceId(currentLease, activeRole); - if (!effectiveServiceId) { - throw new Error(`Missing runtime service id for ${activeRole} lease`); - } + const executionTarget = resolveExecutionTarget({ + lease: currentLease, + pairedTaskStatus: pairedTask?.status, + groupFolder: group.folder, + groupAgentType: group.agentType, + forcedRole: args.forcedRole, + forcedAgentType: args.forcedAgentType, + }); + const { + inferredRole, + canHonorForcedRole, + activeRole, + effectiveServiceId, + reviewerServiceId, + arbiterServiceId, + roleAgentPlan, + configuredAgentType, + effectiveAgentType, + sessionFolder, + } = executionTarget; const reviewerMode = activeRole === 'reviewer'; const arbiterMode = activeRole === 'arbiter'; - const reviewerServiceId = resolveLeaseServiceId(currentLease, 'reviewer'); - const arbiterServiceId = resolveLeaseServiceId(currentLease, 'arbiter'); - const roleAgentPlan = resolveConfiguredRoleAgentPlan( - currentLease.reviewer_agent_type != null, - group.agentType, - ); - - const configuredAgentType = resolveEffectiveAgentType( - activeRole, - group.agentType, - ); - const effectiveAgentType = args.forcedAgentType ?? configuredAgentType; const effectiveGroup = effectiveAgentType !== roleAgentPlan.ownerAgentType ? { ...group, agentType: effectiveAgentType } : group; const isClaudeCodeAgent = effectiveAgentType === 'claude-code'; - const sessionFolder = resolveSessionFolder( - group.folder, - activeRole, - group.agentType, - ); const unsafeHostPairedMode = process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1'; const forceFreshClaudeReviewerSession = diff --git a/src/message-runtime-prompts.ts b/src/message-runtime-prompts.ts new file mode 100644 index 0000000..1903004 --- /dev/null +++ b/src/message-runtime-prompts.ts @@ -0,0 +1,141 @@ +import { buildArbiterContextPrompt } from './arbiter-context.js'; +import { formatMessages } from './router.js'; +import type { NewMessage, PairedTask, PairedTurnOutput } from './types.js'; + +function turnOutputsToMessages( + outputs: PairedTurnOutput[], + chatJid: string, +): NewMessage[] { + return outputs.map((turnOutput) => ({ + id: `turn-${turnOutput.task_id}-${turnOutput.turn_number}`, + chat_jid: chatJid, + sender: turnOutput.role, + sender_name: turnOutput.role, + content: turnOutput.output_text, + timestamp: turnOutput.created_at, + is_bot_message: true as const, + is_from_me: false as const, + })); +} + +function mergeHumanAndTurnOutputMessages( + chatJid: string, + humanMessages: NewMessage[], + turnOutputs: PairedTurnOutput[], +): NewMessage[] { + return [...humanMessages, ...turnOutputsToMessages(turnOutputs, chatJid)].sort( + (a, b) => a.timestamp.localeCompare(b.timestamp), + ); +} + +export function buildPairedTurnPrompt(args: { + taskId: string; + chatJid: string; + timezone: string; + missedMessages: NewMessage[]; + labeledFallbackMessages: NewMessage[]; + turnOutputs: PairedTurnOutput[]; +}): string { + if (args.turnOutputs.length === 0) { + return formatMessages(args.labeledFallbackMessages, args.timezone); + } + + const humanMessages = args.missedMessages.filter((message) => !message.is_bot_message); + return formatMessages( + mergeHumanAndTurnOutputMessages( + args.chatJid, + humanMessages, + args.turnOutputs, + ), + args.timezone, + ); +} + +export function buildReviewerPendingPrompt(args: { + chatJid: string; + timezone: string; + turnOutputs: PairedTurnOutput[]; + recentHumanMessages: NewMessage[]; + lastHumanMessage: string | null | undefined; +}): string { + if (args.turnOutputs.length > 0) { + return formatMessages( + mergeHumanAndTurnOutputMessages( + args.chatJid, + args.recentHumanMessages, + args.turnOutputs, + ), + args.timezone, + ); + } + + if (!args.lastHumanMessage) { + return 'Review the latest owner changes in the workspace.'; + } + + return `User request:\n---\n${args.lastHumanMessage}\n---\n\nReview the latest owner changes in the workspace.`; +} + +export function buildOwnerPendingPrompt(args: { + chatJid: string; + timezone: string; + turnOutputs: PairedTurnOutput[]; + recentHumanMessages: NewMessage[]; + lastHumanMessage: string | null | undefined; +}): string { + if (args.turnOutputs.length > 0) { + return formatMessages( + mergeHumanAndTurnOutputMessages( + args.chatJid, + args.recentHumanMessages, + args.turnOutputs, + ), + args.timezone, + ); + } + + if (!args.lastHumanMessage) { + return 'Continue the owner turn using the latest reviewer or arbiter feedback.'; + } + + return `User request:\n---\n${args.lastHumanMessage}\n---\n\nContinue the owner turn using the latest reviewer or arbiter feedback.`; +} + +export function buildArbiterPromptForTask(args: { + task: PairedTask; + chatJid: string; + timezone: string; + turnOutputs: PairedTurnOutput[]; + recentMessages: NewMessage[]; + labeledRecentMessages: NewMessage[]; +}): string { + const messages = + args.turnOutputs.length > 0 + ? mergeHumanAndTurnOutputMessages( + args.chatJid, + args.recentMessages.filter((message) => !message.is_bot_message), + args.turnOutputs, + ) + : args.labeledRecentMessages; + + return buildArbiterContextPrompt({ + chatJid: args.chatJid, + taskId: args.task.id, + roundTripCount: args.task.round_trip_count, + timezone: args.timezone, + messages, + }); +} + +export function buildFinalizePendingPrompt(args: { + turnOutputs: PairedTurnOutput[]; +}): string { + const lastReviewerOutput = [...args.turnOutputs] + .reverse() + .find((output) => output.role === 'reviewer'); + const reviewerSummary = lastReviewerOutput?.output_text + ? `\n\nReviewer's final assessment:\n${lastReviewerOutput.output_text.slice(0, 2000)}` + : ''; + + return `The reviewer approved your work (DONE). Finalize and report the result.${reviewerSummary}`; +} diff --git a/src/message-runtime-rules.test.ts b/src/message-runtime-rules.test.ts new file mode 100644 index 0000000..9d44c3a --- /dev/null +++ b/src/message-runtime-rules.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest'; + +import { + resolveExecutionTarget, + resolveNextTurnAction, + resolveSessionFolder, +} from './message-runtime-rules.js'; +import { + resolveLeaseServiceId, + type EffectiveChannelLease, +} from './service-routing.js'; + +const baseLease: EffectiveChannelLease = { + chat_jid: 'chat-1', + owner_agent_type: 'claude-code', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: 'codex', + owner_service_id: 'svc-owner', + reviewer_service_id: 'svc-reviewer', + arbiter_service_id: 'svc-arbiter', + owner_failover_active: false, + activated_at: null, + reason: null, + explicit: true, +}; + +describe('message-runtime-rules', () => { + it('maps review_ready to a reviewer turn', () => { + expect( + resolveNextTurnAction({ + taskStatus: 'review_ready', + }), + ).toEqual({ kind: 'reviewer-turn' }); + }); + + it('maps arbiter_requested to an arbiter turn', () => { + expect( + resolveNextTurnAction({ + taskStatus: 'arbiter_requested', + }), + ).toEqual({ kind: 'arbiter-turn' }); + }); + + it('maps merge_ready to an inline finalize owner turn', () => { + expect( + resolveNextTurnAction({ + taskStatus: 'merge_ready', + }), + ).toEqual({ kind: 'finalize-owner-turn' }); + }); + + it('maps active tasks with reviewer output to an owner follow-up', () => { + expect( + resolveNextTurnAction({ + taskStatus: 'active', + lastTurnOutputRole: 'reviewer', + }), + ).toEqual({ kind: 'owner-follow-up' }); + }); + + it('returns none when an active task has no reviewer or arbiter handoff output', () => { + expect( + resolveNextTurnAction({ + taskStatus: 'active', + lastTurnOutputRole: 'owner', + }), + ).toEqual({ kind: 'none' }); + }); + + it('resolves reviewer execution target from review_ready task status', () => { + const resolution = resolveExecutionTarget({ + lease: baseLease, + pairedTaskStatus: 'review_ready', + groupFolder: 'group-1', + groupAgentType: 'claude-code', + }); + + expect(resolution).toMatchObject({ + inferredRole: 'reviewer', + activeRole: 'reviewer', + configuredAgentType: 'claude-code', + effectiveAgentType: 'claude-code', + sessionFolder: 'group-1', + }); + expect(resolution.effectiveServiceId).toBe(resolution.reviewerServiceId); + }); + + it('honors an arbiter forced role only when arbiter is configured', () => { + const resolution = resolveExecutionTarget({ + lease: { ...baseLease, arbiter_agent_type: null, arbiter_service_id: null }, + pairedTaskStatus: 'active', + groupFolder: 'group-1', + groupAgentType: 'claude-code', + forcedRole: 'arbiter', + }); + + expect(resolution).toMatchObject({ + inferredRole: 'owner', + canHonorForcedRole: false, + activeRole: 'owner', + }); + expect(resolution.effectiveServiceId).toBe( + resolveLeaseServiceId(baseLease, 'owner'), + ); + }); + + it('applies forced agent type overrides without changing the configured role target', () => { + const resolution = resolveExecutionTarget({ + lease: baseLease, + pairedTaskStatus: 'review_ready', + groupFolder: 'group-1', + groupAgentType: 'claude-code', + forcedAgentType: 'codex', + }); + + expect(resolution).toMatchObject({ + activeRole: 'reviewer', + configuredAgentType: 'claude-code', + effectiveAgentType: 'codex', + }); + expect(resolution.effectiveServiceId).toBe(resolution.reviewerServiceId); + }); + + it('always gives arbiter a dedicated session folder', () => { + expect( + resolveSessionFolder('group-1', 'arbiter', 'claude-code'), + ).toBe('group-1:arbiter'); + expect( + resolveSessionFolder('group-1', 'arbiter', 'codex'), + ).toBe('group-1:arbiter'); + }); +}); diff --git a/src/message-runtime-rules.ts b/src/message-runtime-rules.ts index 601e6b7..60d21de 100644 --- a/src/message-runtime-rules.ts +++ b/src/message-runtime-rules.ts @@ -4,7 +4,11 @@ import { normalizeStoredSeqCursor } from './message-cursor.js'; import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js'; import { isTaskStatusControlMessage } from './task-watch-status.js'; import { ARBITER_AGENT_TYPE, REVIEWER_AGENT_TYPE } from './config.js'; -import { hasReviewerLease } from './service-routing.js'; +import { + hasReviewerLease, + resolveLeaseServiceId, + type EffectiveChannelLease, +} from './service-routing.js'; import { resolveAgentTypeForRole, resolveRoleAgentPlan, @@ -14,6 +18,8 @@ import { type AgentType, type Channel, type NewMessage, + type PairedRoomRole, + type PairedTaskStatus, type RegisteredGroup, } from './types.js'; @@ -55,6 +61,36 @@ export function resolveActiveRole( } } +export type NextTurnAction = + | { kind: 'none' } + | { kind: 'reviewer-turn' } + | { kind: 'arbiter-turn' } + | { kind: 'owner-follow-up' } + | { kind: 'finalize-owner-turn' }; + +export function resolveNextTurnAction(args: { + taskStatus?: PairedTaskStatus | null; + lastTurnOutputRole?: PairedRoomRole | null; +}): NextTurnAction { + switch (args.taskStatus) { + case 'review_ready': + case 'in_review': + return { kind: 'reviewer-turn' }; + case 'arbiter_requested': + case 'in_arbitration': + return { kind: 'arbiter-turn' }; + case 'merge_ready': + return { kind: 'finalize-owner-turn' }; + case 'active': + return args.lastTurnOutputRole === 'reviewer' || + args.lastTurnOutputRole === 'arbiter' + ? { kind: 'owner-follow-up' } + : { kind: 'none' }; + default: + return { kind: 'none' }; + } +} + /** Cursor key for a role. Owner uses chatJid, others use chatJid:role. */ export function resolveCursorKey( chatJid: string, @@ -102,6 +138,69 @@ export function resolveSessionFolder( return `${groupFolder}:${role}`; } +export interface ExecutionTargetResolution { + inferredRole: PairedRoomRole; + canHonorForcedRole: boolean; + activeRole: PairedRoomRole; + effectiveServiceId: string; + reviewerServiceId: string | null; + arbiterServiceId: string | null; + roleAgentPlan: RoleAgentPlan; + configuredAgentType: AgentType; + effectiveAgentType: AgentType; + sessionFolder: string; +} + +export function resolveExecutionTarget(args: { + lease: EffectiveChannelLease; + pairedTaskStatus?: PairedTaskStatus | null; + groupFolder: string; + groupAgentType?: AgentType; + forcedRole?: PairedRoomRole; + forcedAgentType?: AgentType; +}): ExecutionTargetResolution { + const inferredRole = resolveActiveRole(args.pairedTaskStatus); + const canHonorForcedRole = Boolean( + args.forcedRole === 'owner' || + (args.forcedRole === 'reviewer' && args.lease.reviewer_agent_type) || + (args.forcedRole === 'arbiter' && args.lease.arbiter_agent_type), + ); + const activeRole = canHonorForcedRole ? args.forcedRole! : inferredRole; + const effectiveServiceId = resolveLeaseServiceId(args.lease, activeRole); + if (!effectiveServiceId) { + throw new Error(`Missing runtime service id for ${activeRole} lease`); + } + + const reviewerServiceId = resolveLeaseServiceId(args.lease, 'reviewer'); + const arbiterServiceId = resolveLeaseServiceId(args.lease, 'arbiter'); + const roleAgentPlan = resolveConfiguredRoleAgentPlan( + args.lease.reviewer_agent_type != null, + args.groupAgentType, + ); + const configuredAgentType = resolveEffectiveAgentType( + activeRole, + args.groupAgentType, + ); + const effectiveAgentType = args.forcedAgentType ?? configuredAgentType; + + return { + inferredRole, + canHonorForcedRole, + activeRole, + effectiveServiceId, + reviewerServiceId, + arbiterServiceId, + roleAgentPlan, + configuredAgentType, + effectiveAgentType, + sessionFolder: resolveSessionFolder( + args.groupFolder, + activeRole, + args.groupAgentType, + ), + }; +} + export function createImplicitContinuationTracker(idleTimeout: number) { const implicitContinuationUntil = new Map(); diff --git a/src/message-runtime.ts b/src/message-runtime.ts index d263124..f2733cb 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -33,6 +33,7 @@ import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js'; import { advanceLastAgentCursor, createImplicitContinuationTracker, + resolveNextTurnAction, resolveActiveRole, resolveCursorKey, filterLoopingPairedBotMessages, @@ -40,9 +41,15 @@ import { hasAllowedTrigger, shouldSkipBotOnlyCollaboration, } from './message-runtime-rules.js'; -import { buildArbiterContextPrompt } from './arbiter-context.js'; import { runAgentForGroup } from './message-agent-executor.js'; import { MessageTurnController } from './message-turn-controller.js'; +import { + buildArbiterPromptForTask, + buildFinalizePendingPrompt, + buildOwnerPendingPrompt, + buildPairedTurnPrompt, + buildReviewerPendingPrompt, +} from './message-runtime-prompts.js'; import { extractSessionCommand, handleSessionCommand, @@ -56,7 +63,6 @@ import { type PairedRoomRole, RegisteredGroup, type PairedTask, - type PairedTurnOutput, } from './types.js'; import { createScopedLogger, logger } from './logger.js'; import { resolveGroupIpcPath } from './group-folder.js'; @@ -193,144 +199,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }); }; - /** Convert paired turn outputs to NewMessage format for formatMessages(). */ - const turnOutputsToMessages = ( - outputs: PairedTurnOutput[], - chatJid: string, - ): NewMessage[] => - outputs.map((t) => ({ - id: `turn-${t.task_id}-${t.turn_number}`, - chat_jid: chatJid, - sender: t.role, - sender_name: t.role, - content: t.output_text, - timestamp: t.created_at, - is_bot_message: true as const, - is_from_me: false as const, - })); - - const mergeHumanAndTurnOutputMessages = ( - chatJid: string, - humanMessages: NewMessage[], - turnOutputs: PairedTurnOutput[], - ): NewMessage[] => - [...humanMessages, ...turnOutputsToMessages(turnOutputs, chatJid)].sort( - (a, b) => a.timestamp.localeCompare(b.timestamp), - ); - - /** - * Build a prompt from paired_turn_outputs (Discord-independent) + human messages. - * Falls back to the legacy labelPairedSenders path when no turn outputs exist. - */ - const buildPairedTurnPrompt = ( - taskId: string, - chatJid: string, - timezone: string, - missedMessages: NewMessage[], - ): string => { - const turnOutputs = getPairedTurnOutputs(taskId); - if (turnOutputs.length === 0) { - // No stored outputs yet — fall back to Discord messages - return formatMessages( - labelPairedSenders(chatJid, missedMessages), - timezone, - ); - } - - // Human messages from the missed messages (exclude bot messages) - const humanMessages = missedMessages.filter((m) => !m.is_bot_message); - - return formatMessages( - mergeHumanAndTurnOutputMessages(chatJid, humanMessages, turnOutputs), - timezone, - ); - }; - - const buildReviewerPendingPrompt = ( - task: PairedTask, - chatJid: string, - timezone: string, - ): string => { - const turnOutputs = getPairedTurnOutputs(task.id); - if (turnOutputs.length > 0) { - const humanMessages = getRecentChatMessages(chatJid, 20).filter( - (message) => !message.is_bot_message, - ); - return formatMessages( - mergeHumanAndTurnOutputMessages(chatJid, humanMessages, turnOutputs), - timezone, - ); - } - - const userMessage = getLastHumanMessageContent(chatJid); - if (!userMessage) { - return 'Review the latest owner changes in the workspace.'; - } - - return `User request:\n---\n${userMessage}\n---\n\nReview the latest owner changes in the workspace.`; - }; - - const buildOwnerPendingPrompt = ( - task: PairedTask, - chatJid: string, - timezone: string, - ): string => { - const turnOutputs = getPairedTurnOutputs(task.id); - if (turnOutputs.length > 0) { - const humanMessages = getRecentChatMessages(chatJid, 20).filter( - (message) => !message.is_bot_message, - ); - return formatMessages( - mergeHumanAndTurnOutputMessages(chatJid, humanMessages, turnOutputs), - timezone, - ); - } - - const userMessage = getLastHumanMessageContent(chatJid); - if (!userMessage) { - return 'Continue the owner turn using the latest reviewer or arbiter feedback.'; - } - - return `User request:\n---\n${userMessage}\n---\n\nContinue the owner turn using the latest reviewer or arbiter feedback.`; - }; - - const buildArbiterPromptForTask = ( - task: PairedTask, - chatJid: string, - timezone: string, - ): string => { - const turnOutputs = getPairedTurnOutputs(task.id); - const recentMessages = getRecentChatMessages(chatJid, 20); - const arbiterMessages = - turnOutputs.length > 0 - ? mergeHumanAndTurnOutputMessages( - chatJid, - recentMessages.filter((message) => !message.is_bot_message), - turnOutputs, - ) - : labelPairedSenders(chatJid, recentMessages); - - return buildArbiterContextPrompt({ - chatJid, - taskId: task.id, - roundTripCount: task.round_trip_count, - timezone, - messages: arbiterMessages, - }); - }; - - const buildFinalizePendingPrompt = (task: PairedTask): string => { - const turnOutputs = getPairedTurnOutputs(task.id); - const lastReviewerOutput = [...turnOutputs] - .reverse() - .find((output) => output.role === 'reviewer'); - const reviewerSummary = lastReviewerOutput?.output_text - ? `\n\nReviewer's final assessment:\n${lastReviewerOutput.output_text.slice(0, 2000)}` - : ''; - - return `The reviewer approved your work (DONE). Finalize and report the result.${reviewerSummary}`; - }; - const isBotOnlyPairedRoomTurn = ( chatJid: string, messages: NewMessage[], @@ -370,7 +238,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { const cursor = pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; - if (task.status === 'merge_ready') { + const nextTurnAction = resolveNextTurnAction({ + taskStatus: task.status, + }); + + if (nextTurnAction.kind === 'finalize-owner-turn') { return { kind: 'inline-finalize', task, @@ -379,18 +251,16 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { } if ( - task.status === 'review_ready' || - task.status === 'in_review' || - task.status === 'arbiter_requested' || - task.status === 'in_arbitration' + nextTurnAction.kind === 'reviewer-turn' || + nextTurnAction.kind === 'arbiter-turn' ) { - const nextRole = resolveActiveRole(task.status); return { kind: 'requeue-pending-turn', task, cursor, cursorKey: resolveCursorKey(chatJid, task.status), - nextRole: nextRole === 'arbiter' ? 'arbiter' : 'reviewer', + nextRole: + nextTurnAction.kind === 'arbiter-turn' ? 'arbiter' : 'reviewer', }; } @@ -446,7 +316,9 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { ); const { deliverySucceeded } = await executeTurn({ group, - prompt: buildFinalizePendingPrompt(action.task), + prompt: buildFinalizePendingPrompt({ + turnOutputs: getPairedTurnOutputs(action.task.id), + }), chatJid, runId, channel, @@ -966,11 +838,27 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { const lastRaw = rawMissedMessages[rawMissedMessages.length - 1]; const cursor = lastRaw?.seq ?? lastRaw?.timestamp ?? null; const taskStatus = task.status; - const pendingRole = resolveActiveRole(taskStatus); + const turnOutputs = getPairedTurnOutputs(task.id); + const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; + const nextTurnAction = resolveNextTurnAction({ + taskStatus, + lastTurnOutputRole: lastTurnOutput?.role ?? null, + }); + const recentMessages = getRecentChatMessages(chatJid, 20); + const recentHumanMessages = recentMessages.filter( + (message) => !message.is_bot_message, + ); + const lastHumanMessage = getLastHumanMessageContent(chatJid); - if (pendingRole === 'reviewer') { + if (nextTurnAction.kind === 'reviewer-turn') { return { - prompt: buildReviewerPendingPrompt(task, chatJid, deps.timezone), + prompt: buildReviewerPendingPrompt({ + chatJid, + timezone: deps.timezone, + turnOutputs, + recentHumanMessages, + lastHumanMessage, + }), channel: resolveChannel(taskStatus), cursor, cursorKey: resolveCursorKey(chatJid, taskStatus), @@ -978,9 +866,16 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }; } - if (pendingRole === 'arbiter') { + if (nextTurnAction.kind === 'arbiter-turn') { return { - prompt: buildArbiterPromptForTask(task, chatJid, deps.timezone), + prompt: buildArbiterPromptForTask({ + task, + chatJid, + timezone: deps.timezone, + turnOutputs, + recentMessages, + labeledRecentMessages: labelPairedSenders(chatJid, recentMessages), + }), channel: resolveChannel(taskStatus), cursor, cursorKey: resolveCursorKey(chatJid, taskStatus), @@ -988,28 +883,26 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }; } - if (taskStatus === 'merge_ready') { + if (nextTurnAction.kind === 'finalize-owner-turn') { return { - prompt: buildFinalizePendingPrompt(task), + prompt: buildFinalizePendingPrompt({ turnOutputs }), channel: resolveChannel(taskStatus), cursor, }; } - if (taskStatus === 'active') { - const turnOutputs = getPairedTurnOutputs(task.id); - const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; - if ( - lastTurnOutput && - (lastTurnOutput.role === 'reviewer' || - lastTurnOutput.role === 'arbiter') - ) { - return { - prompt: buildOwnerPendingPrompt(task, chatJid, deps.timezone), - channel: resolveChannel(taskStatus), - cursor, - }; - } + if (nextTurnAction.kind === 'owner-follow-up') { + return { + prompt: buildOwnerPendingPrompt({ + chatJid, + timezone: deps.timezone, + turnOutputs, + recentHumanMessages, + lastHumanMessage, + }), + channel: resolveChannel(taskStatus), + cursor, + }; } return null; @@ -1068,7 +961,9 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }): boolean => hasReviewerLease(chatJid) && args.deliveryRole === 'reviewer' && - args.pendingTask?.status === 'merge_ready'; + resolveNextTurnAction({ + taskStatus: args.pendingTask?.status ?? null, + }).kind === 'finalize-owner-turn'; const enqueueGenericFollowUpAfterDeliveryRetry = (args: { deliveryRole: PairedRoomRole; @@ -1301,18 +1196,24 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { const turnRole = resolveActiveRole(taskStatus); let prompt: string; if (turnRole === 'arbiter' && pendingTaskForChannel) { - prompt = buildArbiterPromptForTask( - pendingTaskForChannel, + const recentMessages = getRecentChatMessages(chatJid, 20); + prompt = buildArbiterPromptForTask({ + task: pendingTaskForChannel, chatJid, - deps.timezone, - ); + timezone: deps.timezone, + turnOutputs: getPairedTurnOutputs(pendingTaskForChannel.id), + recentMessages, + labeledRecentMessages: labelPairedSenders(chatJid, recentMessages), + }); } else if (pendingTaskForChannel) { - prompt = buildPairedTurnPrompt( - pendingTaskForChannel.id, + prompt = buildPairedTurnPrompt({ + taskId: pendingTaskForChannel.id, chatJid, - deps.timezone, + timezone: deps.timezone, missedMessages, - ); + labeledFallbackMessages: labelPairedSenders(chatJid, missedMessages), + turnOutputs: getPairedTurnOutputs(pendingTaskForChannel.id), + }); } else { prompt = formatMessages( labelPairedSenders(chatJid, missedMessages), @@ -1538,12 +1439,17 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { ? pendingMessages : processableGroupMessages; const formatted = loopPendingTask - ? buildPairedTurnPrompt( - loopPendingTask.id, + ? buildPairedTurnPrompt({ + taskId: loopPendingTask.id, chatJid, - deps.timezone, - messagesToSend, - ) + timezone: deps.timezone, + missedMessages: messagesToSend, + labeledFallbackMessages: labelPairedSenders( + chatJid, + messagesToSend, + ), + turnOutputs: getPairedTurnOutputs(loopPendingTask.id), + }) : formatMessages( labelPairedSenders(chatJid, messagesToSend), deps.timezone, From f7aeac99c86107457156f384886d73187ca731a4 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 22:45:04 +0900 Subject: [PATCH 19/29] Format routing simplification files --- src/message-agent-executor.ts | 4 +--- src/message-runtime-prompts.ts | 11 +++++++---- src/message-runtime-rules.test.ts | 18 +++++++++++------- src/message-runtime-rules.ts | 4 ++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 6bca327..d54c678 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -31,9 +31,7 @@ import { preparePairedExecutionContext, } from './paired-execution-context.js'; import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; -import { - resolveExecutionTarget, -} from './message-runtime-rules.js'; +import { resolveExecutionTarget } from './message-runtime-rules.js'; import { isRetryableClaudeSessionFailureAttempt, resolveClaudeRetryTrigger, diff --git a/src/message-runtime-prompts.ts b/src/message-runtime-prompts.ts index 1903004..9745a8a 100644 --- a/src/message-runtime-prompts.ts +++ b/src/message-runtime-prompts.ts @@ -23,9 +23,10 @@ function mergeHumanAndTurnOutputMessages( humanMessages: NewMessage[], turnOutputs: PairedTurnOutput[], ): NewMessage[] { - return [...humanMessages, ...turnOutputsToMessages(turnOutputs, chatJid)].sort( - (a, b) => a.timestamp.localeCompare(b.timestamp), - ); + return [ + ...humanMessages, + ...turnOutputsToMessages(turnOutputs, chatJid), + ].sort((a, b) => a.timestamp.localeCompare(b.timestamp)); } export function buildPairedTurnPrompt(args: { @@ -40,7 +41,9 @@ export function buildPairedTurnPrompt(args: { return formatMessages(args.labeledFallbackMessages, args.timezone); } - const humanMessages = args.missedMessages.filter((message) => !message.is_bot_message); + const humanMessages = args.missedMessages.filter( + (message) => !message.is_bot_message, + ); return formatMessages( mergeHumanAndTurnOutputMessages( args.chatJid, diff --git a/src/message-runtime-rules.test.ts b/src/message-runtime-rules.test.ts index 9d44c3a..41cdeff 100644 --- a/src/message-runtime-rules.test.ts +++ b/src/message-runtime-rules.test.ts @@ -87,7 +87,11 @@ describe('message-runtime-rules', () => { it('honors an arbiter forced role only when arbiter is configured', () => { const resolution = resolveExecutionTarget({ - lease: { ...baseLease, arbiter_agent_type: null, arbiter_service_id: null }, + lease: { + ...baseLease, + arbiter_agent_type: null, + arbiter_service_id: null, + }, pairedTaskStatus: 'active', groupFolder: 'group-1', groupAgentType: 'claude-code', @@ -122,11 +126,11 @@ describe('message-runtime-rules', () => { }); it('always gives arbiter a dedicated session folder', () => { - expect( - resolveSessionFolder('group-1', 'arbiter', 'claude-code'), - ).toBe('group-1:arbiter'); - expect( - resolveSessionFolder('group-1', 'arbiter', 'codex'), - ).toBe('group-1:arbiter'); + expect(resolveSessionFolder('group-1', 'arbiter', 'claude-code')).toBe( + 'group-1:arbiter', + ); + expect(resolveSessionFolder('group-1', 'arbiter', 'codex')).toBe( + 'group-1:arbiter', + ); }); }); diff --git a/src/message-runtime-rules.ts b/src/message-runtime-rules.ts index 60d21de..842875c 100644 --- a/src/message-runtime-rules.ts +++ b/src/message-runtime-rules.ts @@ -162,8 +162,8 @@ export function resolveExecutionTarget(args: { const inferredRole = resolveActiveRole(args.pairedTaskStatus); const canHonorForcedRole = Boolean( args.forcedRole === 'owner' || - (args.forcedRole === 'reviewer' && args.lease.reviewer_agent_type) || - (args.forcedRole === 'arbiter' && args.lease.arbiter_agent_type), + (args.forcedRole === 'reviewer' && args.lease.reviewer_agent_type) || + (args.forcedRole === 'arbiter' && args.lease.arbiter_agent_type), ); const activeRole = canHonorForcedRole ? args.forcedRole! : inferredRole; const effectiveServiceId = resolveLeaseServiceId(args.lease, activeRole); From 03630212186de63216370da7d557602066e797e4 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Mon, 6 Apr 2026 23:28:53 +0900 Subject: [PATCH 20/29] Fix owner follow-up after reviewer bot output --- src/message-runtime.test.ts | 125 +++++++++++++++++++++++++++++++++++- src/message-runtime.ts | 11 +++- 2 files changed, 133 insertions(+), 3 deletions(-) diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 048a022..009bcfe 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -1570,6 +1570,7 @@ describe('createMessageRuntime', () => { const channel = makeChannel(chatJid); const enqueueMessageCheck = vi.fn(); const closeStdin = vi.fn(); + const sendMessage = vi.fn(() => false); const setLastTimestamp = vi.fn(); const stopLoop = new Error('stop-message-loop'); @@ -1648,7 +1649,7 @@ describe('createMessageRuntime', () => { closeStdin, enqueueMessageCheck, notifyIdle: vi.fn(), - sendMessage: vi.fn(() => false), + sendMessage, } as any, getRegisteredGroups: () => ({ [chatJid]: group }), getSessions: () => ({}), @@ -1704,6 +1705,128 @@ describe('createMessageRuntime', () => { ); }); + it('requeues owner follow-ups from reviewer bot-only messages instead of piping them into the active run', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const enqueueMessageCheck = vi.fn(); + const closeStdin = vi.fn(); + const sendMessage = vi.fn(() => false); + const setLastTimestamp = vi.fn(); + const saveState = vi.fn(); + const lastAgentTimestamps: Record = {}; + const stopLoop = new Error('stop-message-loop'); + + vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'task-active-owner-follow-up-loop', + chat_jid: chatJid, + group_folder: group.folder, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: '2026-03-30T00:00:00.000Z', + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + }); + vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ + { + id: 1, + task_id: 'task-active-owner-follow-up-loop', + turn_number: 1, + role: 'owner', + output_text: 'owner 초안', + created_at: '2026-03-30T00:00:01.000Z', + }, + { + id: 2, + task_id: 'task-active-owner-follow-up-loop', + turn_number: 2, + role: 'reviewer', + output_text: 'reviewer 수정 요청', + created_at: '2026-03-30T00:00:02.000Z', + }, + ]); + vi.mocked(db.getNewMessages).mockReturnValue({ + messages: [ + { + id: 'reviewer-bot-message-owner-follow-up-loop', + chat_jid: chatJid, + sender: 'reviewer-bot@test', + sender_name: '리뷰어', + content: 'DONE_WITH_CONCERNS\n\nreviewer direct message', + timestamp: '2026-03-30T00:00:04.000Z', + seq: 42, + is_bot_message: true, + } as any, + ], + newTimestamp: '42', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 60_000, + pollInterval: 123, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin, + enqueueMessageCheck, + notifyIdle: vi.fn(), + sendMessage, + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp, + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const originalSetTimeout = global.setTimeout; + const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation((( + handler: any, + timeout?: number, + ...args: any[] + ) => { + if (timeout === 123) { + throw stopLoop; + } + return (originalSetTimeout as any)(handler, timeout, ...args); + }) as typeof setTimeout); + + try { + await expect(runtime.startMessageLoop()).rejects.toThrow( + stopLoop.message, + ); + } finally { + setTimeoutSpy.mockRestore(); + } + + expect(setLastTimestamp).toHaveBeenCalledWith('42'); + expect(agentRunner.runAgentProcess).not.toHaveBeenCalled(); + expect(closeStdin).toHaveBeenCalledWith( + chatJid, + expect.objectContaining({ reason: 'paired-pending-turn-follow-up' }), + ); + expect(enqueueMessageCheck).toHaveBeenCalledWith( + chatJid, + resolveGroupIpcPath(group.folder), + ); + expect(sendMessage).not.toHaveBeenCalled(); + }); + it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => { const chatJid = 'group@test'; const group = makeGroup('codex'); diff --git a/src/message-runtime.ts b/src/message-runtime.ts index f2733cb..ea9c19b 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -220,7 +220,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { task: PairedTask; cursor: string | number | null; cursorKey: string; - nextRole: 'reviewer' | 'arbiter'; + nextRole: 'owner' | 'reviewer' | 'arbiter'; }; const resolveBotOnlyPairedFollowUpAction = (args: { @@ -237,9 +237,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { const cursor = pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; + const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1); const nextTurnAction = resolveNextTurnAction({ taskStatus: task.status, + lastTurnOutputRole: lastTurnOutput?.role ?? null, }); if (nextTurnAction.kind === 'finalize-owner-turn') { @@ -251,6 +253,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { } if ( + nextTurnAction.kind === 'owner-follow-up' || nextTurnAction.kind === 'reviewer-turn' || nextTurnAction.kind === 'arbiter-turn' ) { @@ -260,7 +263,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { cursor, cursorKey: resolveCursorKey(chatJid, task.status), nextRole: - nextTurnAction.kind === 'arbiter-turn' ? 'arbiter' : 'reviewer', + nextTurnAction.kind === 'owner-follow-up' + ? 'owner' + : nextTurnAction.kind === 'arbiter-turn' + ? 'arbiter' + : 'reviewer', }; } From cbb6bc97c772ccfc9ec32ce37886ed6bc30bd12f Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 00:45:17 +0900 Subject: [PATCH 21/29] Refactor paired execution flow boundaries --- runners/agent-runner/src/verification.ts | 48 +- shared/verification-snapshot.d.ts | 8 + shared/verification-snapshot.js | 50 +++ src/agent-attempt.ts | 71 +++ src/message-agent-executor.test.ts | 32 +- src/message-agent-executor.ts | 311 +++++++------ src/message-runtime-flow.ts | 366 +++++++++++++++ src/message-runtime.ts | 363 +++------------ src/paired-execution-context-arbiter.ts | 81 ++++ src/paired-execution-context-owner.ts | 201 +++++++++ src/paired-execution-context-reviewer.ts | 139 ++++++ src/paired-execution-context-shared.ts | 122 +++++ src/paired-execution-context.ts | 542 +---------------------- src/task-scheduler.ts | 81 ++-- src/verification.ts | 46 +- 15 files changed, 1345 insertions(+), 1116 deletions(-) create mode 100644 shared/verification-snapshot.d.ts create mode 100644 shared/verification-snapshot.js create mode 100644 src/agent-attempt.ts create mode 100644 src/message-runtime-flow.ts create mode 100644 src/paired-execution-context-arbiter.ts create mode 100644 src/paired-execution-context-owner.ts create mode 100644 src/paired-execution-context-reviewer.ts create mode 100644 src/paired-execution-context-shared.ts diff --git a/runners/agent-runner/src/verification.ts b/runners/agent-runner/src/verification.ts index eebf243..b274942 100644 --- a/runners/agent-runner/src/verification.ts +++ b/runners/agent-runner/src/verification.ts @@ -1,6 +1,9 @@ -import { createHash } from 'crypto'; import fs from 'fs'; import path from 'path'; +export { + computeVerificationSnapshotId, + resolveVerificationResponsesDir, +} from '../../../shared/verification-snapshot.js'; export const VERIFICATION_PROFILES = [ 'test', @@ -24,8 +27,6 @@ export interface VerificationResponse { error?: string; } -const SNAPSHOT_EXCLUDE_NAMES = new Set(['.git', 'node_modules', '.env']); - export function isVerificationProfile( value: unknown, ): value is VerificationProfile { @@ -35,47 +36,6 @@ export function isVerificationProfile( ); } -function updateSnapshotHash( - hash: ReturnType, - repoDir: string, - currentPath: string, -): void { - const relPath = path.relative(repoDir, currentPath) || '.'; - const stat = fs.lstatSync(currentPath); - - if (stat.isDirectory()) { - if (relPath !== '.') { - hash.update(`dir\0${relPath}\0`); - } - for (const entry of fs.readdirSync(currentPath).sort()) { - if (SNAPSHOT_EXCLUDE_NAMES.has(entry)) continue; - updateSnapshotHash(hash, repoDir, path.join(currentPath, entry)); - } - return; - } - - if (stat.isSymbolicLink()) { - hash.update(`symlink\0${relPath}\0${fs.readlinkSync(currentPath)}\0`); - return; - } - - if (stat.isFile()) { - hash.update(`file\0${relPath}\0`); - hash.update(fs.readFileSync(currentPath)); - hash.update('\0'); - } -} - -export function computeVerificationSnapshotId(repoDir: string): string { - const hash = createHash('sha256'); - updateSnapshotHash(hash, repoDir, repoDir); - return `fs:${hash.digest('hex').slice(0, 24)}`; -} - -export function resolveVerificationResponsesDir(hostIpcDir: string): string { - return path.join(hostIpcDir, 'verification-responses'); -} - export async function waitForVerificationResponse( responseDir: string, requestId: string, diff --git a/shared/verification-snapshot.d.ts b/shared/verification-snapshot.d.ts new file mode 100644 index 0000000..0102eda --- /dev/null +++ b/shared/verification-snapshot.d.ts @@ -0,0 +1,8 @@ +export declare const VERIFICATION_SNAPSHOT_EXCLUDE_NAMES: ReadonlySet; +export declare function isVerificationSnapshotExcludedName( + name: string, +): boolean; +export declare function computeVerificationSnapshotId(repoDir: string): string; +export declare function resolveVerificationResponsesDir( + hostIpcDir: string, +): string; diff --git a/shared/verification-snapshot.js b/shared/verification-snapshot.js new file mode 100644 index 0000000..9e0a968 --- /dev/null +++ b/shared/verification-snapshot.js @@ -0,0 +1,50 @@ +import { createHash } from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +export const VERIFICATION_SNAPSHOT_EXCLUDE_NAMES = new Set([ + '.git', + 'node_modules', + '.env', +]); + +export function isVerificationSnapshotExcludedName(name) { + return VERIFICATION_SNAPSHOT_EXCLUDE_NAMES.has(name); +} + +function updateVerificationSnapshotHash(hash, repoDir, currentPath) { + const relPath = path.relative(repoDir, currentPath) || '.'; + const stat = fs.lstatSync(currentPath); + + if (stat.isDirectory()) { + if (relPath !== '.') { + hash.update(`dir\0${relPath}\0`); + } + for (const entry of fs.readdirSync(currentPath).sort()) { + if (isVerificationSnapshotExcludedName(entry)) continue; + updateVerificationSnapshotHash(hash, repoDir, path.join(currentPath, entry)); + } + return; + } + + if (stat.isSymbolicLink()) { + hash.update(`symlink\0${relPath}\0${fs.readlinkSync(currentPath)}\0`); + return; + } + + if (stat.isFile()) { + hash.update(`file\0${relPath}\0`); + hash.update(fs.readFileSync(currentPath)); + hash.update('\0'); + } +} + +export function computeVerificationSnapshotId(repoDir) { + const hash = createHash('sha256'); + updateVerificationSnapshotHash(hash, repoDir, repoDir); + return `fs:${hash.digest('hex').slice(0, 24)}`; +} + +export function resolveVerificationResponsesDir(hostIpcDir) { + return path.join(hostIpcDir, 'verification-responses'); +} diff --git a/src/agent-attempt.ts b/src/agent-attempt.ts new file mode 100644 index 0000000..d22c39d --- /dev/null +++ b/src/agent-attempt.ts @@ -0,0 +1,71 @@ +import { + getAgentOutputText, + getStructuredAgentOutput, +} from './agent-output.js'; +import type { AgentOutput } from './agent-runner.js'; +import { + evaluateStreamedOutput, + type EvaluateStreamedOutputOptions, + type EvaluateStreamedOutputResult, + type StreamedOutputState, +} from './streamed-output-evaluator.js'; + +export interface EvaluatedAgentOutput { + output: AgentOutput; + outputText: string | null; + structuredOutput: ReturnType; + evaluation: EvaluateStreamedOutputResult; +} + +export function createInitialStreamedOutputState(): StreamedOutputState { + return { + sawOutput: false, + sawVisibleOutput: false, + sawSuccessNullResultWithoutOutput: false, + }; +} + +export function createEvaluatedOutputHandler(args: { + agentType: EvaluateStreamedOutputOptions['agentType']; + provider: string; + evaluationOptions?: Omit; + onEvaluatedOutput?: ( + output: EvaluatedAgentOutput, + ) => Promise | void; +}): { + handleOutput: (output: AgentOutput) => Promise; + getState: () => StreamedOutputState; + markVisibleOutput: () => void; +} { + let state = createInitialStreamedOutputState(); + + return { + async handleOutput(output: AgentOutput): Promise { + const rawOutputText = getAgentOutputText(output); + const outputText = + typeof rawOutputText === 'string' ? rawOutputText : null; + const structuredOutput = getStructuredAgentOutput(output); + const evaluation = evaluateStreamedOutput(output, state, { + agentType: args.agentType, + provider: args.provider, + ...args.evaluationOptions, + }); + state = evaluation.state; + await args.onEvaluatedOutput?.({ + output, + outputText, + structuredOutput, + evaluation, + }); + }, + getState(): StreamedOutputState { + return state; + }, + markVisibleOutput(): void { + state = { + ...state, + sawVisibleOutput: true, + }; + }, + }; +} diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 0cdf23a..ad0443f 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -288,7 +288,7 @@ describe('runAgentForGroup room memory', () => { memoryBriefing: '## Shared Room Memory\n- remembered context', }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -317,7 +317,7 @@ describe('runAgentForGroup room memory', () => { memoryBriefing: undefined, }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -338,7 +338,7 @@ describe('runAgentForGroup room memory', () => { prompt: 'hello', }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -365,7 +365,7 @@ describe('runAgentForGroup room memory', () => { }), }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -398,7 +398,7 @@ describe('runAgentForGroup room memory', () => { prompt: 'hello', }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -455,7 +455,7 @@ describe('runAgentForGroup room memory', () => { }), }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -513,7 +513,7 @@ describe('runAgentForGroup room memory', () => { }), }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -578,7 +578,7 @@ describe('runAgentForGroup room memory', () => { sessionId: undefined, }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -635,7 +635,7 @@ describe('runAgentForGroup room memory', () => { }), expect.any(Object), expect.any(Function), - undefined, + expect.any(Function), {}, ); }); @@ -772,22 +772,22 @@ describe('runAgentForGroup room memory', () => { roomRoleContext: expect.any(Object), }), expect.any(Function), - undefined, + expect.any(Function), expect.objectContaining({ EJCLAW_WORK_DIR: '/tmp/paired/owner', EJCLAW_PAIRED_TASK_ID: 'paired-task-1', EJCLAW_PAIRED_ROLE: 'owner', }), ); - // Owner produced no visible output (mock doesn't go through streamed - // evaluator) → treated as interrupted, status is 'failed' to prevent - // auto-triggering the reviewer. + // The shared attempt harness now routes runner output through the streamed + // evaluator, so a visible public result is treated as a successful owner + // turn. expect( pairedExecutionContext.completePairedExecutionContext, ).toHaveBeenCalledWith({ taskId: 'paired-task-1', role: 'owner', - status: 'failed', + status: 'succeeded', summary: 'ok', }); }); @@ -1045,7 +1045,7 @@ describe('runAgentForGroup room memory', () => { sessionId: 'reviewer-session', }), expect.any(Function), - undefined, + expect.any(Function), undefined, ); }); @@ -1144,7 +1144,7 @@ describe('runAgentForGroup room memory', () => { sessionId: undefined, }), expect.any(Function), - undefined, + expect.any(Function), expect.objectContaining({ EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1', }), diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index d54c678..07bdd9d 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -5,8 +5,8 @@ import { getErrorMessage } from './utils.js'; import { getAgentOutputText, - getStructuredAgentOutput, } from './agent-output.js'; +import { createEvaluatedOutputHandler } from './agent-attempt.js'; import { AgentOutput, runAgentProcess, @@ -63,10 +63,6 @@ import { activateCodexFailover, getEffectiveChannelLease, } from './service-routing.js'; -import { - evaluateStreamedOutput, - type StreamedOutputState, -} from './streamed-output-evaluator.js'; import { detectCodexRotationTrigger, getCodexAccountCount, @@ -290,21 +286,31 @@ export async function runAgentForGroup( const systemPrompt = readArbiterPrompt(process.cwd()) || 'You are an arbiter.'; - const references = await collectMoaReferences({ - config: moaConfig, - systemPrompt, - contextPrompt: prompt, - }); + try { + const references = await collectMoaReferences({ + config: moaConfig, + systemPrompt, + contextPrompt: prompt, + }); - const moaSection = formatMoaReferencesForPrompt(references); - if (moaSection) { - moaEnrichedPrompt = prompt + '\n' + moaSection; - log.info( + const moaSection = formatMoaReferencesForPrompt(references); + if (moaSection) { + moaEnrichedPrompt = prompt + '\n' + moaSection; + log.info( + { + successCount: references.filter((r) => !r.error).length, + totalCount: references.length, + }, + 'MoA: injected reference opinions into arbiter prompt', + ); + } + } catch (err) { + log.warn( { - successCount: references.filter((r) => !r.error).length, - totalCount: references.length, + err, + models: moaConfig.referenceModels.map((m) => m.name), }, - 'MoA: injected reference opinions into arbiter prompt', + 'MoA: failed to collect reference opinions; continuing without enrichment', ); } } @@ -411,151 +417,142 @@ export async function runAgentForGroup( retryAfterMs?: number; }; }> => { - let streamedState: StreamedOutputState = { - sawOutput: false, - sawVisibleOutput: false, - sawSuccessNullResultWithoutOutput: false, - }; const attemptSessionId = currentSessionId; + const streamedOutputHandler = createEvaluatedOutputHandler({ + agentType: isClaudeCodeAgent ? 'claude-code' : 'codex', + provider, + evaluationOptions: { + suppressClaudeAuthErrorOutput: provider === 'claude', + trackSuccessNullResult: true, + shortCircuitTriggeredErrors: + provider === 'claude' + ? canRetryClaudeCredentials + : getCodexAccountCount() > 1, + }, + onEvaluatedOutput: async ({ + output, + outputText, + structuredOutput, + evaluation, + }) => { + const outputPhase = output.phase ?? 'final'; + if (outputPhase !== 'final') { + log.info( + { + provider, + outputPhase, + outputStatus: output.status, + visibility: structuredOutput?.visibility ?? null, + preview: + outputText && outputText.length > 0 + ? outputText.slice(0, 160) + : null, + errorPreview: + typeof output.error === 'string' && output.error.length > 0 + ? output.error.slice(0, 160) + : null, + activeRole, + effectiveServiceId, + effectiveAgentType, + sessionFolder, + resumedSession: attemptSessionId ?? null, + streamedSessionId: output.newSessionId ?? null, + roomRoleServiceId: roomRoleContext?.serviceId ?? null, + roomRole: roomRoleContext?.role ?? null, + pairedTaskId: pairedExecutionContext?.task.id ?? null, + workspaceDir: + pairedExecutionContext?.workspace?.workspace_dir ?? + group.workDir ?? + null, + }, + 'Observed streamed agent activity', + ); + } + if ( + isClaudeCodeAgent && + provider === 'claude' && + shouldResetSessionOnAgentFailure(output) + ) { + resetSessionRequested = true; + } + if ( + output.newSessionId && + !resetSessionRequested && + shouldPersistSession + ) { + deps.persistSession(sessionFolder, output.newSessionId); + currentSessionId = output.newSessionId; + } - const wrappedOnOutput = onOutput - ? async (output: AgentOutput) => { - const outputPhase = output.phase ?? 'final'; - const outputText = getAgentOutputText(output); - const structuredOutput = getStructuredAgentOutput(output); - if (outputPhase !== 'final') { - log.info( - { - provider, - outputPhase, - outputStatus: output.status, - visibility: structuredOutput?.visibility ?? null, - preview: - typeof outputText === 'string' && outputText.length > 0 - ? outputText.slice(0, 160) - : null, - errorPreview: - typeof output.error === 'string' && output.error.length > 0 - ? output.error.slice(0, 160) - : null, - activeRole, - effectiveServiceId, - effectiveAgentType, - sessionFolder, - resumedSession: attemptSessionId ?? null, - streamedSessionId: output.newSessionId ?? null, - roomRoleServiceId: roomRoleContext?.serviceId ?? null, - roomRole: roomRoleContext?.role ?? null, - pairedTaskId: pairedExecutionContext?.task.id ?? null, - workspaceDir: - pairedExecutionContext?.workspace?.workspace_dir ?? - group.workDir ?? - null, - }, - 'Observed streamed agent activity', - ); - } - if ( - isClaudeCodeAgent && - provider === 'claude' && - shouldResetSessionOnAgentFailure(output) - ) { - resetSessionRequested = true; - } - if ( - output.newSessionId && - !resetSessionRequested && - shouldPersistSession - ) { - deps.persistSession(sessionFolder, output.newSessionId); - currentSessionId = output.newSessionId; - } - const evaluation = evaluateStreamedOutput(output, streamedState, { - agentType: isClaudeCodeAgent ? 'claude-code' : 'codex', - provider, - suppressClaudeAuthErrorOutput: provider === 'claude', - trackSuccessNullResult: true, - shortCircuitTriggeredErrors: - provider === 'claude' - ? canRetryClaudeCredentials - : getCodexAccountCount() > 1, - }); - streamedState = evaluation.state; + if (outputText && outputText.length > 0) { + pairedExecutionSummary = outputText.slice(0, 500); + pairedFullOutput = outputText; + } else if ( + typeof output.error === 'string' && + output.error.length > 0 + ) { + pairedExecutionSummary = output.error.slice(0, 500); + } + if (evaluation.newTrigger && outputText && output.status === 'success') { + log.warn( + { + reason: evaluation.newTrigger.reason, + resultPreview: outputText.slice(0, 120), + }, + 'Detected Claude rotation trigger in successful output', + ); + } else if ( + evaluation.newTrigger && + typeof output.error === 'string' + ) { + log.warn( + { + reason: evaluation.newTrigger.reason, + errorPreview: output.error.slice(0, 120), + }, + provider === 'claude' + ? 'Detected Claude rotation trigger in streamed error output' + : 'Detected Codex rotation trigger in streamed error output', + ); + } - if (typeof outputText === 'string' && outputText.length > 0) { - pairedExecutionSummary = outputText.slice(0, 500); - pairedFullOutput = outputText; - } else if ( - typeof output.error === 'string' && - output.error.length > 0 - ) { - pairedExecutionSummary = output.error.slice(0, 500); - } - if ( - evaluation.newTrigger && - typeof outputText === 'string' && - output.status === 'success' - ) { - log.warn( - { - reason: evaluation.newTrigger.reason, - resultPreview: outputText.slice(0, 120), - }, - 'Detected Claude rotation trigger in successful output', - ); - } else if ( - evaluation.newTrigger && - typeof output.error === 'string' - ) { - log.warn( - { - reason: evaluation.newTrigger.reason, - errorPreview: output.error.slice(0, 120), - }, - provider === 'claude' - ? 'Detected Claude rotation trigger in streamed error output' - : 'Detected Codex rotation trigger in streamed error output', - ); - } + if (evaluation.suppressedAuthError) { + log.warn( + { + resultPreview: outputText ? outputText.slice(0, 120) : undefined, + }, + 'Suppressed Claude 401 auth error from chat output', + ); + return; + } - if (evaluation.suppressedAuthError) { - log.warn( - { - resultPreview: - typeof outputText === 'string' - ? outputText.slice(0, 120) - : undefined, - }, - 'Suppressed Claude 401 auth error from chat output', - ); - return; - } + if (evaluation.suppressedRetryableSessionFailure) { + log.warn( + { + resultPreview: outputText + ? outputText.slice(0, 160) + : output.error?.slice(0, 160), + }, + 'Suppressed retryable Claude session failure from chat output', + ); + return; + } - if (evaluation.suppressedRetryableSessionFailure) { - log.warn( - { - resultPreview: - typeof outputText === 'string' - ? outputText.slice(0, 160) - : output.error?.slice(0, 160), - }, - 'Suppressed retryable Claude session failure from chat output', - ); - return; - } - - if (!evaluation.shouldForwardOutput) { - return; - } - if (typeof outputText === 'string' && outputText.length > 0) { - streamedState = { - ...evaluation.state, - sawVisibleOutput: true, - }; - } + if (!evaluation.shouldForwardOutput) { + return; + } + if (outputText && outputText.length > 0) { + streamedOutputHandler.markVisibleOutput(); + } + if (onOutput) { await onOutput(output); } - : undefined; + }, + }); + + const wrappedOnOutput = async (output: AgentOutput) => { + await streamedOutputHandler.handleOutput(output); + }; const providerLog = log.child({ provider, @@ -584,11 +581,12 @@ export async function runAgentForGroup( providerLog.info( { status: output.status, - sawOutput: streamedState.sawOutput, + sawOutput: streamedOutputHandler.getState().sawOutput, }, `Provider response completed (provider: ${provider})`, ); + const streamedState = streamedOutputHandler.getState(); return { output, sawOutput: streamedState.sawOutput, @@ -600,6 +598,7 @@ export async function runAgentForGroup( streamedTriggerReason: streamedState.streamedTriggerReason, }; } catch (error) { + const streamedState = streamedOutputHandler.getState(); return { error, sawOutput: streamedState.sawOutput, diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts new file mode 100644 index 0000000..b808fc4 --- /dev/null +++ b/src/message-runtime-flow.ts @@ -0,0 +1,366 @@ +import { + getLastHumanMessageContent, + getPairedTurnOutputs, + getRecentChatMessages, +} from './db.js'; +import { logger } from './logger.js'; +import { + buildArbiterPromptForTask, + buildFinalizePendingPrompt, + buildOwnerPendingPrompt, + buildReviewerPendingPrompt, +} from './message-runtime-prompts.js'; +import { + advanceLastAgentCursor, + resolveCursorKey, + resolveNextTurnAction, +} from './message-runtime-rules.js'; +import { hasReviewerLease } from './service-routing.js'; +import type { + Channel, + PairedTask, + PairedRoomRole, + RegisteredGroup, +} from './types.js'; + +export type PendingPairedTurn = { + prompt: string; + channel: Channel | null; + cursor: string | number | null; + cursorKey?: string; + role?: 'reviewer' | 'arbiter'; +} | null; + +export type BotOnlyPairedFollowUpAction = + | { kind: 'none' } + | { + kind: 'inline-finalize'; + task: PairedTask; + cursor: string | number | null; + } + | { + kind: 'requeue-pending-turn'; + task: PairedTask; + cursor: string | number | null; + cursorKey: string; + nextRole: 'owner' | 'reviewer' | 'arbiter'; + }; + +export function buildPendingPairedTurn(args: { + chatJid: string; + timezone: string; + task: PairedTask; + rawMissedMessages: Array<{ seq?: number | null; timestamp?: string | null }>; + recentHumanMessages: Parameters[0]['recentHumanMessages']; + labeledRecentMessages: Parameters[0]['labeledRecentMessages']; + resolveChannel: (taskStatus?: string | null) => Channel | null; +}): PendingPairedTurn { + const { + chatJid, + timezone, + task, + rawMissedMessages, + recentHumanMessages, + labeledRecentMessages, + resolveChannel, + } = args; + const lastRaw = rawMissedMessages[rawMissedMessages.length - 1]; + const cursor = lastRaw?.seq ?? lastRaw?.timestamp ?? null; + const taskStatus = task.status; + const turnOutputs = getPairedTurnOutputs(task.id); + const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; + const nextTurnAction = resolveNextTurnAction({ + taskStatus, + lastTurnOutputRole: lastTurnOutput?.role ?? null, + }); + const recentMessages = getRecentChatMessages(chatJid, 20); + const lastHumanMessage = getLastHumanMessageContent(chatJid); + + if (nextTurnAction.kind === 'reviewer-turn') { + return { + prompt: buildReviewerPendingPrompt({ + chatJid, + timezone, + turnOutputs, + recentHumanMessages, + lastHumanMessage, + }), + channel: resolveChannel(taskStatus), + cursor, + cursorKey: resolveCursorKey(chatJid, taskStatus), + role: 'reviewer', + }; + } + + if (nextTurnAction.kind === 'arbiter-turn') { + return { + prompt: buildArbiterPromptForTask({ + task, + chatJid, + timezone, + turnOutputs, + recentMessages, + labeledRecentMessages, + }), + channel: resolveChannel(taskStatus), + cursor, + cursorKey: resolveCursorKey(chatJid, taskStatus), + role: 'arbiter', + }; + } + + if (nextTurnAction.kind === 'finalize-owner-turn') { + return { + prompt: buildFinalizePendingPrompt({ turnOutputs }), + channel: resolveChannel(taskStatus), + cursor, + }; + } + + if (nextTurnAction.kind === 'owner-follow-up') { + return { + prompt: buildOwnerPendingPrompt({ + chatJid, + timezone, + turnOutputs, + recentHumanMessages, + lastHumanMessage, + }), + channel: resolveChannel(taskStatus), + cursor, + }; + } + + return null; +} + +export async function executePendingPairedTurn(args: { + pendingTurn: Exclude; + chatJid: string; + group: RegisteredGroup; + runId: string; + log: typeof logger; + saveState: () => void; + lastAgentTimestamps: Record; + executeTurn: (args: { + group: RegisteredGroup; + prompt: string; + chatJid: string; + runId: string; + channel: Channel; + startSeq: number | null; + endSeq: number | null; + deliveryRole?: PairedRoomRole; + }) => Promise<{ deliverySucceeded: boolean }>; + getFixedRoleChannelName: (role: 'reviewer' | 'arbiter') => string; +}): Promise { + const { + pendingTurn, + chatJid, + group, + runId, + log, + saveState, + lastAgentTimestamps, + executeTurn, + getFixedRoleChannelName, + } = args; + + if (!pendingTurn.channel) { + const missingRole = pendingTurn.role ?? 'reviewer'; + log.error( + { + role: missingRole, + requiredChannel: getFixedRoleChannelName(missingRole), + }, + 'Skipping paired turn because the dedicated Discord role channel is not configured', + ); + return false; + } + + if (pendingTurn.cursor != null) { + advanceLastAgentCursor( + lastAgentTimestamps, + saveState, + chatJid, + pendingTurn.cursor, + pendingTurn.cursorKey, + ); + } + + const { deliverySucceeded } = await executeTurn({ + group, + prompt: pendingTurn.prompt, + chatJid, + runId, + channel: pendingTurn.channel, + deliveryRole: pendingTurn.role, + startSeq: null, + endSeq: null, + }); + + return deliverySucceeded; +} + +export function resolveBotOnlyPairedFollowUpAction(args: { + chatJid: string; + task: PairedTask | null | undefined; + isBotOnlyPairedFollowUp: boolean; + pendingCursorSource: + | { seq?: number | null; timestamp?: string | null } + | undefined; +}): BotOnlyPairedFollowUpAction { + const { chatJid, task, isBotOnlyPairedFollowUp, pendingCursorSource } = args; + if (!task || !isBotOnlyPairedFollowUp) { + return { kind: 'none' }; + } + + const cursor = pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; + const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1); + const nextTurnAction = resolveNextTurnAction({ + taskStatus: task.status, + lastTurnOutputRole: lastTurnOutput?.role ?? null, + }); + + if (nextTurnAction.kind === 'finalize-owner-turn') { + return { + kind: 'inline-finalize', + task, + cursor, + }; + } + + if ( + nextTurnAction.kind === 'owner-follow-up' || + nextTurnAction.kind === 'reviewer-turn' || + nextTurnAction.kind === 'arbiter-turn' + ) { + return { + kind: 'requeue-pending-turn', + task, + cursor, + cursorKey: resolveCursorKey(chatJid, task.status), + nextRole: + nextTurnAction.kind === 'owner-follow-up' + ? 'owner' + : nextTurnAction.kind === 'arbiter-turn' + ? 'arbiter' + : 'reviewer', + }; + } + + return { kind: 'none' }; +} + +export async function executeBotOnlyPairedFollowUpAction(args: { + action: BotOnlyPairedFollowUpAction; + chatJid: string; + group: RegisteredGroup; + runId: string; + channel: Channel; + log: typeof logger; + saveState: () => void; + lastAgentTimestamps: Record; + executeTurn: (args: { + group: RegisteredGroup; + prompt: string; + chatJid: string; + runId: string; + channel: Channel; + startSeq: number | null; + endSeq: number | null; + }) => Promise<{ deliverySucceeded: boolean }>; + enqueueGroupMessageCheck: () => void; + closeStdin: () => void; +}): Promise { + const { + action, + chatJid, + group, + runId, + channel, + log, + saveState, + lastAgentTimestamps, + executeTurn, + enqueueGroupMessageCheck, + closeStdin, + } = args; + + if (action.kind === 'none') { + return false; + } + + if (action.cursor != null) { + advanceLastAgentCursor( + lastAgentTimestamps, + saveState, + chatJid, + action.cursor, + action.kind === 'requeue-pending-turn' ? action.cursorKey : undefined, + ); + } + + if (action.kind === 'inline-finalize') { + log.info( + { + chatJid, + group: group.name, + groupFolder: group.folder, + taskId: action.task.id, + taskStatus: action.task.status, + handoffMode: 'inline-finalize', + nextRole: 'owner', + cursor: action.cursor, + }, + 'Executing merge_ready finalize turn inline after bot-only reviewer follow-up', + ); + const { deliverySucceeded } = await executeTurn({ + group, + prompt: buildFinalizePendingPrompt({ + turnOutputs: getPairedTurnOutputs(action.task.id), + }), + chatJid, + runId, + channel, + startSeq: null, + endSeq: null, + }); + if (!deliverySucceeded) { + enqueueGroupMessageCheck(); + } + return true; + } + + closeStdin(); + enqueueGroupMessageCheck(); + log.info( + { + chatJid, + group: group.name, + groupFolder: group.folder, + taskId: action.task.id, + taskStatus: action.task.status, + handoffMode: 'requeue', + nextRole: action.nextRole, + cursor: action.cursor, + cursorKey: action.cursorKey, + }, + 'Queued fresh paired pending turn instead of piping bot-only follow-up into the active agent', + ); + return true; +} + +export function shouldSkipGenericFollowUpAfterDeliveryRetry(args: { + chatJid: string; + deliveryRole: PairedRoomRole; + pendingTask: PairedTask | null | undefined; +}): boolean { + return ( + hasReviewerLease(args.chatJid) && + args.deliveryRole === 'reviewer' && + resolveNextTurnAction({ + taskStatus: args.pendingTask?.status ?? null, + }).kind === 'finalize-owner-turn' + ); +} diff --git a/src/message-runtime.ts b/src/message-runtime.ts index ea9c19b..70b0d57 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -42,6 +42,13 @@ import { shouldSkipBotOnlyCollaboration, } from './message-runtime-rules.js'; import { runAgentForGroup } from './message-agent-executor.js'; +import { + buildPendingPairedTurn, + executeBotOnlyPairedFollowUpAction, + executePendingPairedTurn, + resolveBotOnlyPairedFollowUpAction, + shouldSkipGenericFollowUpAfterDeliveryRetry, +} from './message-runtime-flow.js'; import { MessageTurnController } from './message-turn-controller.js'; import { buildArbiterPromptForTask, @@ -208,157 +215,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { (message) => message.is_from_me === true || !!message.is_bot_message, ); - type BotOnlyPairedFollowUpAction = - | { kind: 'none' } - | { - kind: 'inline-finalize'; - task: PairedTask; - cursor: string | number | null; - } - | { - kind: 'requeue-pending-turn'; - task: PairedTask; - cursor: string | number | null; - cursorKey: string; - nextRole: 'owner' | 'reviewer' | 'arbiter'; - }; - - const resolveBotOnlyPairedFollowUpAction = (args: { - chatJid: string; - task: PairedTask | null | undefined; - isBotOnlyPairedFollowUp: boolean; - pendingCursorSource: NewMessage | undefined; - }): BotOnlyPairedFollowUpAction => { - const { chatJid, task, isBotOnlyPairedFollowUp, pendingCursorSource } = - args; - if (!task || !isBotOnlyPairedFollowUp) { - return { kind: 'none' }; - } - - const cursor = - pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; - const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1); - - const nextTurnAction = resolveNextTurnAction({ - taskStatus: task.status, - lastTurnOutputRole: lastTurnOutput?.role ?? null, - }); - - if (nextTurnAction.kind === 'finalize-owner-turn') { - return { - kind: 'inline-finalize', - task, - cursor, - }; - } - - if ( - nextTurnAction.kind === 'owner-follow-up' || - nextTurnAction.kind === 'reviewer-turn' || - nextTurnAction.kind === 'arbiter-turn' - ) { - return { - kind: 'requeue-pending-turn', - task, - cursor, - cursorKey: resolveCursorKey(chatJid, task.status), - nextRole: - nextTurnAction.kind === 'owner-follow-up' - ? 'owner' - : nextTurnAction.kind === 'arbiter-turn' - ? 'arbiter' - : 'reviewer', - }; - } - - return { kind: 'none' }; - }; - - const executeBotOnlyPairedFollowUpAction = async (args: { - action: BotOnlyPairedFollowUpAction; - chatJid: string; - group: RegisteredGroup; - runId: string; - channel: Channel; - log: typeof logger; - enqueueGroupMessageCheck: () => void; - }): Promise => { - const { - action, - chatJid, - group, - runId, - channel, - log, - enqueueGroupMessageCheck, - } = args; - - if (action.kind === 'none') { - return false; - } - - if (action.cursor != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - action.cursor, - action.kind === 'requeue-pending-turn' ? action.cursorKey : undefined, - ); - } - - if (action.kind === 'inline-finalize') { - log.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - taskId: action.task.id, - taskStatus: action.task.status, - handoffMode: 'inline-finalize', - nextRole: 'owner', - cursor: action.cursor, - }, - 'Executing merge_ready finalize turn inline after bot-only reviewer follow-up', - ); - const { deliverySucceeded } = await executeTurn({ - group, - prompt: buildFinalizePendingPrompt({ - turnOutputs: getPairedTurnOutputs(action.task.id), - }), - chatJid, - runId, - channel, - startSeq: null, - endSeq: null, - }); - if (!deliverySucceeded) { - enqueueGroupMessageCheck(); - } - return true; - } - - deps.queue.closeStdin(chatJid, { - reason: 'paired-pending-turn-follow-up', - }); - enqueueGroupMessageCheck(); - log.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - taskId: action.task.id, - taskStatus: action.task.status, - handoffMode: 'requeue', - nextRole: action.nextRole, - cursor: action.cursor, - cursorKey: action.cursorKey, - }, - 'Queued fresh paired pending turn instead of piping bot-only follow-up into the active agent', - ); - return true; - }; - const enqueueScopedGroupMessageCheck = ( chatJid: string, groupFolder: string, @@ -832,152 +688,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { return role === 'owner' ? channel : roleToChannel[role]; }; - const buildPendingPairedTurn = ( - task: PairedTask, - rawMissedMessages: NewMessage[], - ): { - prompt: string; - channel: Channel | null; - cursor: string | number | null; - cursorKey?: string; - role?: 'reviewer' | 'arbiter'; - } | null => { - const lastRaw = rawMissedMessages[rawMissedMessages.length - 1]; - const cursor = lastRaw?.seq ?? lastRaw?.timestamp ?? null; - const taskStatus = task.status; - const turnOutputs = getPairedTurnOutputs(task.id); - const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; - const nextTurnAction = resolveNextTurnAction({ - taskStatus, - lastTurnOutputRole: lastTurnOutput?.role ?? null, - }); - const recentMessages = getRecentChatMessages(chatJid, 20); - const recentHumanMessages = recentMessages.filter( - (message) => !message.is_bot_message, - ); - const lastHumanMessage = getLastHumanMessageContent(chatJid); - - if (nextTurnAction.kind === 'reviewer-turn') { - return { - prompt: buildReviewerPendingPrompt({ - chatJid, - timezone: deps.timezone, - turnOutputs, - recentHumanMessages, - lastHumanMessage, - }), - channel: resolveChannel(taskStatus), - cursor, - cursorKey: resolveCursorKey(chatJid, taskStatus), - role: 'reviewer', - }; - } - - if (nextTurnAction.kind === 'arbiter-turn') { - return { - prompt: buildArbiterPromptForTask({ - task, - chatJid, - timezone: deps.timezone, - turnOutputs, - recentMessages, - labeledRecentMessages: labelPairedSenders(chatJid, recentMessages), - }), - channel: resolveChannel(taskStatus), - cursor, - cursorKey: resolveCursorKey(chatJid, taskStatus), - role: 'arbiter', - }; - } - - if (nextTurnAction.kind === 'finalize-owner-turn') { - return { - prompt: buildFinalizePendingPrompt({ turnOutputs }), - channel: resolveChannel(taskStatus), - cursor, - }; - } - - if (nextTurnAction.kind === 'owner-follow-up') { - return { - prompt: buildOwnerPendingPrompt({ - chatJid, - timezone: deps.timezone, - turnOutputs, - recentHumanMessages, - lastHumanMessage, - }), - channel: resolveChannel(taskStatus), - cursor, - }; - } - - return null; - }; - - const executePendingPairedTurn = async (args: { - prompt: string; - channel: Channel | null; - cursor: string | number | null; - cursorKey?: string; - role?: 'reviewer' | 'arbiter'; - }): Promise => { - if (!args.channel) { - const missingRole = args.role ?? 'reviewer'; - log.error( - { - role: missingRole, - requiredChannel: getFixedRoleChannelName(missingRole), - }, - 'Skipping paired turn because the dedicated Discord role channel is not configured', - ); - return false; - } - - if (args.cursor != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - args.cursor, - args.cursorKey, - ); - } - - const { deliverySucceeded } = await executeTurn({ - group, - prompt: args.prompt, - chatJid, - runId, - channel: args.channel, - deliveryRole: args.role, - startSeq: null, - endSeq: null, - }); - - return deliverySucceeded; - }; - const enqueueGroupMessageCheck = (): void => { enqueueScopedGroupMessageCheck(chatJid, group.folder); }; - const shouldSkipGenericFollowUpAfterDeliveryRetry = (args: { - deliveryRole: PairedRoomRole; - pendingTask: PairedTask | null | undefined; - }): boolean => - hasReviewerLease(chatJid) && - args.deliveryRole === 'reviewer' && - resolveNextTurnAction({ - taskStatus: args.pendingTask?.status ?? null, - }).kind === 'finalize-owner-turn'; - const enqueueGenericFollowUpAfterDeliveryRetry = (args: { deliveryRole: PairedRoomRole; pendingTask: PairedTask | null | undefined; workItemId: string | number; }): void => { - if (shouldSkipGenericFollowUpAfterDeliveryRetry(args)) { + if ( + shouldSkipGenericFollowUpAfterDeliveryRetry({ + chatJid, + deliveryRole: args.deliveryRole, + pendingTask: args.pendingTask, + }) + ) { log.info( { workItemId: args.workItemId, @@ -1074,10 +800,33 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { ? getLatestOpenPairedTaskForChat(chatJid) : null; const pendingTurn = pendingReviewTask - ? buildPendingPairedTurn(pendingReviewTask, rawMissedMessages) + ? buildPendingPairedTurn({ + chatJid, + timezone: deps.timezone, + task: pendingReviewTask, + rawMissedMessages, + recentHumanMessages: getRecentChatMessages(chatJid, 20).filter( + (message) => !message.is_bot_message, + ), + labeledRecentMessages: labelPairedSenders( + chatJid, + getRecentChatMessages(chatJid, 20), + ), + resolveChannel, + }) : null; if (pendingTurn) { - return executePendingPairedTurn(pendingTurn); + return executePendingPairedTurn({ + pendingTurn, + chatJid, + group, + runId, + log, + saveState: deps.saveState, + lastAgentTimestamps: deps.getLastAgentTimestamps(), + executeTurn, + getFixedRoleChannelName, + }); } const lastIgnored = rawMissedMessages[rawMissedMessages.length - 1]; @@ -1097,10 +846,33 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { : null; const botOnlyPendingTurn = botOnlyPendingTask && isBotOnlyPairedRoomTurn(chatJid, missedMessages) - ? buildPendingPairedTurn(botOnlyPendingTask, rawMissedMessages) + ? buildPendingPairedTurn({ + chatJid, + timezone: deps.timezone, + task: botOnlyPendingTask, + rawMissedMessages, + recentHumanMessages: getRecentChatMessages(chatJid, 20).filter( + (message) => !message.is_bot_message, + ), + labeledRecentMessages: labelPairedSenders( + chatJid, + getRecentChatMessages(chatJid, 20), + ), + resolveChannel, + }) : null; if (botOnlyPendingTurn) { - return executePendingPairedTurn(botOnlyPendingTurn); + return executePendingPairedTurn({ + pendingTurn: botOnlyPendingTurn, + chatJid, + group, + runId, + log, + saveState: deps.saveState, + lastAgentTimestamps: deps.getLastAgentTimestamps(), + executeTurn, + getFixedRoleChannelName, + }); } if (shouldSkipBotOnlyCollaboration(chatJid, missedMessages)) { @@ -1483,8 +1255,15 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { runId: `loop-merge-ready-${Date.now().toString(36)}`, channel, log: logger, + saveState: deps.saveState, + lastAgentTimestamps: deps.getLastAgentTimestamps(), + executeTurn, enqueueGroupMessageCheck: () => enqueueScopedGroupMessageCheck(chatJid, group.folder), + closeStdin: () => + deps.queue.closeStdin(chatJid, { + reason: 'paired-pending-turn-follow-up', + }), }) ) { continue; diff --git a/src/paired-execution-context-arbiter.ts b/src/paired-execution-context-arbiter.ts new file mode 100644 index 0000000..de46e5b --- /dev/null +++ b/src/paired-execution-context-arbiter.ts @@ -0,0 +1,81 @@ +import { ARBITER_DEADLOCK_THRESHOLD } from './config.js'; +import { updatePairedTask } from './db.js'; +import { logger } from './logger.js'; +import { classifyArbiterVerdict } from './paired-execution-context-shared.js'; +import type { PairedTask } from './types.js'; + +export function handleFailedArbiterExecution(args: { + task: PairedTask; + taskId: string; +}): void { + const { task, taskId } = args; + const now = new Date().toISOString(); + const fallbackStatus = + task.status === 'in_arbitration' || task.status === 'arbiter_requested' + ? 'arbiter_requested' + : task.status; + if (fallbackStatus !== task.status) { + updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); + logger.warn( + { + taskId, + role: 'arbiter', + previousStatus: task.status, + nextStatus: fallbackStatus, + }, + 'Preserved arbiter task in arbitration-requested state after failed execution', + ); + } +} + +export function handleArbiterCompletion(args: { + taskId: string; + summary?: string | null; +}): void { + const { taskId, summary } = args; + const now = new Date().toISOString(); + const arbiterVerdict = classifyArbiterVerdict(summary); + + logger.info( + { taskId, arbiterVerdict, summary: summary?.slice(0, 200) }, + 'Arbiter verdict rendered', + ); + + switch (arbiterVerdict) { + case 'proceed': + case 'revise': + case 'reset': + updatePairedTask(taskId, { + status: 'active', + round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), + arbiter_verdict: arbiterVerdict, + updated_at: now, + }); + logger.info( + { taskId, arbiterVerdict }, + 'Arbiter resolved deadlock — resuming ping-pong', + ); + return; + case 'escalate': + updatePairedTask(taskId, { + status: 'completed', + arbiter_verdict: 'escalate', + completion_reason: 'arbiter_escalated', + updated_at: now, + }); + logger.info({ taskId }, 'Arbiter escalated to user — task completed'); + return; + default: + updatePairedTask(taskId, { + status: 'active', + round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), + arbiter_verdict: 'unknown', + updated_at: now, + }); + logger.warn( + { taskId, summary: summary?.slice(0, 200) }, + 'Arbiter verdict unrecognized — falling back to proceed', + ); + return; + } +} diff --git a/src/paired-execution-context-owner.ts b/src/paired-execution-context-owner.ts new file mode 100644 index 0000000..186b3f4 --- /dev/null +++ b/src/paired-execution-context-owner.ts @@ -0,0 +1,201 @@ +import { + ARBITER_DEADLOCK_THRESHOLD, + PAIRED_MAX_ROUND_TRIPS, +} from './config.js'; +import { + getPairedWorkspace, + hasActiveCiWatcherForChat, + updatePairedTask, +} from './db.js'; +import { logger } from './logger.js'; +import { markPairedTaskReviewReady } from './paired-workspace-manager.js'; +import { + classifyVerdict, + hasCodeChangesSinceRef, + requestArbiterOrEscalate, + resolveCanonicalSourceRef, +} from './paired-execution-context-shared.js'; +import type { PairedTask } from './types.js'; + +export function handleFailedOwnerExecution(args: { + task: PairedTask; + taskId: string; +}): void { + const { task, taskId } = args; + if (task.status !== 'active') { + const now = new Date().toISOString(); + updatePairedTask(taskId, { status: 'active', updated_at: now }); + logger.info( + { taskId, role: 'owner', previousStatus: task.status }, + 'Reset task to active after failed execution', + ); + } +} + +function handleOwnerFinalizeCompletion(args: { + task: PairedTask; + taskId: string; + summary?: string | null; + now: string; +}): boolean { + const { task, taskId, summary, now } = args; + const ownerVerdict = classifyVerdict(summary); + + if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter', + escalateLogMessage: 'Owner blocked during finalize — escalating to user', + logContext: { + taskId, + ownerVerdict, + summary: summary?.slice(0, 100), + }, + }); + return true; + } + + if (ownerVerdict === 'done_with_concerns') { + if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter', + escalateLogMessage: 'Owner finalize loop detected — escalating to user', + logContext: { + taskId, + ownerVerdict, + roundTrips: task.round_trip_count, + }, + }); + return true; + } + updatePairedTask(taskId, { status: 'active', updated_at: now }); + logger.info( + { + taskId, + ownerVerdict, + summary: summary?.slice(0, 100), + }, + 'Owner raised concerns during finalize — task set back to active', + ); + return false; + } + + const workspace = getPairedWorkspace(task.id, 'owner'); + const hasNewChanges = workspace?.workspace_dir + ? hasCodeChangesSinceRef(workspace.workspace_dir, task.source_ref) + : null; + + if (hasNewChanges === true) { + if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: + 'Owner finalize DONE loop detected — requesting arbiter', + escalateLogMessage: + 'Owner finalize DONE loop detected — escalating to user', + logContext: { + taskId, + roundTrips: task.round_trip_count, + hasNewChanges, + }, + }); + return true; + } + logger.info( + { + taskId, + sourceRef: task.source_ref, + hasNewChanges, + }, + 'Owner made changes after reviewer approval — re-triggering review', + ); + return false; + } + + updatePairedTask(taskId, { + status: 'completed', + completion_reason: 'done', + updated_at: now, + }); + logger.info( + { taskId, hasNewChanges, summary: summary?.slice(0, 100) }, + 'Owner finalized after reviewer approval — task completed', + ); + return true; +} + +export function handleOwnerCompletion(args: { + task: PairedTask; + taskId: string; + summary?: string | null; +}): void { + const { task, taskId, summary } = args; + const now = new Date().toISOString(); + + if (task.status === 'merge_ready') { + const shouldStop = handleOwnerFinalizeCompletion({ + task, + taskId, + summary, + now, + }); + if (shouldStop) { + return; + } + } + + if (hasActiveCiWatcherForChat(task.chat_jid)) { + logger.info( + { taskId, chatJid: task.chat_jid }, + 'Active CI watcher found, deferring auto-review until watcher completes', + ); + return; + } + + if (task.status !== 'merge_ready') { + const ownerVerdict = classifyVerdict(summary); + if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter', + escalateLogMessage: 'Owner blocked/needs_context — escalating to user', + logContext: { + taskId, + ownerVerdict, + summary: summary?.slice(0, 100), + }, + }); + return; + } + } + + if (task.round_trip_count >= PAIRED_MAX_ROUND_TRIPS) { + logger.info( + { + taskId, + roundTrips: task.round_trip_count, + max: PAIRED_MAX_ROUND_TRIPS, + }, + 'Round trip limit reached, skipping auto-review', + ); + return; + } + + const result = markPairedTaskReviewReady(taskId); + if (result) { + updatePairedTask(taskId, { + round_trip_count: task.round_trip_count + 1, + review_requested_at: now, + updated_at: now, + }); + logger.info( + { taskId, roundTrip: task.round_trip_count + 1 }, + 'Auto-triggered reviewer after owner completion', + ); + } +} diff --git a/src/paired-execution-context-reviewer.ts b/src/paired-execution-context-reviewer.ts new file mode 100644 index 0000000..2ebfe19 --- /dev/null +++ b/src/paired-execution-context-reviewer.ts @@ -0,0 +1,139 @@ +import { ARBITER_DEADLOCK_THRESHOLD } from './config.js'; +import { getPairedWorkspace, updatePairedTask } from './db.js'; +import { logger } from './logger.js'; +import { + classifyVerdict, + requestArbiterOrEscalate, + resolveCanonicalSourceRef, +} from './paired-execution-context-shared.js'; +import type { PairedTask } from './types.js'; + +export function handleFailedReviewerExecution(args: { + task: PairedTask; + taskId: string; + summary?: string | null; +}): void { + const { task, taskId, summary } = args; + const now = new Date().toISOString(); + + if (summary) { + const verdict = classifyVerdict(summary); + if ( + verdict === 'done' || + verdict === 'blocked' || + verdict === 'needs_context' + ) { + const ownerWs = + verdict === 'done' ? getPairedWorkspace(taskId, 'owner') : null; + const approvedSourceRef = + verdict === 'done' && ownerWs?.workspace_dir + ? resolveCanonicalSourceRef(ownerWs.workspace_dir) + : task.source_ref; + updatePairedTask(taskId, { + status: verdict === 'done' ? 'merge_ready' : 'completed', + ...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), + ...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), + updated_at: now, + }); + logger.info( + { + taskId, + verdict, + approvedSourceRef, + summary: summary.slice(0, 100), + }, + 'Reviewer verdict detected from failed execution — stopping ping-pong', + ); + return; + } + } + + const fallbackStatus = + task.status === 'in_review' || task.status === 'review_ready' + ? 'review_ready' + : task.status; + if (fallbackStatus !== task.status) { + updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); + logger.warn( + { + taskId, + role: 'reviewer', + previousStatus: task.status, + nextStatus: fallbackStatus, + }, + 'Preserved reviewer task in review-ready state after failed execution', + ); + } +} + +export function handleReviewerCompletion(args: { + task: PairedTask; + taskId: string; + summary?: string | null; +}): void { + const { task, taskId, summary } = args; + const now = new Date().toISOString(); + const verdict = classifyVerdict(summary); + + switch (verdict) { + case 'done': { + const ownerWs = getPairedWorkspace(taskId, 'owner'); + const approvedSourceRef = ownerWs?.workspace_dir + ? resolveCanonicalSourceRef(ownerWs.workspace_dir) + : task.source_ref; + updatePairedTask(taskId, { + status: 'merge_ready', + source_ref: approvedSourceRef, + updated_at: now, + }); + logger.info( + { + taskId, + verdict, + approvedSourceRef, + summary: summary?.slice(0, 100), + }, + 'Reviewer approved — owner gets final turn to finalize', + ); + return; + } + + case 'blocked': + case 'needs_context': + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: + 'Reviewer blocked/needs_context — requesting arbiter before escalating', + escalateLogMessage: 'Reviewer escalated to user — ping-pong stopped', + logContext: { taskId, verdict, summary: summary?.slice(0, 100) }, + }); + return; + + case 'done_with_concerns': + case 'continue': + default: + if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { + requestArbiterOrEscalate({ + taskId, + now, + arbiterLogMessage: + 'Deadlock detected — requesting arbiter intervention', + escalateLogMessage: + 'Stopped ping-pong — escalating to user (arbiter not configured)', + logContext: { + taskId, + verdict, + roundTrips: task.round_trip_count, + }, + }); + return; + } + updatePairedTask(taskId, { status: 'active', updated_at: now }); + logger.info( + { taskId, verdict }, + 'Reviewer has feedback, task set back to active for owner', + ); + return; + } +} diff --git a/src/paired-execution-context-shared.ts b/src/paired-execution-context-shared.ts new file mode 100644 index 0000000..349e2c1 --- /dev/null +++ b/src/paired-execution-context-shared.ts @@ -0,0 +1,122 @@ +import { execFileSync } from 'child_process'; + +import { isArbiterEnabled } from './config.js'; +import { updatePairedTask } from './db.js'; +import { logger } from './logger.js'; + +export type Verdict = + | 'done' + | 'done_with_concerns' + | 'blocked' + | 'needs_context' + | 'continue'; + +export function classifyVerdict(summary: string | null | undefined): Verdict { + if (!summary) return 'continue'; + const cleaned = summary.replace(/[\s\S]*?<\/internal>/g, '').trim(); + if (!cleaned) return 'continue'; + const firstLine = cleaned.split('\n')[0].trim(); + if (/^\*{0,2}BLOCKED\*{0,2}\b/i.test(firstLine)) return 'blocked'; + if (/^\*{0,2}NEEDS_CONTEXT\*{0,2}\b/i.test(firstLine)) + return 'needs_context'; + if (/^\*{0,2}DONE_WITH_CONCERNS\*{0,2}\b/i.test(firstLine)) + return 'done_with_concerns'; + if (/^\*{0,2}DONE\*{0,2}\b/i.test(firstLine)) return 'done'; + if (/^\*{0,2}Approved\.?\*{0,2}/i.test(firstLine)) return 'done'; + if (/^\*{0,2}LGTM\*{0,2}/i.test(firstLine)) return 'done'; + return 'continue'; +} + +export type ArbiterVerdictResult = + | 'proceed' + | 'revise' + | 'reset' + | 'escalate' + | 'unknown'; + +export function classifyArbiterVerdict( + summary: string | null | undefined, +): ArbiterVerdictResult { + if (!summary) return 'unknown'; + const cleaned = summary.replace(/[\s\S]*?<\/internal>/g, '').trim(); + if (!cleaned) return 'unknown'; + const firstLine = cleaned.split('\n')[0].trim(); + const verdictMatch = firstLine.match( + /\*{0,2}(?:VERDICT\s*[:—-]\s*)?(PROCEED|REVISE|RESET|ESCALATE)\*{0,2}/i, + ); + if (verdictMatch) { + return verdictMatch[1].toLowerCase() as ArbiterVerdictResult; + } + return 'unknown'; +} + +export function resolveCanonicalSourceRef(workDir: string): string { + const treeHash = resolveCanonicalTreeHash(workDir); + return treeHash || 'HEAD'; +} + +function resolveCanonicalTreeHash(workDir: string): string | null { + try { + const treeHash = execFileSync('git', ['rev-parse', 'HEAD^{tree}'], { + cwd: workDir, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + return treeHash || null; + } catch { + return null; + } +} + +export function hasCodeChangesSinceRef( + workDir: string, + sourceRef: string | null | undefined, +): boolean | null { + if (!sourceRef) return null; + try { + execFileSync('git', ['diff', '--quiet', sourceRef, 'HEAD'], { + cwd: workDir, + stdio: ['ignore', 'pipe', 'pipe'], + }); + return false; + } catch (error) { + const exitCode = + typeof error === 'object' && + error !== null && + 'status' in error && + typeof (error as { status?: unknown }).status === 'number' + ? (error as { status: number }).status + : null; + if (exitCode === 1) { + return true; + } + return null; + } +} + +export function requestArbiterOrEscalate(args: { + taskId: string; + now: string; + arbiterLogMessage: string; + escalateLogMessage: string; + logContext?: Record; +}): void { + const { taskId, now, arbiterLogMessage, escalateLogMessage, logContext } = + args; + if (isArbiterEnabled()) { + updatePairedTask(taskId, { + status: 'arbiter_requested', + arbiter_requested_at: now, + updated_at: now, + }); + logger.info(logContext ?? { taskId }, arbiterLogMessage); + return; + } + + updatePairedTask(taskId, { + status: 'completed', + completion_reason: 'escalated', + updated_at: now, + }); + logger.info(logContext ?? { taskId }, escalateLogMessage); +} diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index 639b499..38d8094 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -1,4 +1,3 @@ -import { execFileSync } from 'child_process'; import crypto from 'crypto'; import fs from 'fs'; import path from 'path'; @@ -12,7 +11,6 @@ import { DATA_DIR, PAIRED_MAX_ROUND_TRIPS, REVIEWER_AGENT_TYPE, - isArbiterEnabled, } from './config.js'; import { createPairedTask, @@ -25,6 +23,22 @@ import { upsertPairedProject, } from './db.js'; import { logger } from './logger.js'; +import { + handleArbiterCompletion, + handleFailedArbiterExecution, +} from './paired-execution-context-arbiter.js'; +import { + handleFailedOwnerExecution, + handleOwnerCompletion, +} from './paired-execution-context-owner.js'; +import { + handleFailedReviewerExecution, + handleReviewerCompletion, +} from './paired-execution-context-reviewer.js'; +import { + resolveCanonicalSourceRef, + requestArbiterOrEscalate, +} from './paired-execution-context-shared.js'; import { markPairedTaskReviewReady, prepareReviewerWorkspaceForExecution, @@ -40,137 +54,6 @@ import type { } from './types.js'; import { resolveRoleAgentPlan } from './role-agent-plan.js'; -// --------------------------------------------------------------------------- -// Reviewer verdict detection -// --------------------------------------------------------------------------- - -type Verdict = - | 'done' - | 'done_with_concerns' - | 'blocked' - | 'needs_context' - | 'continue'; - -function classifyVerdict(summary: string | null | undefined): Verdict { - if (!summary) return 'continue'; - // Strip ... tags — these are internal reasoning, not the verdict - const cleaned = summary.replace(/[\s\S]*?<\/internal>/g, '').trim(); - if (!cleaned) return 'continue'; - const firstLine = cleaned.split('\n')[0].trim(); - // Match verdict markers at the start of the first visible line - if (/^\*{0,2}BLOCKED\*{0,2}\b/i.test(firstLine)) return 'blocked'; - if (/^\*{0,2}NEEDS_CONTEXT\*{0,2}\b/i.test(firstLine)) return 'needs_context'; - if (/^\*{0,2}DONE_WITH_CONCERNS\*{0,2}\b/i.test(firstLine)) - return 'done_with_concerns'; - if (/^\*{0,2}DONE\*{0,2}\b/i.test(firstLine)) return 'done'; - if (/^\*{0,2}Approved\.?\*{0,2}/i.test(firstLine)) return 'done'; - if (/^\*{0,2}LGTM\*{0,2}/i.test(firstLine)) return 'done'; - return 'continue'; -} - -// --------------------------------------------------------------------------- -// Arbiter verdict detection -// --------------------------------------------------------------------------- - -type ArbiterVerdictResult = - | 'proceed' - | 'revise' - | 'reset' - | 'escalate' - | 'unknown'; - -function classifyArbiterVerdict( - summary: string | null | undefined, -): ArbiterVerdictResult { - if (!summary) return 'unknown'; - const cleaned = summary.replace(/[\s\S]*?<\/internal>/g, '').trim(); - if (!cleaned) return 'unknown'; - const firstLine = cleaned.split('\n')[0].trim(); - // Match verdict keywords with optional prefix like "VERDICT:" and markdown bold - const verdictMatch = firstLine.match( - /\*{0,2}(?:VERDICT\s*[:—-]\s*)?(PROCEED|REVISE|RESET|ESCALATE)\*{0,2}/i, - ); - if (verdictMatch) { - return verdictMatch[1].toLowerCase() as ArbiterVerdictResult; - } - return 'unknown'; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function resolveCanonicalSourceRef(workDir: string): string { - const treeHash = resolveCanonicalTreeHash(workDir); - return treeHash || 'HEAD'; -} - -function resolveCanonicalTreeHash(workDir: string): string | null { - try { - const treeHash = execFileSync('git', ['rev-parse', 'HEAD^{tree}'], { - cwd: workDir, - encoding: 'utf-8', - stdio: ['ignore', 'pipe', 'pipe'], - }).trim(); - return treeHash || null; - } catch { - return null; - } -} - -function hasCodeChangesSinceRef( - workDir: string, - sourceRef: string | null | undefined, -): boolean | null { - if (!sourceRef) return null; - try { - execFileSync('git', ['diff', '--quiet', sourceRef, 'HEAD'], { - cwd: workDir, - stdio: ['ignore', 'pipe', 'pipe'], - }); - return false; - } catch (error) { - const exitCode = - typeof error === 'object' && - error !== null && - 'status' in error && - typeof (error as { status?: unknown }).status === 'number' - ? (error as { status: number }).status - : null; - if (exitCode === 1) { - return true; - } - return null; - } -} - -function requestArbiterOrEscalate(args: { - taskId: string; - now: string; - arbiterLogMessage: string; - escalateLogMessage: string; - logContext?: Record; -}): void { - const { taskId, now, arbiterLogMessage, escalateLogMessage, logContext } = - args; - if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, - }); - logger.info(logContext ?? { taskId }, arbiterLogMessage); - return; - } - - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, - }); - logger.info(logContext ?? { taskId }, escalateLogMessage); -} - function ensurePairedProject( group: RegisteredGroup, chatJid: string, @@ -444,399 +327,6 @@ export function preparePairedExecutionContext(args: { }; } -// --------------------------------------------------------------------------- -// completePairedExecutionContext -// --------------------------------------------------------------------------- - -function handleFailedReviewerExecution(args: { - task: PairedTask; - taskId: string; - summary?: string | null; -}): void { - const { task, taskId, summary } = args; - const now = new Date().toISOString(); - - if (summary) { - const verdict = classifyVerdict(summary); - if ( - verdict === 'done' || - verdict === 'blocked' || - verdict === 'needs_context' - ) { - const ownerWs = - verdict === 'done' ? getPairedWorkspace(taskId, 'owner') : null; - const approvedSourceRef = - verdict === 'done' && ownerWs?.workspace_dir - ? resolveCanonicalSourceRef(ownerWs.workspace_dir) - : task.source_ref; - updatePairedTask(taskId, { - status: verdict === 'done' ? 'merge_ready' : 'completed', - ...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), - ...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), - updated_at: now, - }); - logger.info( - { - taskId, - verdict, - approvedSourceRef, - summary: summary.slice(0, 100), - }, - 'Reviewer verdict detected from failed execution — stopping ping-pong', - ); - return; - } - } - - const fallbackStatus = - task.status === 'in_review' || task.status === 'review_ready' - ? 'review_ready' - : task.status; - if (fallbackStatus !== task.status) { - updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); - logger.warn( - { - taskId, - role: 'reviewer', - previousStatus: task.status, - nextStatus: fallbackStatus, - }, - 'Preserved reviewer task in review-ready state after failed execution', - ); - } -} - -function handleFailedArbiterExecution(args: { - task: PairedTask; - taskId: string; -}): void { - const { task, taskId } = args; - const now = new Date().toISOString(); - const fallbackStatus = - task.status === 'in_arbitration' || task.status === 'arbiter_requested' - ? 'arbiter_requested' - : task.status; - if (fallbackStatus !== task.status) { - updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); - logger.warn( - { - taskId, - role: 'arbiter', - previousStatus: task.status, - nextStatus: fallbackStatus, - }, - 'Preserved arbiter task in arbitration-requested state after failed execution', - ); - } -} - -function handleFailedOwnerExecution(args: { - task: PairedTask; - taskId: string; -}): void { - const { task, taskId } = args; - if (task.status !== 'active') { - const now = new Date().toISOString(); - updatePairedTask(taskId, { status: 'active', updated_at: now }); - logger.info( - { taskId, role: 'owner', previousStatus: task.status }, - 'Reset task to active after failed execution', - ); - } -} - -function handleOwnerFinalizeCompletion(args: { - task: PairedTask; - taskId: string; - summary?: string | null; - now: string; -}): boolean { - const { task, taskId, summary, now } = args; - const ownerVerdict = classifyVerdict(summary); - - if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { - requestArbiterOrEscalate({ - taskId, - now, - arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter', - escalateLogMessage: 'Owner blocked during finalize — escalating to user', - logContext: { - taskId, - ownerVerdict, - summary: summary?.slice(0, 100), - }, - }); - return true; - } - - if (ownerVerdict === 'done_with_concerns') { - if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { - requestArbiterOrEscalate({ - taskId, - now, - arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter', - escalateLogMessage: 'Owner finalize loop detected — escalating to user', - logContext: { - taskId, - ownerVerdict, - roundTrips: task.round_trip_count, - }, - }); - return true; - } - updatePairedTask(taskId, { status: 'active', updated_at: now }); - logger.info( - { - taskId, - ownerVerdict, - summary: summary?.slice(0, 100), - }, - 'Owner raised concerns during finalize — task set back to active', - ); - return false; - } - - const workspace = getPairedWorkspace(task.id, 'owner'); - const hasNewChanges = workspace?.workspace_dir - ? hasCodeChangesSinceRef(workspace.workspace_dir, task.source_ref) - : null; - - if (hasNewChanges === true) { - if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { - requestArbiterOrEscalate({ - taskId, - now, - arbiterLogMessage: - 'Owner finalize DONE loop detected — requesting arbiter', - escalateLogMessage: - 'Owner finalize DONE loop detected — escalating to user', - logContext: { - taskId, - roundTrips: task.round_trip_count, - hasNewChanges, - }, - }); - return true; - } - logger.info( - { - taskId, - sourceRef: task.source_ref, - hasNewChanges, - }, - 'Owner made changes after reviewer approval — re-triggering review', - ); - return false; - } - - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'done', - updated_at: now, - }); - logger.info( - { taskId, hasNewChanges, summary: summary?.slice(0, 100) }, - 'Owner finalized after reviewer approval — task completed', - ); - return true; -} - -function handleOwnerCompletion(args: { - task: PairedTask; - taskId: string; - summary?: string | null; -}): void { - const { task, taskId, summary } = args; - const now = new Date().toISOString(); - - if (task.status === 'merge_ready') { - const shouldStop = handleOwnerFinalizeCompletion({ - task, - taskId, - summary, - now, - }); - if (shouldStop) { - return; - } - } - - if (hasActiveCiWatcherForChat(task.chat_jid)) { - logger.info( - { taskId, chatJid: task.chat_jid }, - 'Active CI watcher found, deferring auto-review until watcher completes', - ); - return; - } - - if (task.status !== 'merge_ready') { - const ownerVerdict = classifyVerdict(summary); - if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { - requestArbiterOrEscalate({ - taskId, - now, - arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter', - escalateLogMessage: 'Owner blocked/needs_context — escalating to user', - logContext: { - taskId, - ownerVerdict, - summary: summary?.slice(0, 100), - }, - }); - return; - } - } - - if (task.round_trip_count >= PAIRED_MAX_ROUND_TRIPS) { - logger.info( - { - taskId, - roundTrips: task.round_trip_count, - max: PAIRED_MAX_ROUND_TRIPS, - }, - 'Round trip limit reached, skipping auto-review', - ); - return; - } - - const result = markPairedTaskReviewReady(taskId); - if (result) { - updatePairedTask(taskId, { - round_trip_count: task.round_trip_count + 1, - review_requested_at: now, - updated_at: now, - }); - logger.info( - { taskId, roundTrip: task.round_trip_count + 1 }, - 'Auto-triggered reviewer after owner completion', - ); - } -} - -function handleReviewerCompletion(args: { - task: PairedTask; - taskId: string; - summary?: string | null; -}): void { - const { task, taskId, summary } = args; - const now = new Date().toISOString(); - const verdict = classifyVerdict(summary); - - switch (verdict) { - case 'done': { - const ownerWs = getPairedWorkspace(taskId, 'owner'); - const approvedSourceRef = ownerWs?.workspace_dir - ? resolveCanonicalSourceRef(ownerWs.workspace_dir) - : task.source_ref; - updatePairedTask(taskId, { - status: 'merge_ready', - source_ref: approvedSourceRef, - updated_at: now, - }); - logger.info( - { - taskId, - verdict, - approvedSourceRef, - summary: summary?.slice(0, 100), - }, - 'Reviewer approved — owner gets final turn to finalize', - ); - return; - } - - case 'blocked': - case 'needs_context': - requestArbiterOrEscalate({ - taskId, - now, - arbiterLogMessage: - 'Reviewer blocked/needs_context — requesting arbiter before escalating', - escalateLogMessage: 'Reviewer escalated to user — ping-pong stopped', - logContext: { taskId, verdict, summary: summary?.slice(0, 100) }, - }); - return; - - case 'done_with_concerns': - case 'continue': - default: - if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { - requestArbiterOrEscalate({ - taskId, - now, - arbiterLogMessage: - 'Deadlock detected — requesting arbiter intervention', - escalateLogMessage: - 'Stopped ping-pong — escalating to user (arbiter not configured)', - logContext: { - taskId, - verdict, - roundTrips: task.round_trip_count, - }, - }); - return; - } - updatePairedTask(taskId, { status: 'active', updated_at: now }); - logger.info( - { taskId, verdict }, - 'Reviewer has feedback, task set back to active for owner', - ); - return; - } -} - -function handleArbiterCompletion(args: { - taskId: string; - summary?: string | null; -}): void { - const { taskId, summary } = args; - const now = new Date().toISOString(); - const arbiterVerdict = classifyArbiterVerdict(summary); - - logger.info( - { taskId, arbiterVerdict, summary: summary?.slice(0, 200) }, - 'Arbiter verdict rendered', - ); - - switch (arbiterVerdict) { - case 'proceed': - case 'revise': - case 'reset': - updatePairedTask(taskId, { - status: 'active', - round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), - arbiter_verdict: arbiterVerdict, - updated_at: now, - }); - logger.info( - { taskId, arbiterVerdict }, - 'Arbiter resolved deadlock — resuming ping-pong', - ); - return; - case 'escalate': - updatePairedTask(taskId, { - status: 'completed', - arbiter_verdict: 'escalate', - completion_reason: 'arbiter_escalated', - updated_at: now, - }); - logger.info({ taskId }, 'Arbiter escalated to user — task completed'); - return; - default: - updatePairedTask(taskId, { - status: 'active', - round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), - arbiter_verdict: 'unknown', - updated_at: now, - }); - logger.warn( - { taskId, summary: summary?.slice(0, 200) }, - 'Arbiter verdict unrecognized — falling back to proceed', - ); - return; - } -} - export function completePairedExecutionContext(args: { taskId: string; role: PairedRoomRole; diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 0154c7f..2e0507c 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -2,6 +2,7 @@ import { ChildProcess } from 'child_process'; import { CronExpressionParser } from 'cron-parser'; import fs from 'fs'; import { getAgentOutputText } from './agent-output.js'; +import { createEvaluatedOutputHandler } from './agent-attempt.js'; import { getErrorMessage } from './utils.js'; import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js'; @@ -53,10 +54,6 @@ import { formatSuspensionNotice, suspendTask, } from './task-suspension.js'; -import { - evaluateStreamedOutput, - type StreamedOutputState, -} from './streamed-output-evaluator.js'; import { extractWatchCiTarget, getTaskQueueJid, @@ -339,58 +336,31 @@ async function runTask( attemptResult: string | null; attemptError: string | null; }> => { - let streamedState: StreamedOutputState = { - sawOutput: false, - sawVisibleOutput: false, - sawSuccessNullResultWithoutOutput: false, - }; let attemptResult: string | null = null; let attemptError: string | null = null; - - const output = await runAgentProcess( - context.group, - { - prompt: task.prompt, - sessionId: context.sessionId, - groupFolder: task.group_folder, - chatJid: task.chat_jid, - isMain: context.isMain, - isScheduledTask: true, - runtimeTaskId: context.runtimeTaskId, - useTaskScopedSession: context.useTaskScopedSession, - assistantName: ASSISTANT_NAME, + const streamedOutputHandler = createEvaluatedOutputHandler({ + agentType: isClaudeAgent ? 'claude-code' : 'codex', + provider, + evaluationOptions: { + shortCircuitTriggeredErrors: true, }, - (proc, processName) => - deps.onProcess( - context.queueJid, - proc, - processName, - context.runtimeIpcDir, - ), - async (streamedOutput: AgentOutput) => { + onEvaluatedOutput: async ({ + output: streamedOutput, + outputText, + evaluation, + }) => { if (streamedOutput.phase === 'progress') { return; } - const evaluation = evaluateStreamedOutput( - streamedOutput, - streamedState, - { - agentType: isClaudeAgent ? 'claude-code' : 'codex', - provider, - shortCircuitTriggeredErrors: true, - }, - ); - streamedState = evaluation.state; - if ( evaluation.newTrigger && - typeof streamedOutput.result === 'string' && + outputText && streamedOutput.status === 'success' ) { log.warn( { reason: evaluation.newTrigger.reason, - resultPreview: streamedOutput.result.slice(0, 120), + resultPreview: outputText.slice(0, 120), }, 'Detected Claude rotation trigger during scheduled task output', ); @@ -416,7 +386,6 @@ async function runTask( return; } - const outputText = getAgentOutputText(streamedOutput); if (outputText) { attemptResult = outputText; // Paired-room scheduler output must use the reviewer bot slot. @@ -427,6 +396,29 @@ async function runTask( attemptError = streamedOutput.error || 'Unknown error'; } }, + }); + + const output = await runAgentProcess( + context.group, + { + prompt: task.prompt, + sessionId: context.sessionId, + groupFolder: task.group_folder, + chatJid: task.chat_jid, + isMain: context.isMain, + isScheduledTask: true, + runtimeTaskId: context.runtimeTaskId, + useTaskScopedSession: context.useTaskScopedSession, + assistantName: ASSISTANT_NAME, + }, + (proc, processName) => + deps.onProcess( + context.queueJid, + proc, + processName, + context.runtimeIpcDir, + ), + streamedOutputHandler.handleOutput, undefined, ); @@ -439,6 +431,7 @@ async function runTask( } } + const streamedState = streamedOutputHandler.getState(); return { output, sawOutput: streamedState.sawOutput, diff --git a/src/verification.ts b/src/verification.ts index e6bd918..af2efa0 100644 --- a/src/verification.ts +++ b/src/verification.ts @@ -1,5 +1,4 @@ import { execFile, execFileSync } from 'child_process'; -import { createHash } from 'crypto'; import fs from 'fs'; import os from 'os'; import path from 'path'; @@ -18,6 +17,11 @@ import { detectPnpmStorePath, hasInstalledNodeModules, } from './workspace-package-manager.js'; +import { + computeVerificationSnapshotId, + isVerificationSnapshotExcludedName, + resolveVerificationResponsesDir, +} from '../shared/verification-snapshot.js'; export const VERIFICATION_PROFILES = ['test', 'typecheck', 'build'] as const; @@ -58,7 +62,6 @@ interface VerificationCommandSpec { } const PRIMARY_PROJECT_MOUNT = '/workspace/project'; -const SNAPSHOT_EXCLUDE_NAMES = new Set(['.git', 'node_modules', '.env']); const MAX_OUTPUT_CHARS = 24_000; const COMMAND_TIMEOUT_MS = 20 * 60 * 1000; const COMMAND_MAX_BUFFER = 20 * 1024 * 1024; @@ -92,47 +95,14 @@ export function buildVerificationCommand( } function shouldExcludePath(name: string): boolean { - return SNAPSHOT_EXCLUDE_NAMES.has(name); -} - -function updateSnapshotHash( - hash: ReturnType, - repoDir: string, - currentPath: string, -): void { - const relPath = path.relative(repoDir, currentPath) || '.'; - const stat = fs.lstatSync(currentPath); - - if (stat.isDirectory()) { - if (relPath !== '.') { - hash.update(`dir\0${relPath}\0`); - } - for (const entry of fs.readdirSync(currentPath).sort()) { - if (shouldExcludePath(entry)) continue; - updateSnapshotHash(hash, repoDir, path.join(currentPath, entry)); - } - return; - } - - if (stat.isSymbolicLink()) { - hash.update(`symlink\0${relPath}\0${fs.readlinkSync(currentPath)}\0`); - return; - } - - if (stat.isFile()) { - hash.update(`file\0${relPath}\0`); - hash.update(fs.readFileSync(currentPath)); - hash.update('\0'); - } + return isVerificationSnapshotExcludedName(name); } export function computeVerificationSnapshot( repoDir: string, ): VerificationSnapshot { - const hash = createHash('sha256'); - updateSnapshotHash(hash, repoDir, repoDir); return { - snapshotId: `fs:${hash.digest('hex').slice(0, 24)}`, + snapshotId: computeVerificationSnapshotId(repoDir), }; } @@ -403,7 +373,7 @@ export async function runVerificationRequest( } export function resolveVerificationResponseDir(groupFolder: string): string { - return path.join(resolveGroupIpcPath(groupFolder), 'verification-responses'); + return resolveVerificationResponsesDir(resolveGroupIpcPath(groupFolder)); } export function writeVerificationResponse( From 48592d97edbdec73a41269b9ffd206ddb5c74b75 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 00:45:55 +0900 Subject: [PATCH 22/29] Format paired execution refactor files --- src/agent-attempt.ts | 9 +++++---- src/message-agent-executor.ts | 15 +++++++-------- src/message-runtime-flow.ts | 11 ++++++++--- src/paired-execution-context-shared.ts | 3 +-- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/agent-attempt.ts b/src/agent-attempt.ts index d22c39d..2627e97 100644 --- a/src/agent-attempt.ts +++ b/src/agent-attempt.ts @@ -28,10 +28,11 @@ export function createInitialStreamedOutputState(): StreamedOutputState { export function createEvaluatedOutputHandler(args: { agentType: EvaluateStreamedOutputOptions['agentType']; provider: string; - evaluationOptions?: Omit; - onEvaluatedOutput?: ( - output: EvaluatedAgentOutput, - ) => Promise | void; + evaluationOptions?: Omit< + EvaluateStreamedOutputOptions, + 'agentType' | 'provider' + >; + onEvaluatedOutput?: (output: EvaluatedAgentOutput) => Promise | void; }): { handleOutput: (output: AgentOutput) => Promise; getState: () => StreamedOutputState; diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 07bdd9d..caf23c0 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -3,9 +3,7 @@ import path from 'path'; import { getErrorMessage } from './utils.js'; -import { - getAgentOutputText, -} from './agent-output.js'; +import { getAgentOutputText } from './agent-output.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js'; import { AgentOutput, @@ -493,7 +491,11 @@ export async function runAgentForGroup( ) { pairedExecutionSummary = output.error.slice(0, 500); } - if (evaluation.newTrigger && outputText && output.status === 'success') { + if ( + evaluation.newTrigger && + outputText && + output.status === 'success' + ) { log.warn( { reason: evaluation.newTrigger.reason, @@ -501,10 +503,7 @@ export async function runAgentForGroup( }, 'Detected Claude rotation trigger in successful output', ); - } else if ( - evaluation.newTrigger && - typeof output.error === 'string' - ) { + } else if (evaluation.newTrigger && typeof output.error === 'string') { log.warn( { reason: evaluation.newTrigger.reason, diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts index b808fc4..4ab28fe 100644 --- a/src/message-runtime-flow.ts +++ b/src/message-runtime-flow.ts @@ -51,8 +51,12 @@ export function buildPendingPairedTurn(args: { timezone: string; task: PairedTask; rawMissedMessages: Array<{ seq?: number | null; timestamp?: string | null }>; - recentHumanMessages: Parameters[0]['recentHumanMessages']; - labeledRecentMessages: Parameters[0]['labeledRecentMessages']; + recentHumanMessages: Parameters< + typeof buildReviewerPendingPrompt + >[0]['recentHumanMessages']; + labeledRecentMessages: Parameters< + typeof buildArbiterPromptForTask + >[0]['labeledRecentMessages']; resolveChannel: (taskStatus?: string | null) => Channel | null; }): PendingPairedTurn { const { @@ -215,7 +219,8 @@ export function resolveBotOnlyPairedFollowUpAction(args: { return { kind: 'none' }; } - const cursor = pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; + const cursor = + pendingCursorSource?.seq ?? pendingCursorSource?.timestamp ?? null; const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1); const nextTurnAction = resolveNextTurnAction({ taskStatus: task.status, diff --git a/src/paired-execution-context-shared.ts b/src/paired-execution-context-shared.ts index 349e2c1..a2e7a2f 100644 --- a/src/paired-execution-context-shared.ts +++ b/src/paired-execution-context-shared.ts @@ -17,8 +17,7 @@ export function classifyVerdict(summary: string | null | undefined): Verdict { if (!cleaned) return 'continue'; const firstLine = cleaned.split('\n')[0].trim(); if (/^\*{0,2}BLOCKED\*{0,2}\b/i.test(firstLine)) return 'blocked'; - if (/^\*{0,2}NEEDS_CONTEXT\*{0,2}\b/i.test(firstLine)) - return 'needs_context'; + if (/^\*{0,2}NEEDS_CONTEXT\*{0,2}\b/i.test(firstLine)) return 'needs_context'; if (/^\*{0,2}DONE_WITH_CONCERNS\*{0,2}\b/i.test(firstLine)) return 'done_with_concerns'; if (/^\*{0,2}DONE\*{0,2}\b/i.test(firstLine)) return 'done'; From 9d8c97a8e2f82c7056bc6e54a10e0a7826476502 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 01:22:27 +0900 Subject: [PATCH 23/29] Fix reviewer-to-owner follow-up race --- src/message-agent-executor.test.ts | 101 ++++++++++++++++++++++++ src/message-agent-executor.ts | 14 ++-- src/message-runtime-flow.ts | 32 +++++++- src/message-runtime.test.ts | 118 +++++++++++++++++++++++++++++ src/message-runtime.ts | 17 +++-- 5 files changed, 265 insertions(+), 17 deletions(-) diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index ad0443f..11f30cd 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1315,6 +1315,107 @@ describe('runAgentForGroup room memory', () => { expect(deps.queue.enqueueMessageCheck).toHaveBeenCalledWith('group@test'); }); + it('stores reviewer turn output before transitioning the paired task back to active', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + const deps = makeDeps(); + + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_agent_type: 'codex', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + arbiter_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }); + vi.mocked( + pairedExecutionContext.preparePairedExecutionContext, + ).mockReturnValue({ + task: { + id: 'paired-task-reviewer-output-order', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'in_review', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }, + workspace: null, + envOverrides: {}, + }); + vi.mocked(db.getPairedTaskById).mockReturnValue({ + id: 'paired-task-reviewer-output-order', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + round_trip_count: 1, + review_requested_at: '2026-03-31T00:00:00.000Z', + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-31T00:00:00.000Z', + updated_at: '2026-03-31T00:00:00.000Z', + }); + vi.mocked(agentRunner.runAgentProcess).mockImplementation( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + result: 'DONE_WITH_CONCERNS\nreviewer feedback', + output: { + visibility: 'public', + text: 'DONE_WITH_CONCERNS\nreviewer feedback', + }, + phase: 'final', + }); + return { + status: 'success', + result: 'DONE_WITH_CONCERNS\nreviewer feedback', + }; + }, + ); + + const result = await runAgentForGroup(deps, { + group, + prompt: 'please review', + chatJid: 'group@test', + runId: 'run-reviewer-output-order', + forcedRole: 'reviewer', + onOutput: async () => {}, + }); + + expect(result).toBe('success'); + expect(db.insertPairedTurnOutput).toHaveBeenCalledWith( + 'paired-task-reviewer-output-order', + 1, + 'reviewer', + 'DONE_WITH_CONCERNS\nreviewer feedback', + ); + expect( + vi.mocked(db.insertPairedTurnOutput).mock.invocationCallOrder[0], + ).toBeLessThan( + vi.mocked( + pairedExecutionContext.completePairedExecutionContext, + ).mock.invocationCallOrder[0], + ); + }); + it('does not enqueue a generic follow-up when reviewer output already returned the task to active', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index caf23c0..003a9e2 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -939,13 +939,6 @@ export async function runAgentForGroup( !pairedSawOutput ? 'failed' : completionStatus; - completePairedExecutionContext({ - taskId: pairedExecutionContext.task.id, - role: completedRole, - status: effectiveStatus, - summary: pairedExecutionSummary, - }); - // Store full output for direct inter-agent data passing (Discord-independent). if (pairedFullOutput && effectiveStatus === 'succeeded') { try { @@ -964,6 +957,13 @@ export async function runAgentForGroup( ); } } + + completePairedExecutionContext({ + taskId: pairedExecutionContext.task.id, + role: completedRole, + status: effectiveStatus, + summary: pairedExecutionSummary, + }); } // Notify user when paired task reaches a terminal state that requires attention. diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts index 4ab28fe..b6c29fe 100644 --- a/src/message-runtime-flow.ts +++ b/src/message-runtime-flow.ts @@ -18,6 +18,7 @@ import { import { hasReviewerLease } from './service-routing.js'; import type { Channel, + NewMessage, PairedTask, PairedRoomRole, RegisteredGroup, @@ -46,6 +47,25 @@ export type BotOnlyPairedFollowUpAction = nextRole: 'owner' | 'reviewer' | 'arbiter'; }; +export function resolveLastDeliveredBotRole( + messages: NewMessage[], +): PairedRoomRole | null { + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (!message?.is_bot_message) { + continue; + } + if ( + message.sender_name === 'owner' || + message.sender_name === 'reviewer' || + message.sender_name === 'arbiter' + ) { + return message.sender_name; + } + } + return null; +} + export function buildPendingPairedTurn(args: { chatJid: string; timezone: string; @@ -57,6 +77,7 @@ export function buildPendingPairedTurn(args: { labeledRecentMessages: Parameters< typeof buildArbiterPromptForTask >[0]['labeledRecentMessages']; + lastDeliveredMessages?: NewMessage[]; resolveChannel: (taskStatus?: string | null) => Channel | null; }): PendingPairedTurn { const { @@ -75,7 +96,10 @@ export function buildPendingPairedTurn(args: { const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; const nextTurnAction = resolveNextTurnAction({ taskStatus, - lastTurnOutputRole: lastTurnOutput?.role ?? null, + lastTurnOutputRole: + resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ?? + lastTurnOutput?.role ?? + null, }); const recentMessages = getRecentChatMessages(chatJid, 20); const lastHumanMessage = getLastHumanMessageContent(chatJid); @@ -210,6 +234,7 @@ export function resolveBotOnlyPairedFollowUpAction(args: { chatJid: string; task: PairedTask | null | undefined; isBotOnlyPairedFollowUp: boolean; + lastDeliveredMessages?: NewMessage[]; pendingCursorSource: | { seq?: number | null; timestamp?: string | null } | undefined; @@ -224,7 +249,10 @@ export function resolveBotOnlyPairedFollowUpAction(args: { const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1); const nextTurnAction = resolveNextTurnAction({ taskStatus: task.status, - lastTurnOutputRole: lastTurnOutput?.role ?? null, + lastTurnOutputRole: + resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ?? + lastTurnOutput?.role ?? + null, }); if (nextTurnAction.kind === 'finalize-owner-turn') { diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 009bcfe..5199a6e 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -1827,6 +1827,124 @@ describe('createMessageRuntime', () => { expect(sendMessage).not.toHaveBeenCalled(); }); + it('requeues owner follow-ups from reviewer bot-only messages even when paired turn output storage is stale', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const ownerChannel = makeChannel(chatJid); + const reviewerChannel = { + ...makeChannel(chatJid, 'discord-review', false), + isOwnMessage: vi.fn((message) => message.sender === 'reviewer-bot@test'), + }; + const enqueueMessageCheck = vi.fn(); + const closeStdin = vi.fn(); + const sendMessage = vi.fn(() => false); + const setLastTimestamp = vi.fn(); + const saveState = vi.fn(); + const lastAgentTimestamps: Record = {}; + const stopLoop = new Error('stop-message-loop'); + + vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); + vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ + id: 'task-active-owner-follow-up-stale-output', + chat_jid: chatJid, + group_folder: group.folder, + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: '2026-03-30T00:00:00.000Z', + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + }); + vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ + { + id: 1, + task_id: 'task-active-owner-follow-up-stale-output', + turn_number: 1, + role: 'owner', + output_text: 'owner 초안', + created_at: '2026-03-30T00:00:01.000Z', + }, + ]); + vi.mocked(db.getNewMessages).mockReturnValue({ + messages: [ + { + id: 'reviewer-bot-message-owner-follow-up-stale-output', + chat_jid: chatJid, + sender: 'reviewer-bot@test', + sender_name: '리뷰어', + content: 'DONE_WITH_CONCERNS\n\nreviewer direct message', + timestamp: '2026-03-30T00:00:04.000Z', + seq: 42, + is_bot_message: true, + } as any, + ], + newTimestamp: '42', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 60_000, + pollInterval: 123, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [ownerChannel, reviewerChannel], + queue: { + registerProcess: vi.fn(), + closeStdin, + enqueueMessageCheck, + notifyIdle: vi.fn(), + sendMessage, + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp, + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const originalSetTimeout = global.setTimeout; + const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation((( + handler: any, + timeout?: number, + ...args: any[] + ) => { + if (timeout === 123) { + throw stopLoop; + } + return (originalSetTimeout as any)(handler, timeout, ...args); + }) as typeof setTimeout); + + try { + await expect(runtime.startMessageLoop()).rejects.toThrow( + stopLoop.message, + ); + } finally { + setTimeoutSpy.mockRestore(); + } + + expect(setLastTimestamp).toHaveBeenCalledWith('42'); + expect(agentRunner.runAgentProcess).not.toHaveBeenCalled(); + expect(closeStdin).toHaveBeenCalledWith( + chatJid, + expect.objectContaining({ reason: 'paired-pending-turn-follow-up' }), + ); + expect(enqueueMessageCheck).toHaveBeenCalledWith( + chatJid, + resolveGroupIpcPath(group.folder), + ); + expect(sendMessage).not.toHaveBeenCalled(); + }); + it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => { const chatJid = 'group@test'; const group = makeGroup('codex'); diff --git a/src/message-runtime.ts b/src/message-runtime.ts index 70b0d57..fde7752 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -47,6 +47,7 @@ import { executeBotOnlyPairedFollowUpAction, executePendingPairedTurn, resolveBotOnlyPairedFollowUpAction, + resolveLastDeliveredBotRole, shouldSkipGenericFollowUpAfterDeliveryRetry, } from './message-runtime-flow.js'; import { MessageTurnController } from './message-turn-controller.js'; @@ -858,6 +859,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { chatJid, getRecentChatMessages(chatJid, 20), ), + lastDeliveredMessages: labelPairedSenders(chatJid, missedMessages), resolveChannel, }) : null; @@ -1217,22 +1219,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { pendingMessages.length > 0 ? pendingMessages : processableGroupMessages; + const labeledMessagesToSend = labelPairedSenders( + chatJid, + messagesToSend, + ); const formatted = loopPendingTask ? buildPairedTurnPrompt({ taskId: loopPendingTask.id, chatJid, timezone: deps.timezone, missedMessages: messagesToSend, - labeledFallbackMessages: labelPairedSenders( - chatJid, - messagesToSend, - ), + labeledFallbackMessages: labeledMessagesToSend, turnOutputs: getPairedTurnOutputs(loopPendingTask.id), }) - : formatMessages( - labelPairedSenders(chatJid, messagesToSend), - deps.timezone, - ); + : formatMessages(labeledMessagesToSend, deps.timezone); const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn( chatJid, messagesToSend, @@ -1245,6 +1245,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { chatJid, task: loopPendingTask, isBotOnlyPairedFollowUp, + lastDeliveredMessages: labeledMessagesToSend, pendingCursorSource, }); if ( From cd6aa321579cf0f52f34c4c996b05f986c685a1f Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 02:00:57 +0900 Subject: [PATCH 24/29] Separate runtime-only deploy from container build --- CLAUDE.md | 6 +++++- package.json | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cdcfc56..10c2426 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,8 @@ Run commands directly—don't tell the user to run them. ```bash bun run build # Build main project bun run build:runners # Install + build both runners -bun run build:container # Rebuild reviewer Docker image +bun run build:runtime # Build host runtime only +bun run build:container # Optional: rebuild reviewer Docker image bun run dev # Dev mode with hot reload ``` @@ -53,6 +54,9 @@ Deploy: bun run deploy ``` +`deploy` rebuilds only the host runtime. Rebuild the reviewer Docker image +separately with `bun run build:container` only when you need the container path. + ## Service Stack Architecture Single unified service manages all three Discord bots in one process: diff --git a/package.json b/package.json index f7ec7d2..797dd1e 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,10 @@ "scripts": { "build": "tsc", "build:runners": "bun install --cwd runners/agent-runner && bun run --cwd runners/agent-runner build && bun install --cwd runners/codex-runner && bun run --cwd runners/codex-runner build", + "build:runtime": "bun run build && bun run build:runners", "build:container": "docker build -f container/Dockerfile -t ejclaw-reviewer:latest .", - "build:all": "bun run build && bun run build:runners && bun run build:container", - "deploy": "git pull --ff-only && bun run build:all && systemctl --user restart ejclaw", + "build:all": "bun run build:runtime && bun run build:container", + "deploy": "git pull --ff-only && bun run build:runtime && systemctl --user restart ejclaw", "start": "bun dist/index.js", "dev": "bun --watch src/index.ts", "test": "vitest run", From 4511ca569242e51a0f9e109cb47fb96bfaee907d Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 02:13:22 +0900 Subject: [PATCH 25/29] Clarify reviewer verification limits --- prompts/arbiter-paired-room.md | 2 ++ prompts/claude-paired-room.md | 2 ++ .../codex-runner/test/reviewer-runtime.test.ts | 18 +++++++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/prompts/arbiter-paired-room.md b/prompts/arbiter-paired-room.md index 48af0bd..f620a39 100644 --- a/prompts/arbiter-paired-room.md +++ b/prompts/arbiter-paired-room.md @@ -34,6 +34,8 @@ You may receive reference opinions from external models appended to your prompt. ## Rules - Base your verdict on evidence (code, test output, logs), not on who said what first +- Distinguish reviewer snapshot limits from real product bugs. Reviewer workspaces may intentionally omit heavy artifacts like `node_modules`, `dist`, and `build`; inability to run direct local test/typecheck/build there is not, by itself, a blocker if dedicated verification evidence exists +- When verification evidence exists from the dedicated verification path, judge that evidence on its merits instead of requiring the reviewer to reproduce the same result from the lightweight reviewer snapshot - Your verdict is final for this deadlock cycle — after it, work resumes normally - You do NOT implement or review code — you only judge the disagreement - Keep your verdict concise — state the decision, the evidence, and the required action diff --git a/prompts/claude-paired-room.md b/prompts/claude-paired-room.md index e357773..8205400 100644 --- a/prompts/claude-paired-room.md +++ b/prompts/claude-paired-room.md @@ -30,6 +30,8 @@ Push back with evidence when the owner is wrong. Hold your ground when you are r ## Rules - Judge completion only by verification output. "It should work now" means run it. "I'm confident" means nothing — confidence is not evidence. "I tested earlier" means test again if code changed since. "It's a trivial change" means verify anyway +- Reviewer workspaces may intentionally exclude heavy runtime artifacts such as `node_modules`, `dist`, and `build`. Do not treat the inability to run direct local test/typecheck/build from the reviewer snapshot as a product bug by itself +- When test/typecheck/build evidence is needed, prefer the dedicated verification path (`run_verification`) over assuming the reviewer snapshot can execute the full project locally - Stagnation: **Spinning** (same error 3+), **Oscillation** (alternating approaches), **Diminishing returns** (shrinking improvement), **No progress** (discussion without change) — name the pattern and report: **Status**, **Attempted**, **Recommendation** - Implementation, commits, and pushes require agreement from both sides. Either can veto - Keep reviews concise — approve quickly when there is nothing to critique diff --git a/runners/codex-runner/test/reviewer-runtime.test.ts b/runners/codex-runner/test/reviewer-runtime.test.ts index 27f6f40..fb54d88 100644 --- a/runners/codex-runner/test/reviewer-runtime.test.ts +++ b/runners/codex-runner/test/reviewer-runtime.test.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { assertReadonlyWorkspaceRepoConnectivity, @@ -11,6 +11,22 @@ import { isReviewerRuntime, } from '../src/reviewer-runtime.js'; +const ORIGINAL_UNSAFE_HOST_PAIRED_MODE = + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + +afterEach(() => { + if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; + } else { + process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = + ORIGINAL_UNSAFE_HOST_PAIRED_MODE; + } +}); + +beforeEach(() => { + delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE; +}); + function createTempRepo(prefix: string): string { const cwd = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); execFileSync('git', ['init'], { From b8e8c9dabf9fd898992e72f87681b6e37119670b Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 02:49:02 +0900 Subject: [PATCH 26/29] Unify paired status transitions and retry handling --- src/agent-attempt-retry.ts | 164 +++++++++++++++++++++++ src/message-agent-executor-rules.test.ts | 42 ++++++ src/message-agent-executor-rules.ts | 120 ++--------------- src/message-agent-executor.ts | 32 +++-- src/message-runtime-flow.ts | 69 ++++++++++ src/message-runtime.ts | 66 ++++----- src/paired-execution-context-arbiter.ts | 55 +++++--- src/paired-execution-context-owner.ts | 31 ++++- src/paired-execution-context-reviewer.ts | 43 ++++-- src/paired-execution-context-shared.ts | 100 ++++++++++++-- src/paired-execution-context.ts | 38 ++++-- src/paired-workspace-manager.ts | 13 +- src/task-scheduler.ts | 81 ++++------- 13 files changed, 574 insertions(+), 280 deletions(-) create mode 100644 src/agent-attempt-retry.ts diff --git a/src/agent-attempt-retry.ts b/src/agent-attempt-retry.ts new file mode 100644 index 0000000..49b5491 --- /dev/null +++ b/src/agent-attempt-retry.ts @@ -0,0 +1,164 @@ +import { + classifyRotationTrigger, + type AgentTriggerReason, + type CodexRotationReason, + isCodexRotationReason, +} from './agent-error-detection.js'; +import { detectCodexRotationTrigger } from './codex-token-rotation.js'; +import { getErrorMessage } from './utils.js'; + +export interface AttemptStreamedTrigger { + reason: AgentTriggerReason; + retryAfterMs?: number; +} + +export interface AttemptRetryState { + sawOutput: boolean; + retryableSessionFailureDetected?: boolean; + streamedTriggerReason?: AttemptStreamedTrigger; + error?: unknown; + outputError?: string | null; +} + +export function isRetryableClaudeSessionFailureAttempt(args: { + attempt: AttemptRetryState; + isClaudeCodeAgent: boolean; + provider: 'claude' | 'codex'; + shouldRetryFreshSessionOnAgentFailure: (args: { + result: null; + error: string; + }) => boolean; +}): boolean { + if ( + !args.isClaudeCodeAgent || + args.provider !== 'claude' || + args.attempt.sawOutput + ) { + return false; + } + + if (args.attempt.retryableSessionFailureDetected === true) { + return true; + } + + if (args.attempt.error == null) { + return false; + } + + return args.shouldRetryFreshSessionOnAgentFailure({ + result: null, + error: getErrorMessage(args.attempt.error), + }); +} + +export function resolveClaudeRetryTrigger(args: { + canRetryClaudeCredentials: boolean; + provider: 'claude' | 'codex'; + attempt: Pick; + fallbackMessage?: string | null; +}): { reason: AgentTriggerReason; retryAfterMs?: number } | null { + if ( + !( + args.canRetryClaudeCredentials && + args.provider === 'claude' && + !args.attempt.sawOutput + ) + ) { + return null; + } + + if (args.attempt.streamedTriggerReason) { + return { + reason: args.attempt.streamedTriggerReason.reason, + retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs, + }; + } + + const trigger = classifyRotationTrigger(args.fallbackMessage); + if (!trigger.shouldRetry) { + return null; + } + + return { + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }; +} + +export function resolveCodexRetryTrigger(args: { + canRetryCodex: boolean; + attempt: Pick; + rotationMessage?: string | null; +}): { reason: CodexRotationReason } | null { + if (!args.canRetryCodex) { + return null; + } + + if (args.attempt.streamedTriggerReason) { + if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) { + return null; + } + return { + reason: args.attempt.streamedTriggerReason.reason, + }; + } + + const trigger = detectCodexRotationTrigger(args.rotationMessage); + if (!trigger.shouldRotate) { + return null; + } + + return { reason: trigger.reason }; +} + +export type AttemptRetryAction = + | { + kind: 'claude'; + trigger: { reason: AgentTriggerReason; retryAfterMs?: number }; + rotationMessage?: string; + } + | { + kind: 'codex'; + trigger: { reason: CodexRotationReason }; + rotationMessage?: string; + } + | { kind: 'none' }; + +export function resolveAttemptRetryAction(args: { + provider: 'claude' | 'codex'; + canRetryClaudeCredentials: boolean; + canRetryCodex: boolean; + attempt: Pick; + rotationMessage?: string | null; +}): AttemptRetryAction { + const normalizedRotationMessage = args.rotationMessage ?? undefined; + + const claudeTrigger = resolveClaudeRetryTrigger({ + canRetryClaudeCredentials: args.canRetryClaudeCredentials, + provider: args.provider, + attempt: args.attempt, + fallbackMessage: normalizedRotationMessage, + }); + if (claudeTrigger) { + return { + kind: 'claude', + trigger: claudeTrigger, + rotationMessage: normalizedRotationMessage, + }; + } + + const codexTrigger = resolveCodexRetryTrigger({ + canRetryCodex: args.canRetryCodex, + attempt: args.attempt, + rotationMessage: normalizedRotationMessage, + }); + if (codexTrigger) { + return { + kind: 'codex', + trigger: codexTrigger, + rotationMessage: normalizedRotationMessage, + }; + } + + return { kind: 'none' }; +} diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts index 315461e..bb14e43 100644 --- a/src/message-agent-executor-rules.test.ts +++ b/src/message-agent-executor-rules.test.ts @@ -23,6 +23,7 @@ import { classifyRotationTrigger } from './agent-error-detection.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import { isRetryableClaudeSessionFailureAttempt, + resolveAttemptRetryAction, resolveClaudeRetryTrigger, resolveCodexRetryTrigger, resolvePairedFollowUpQueueAction, @@ -88,6 +89,47 @@ describe('message-agent-executor-rules', () => { expect(detectCodexRotationTrigger).not.toHaveBeenCalled(); }); + it('resolves a shared Claude retry action from fallback output text', () => { + vi.mocked(classifyRotationTrigger).mockReturnValue({ + shouldRetry: true, + reason: '429', + retryAfterMs: 30000, + }); + + expect( + resolveAttemptRetryAction({ + provider: 'claude', + canRetryClaudeCredentials: true, + canRetryCodex: false, + attempt: { sawOutput: false }, + rotationMessage: '429 rate limit', + }), + ).toEqual({ + kind: 'claude', + trigger: { reason: '429', retryAfterMs: 30000 }, + rotationMessage: '429 rate limit', + }); + }); + + it('resolves a shared Codex retry action from streamed state', () => { + expect( + resolveAttemptRetryAction({ + provider: 'codex', + canRetryClaudeCredentials: false, + canRetryCodex: true, + attempt: { + sawOutput: false, + streamedTriggerReason: { reason: 'auth-expired' }, + }, + rotationMessage: 'oauth token expired', + }), + ).toEqual({ + kind: 'codex', + trigger: { reason: 'auth-expired' }, + rotationMessage: 'oauth token expired', + }); + }); + it('detects retryable Claude session failures from either flag or classifier', () => { expect( isRetryableClaudeSessionFailureAttempt({ diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index de2ca9b..daef4f8 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -1,116 +1,14 @@ -import { - classifyRotationTrigger, - type AgentTriggerReason, - type CodexRotationReason, - isCodexRotationReason, -} from './agent-error-detection.js'; -import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import { resolveNextTurnAction } from './message-runtime-rules.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js'; -import { getErrorMessage } from './utils.js'; - -export interface ExecutorStreamedTrigger { - reason: AgentTriggerReason; - retryAfterMs?: number; -} - -export interface ExecutorAttemptState { - sawOutput: boolean; - retryableSessionFailureDetected?: boolean; - streamedTriggerReason?: ExecutorStreamedTrigger; - error?: unknown; -} - -export function isRetryableClaudeSessionFailureAttempt(args: { - attempt: ExecutorAttemptState; - isClaudeCodeAgent: boolean; - provider: 'claude' | 'codex'; - shouldRetryFreshSessionOnAgentFailure: (args: { - result: null; - error: string; - }) => boolean; -}): boolean { - if ( - !args.isClaudeCodeAgent || - args.provider !== 'claude' || - args.attempt.sawOutput - ) { - return false; - } - - if (args.attempt.retryableSessionFailureDetected === true) { - return true; - } - - if (args.attempt.error == null) { - return false; - } - - return args.shouldRetryFreshSessionOnAgentFailure({ - result: null, - error: getErrorMessage(args.attempt.error), - }); -} - -export function resolveClaudeRetryTrigger(args: { - canRetryClaudeCredentials: boolean; - provider: 'claude' | 'codex'; - attempt: Pick; - fallbackMessage?: string | null; -}): { reason: AgentTriggerReason; retryAfterMs?: number } | null { - if ( - !( - args.canRetryClaudeCredentials && - args.provider === 'claude' && - !args.attempt.sawOutput - ) - ) { - return null; - } - - if (args.attempt.streamedTriggerReason) { - return { - reason: args.attempt.streamedTriggerReason.reason, - retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs, - }; - } - - const trigger = classifyRotationTrigger(args.fallbackMessage); - if (!trigger.shouldRetry) { - return null; - } - - return { - reason: trigger.reason, - retryAfterMs: trigger.retryAfterMs, - }; -} - -export function resolveCodexRetryTrigger(args: { - canRetryCodex: boolean; - attempt: Pick; - rotationMessage?: string | null; -}): { reason: CodexRotationReason } | null { - if (!args.canRetryCodex) { - return null; - } - - if (args.attempt.streamedTriggerReason) { - if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) { - return null; - } - return { - reason: args.attempt.streamedTriggerReason.reason, - }; - } - - const trigger = detectCodexRotationTrigger(args.rotationMessage); - if (!trigger.shouldRotate) { - return null; - } - - return { reason: trigger.reason }; -} +export { + isRetryableClaudeSessionFailureAttempt, + resolveAttemptRetryAction, + resolveClaudeRetryTrigger, + resolveCodexRetryTrigger, + type AttemptRetryAction, + type AttemptRetryState as ExecutorAttemptState, + type AttemptStreamedTrigger as ExecutorStreamedTrigger, +} from './agent-attempt-retry.js'; export type PairedFollowUpQueueAction = | 'generic' diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 003a9e2..37db074 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -5,6 +5,10 @@ import { getErrorMessage } from './utils.js'; import { getAgentOutputText } from './agent-output.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js'; +import { + isRetryableClaudeSessionFailureAttempt, + resolveAttemptRetryAction, +} from './agent-attempt-retry.js'; import { AgentOutput, runAgentProcess, @@ -31,9 +35,6 @@ import { import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; import { resolveExecutionTarget } from './message-runtime-rules.js'; import { - isRetryableClaudeSessionFailureAttempt, - resolveClaudeRetryTrigger, - resolveCodexRetryTrigger, resolvePairedFollowUpQueueAction, } from './message-agent-executor-rules.js'; import { buildRoomRoleContext } from './room-role-context.js'; @@ -683,22 +684,23 @@ export async function runAgentForGroup( attempt: AgentAttempt, rotationMessage?: string | null, ): Promise<'success' | 'error' | null> => { - const trigger = resolveClaudeRetryTrigger({ - canRetryClaudeCredentials, + const retryAction = resolveAttemptRetryAction({ provider, + canRetryClaudeCredentials, + canRetryCodex: false, attempt, - fallbackMessage: rotationMessage, + rotationMessage, }); - if (!trigger) { + if (retryAction.kind !== 'claude') { return null; } const result = await retryClaudeWithRotation( - trigger, - rotationMessage ?? undefined, + retryAction.trigger, + retryAction.rotationMessage, ); if (result === 'error') { - return maybeHandoffAfterError(trigger.reason, attempt); + return maybeHandoffAfterError(retryAction.trigger.reason, attempt); } pairedExecutionStatus = 'succeeded'; @@ -709,18 +711,20 @@ export async function runAgentForGroup( attempt: AgentAttempt, rotationMessage?: string | null, ): Promise<'success' | 'error' | null> => { - const trigger = resolveCodexRetryTrigger({ + const retryAction = resolveAttemptRetryAction({ + provider, + canRetryClaudeCredentials: false, canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1, attempt, rotationMessage, }); - if (!trigger) { + if (retryAction.kind !== 'codex') { return null; } const result = await retryCodexWithRotation( - { reason: trigger.reason }, - rotationMessage ?? undefined, + retryAction.trigger, + retryAction.rotationMessage, ); if (result === 'success') { pairedExecutionStatus = 'succeeded'; diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts index b6c29fe..7f7311a 100644 --- a/src/message-runtime-flow.ts +++ b/src/message-runtime-flow.ts @@ -8,6 +8,7 @@ import { buildArbiterPromptForTask, buildFinalizePendingPrompt, buildOwnerPendingPrompt, + buildPairedTurnPrompt, buildReviewerPendingPrompt, } from './message-runtime-prompts.js'; import { @@ -47,6 +48,26 @@ export type BotOnlyPairedFollowUpAction = nextRole: 'owner' | 'reviewer' | 'arbiter'; }; +export type QueuedTurnDispatch = { + formatted: string; + botOnlyFollowUpAction: BotOnlyPairedFollowUpAction; + isBotOnlyPairedFollowUp: boolean; + loopCursorKey: string; + endSeq: number | null; +}; + +export function isBotOnlyPairedRoomTurn( + chatJid: string, + messages: NewMessage[], +): boolean { + return ( + hasReviewerLease(chatJid) && + messages.every( + (message) => message.is_from_me === true || !!message.is_bot_message, + ) + ); +} + export function resolveLastDeliveredBotRole( messages: NewMessage[], ): PairedRoomRole | null { @@ -384,6 +405,54 @@ export async function executeBotOnlyPairedFollowUpAction(args: { return true; } +export function buildQueuedTurnDispatch(args: { + chatJid: string; + timezone: string; + loopPendingTask: PairedTask | null | undefined; + rawPendingMessages: NewMessage[]; + messagesToSend: NewMessage[]; + labeledMessagesToSend: NewMessage[]; + formatMessages: (messages: NewMessage[], timezone: string) => string; +}): QueuedTurnDispatch { + const loopCursorKey = resolveCursorKey( + args.chatJid, + args.loopPendingTask?.status, + ); + const formatted = args.loopPendingTask + ? buildPairedTurnPrompt({ + taskId: args.loopPendingTask.id, + chatJid: args.chatJid, + timezone: args.timezone, + missedMessages: args.messagesToSend, + labeledFallbackMessages: args.labeledMessagesToSend, + turnOutputs: getPairedTurnOutputs(args.loopPendingTask.id), + }) + : args.formatMessages(args.labeledMessagesToSend, args.timezone); + const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn( + args.chatJid, + args.messagesToSend, + ); + const pendingCursorSource = + args.rawPendingMessages.length > 0 + ? args.rawPendingMessages[args.rawPendingMessages.length - 1] + : args.messagesToSend[args.messagesToSend.length - 1]; + const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({ + chatJid: args.chatJid, + task: args.loopPendingTask, + isBotOnlyPairedFollowUp, + lastDeliveredMessages: args.labeledMessagesToSend, + pendingCursorSource, + }); + + return { + formatted, + botOnlyFollowUpAction, + isBotOnlyPairedFollowUp, + loopCursorKey, + endSeq: args.messagesToSend[args.messagesToSend.length - 1]?.seq ?? null, + }; +} + export function shouldSkipGenericFollowUpAfterDeliveryRetry(args: { chatJid: string; deliveryRole: PairedRoomRole; diff --git a/src/message-runtime.ts b/src/message-runtime.ts index fde7752..32a8880 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -43,11 +43,11 @@ import { } from './message-runtime-rules.js'; import { runAgentForGroup } from './message-agent-executor.js'; import { + buildQueuedTurnDispatch, buildPendingPairedTurn, executeBotOnlyPairedFollowUpAction, executePendingPairedTurn, - resolveBotOnlyPairedFollowUpAction, - resolveLastDeliveredBotRole, + isBotOnlyPairedRoomTurn, shouldSkipGenericFollowUpAfterDeliveryRetry, } from './message-runtime-flow.js'; import { MessageTurnController } from './message-turn-controller.js'; @@ -58,6 +58,7 @@ import { buildPairedTurnPrompt, buildReviewerPendingPrompt, } from './message-runtime-prompts.js'; +import { transitionPairedTaskStatus } from './paired-execution-context-shared.js'; import { extractSessionCommand, handleSessionCommand, @@ -207,15 +208,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }); }; - const isBotOnlyPairedRoomTurn = ( - chatJid: string, - messages: NewMessage[], - ): boolean => - hasReviewerLease(chatJid) && - messages.every( - (message) => message.is_from_me === true || !!message.is_bot_message, - ); - const enqueueScopedGroupMessageCheck = ( chatJid: string, groupFolder: string, @@ -937,10 +929,15 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { if (hasReviewerLease(chatJid)) { const task = getLatestOpenPairedTaskForChat(chatJid); if (task) { - updatePairedTask(task.id, { - status: 'completed', - completion_reason: 'stopped', - updated_at: new Date().toISOString(), + const now = new Date().toISOString(); + transitionPairedTaskStatus({ + taskId: task.id, + currentStatus: task.status, + nextStatus: 'completed', + updatedAt: now, + patch: { + completion_reason: 'stopped', + }, }); } } @@ -1223,30 +1220,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { chatJid, messagesToSend, ); - const formatted = loopPendingTask - ? buildPairedTurnPrompt({ - taskId: loopPendingTask.id, - chatJid, - timezone: deps.timezone, - missedMessages: messagesToSend, - labeledFallbackMessages: labeledMessagesToSend, - turnOutputs: getPairedTurnOutputs(loopPendingTask.id), - }) - : formatMessages(labeledMessagesToSend, deps.timezone); - const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn( - chatJid, - messagesToSend, - ); - const pendingCursorSource = - rawPendingMessages.length > 0 - ? rawPendingMessages[rawPendingMessages.length - 1] - : messagesToSend[messagesToSend.length - 1]; - const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({ - chatJid, - task: loopPendingTask, + const { + formatted, + botOnlyFollowUpAction, isBotOnlyPairedFollowUp, - lastDeliveredMessages: labeledMessagesToSend, - pendingCursorSource, + loopCursorKey: dispatchCursorKey, + endSeq, + } = buildQueuedTurnDispatch({ + chatJid, + timezone: deps.timezone, + loopPendingTask, + rawPendingMessages, + messagesToSend, + labeledMessagesToSend, + formatMessages, }); if ( await executeBotOnlyPairedFollowUpAction({ @@ -1271,14 +1258,13 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { } if (deps.queue.sendMessage(chatJid, formatted)) { - const endSeq = messagesToSend[messagesToSend.length - 1]?.seq; if (endSeq != null) { advanceLastAgentCursor( deps.getLastAgentTimestamps(), deps.saveState, chatJid, endSeq, - loopCursorKey, + dispatchCursorKey, ); } logger.debug( diff --git a/src/paired-execution-context-arbiter.ts b/src/paired-execution-context-arbiter.ts index de46e5b..04c33f1 100644 --- a/src/paired-execution-context-arbiter.ts +++ b/src/paired-execution-context-arbiter.ts @@ -1,7 +1,9 @@ import { ARBITER_DEADLOCK_THRESHOLD } from './config.js'; -import { updatePairedTask } from './db.js'; import { logger } from './logger.js'; -import { classifyArbiterVerdict } from './paired-execution-context-shared.js'; +import { + classifyArbiterVerdict, + transitionPairedTaskStatus, +} from './paired-execution-context-shared.js'; import type { PairedTask } from './types.js'; export function handleFailedArbiterExecution(args: { @@ -15,7 +17,12 @@ export function handleFailedArbiterExecution(args: { ? 'arbiter_requested' : task.status; if (fallbackStatus !== task.status) { - updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: fallbackStatus, + updatedAt: now, + }); logger.warn( { taskId, @@ -45,11 +52,15 @@ export function handleArbiterCompletion(args: { case 'proceed': case 'revise': case 'reset': - updatePairedTask(taskId, { - status: 'active', - round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), - arbiter_verdict: arbiterVerdict, - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus: 'in_arbitration', + nextStatus: 'active', + updatedAt: now, + patch: { + round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), + arbiter_verdict: arbiterVerdict, + }, }); logger.info( { taskId, arbiterVerdict }, @@ -57,20 +68,28 @@ export function handleArbiterCompletion(args: { ); return; case 'escalate': - updatePairedTask(taskId, { - status: 'completed', - arbiter_verdict: 'escalate', - completion_reason: 'arbiter_escalated', - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus: 'in_arbitration', + nextStatus: 'completed', + updatedAt: now, + patch: { + arbiter_verdict: 'escalate', + completion_reason: 'arbiter_escalated', + }, }); logger.info({ taskId }, 'Arbiter escalated to user — task completed'); return; default: - updatePairedTask(taskId, { - status: 'active', - round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), - arbiter_verdict: 'unknown', - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus: 'in_arbitration', + nextStatus: 'active', + updatedAt: now, + patch: { + round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), + arbiter_verdict: 'unknown', + }, }); logger.warn( { taskId, summary: summary?.slice(0, 200) }, diff --git a/src/paired-execution-context-owner.ts b/src/paired-execution-context-owner.ts index 186b3f4..a7dca12 100644 --- a/src/paired-execution-context-owner.ts +++ b/src/paired-execution-context-owner.ts @@ -14,6 +14,7 @@ import { hasCodeChangesSinceRef, requestArbiterOrEscalate, resolveCanonicalSourceRef, + transitionPairedTaskStatus, } from './paired-execution-context-shared.js'; import type { PairedTask } from './types.js'; @@ -24,7 +25,12 @@ export function handleFailedOwnerExecution(args: { const { task, taskId } = args; if (task.status !== 'active') { const now = new Date().toISOString(); - updatePairedTask(taskId, { status: 'active', updated_at: now }); + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: 'active', + updatedAt: now, + }); logger.info( { taskId, role: 'owner', previousStatus: task.status }, 'Reset task to active after failed execution', @@ -44,6 +50,7 @@ function handleOwnerFinalizeCompletion(args: { if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { requestArbiterOrEscalate({ taskId, + currentStatus: task.status, now, arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter', escalateLogMessage: 'Owner blocked during finalize — escalating to user', @@ -60,6 +67,7 @@ function handleOwnerFinalizeCompletion(args: { if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { requestArbiterOrEscalate({ taskId, + currentStatus: task.status, now, arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter', escalateLogMessage: 'Owner finalize loop detected — escalating to user', @@ -71,7 +79,12 @@ function handleOwnerFinalizeCompletion(args: { }); return true; } - updatePairedTask(taskId, { status: 'active', updated_at: now }); + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: 'active', + updatedAt: now, + }); logger.info( { taskId, @@ -92,6 +105,7 @@ function handleOwnerFinalizeCompletion(args: { if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { requestArbiterOrEscalate({ taskId, + currentStatus: task.status, now, arbiterLogMessage: 'Owner finalize DONE loop detected — requesting arbiter', @@ -116,10 +130,14 @@ function handleOwnerFinalizeCompletion(args: { return false; } - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'done', - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: 'completed', + updatedAt: now, + patch: { + completion_reason: 'done', + }, }); logger.info( { taskId, hasNewChanges, summary: summary?.slice(0, 100) }, @@ -161,6 +179,7 @@ export function handleOwnerCompletion(args: { if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { requestArbiterOrEscalate({ taskId, + currentStatus: task.status, now, arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter', escalateLogMessage: 'Owner blocked/needs_context — escalating to user', diff --git a/src/paired-execution-context-reviewer.ts b/src/paired-execution-context-reviewer.ts index 2ebfe19..b3dc496 100644 --- a/src/paired-execution-context-reviewer.ts +++ b/src/paired-execution-context-reviewer.ts @@ -5,6 +5,7 @@ import { classifyVerdict, requestArbiterOrEscalate, resolveCanonicalSourceRef, + transitionPairedTaskStatus, } from './paired-execution-context-shared.js'; import type { PairedTask } from './types.js'; @@ -29,11 +30,15 @@ export function handleFailedReviewerExecution(args: { verdict === 'done' && ownerWs?.workspace_dir ? resolveCanonicalSourceRef(ownerWs.workspace_dir) : task.source_ref; - updatePairedTask(taskId, { - status: verdict === 'done' ? 'merge_ready' : 'completed', - ...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), - ...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: verdict === 'done' ? 'merge_ready' : 'completed', + updatedAt: now, + patch: { + ...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), + ...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), + }, }); logger.info( { @@ -53,7 +58,12 @@ export function handleFailedReviewerExecution(args: { ? 'review_ready' : task.status; if (fallbackStatus !== task.status) { - updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: fallbackStatus, + updatedAt: now, + }); logger.warn( { taskId, @@ -81,10 +91,14 @@ export function handleReviewerCompletion(args: { const approvedSourceRef = ownerWs?.workspace_dir ? resolveCanonicalSourceRef(ownerWs.workspace_dir) : task.source_ref; - updatePairedTask(taskId, { - status: 'merge_ready', - source_ref: approvedSourceRef, - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: 'merge_ready', + updatedAt: now, + patch: { + source_ref: approvedSourceRef, + }, }); logger.info( { @@ -102,6 +116,7 @@ export function handleReviewerCompletion(args: { case 'needs_context': requestArbiterOrEscalate({ taskId, + currentStatus: task.status, now, arbiterLogMessage: 'Reviewer blocked/needs_context — requesting arbiter before escalating', @@ -116,6 +131,7 @@ export function handleReviewerCompletion(args: { if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { requestArbiterOrEscalate({ taskId, + currentStatus: task.status, now, arbiterLogMessage: 'Deadlock detected — requesting arbiter intervention', @@ -129,7 +145,12 @@ export function handleReviewerCompletion(args: { }); return; } - updatePairedTask(taskId, { status: 'active', updated_at: now }); + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: 'active', + updatedAt: now, + }); logger.info( { taskId, verdict }, 'Reviewer has feedback, task set back to active for owner', diff --git a/src/paired-execution-context-shared.ts b/src/paired-execution-context-shared.ts index a2e7a2f..59a1fd8 100644 --- a/src/paired-execution-context-shared.ts +++ b/src/paired-execution-context-shared.ts @@ -3,6 +3,7 @@ import { execFileSync } from 'child_process'; import { isArbiterEnabled } from './config.js'; import { updatePairedTask } from './db.js'; import { logger } from './logger.js'; +import type { PairedTaskStatus } from './types.js'; export type Verdict = | 'done' @@ -93,29 +94,108 @@ export function hasCodeChangesSinceRef( } } +const ALLOWED_PAIRED_STATUS_TRANSITIONS: Record< + PairedTaskStatus, + ReadonlySet +> = { + active: new Set(['review_ready', 'arbiter_requested', 'completed']), + review_ready: new Set([ + 'active', + 'in_review', + 'arbiter_requested', + 'completed', + ]), + in_review: new Set([ + 'active', + 'review_ready', + 'merge_ready', + 'arbiter_requested', + 'completed', + ]), + merge_ready: new Set(['active', 'arbiter_requested', 'completed']), + completed: new Set(), + arbiter_requested: new Set(['in_arbitration', 'completed']), + in_arbitration: new Set(['active', 'arbiter_requested', 'completed']), +}; + +export function assertPairedTaskStatusTransition(args: { + currentStatus: PairedTaskStatus; + nextStatus: PairedTaskStatus; +}): void { + const { currentStatus, nextStatus } = args; + if (currentStatus === nextStatus) { + return; + } + + if (ALLOWED_PAIRED_STATUS_TRANSITIONS[currentStatus].has(nextStatus)) { + return; + } + + throw new Error( + `Invalid paired task status transition: ${currentStatus} -> ${nextStatus}`, + ); +} + +export function transitionPairedTaskStatus(args: { + taskId: string; + currentStatus: PairedTaskStatus; + nextStatus: PairedTaskStatus; + updatedAt: string; + patch?: Omit< + Parameters[1], + 'status' | 'updated_at' + >; +}): void { + assertPairedTaskStatusTransition({ + currentStatus: args.currentStatus, + nextStatus: args.nextStatus, + }); + + updatePairedTask(args.taskId, { + ...args.patch, + status: args.nextStatus, + updated_at: args.updatedAt, + }); +} + export function requestArbiterOrEscalate(args: { taskId: string; + currentStatus: PairedTaskStatus; now: string; arbiterLogMessage: string; escalateLogMessage: string; logContext?: Record; }): void { - const { taskId, now, arbiterLogMessage, escalateLogMessage, logContext } = - args; + const { + taskId, + currentStatus, + now, + arbiterLogMessage, + escalateLogMessage, + logContext, + } = args; if (isArbiterEnabled()) { - updatePairedTask(taskId, { - status: 'arbiter_requested', - arbiter_requested_at: now, - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus, + nextStatus: 'arbiter_requested', + updatedAt: now, + patch: { + arbiter_requested_at: now, + }, }); logger.info(logContext ?? { taskId }, arbiterLogMessage); return; } - updatePairedTask(taskId, { - status: 'completed', - completion_reason: 'escalated', - updated_at: now, + transitionPairedTaskStatus({ + taskId, + currentStatus, + nextStatus: 'completed', + updatedAt: now, + patch: { + completion_reason: 'escalated', + }, }); logger.info(logContext ?? { taskId }, escalateLogMessage); } diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index 38d8094..ff07c62 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -38,6 +38,7 @@ import { import { resolveCanonicalSourceRef, requestArbiterOrEscalate, + transitionPairedTaskStatus, } from './paired-execution-context-shared.js'; import { markPairedTaskReviewReady, @@ -226,11 +227,22 @@ export function preparePairedExecutionContext(args: { latestTask.status === 'review_ready' || latestTask.status === 'in_review'; if (hasHuman || needsStatusReset) { - updatePairedTask(latestTask.id, { - ...(hasHuman ? { round_trip_count: 0 } : {}), - ...(needsStatusReset ? { status: 'active' as const } : {}), - updated_at: now, - }); + if (needsStatusReset) { + transitionPairedTaskStatus({ + taskId: latestTask.id, + currentStatus: latestTask.status, + nextStatus: 'active', + updatedAt: now, + patch: { + ...(hasHuman ? { round_trip_count: 0 } : {}), + }, + }); + } else { + updatePairedTask(latestTask.id, { + ...(hasHuman ? { round_trip_count: 0 } : {}), + updated_at: now, + }); + } } // Use a stable per-channel worktree (not per-task) so the Claude SDK // session persists across tasks. Different channels still get isolation. @@ -253,9 +265,11 @@ export function preparePairedExecutionContext(args: { blockMessage = reviewerWorkspace.blockMessage; const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask; if (workspace && refreshedTask.status === 'review_ready') { - updatePairedTask(latestTask.id, { - status: 'in_review', - updated_at: now, + transitionPairedTaskStatus({ + taskId: latestTask.id, + currentStatus: refreshedTask.status, + nextStatus: 'in_review', + updatedAt: now, }); } } else if (roomRoleContext.role === 'arbiter') { @@ -265,9 +279,11 @@ export function preparePairedExecutionContext(args: { blockMessage = reviewerWorkspace.blockMessage; const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask; if (workspace && refreshedTask.status === 'arbiter_requested') { - updatePairedTask(latestTask.id, { - status: 'in_arbitration', - updated_at: now, + transitionPairedTaskStatus({ + taskId: latestTask.id, + currentStatus: refreshedTask.status, + nextStatus: 'in_arbitration', + updatedAt: now, }); } } diff --git a/src/paired-workspace-manager.ts b/src/paired-workspace-manager.ts index 3bfa486..327cfe8 100644 --- a/src/paired-workspace-manager.ts +++ b/src/paired-workspace-manager.ts @@ -13,6 +13,7 @@ import { } from './db.js'; import { resolvePairedTaskWorkspacePath } from './group-folder.js'; import { logger } from './logger.js'; +import { transitionPairedTaskStatus } from './paired-execution-context-shared.js'; import type { PairedTask, PairedWorkspace } from './types.js'; import { ensureWorkspaceDependenciesInstalled } from './workspace-package-manager.js'; @@ -798,9 +799,15 @@ export function markPairedTaskReviewReady(taskId: string): { ); const now = new Date().toISOString(); - updatePairedTask(taskId, { - status: 'review_ready', - updated_at: now, + const task = getPairedTaskById(taskId); + if (!task) { + return null; + } + transitionPairedTaskStatus({ + taskId, + currentStatus: task.status, + nextStatus: 'review_ready', + updatedAt: now, }); return { ownerWorkspace, reviewerWorkspace }; diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 2e0507c..a79bcc1 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -3,6 +3,7 @@ import { CronExpressionParser } from 'cron-parser'; import fs from 'fs'; import { getAgentOutputText } from './agent-output.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js'; +import { resolveAttemptRetryAction } from './agent-attempt-retry.js'; import { getErrorMessage } from './utils.js'; import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js'; @@ -42,7 +43,6 @@ import { classifyRotationTrigger, type AgentTriggerReason, type CodexRotationReason, - isCodexRotationReason, } from './agent-error-detection.js'; import { getTokenCount, @@ -512,65 +512,34 @@ async function runTask( } }; - const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude'; + const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude'; - { - const attempt = await runTaskAttempt(provider); - result = attempt.attemptResult; - error = attempt.attemptError; + { + const attempt = await runTaskAttempt(provider); + result = attempt.attemptResult; + error = attempt.attemptError; - if ( - provider === 'claude' && - attempt.streamedTriggerReason && - !attempt.sawOutput - ) { - await retryClaudeTaskWithRotation(attempt.streamedTriggerReason); - } else if ( - provider === 'codex' && - attempt.streamedTriggerReason && - !attempt.sawOutput && - isCodexRotationReason(attempt.streamedTriggerReason.reason) - ) { - await retryCodexTaskWithRotation( - { - reason: attempt.streamedTriggerReason.reason, - }, - typeof attempt.output.error === 'string' - ? attempt.output.error - : undefined, - ); - } else if (attempt.output.status === 'error' && provider === 'claude') { - const trigger = attempt.streamedTriggerReason - ? { - shouldRetry: true, - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - } - : classifyRotationTrigger(error); - if (trigger.shouldRetry) { - await retryClaudeTaskWithRotation({ - reason: trigger.reason, - retryAfterMs: trigger.retryAfterMs, - }); - } - } else if (attempt.output.status === 'error' && provider === 'codex') { - const trigger = - attempt.streamedTriggerReason && - isCodexRotationReason(attempt.streamedTriggerReason.reason) - ? { - shouldRotate: true as const, - reason: attempt.streamedTriggerReason.reason, - } - : detectCodexRotationTrigger(error); - if (trigger.shouldRotate) { - await retryCodexTaskWithRotation( - { reason: trigger.reason }, - error || undefined, + const retryAction = resolveAttemptRetryAction({ + provider, + canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0, + canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1, + attempt, + rotationMessage: error, + }); + + if (retryAction.kind === 'claude') { + await retryClaudeTaskWithRotation( + retryAction.trigger, + retryAction.rotationMessage, ); + } else if (retryAction.kind === 'codex') { + await retryCodexTaskWithRotation( + retryAction.trigger, + retryAction.rotationMessage, + ); + } else if (attempt.output.status === 'error') { + error = attempt.attemptError || 'Unknown error'; } - } else if (attempt.output.status === 'error') { - error = attempt.attemptError || 'Unknown error'; - } } // end else (non-exhausted path) log.info( From 58b20fad6cba610909dc53868ac9063acdae342c Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 02:50:08 +0900 Subject: [PATCH 27/29] Format retry transition refactor files --- src/message-agent-executor.test.ts | 5 ++- src/message-agent-executor.ts | 4 +-- src/message-runtime.ts | 5 ++- src/paired-execution-context-shared.ts | 5 +-- src/task-scheduler.ts | 50 +++++++++++++------------- 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 11f30cd..8573b0b 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1410,9 +1410,8 @@ describe('runAgentForGroup room memory', () => { expect( vi.mocked(db.insertPairedTurnOutput).mock.invocationCallOrder[0], ).toBeLessThan( - vi.mocked( - pairedExecutionContext.completePairedExecutionContext, - ).mock.invocationCallOrder[0], + vi.mocked(pairedExecutionContext.completePairedExecutionContext).mock + .invocationCallOrder[0], ); }); diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 37db074..205dea0 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -34,9 +34,7 @@ import { } from './paired-execution-context.js'; import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; import { resolveExecutionTarget } from './message-runtime-rules.js'; -import { - resolvePairedFollowUpQueueAction, -} from './message-agent-executor-rules.js'; +import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js'; import { buildRoomRoleContext } from './room-role-context.js'; import { type AgentTriggerReason } from './agent-error-detection.js'; import { diff --git a/src/message-runtime.ts b/src/message-runtime.ts index 32a8880..d822a76 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -851,7 +851,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { chatJid, getRecentChatMessages(chatJid, 20), ), - lastDeliveredMessages: labelPairedSenders(chatJid, missedMessages), + lastDeliveredMessages: labelPairedSenders( + chatJid, + missedMessages, + ), resolveChannel, }) : null; diff --git a/src/paired-execution-context-shared.ts b/src/paired-execution-context-shared.ts index 59a1fd8..6c1266b 100644 --- a/src/paired-execution-context-shared.ts +++ b/src/paired-execution-context-shared.ts @@ -141,10 +141,7 @@ export function transitionPairedTaskStatus(args: { currentStatus: PairedTaskStatus; nextStatus: PairedTaskStatus; updatedAt: string; - patch?: Omit< - Parameters[1], - 'status' | 'updated_at' - >; + patch?: Omit[1], 'status' | 'updated_at'>; }): void { assertPairedTaskStatusTransition({ currentStatus: args.currentStatus, diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index a79bcc1..ff44b0a 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -512,34 +512,34 @@ async function runTask( } }; - const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude'; + const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude'; - { - const attempt = await runTaskAttempt(provider); - result = attempt.attemptResult; - error = attempt.attemptError; + { + const attempt = await runTaskAttempt(provider); + result = attempt.attemptResult; + error = attempt.attemptError; - const retryAction = resolveAttemptRetryAction({ - provider, - canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0, - canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1, - attempt, - rotationMessage: error, - }); + const retryAction = resolveAttemptRetryAction({ + provider, + canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0, + canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1, + attempt, + rotationMessage: error, + }); - if (retryAction.kind === 'claude') { - await retryClaudeTaskWithRotation( - retryAction.trigger, - retryAction.rotationMessage, - ); - } else if (retryAction.kind === 'codex') { - await retryCodexTaskWithRotation( - retryAction.trigger, - retryAction.rotationMessage, - ); - } else if (attempt.output.status === 'error') { - error = attempt.attemptError || 'Unknown error'; - } + if (retryAction.kind === 'claude') { + await retryClaudeTaskWithRotation( + retryAction.trigger, + retryAction.rotationMessage, + ); + } else if (retryAction.kind === 'codex') { + await retryCodexTaskWithRotation( + retryAction.trigger, + retryAction.rotationMessage, + ); + } else if (attempt.output.status === 'error') { + error = attempt.attemptError || 'Unknown error'; + } } // end else (non-exhausted path) log.info( From ae12b157be8ea937bae952ec942ea41a8a3cc2dc Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 03:38:16 +0900 Subject: [PATCH 28/29] Persist paired turn output before final delivery --- src/message-agent-executor.test.ts | 8 ++++- src/message-agent-executor.ts | 51 +++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 8573b0b..75b2a10 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -1318,6 +1318,7 @@ describe('runAgentForGroup room memory', () => { it('stores reviewer turn output before transitioning the paired task back to active', async () => { const group = { ...makeGroup(), folder: 'test-group' }; const deps = makeDeps(); + const onOutput = vi.fn(async () => {}); vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ chat_jid: 'group@test', @@ -1397,7 +1398,7 @@ describe('runAgentForGroup room memory', () => { chatJid: 'group@test', runId: 'run-reviewer-output-order', forcedRole: 'reviewer', - onOutput: async () => {}, + onOutput, }); expect(result).toBe('success'); @@ -1407,6 +1408,11 @@ describe('runAgentForGroup room memory', () => { 'reviewer', 'DONE_WITH_CONCERNS\nreviewer feedback', ); + expect( + vi.mocked(db.insertPairedTurnOutput).mock.invocationCallOrder[0], + ).toBeLessThan( + onOutput.mock.invocationCallOrder[0], + ); expect( vi.mocked(db.insertPairedTurnOutput).mock.invocationCallOrder[0], ).toBeLessThan( diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 205dea0..b165207 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -315,9 +315,29 @@ export async function runAgentForGroup( const effectivePrompt = moaEnrichedPrompt; let pairedExecutionStatus: 'succeeded' | 'failed' = 'failed'; let pairedExecutionSummary: string | null = null; - let pairedFullOutput: string | null = null; + let pairedFinalOutput: string | null = null; let pairedExecutionCompleted = false; let pairedSawOutput = false; + let pairedTurnOutputPersisted = false; + + const persistPairedTurnOutputIfNeeded = (completedRole: PairedRoomRole) => { + if ( + !pairedExecutionContext || + pairedTurnOutputPersisted || + !pairedFinalOutput || + pairedFinalOutput.length === 0 + ) { + return; + } + const turnNumber = getLatestTurnNumber(pairedExecutionContext.task.id) + 1; + insertPairedTurnOutput( + pairedExecutionContext.task.id, + turnNumber, + completedRole, + pairedFinalOutput, + ); + pairedTurnOutputPersisted = true; + }; const maybeHandoffToCodex = ( reason: AgentTriggerReason, @@ -483,7 +503,6 @@ export async function runAgentForGroup( if (outputText && outputText.length > 0) { pairedExecutionSummary = outputText.slice(0, 500); - pairedFullOutput = outputText; } else if ( typeof output.error === 'string' && output.error.length > 0 @@ -542,6 +561,22 @@ export async function runAgentForGroup( if (outputText && outputText.length > 0) { streamedOutputHandler.markVisibleOutput(); } + if ( + outputPhase === 'final' && + output.status === 'success' && + outputText && + outputText.length > 0 + ) { + pairedFinalOutput = outputText; + try { + persistPairedTurnOutputIfNeeded(roomRoleContext?.role ?? 'owner'); + } catch (err) { + log.warn( + { pairedTaskId: pairedExecutionContext?.task.id ?? null, err }, + 'Failed to persist paired turn output before delivery', + ); + } + } if (onOutput) { await onOutput(output); } @@ -941,17 +976,9 @@ export async function runAgentForGroup( !pairedSawOutput ? 'failed' : completionStatus; - // Store full output for direct inter-agent data passing (Discord-independent). - if (pairedFullOutput && effectiveStatus === 'succeeded') { + if (effectiveStatus === 'succeeded') { try { - const turnNumber = - getLatestTurnNumber(pairedExecutionContext.task.id) + 1; - insertPairedTurnOutput( - pairedExecutionContext.task.id, - turnNumber, - completedRole, - pairedFullOutput, - ); + persistPairedTurnOutputIfNeeded(completedRole); } catch (err) { log.warn( { pairedTaskId: pairedExecutionContext.task.id, err }, From 10e10b499ca624e959628cb83ba9b9a957959521 Mon Sep 17 00:00:00 2001 From: ejclaw Date: Tue, 7 Apr 2026 05:12:22 +0900 Subject: [PATCH 29/29] Complete paired-room SSOT cleanup and logger singleton --- src/agent-attempt-orchestration.ts | 182 +++++++++++++ src/codex-token-rotation.test.ts | 21 ++ src/codex-token-rotation.ts | 7 +- src/logger.test.ts | 33 +++ src/logger.ts | 46 +++- src/message-agent-executor.ts | 82 ++---- src/message-runtime-flow.ts | 32 +-- src/message-runtime.test.ts | 118 -------- src/message-runtime.ts | 363 +++++++++++++------------ src/paired-execution-context-owner.ts | 12 +- src/paired-execution-context-shared.ts | 11 + src/paired-execution-context.test.ts | 1 - src/paired-execution-context.ts | 20 +- src/paired-workspace-manager.test.ts | 25 ++ src/paired-workspace-manager.ts | 11 +- src/task-scheduler.ts | 86 +++--- src/token-rotation.test.ts | 40 +++ src/token-rotation.ts | 31 ++- 18 files changed, 648 insertions(+), 473 deletions(-) create mode 100644 src/agent-attempt-orchestration.ts create mode 100644 src/logger.test.ts diff --git a/src/agent-attempt-orchestration.ts b/src/agent-attempt-orchestration.ts new file mode 100644 index 0000000..acf56b6 --- /dev/null +++ b/src/agent-attempt-orchestration.ts @@ -0,0 +1,182 @@ +import type { AgentOutput } from './agent-runner.js'; +import { + resolveAttemptRetryAction, + type AttemptRetryAction, + type AttemptRetryState, +} from './agent-attempt-retry.js'; +import type { + AgentTriggerReason, + CodexRotationReason, +} from './agent-error-detection.js'; +import { + runClaudeRotationLoop, + runCodexRotationLoop, +} from './provider-retry.js'; + +type RotationLogContext = Record; + +type ClaudeRetryAttemptState = Pick< + AttemptRetryState, + 'sawOutput' | 'streamedTriggerReason' +> & { + output?: Pick; + error?: unknown; + sawSuccessNullResultWithoutOutput?: boolean; +}; + +type CodexRetryAttemptState = Pick< + AttemptRetryState, + 'sawOutput' | 'streamedTriggerReason' +> & { + output?: Pick; + error?: unknown; +}; + +export type ExecutedAttemptRetryAction = + | { kind: 'none' } + | { + kind: 'claude'; + trigger: { reason: AgentTriggerReason; retryAfterMs?: number }; + rotationMessage?: string; + result: 'success' | 'error'; + } + | { + kind: 'codex'; + trigger: { reason: CodexRotationReason }; + rotationMessage?: string; + result: 'success' | 'error'; + }; + +export async function runClaudeAttemptWithRotation< + TAttempt extends ClaudeRetryAttemptState, +>(args: { + initialTrigger: { + reason: AgentTriggerReason; + retryAfterMs?: number; + }; + runAttempt: () => Promise; + logContext: RotationLogContext; + rotationMessage?: string; + afterAttempt?: (attempt: TAttempt) => Promise | void; + onSuccess?: (outcome: { sawOutput: boolean }) => Promise | void; +}): Promise<'success' | 'error'> { + const outcome = await runClaudeRotationLoop( + args.initialTrigger, + async () => { + const attempt = await args.runAttempt(); + await args.afterAttempt?.(attempt); + return { + output: attempt.output, + thrownError: attempt.error, + sawOutput: attempt.sawOutput, + sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput, + streamedTriggerReason: attempt.streamedTriggerReason, + }; + }, + args.logContext, + args.rotationMessage, + ); + + if (outcome.type === 'success') { + await args.onSuccess?.({ sawOutput: outcome.sawOutput }); + return 'success'; + } + + return 'error'; +} + +export async function runCodexAttemptWithRotation< + TAttempt extends CodexRetryAttemptState, +>(args: { + initialTrigger: { reason: CodexRotationReason }; + runAttempt: () => Promise; + logContext: RotationLogContext; + rotationMessage?: string; + afterAttempt?: (attempt: TAttempt) => Promise | void; +}): Promise<'success' | 'error'> { + const outcome = await runCodexRotationLoop( + args.initialTrigger, + async () => { + const attempt = await args.runAttempt(); + await args.afterAttempt?.(attempt); + return { + output: attempt.output, + thrownError: attempt.error, + sawOutput: attempt.sawOutput, + streamedTriggerReason: attempt.streamedTriggerReason, + }; + }, + args.logContext, + args.rotationMessage, + ); + + return outcome.type === 'success' ? 'success' : 'error'; +} + +export async function executeAttemptRetryAction(args: { + provider: 'claude' | 'codex'; + canRetryClaudeCredentials: boolean; + canRetryCodex: boolean; + attempt: Pick; + rotationMessage?: string | null; + runClaude: ( + trigger: { reason: AgentTriggerReason; retryAfterMs?: number }, + rotationMessage?: string, + ) => Promise<'success' | 'error'>; + runCodex: ( + trigger: { reason: CodexRotationReason }, + rotationMessage?: string, + ) => Promise<'success' | 'error'>; +}): Promise { + const retryAction = resolveAttemptRetryAction({ + provider: args.provider, + canRetryClaudeCredentials: args.canRetryClaudeCredentials, + canRetryCodex: args.canRetryCodex, + attempt: args.attempt, + rotationMessage: args.rotationMessage, + }); + + return executeResolvedAttemptRetryAction({ + retryAction, + runClaude: args.runClaude, + runCodex: args.runCodex, + }); +} + +async function executeResolvedAttemptRetryAction(args: { + retryAction: AttemptRetryAction; + runClaude: ( + trigger: { reason: AgentTriggerReason; retryAfterMs?: number }, + rotationMessage?: string, + ) => Promise<'success' | 'error'>; + runCodex: ( + trigger: { reason: CodexRotationReason }, + rotationMessage?: string, + ) => Promise<'success' | 'error'>; +}): Promise { + if (args.retryAction.kind === 'claude') { + return { + kind: 'claude', + trigger: args.retryAction.trigger, + rotationMessage: args.retryAction.rotationMessage, + result: await args.runClaude( + args.retryAction.trigger, + args.retryAction.rotationMessage, + ), + }; + } + + if (args.retryAction.kind === 'codex') { + return { + kind: 'codex', + trigger: args.retryAction.trigger, + rotationMessage: args.retryAction.rotationMessage, + result: await args.runCodex( + args.retryAction.trigger, + args.retryAction.rotationMessage, + ), + }; + } + + return { kind: 'none' }; +} diff --git a/src/codex-token-rotation.test.ts b/src/codex-token-rotation.test.ts index bdb89d5..8762a33 100644 --- a/src/codex-token-rotation.test.ts +++ b/src/codex-token-rotation.test.ts @@ -121,4 +121,25 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => { expect(active).toBeDefined(); expect(active!.index).toBe(1); // fallback picks next non-rate-limited }); + + it('warns when Codex rotation state cannot be persisted', async () => { + const mod = await import('./codex-token-rotation.js'); + const utils = await import('./utils.js'); + const { logger } = await import('./logger.js'); + + vi.mocked(utils.writeJsonFile).mockImplementation(() => { + throw new Error('disk full'); + }); + + mod.initCodexTokenRotation(); + expect(mod.rotateCodexToken('rate limit')).toBe(true); + + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + stateFile: '/tmp/ejclaw-codex-rot-data/codex-rotation-state.json', + err: expect.any(Error), + }), + 'Failed to persist Codex rotation state', + ); + }); }); diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index b0badc1..3c1c039 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -138,8 +138,11 @@ function saveCodexState(): void { resetD7Ats: accounts.map((a) => a.resetD7At ?? null), }; writeJsonFile(STATE_FILE, state); - } catch { - /* best effort */ + } catch (err) { + logger.warn( + { stateFile: STATE_FILE, err }, + 'Failed to persist Codex rotation state', + ); } } diff --git a/src/logger.test.ts b/src/logger.test.ts new file mode 100644 index 0000000..0428691 --- /dev/null +++ b/src/logger.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from 'vitest'; + +describe('logger singleton', () => { + it('reuses the root logger and does not add duplicate process listeners across module reloads', async () => { + vi.resetModules(); + + const beforeUncaught = process.listeners('uncaughtException').length; + const beforeUnhandled = process.listeners('unhandledRejection').length; + const beforeExit = process.listeners('exit').length; + + const first = await import('./logger.js'); + + const afterFirstUncaught = process.listeners('uncaughtException').length; + const afterFirstUnhandled = process.listeners('unhandledRejection').length; + const afterFirstExit = process.listeners('exit').length; + + vi.resetModules(); + + const second = await import('./logger.js'); + + const afterSecondUncaught = process.listeners('uncaughtException').length; + const afterSecondUnhandled = process.listeners('unhandledRejection').length; + const afterSecondExit = process.listeners('exit').length; + + expect(second.logger).toBe(first.logger); + expect(afterFirstUncaught).toBeGreaterThanOrEqual(beforeUncaught); + expect(afterFirstUnhandled).toBeGreaterThanOrEqual(beforeUnhandled); + expect(afterSecondUncaught).toBe(afterFirstUncaught); + expect(afterSecondUnhandled).toBe(afterFirstUnhandled); + expect(afterFirstExit).toBeGreaterThanOrEqual(beforeExit); + expect(afterSecondExit).toBe(afterFirstExit); + }); +}); diff --git a/src/logger.ts b/src/logger.ts index 398583b..2da75a8 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,12 +1,34 @@ import pino, { type Logger } from 'pino'; const serviceName = (process.env.ASSISTANT_NAME || 'claude').toLowerCase(); +const isTestEnv = + process.env.VITEST === 'true' || process.env.NODE_ENV === 'test'; -export const logger = pino({ - level: process.env.LOG_LEVEL || 'info', - name: serviceName, - transport: { target: 'pino-pretty', options: { colorize: true } }, -}); +type LoggerGlobalState = typeof globalThis & { + __ejclawLogger?: Logger; + __ejclawProcessHandlersInstalled?: boolean; +}; + +const globalState = globalThis as LoggerGlobalState; + +function createRootLogger(): Logger { + const baseOptions = { + level: process.env.LOG_LEVEL || 'info', + name: serviceName, + }; + + if (isTestEnv) { + return pino(baseOptions); + } + + return pino({ + ...baseOptions, + transport: { target: 'pino-pretty', options: { colorize: true } }, + }); +} + +export const logger = + globalState.__ejclawLogger ?? (globalState.__ejclawLogger = createRootLogger()); type LogBindings = Record; @@ -30,11 +52,17 @@ export function createScopedLogger(bindings: LogBindings): Logger { } // Route uncaught errors through pino so they get timestamps in stderr -process.on('uncaughtException', (err) => { +function handleUncaughtException(err: unknown): void { logger.fatal({ err }, 'Uncaught exception'); process.exit(1); -}); +} -process.on('unhandledRejection', (reason) => { +function handleUnhandledRejection(reason: unknown): void { logger.error({ err: reason }, 'Unhandled rejection'); -}); +} + +if (!globalState.__ejclawProcessHandlersInstalled) { + process.on('uncaughtException', handleUncaughtException); + process.on('unhandledRejection', handleUnhandledRejection); + globalState.__ejclawProcessHandlersInstalled = true; +} diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index b165207..28ef15a 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -6,9 +6,11 @@ import { getErrorMessage } from './utils.js'; import { getAgentOutputText } from './agent-output.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js'; import { - isRetryableClaudeSessionFailureAttempt, - resolveAttemptRetryAction, -} from './agent-attempt-retry.js'; + executeAttemptRetryAction, + runClaudeAttemptWithRotation, + runCodexAttemptWithRotation, +} from './agent-attempt-orchestration.js'; +import { isRetryableClaudeSessionFailureAttempt } from './agent-attempt-retry.js'; import { AgentOutput, runAgentProcess, @@ -37,10 +39,6 @@ import { resolveExecutionTarget } from './message-runtime-rules.js'; import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js'; import { buildRoomRoleContext } from './room-role-context.js'; import { type AgentTriggerReason } from './agent-error-detection.js'; -import { - runClaudeRotationLoop, - runCodexRotationLoop, -} from './provider-retry.js'; import { shouldResetSessionOnAgentFailure, shouldRetryFreshSessionOnAgentFailure, @@ -649,27 +647,17 @@ export async function runAgentForGroup( initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, ): Promise<'success' | 'error'> => { - const outcome = await runCodexRotationLoop( + return runCodexAttemptWithRotation({ initialTrigger, - async () => { - const attempt = await runAttempt('codex'); - return { - output: attempt.output, - thrownError: attempt.error, - sawOutput: attempt.sawOutput, - streamedTriggerReason: attempt.streamedTriggerReason, - }; - }, - { + runAttempt: () => runAttempt('codex'), + logContext: { chatJid, group: group.name, groupFolder: group.folder, runId, }, rotationMessage, - ); - - return outcome.type === 'success' ? 'success' : 'error'; + }); }; type AgentAttempt = Awaited>; @@ -688,81 +676,63 @@ export async function runAgentForGroup( runId, }; - const outcome = await runClaudeRotationLoop( + return runClaudeAttemptWithRotation({ initialTrigger, - async () => { - const attempt = await runAttempt('claude'); - return { - output: attempt.output, - thrownError: attempt.error, - sawOutput: attempt.sawOutput, - sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput, - streamedTriggerReason: attempt.streamedTriggerReason, - }; - }, - logCtx, + runAttempt: () => runAttempt('claude'), + logContext: logCtx, rotationMessage, - ); - - switch (outcome.type) { - case 'success': - pairedSawOutput = outcome.sawOutput; - return 'success'; - case 'error': - return 'error'; - } + onSuccess: ({ sawOutput }) => { + pairedSawOutput = sawOutput; + }, + }); }; const retryClaudeAttemptIfNeeded = async ( attempt: AgentAttempt, rotationMessage?: string | null, ): Promise<'success' | 'error' | null> => { - const retryAction = resolveAttemptRetryAction({ + const retryAction = await executeAttemptRetryAction({ provider, canRetryClaudeCredentials, canRetryCodex: false, attempt, rotationMessage, + runClaude: retryClaudeWithRotation, + runCodex: retryCodexWithRotation, }); if (retryAction.kind !== 'claude') { return null; } - const result = await retryClaudeWithRotation( - retryAction.trigger, - retryAction.rotationMessage, - ); - if (result === 'error') { + if (retryAction.result === 'error') { return maybeHandoffAfterError(retryAction.trigger.reason, attempt); } pairedExecutionStatus = 'succeeded'; - return result; + return retryAction.result; }; const retryCodexAttemptIfNeeded = async ( attempt: AgentAttempt, rotationMessage?: string | null, ): Promise<'success' | 'error' | null> => { - const retryAction = resolveAttemptRetryAction({ + const retryAction = await executeAttemptRetryAction({ provider, canRetryClaudeCredentials: false, canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1, attempt, rotationMessage, + runClaude: retryClaudeWithRotation, + runCodex: retryCodexWithRotation, }); if (retryAction.kind !== 'codex') { return null; } - const result = await retryCodexWithRotation( - retryAction.trigger, - retryAction.rotationMessage, - ); - if (result === 'success') { + if (retryAction.result === 'success') { pairedExecutionStatus = 'succeeded'; } - return result; + return retryAction.result; }; const maybeHandoffAfterError = ( diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts index 7f7311a..261b43a 100644 --- a/src/message-runtime-flow.ts +++ b/src/message-runtime-flow.ts @@ -68,25 +68,6 @@ export function isBotOnlyPairedRoomTurn( ); } -export function resolveLastDeliveredBotRole( - messages: NewMessage[], -): PairedRoomRole | null { - for (let index = messages.length - 1; index >= 0; index -= 1) { - const message = messages[index]; - if (!message?.is_bot_message) { - continue; - } - if ( - message.sender_name === 'owner' || - message.sender_name === 'reviewer' || - message.sender_name === 'arbiter' - ) { - return message.sender_name; - } - } - return null; -} - export function buildPendingPairedTurn(args: { chatJid: string; timezone: string; @@ -98,7 +79,6 @@ export function buildPendingPairedTurn(args: { labeledRecentMessages: Parameters< typeof buildArbiterPromptForTask >[0]['labeledRecentMessages']; - lastDeliveredMessages?: NewMessage[]; resolveChannel: (taskStatus?: string | null) => Channel | null; }): PendingPairedTurn { const { @@ -117,10 +97,7 @@ export function buildPendingPairedTurn(args: { const lastTurnOutput = turnOutputs[turnOutputs.length - 1]; const nextTurnAction = resolveNextTurnAction({ taskStatus, - lastTurnOutputRole: - resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ?? - lastTurnOutput?.role ?? - null, + lastTurnOutputRole: lastTurnOutput?.role ?? null, }); const recentMessages = getRecentChatMessages(chatJid, 20); const lastHumanMessage = getLastHumanMessageContent(chatJid); @@ -255,7 +232,6 @@ export function resolveBotOnlyPairedFollowUpAction(args: { chatJid: string; task: PairedTask | null | undefined; isBotOnlyPairedFollowUp: boolean; - lastDeliveredMessages?: NewMessage[]; pendingCursorSource: | { seq?: number | null; timestamp?: string | null } | undefined; @@ -270,10 +246,7 @@ export function resolveBotOnlyPairedFollowUpAction(args: { const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1); const nextTurnAction = resolveNextTurnAction({ taskStatus: task.status, - lastTurnOutputRole: - resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ?? - lastTurnOutput?.role ?? - null, + lastTurnOutputRole: lastTurnOutput?.role ?? null, }); if (nextTurnAction.kind === 'finalize-owner-turn') { @@ -440,7 +413,6 @@ export function buildQueuedTurnDispatch(args: { chatJid: args.chatJid, task: args.loopPendingTask, isBotOnlyPairedFollowUp, - lastDeliveredMessages: args.labeledMessagesToSend, pendingCursorSource, }); diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 5199a6e..009bcfe 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -1827,124 +1827,6 @@ describe('createMessageRuntime', () => { expect(sendMessage).not.toHaveBeenCalled(); }); - it('requeues owner follow-ups from reviewer bot-only messages even when paired turn output storage is stale', async () => { - const chatJid = 'group@test'; - const group = makeGroup('codex'); - const ownerChannel = makeChannel(chatJid); - const reviewerChannel = { - ...makeChannel(chatJid, 'discord-review', false), - isOwnMessage: vi.fn((message) => message.sender === 'reviewer-bot@test'), - }; - const enqueueMessageCheck = vi.fn(); - const closeStdin = vi.fn(); - const sendMessage = vi.fn(() => false); - const setLastTimestamp = vi.fn(); - const saveState = vi.fn(); - const lastAgentTimestamps: Record = {}; - const stopLoop = new Error('stop-message-loop'); - - vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); - vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({ - id: 'task-active-owner-follow-up-stale-output', - chat_jid: chatJid, - group_folder: group.folder, - owner_service_id: 'codex-main', - reviewer_service_id: 'claude', - title: null, - source_ref: 'HEAD', - plan_notes: null, - review_requested_at: '2026-03-30T00:00:00.000Z', - round_trip_count: 1, - status: 'active', - arbiter_verdict: null, - arbiter_requested_at: null, - completion_reason: null, - created_at: '2026-03-30T00:00:00.000Z', - updated_at: '2026-03-30T00:00:00.000Z', - }); - vi.mocked(db.getPairedTurnOutputs).mockReturnValue([ - { - id: 1, - task_id: 'task-active-owner-follow-up-stale-output', - turn_number: 1, - role: 'owner', - output_text: 'owner 초안', - created_at: '2026-03-30T00:00:01.000Z', - }, - ]); - vi.mocked(db.getNewMessages).mockReturnValue({ - messages: [ - { - id: 'reviewer-bot-message-owner-follow-up-stale-output', - chat_jid: chatJid, - sender: 'reviewer-bot@test', - sender_name: '리뷰어', - content: 'DONE_WITH_CONCERNS\n\nreviewer direct message', - timestamp: '2026-03-30T00:00:04.000Z', - seq: 42, - is_bot_message: true, - } as any, - ], - newTimestamp: '42', - }); - - const runtime = createMessageRuntime({ - assistantName: 'Andy', - idleTimeout: 60_000, - pollInterval: 123, - timezone: 'UTC', - triggerPattern: /^@Andy\b/i, - channels: [ownerChannel, reviewerChannel], - queue: { - registerProcess: vi.fn(), - closeStdin, - enqueueMessageCheck, - notifyIdle: vi.fn(), - sendMessage, - } as any, - getRegisteredGroups: () => ({ [chatJid]: group }), - getSessions: () => ({}), - getLastTimestamp: () => '', - setLastTimestamp, - getLastAgentTimestamps: () => lastAgentTimestamps, - saveState, - persistSession: vi.fn(), - clearSession: vi.fn(), - }); - - const originalSetTimeout = global.setTimeout; - const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation((( - handler: any, - timeout?: number, - ...args: any[] - ) => { - if (timeout === 123) { - throw stopLoop; - } - return (originalSetTimeout as any)(handler, timeout, ...args); - }) as typeof setTimeout); - - try { - await expect(runtime.startMessageLoop()).rejects.toThrow( - stopLoop.message, - ); - } finally { - setTimeoutSpy.mockRestore(); - } - - expect(setLastTimestamp).toHaveBeenCalledWith('42'); - expect(agentRunner.runAgentProcess).not.toHaveBeenCalled(); - expect(closeStdin).toHaveBeenCalledWith( - chatJid, - expect.objectContaining({ reason: 'paired-pending-turn-follow-up' }), - ); - expect(enqueueMessageCheck).toHaveBeenCalledWith( - chatJid, - resolveGroupIpcPath(group.folder), - ); - expect(sendMessage).not.toHaveBeenCalled(); - }); - it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => { const chatJid = 'group@test'; const group = makeGroup('codex'); diff --git a/src/message-runtime.ts b/src/message-runtime.ts index d822a76..9e53d75 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -851,10 +851,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { chatJid, getRecentChatMessages(chatJid, 20), ), - lastDeliveredMessages: labelPairedSenders( - chatJid, - missedMessages, - ), resolveChannel, }) : null; @@ -1072,6 +1068,191 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { } }; + const processQueuedGroupDispatch = async (args: { + chatJid: string; + group: RegisteredGroup; + channel: Channel; + processableGroupMessages: NewMessage[]; + }): Promise => { + const { chatJid, group, channel, processableGroupMessages } = args; + const loopPendingTask = hasReviewerLease(chatJid) + ? getLatestOpenPairedTaskForChat(chatJid) + : null; + const loopCursorKey = resolveCursorKey(chatJid, loopPendingTask?.status); + const rawPendingMessages = getMessagesSinceSeq( + chatJid, + deps.getLastAgentTimestamps()[loopCursorKey] || '0', + deps.assistantName, + ); + const pendingMessages = filterLoopingPairedBotMessages( + chatJid, + getProcessableMessages(chatJid, rawPendingMessages, channel), + FAILURE_FINAL_TEXT, + ); + const messagesToSend = + pendingMessages.length > 0 ? pendingMessages : processableGroupMessages; + const labeledMessagesToSend = labelPairedSenders(chatJid, messagesToSend); + const { + formatted, + botOnlyFollowUpAction, + isBotOnlyPairedFollowUp, + loopCursorKey: dispatchCursorKey, + endSeq, + } = buildQueuedTurnDispatch({ + chatJid, + timezone: deps.timezone, + loopPendingTask, + rawPendingMessages, + messagesToSend, + labeledMessagesToSend, + formatMessages, + }); + + if ( + await executeBotOnlyPairedFollowUpAction({ + action: botOnlyFollowUpAction, + chatJid, + group, + runId: `loop-merge-ready-${Date.now().toString(36)}`, + channel, + log: logger, + saveState: deps.saveState, + lastAgentTimestamps: deps.getLastAgentTimestamps(), + executeTurn, + enqueueGroupMessageCheck: () => + enqueueScopedGroupMessageCheck(chatJid, group.folder), + closeStdin: () => + deps.queue.closeStdin(chatJid, { + reason: 'paired-pending-turn-follow-up', + }), + }) + ) { + return; + } + + if (deps.queue.sendMessage(chatJid, formatted)) { + if (endSeq != null) { + advanceLastAgentCursor( + deps.getLastAgentTimestamps(), + deps.saveState, + chatJid, + endSeq, + dispatchCursorKey, + ); + } + logger.debug( + { + transition: 'typing:on', + source: 'follow-up-queued', + chatJid, + group: group.name, + groupFolder: group.folder, + endSeq: endSeq ?? null, + suppressed: isBotOnlyPairedFollowUp, + }, + 'Typing indicator transition', + ); + if (!isBotOnlyPairedFollowUp) { + await channel + .setTyping?.(chatJid, true) + ?.catch((err) => + logger.warn( + { chatJid, err }, + 'Failed to set typing indicator', + ), + ); + } + return; + } + + enqueueScopedGroupMessageCheck(chatJid, group.folder); + }; + + const processLoopGroupMessages = async (args: { + chatJid: string; + group: RegisteredGroup; + groupMessages: NewMessage[]; + channel: Channel; + }): Promise => { + const { chatJid, group, groupMessages, channel } = args; + const isMainGroup = group.isMain === true; + const processableGroupMessages = getProcessableMessages( + chatJid, + groupMessages, + channel, + ); + + if (processableGroupMessages.length === 0) { + const lastIgnored = groupMessages[groupMessages.length - 1]; + if (lastIgnored?.seq != null) { + advanceLastAgentCursor( + deps.getLastAgentTimestamps(), + deps.saveState, + chatJid, + lastIgnored.seq, + ); + } + return; + } + + if (shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)) { + const lastIgnored = + processableGroupMessages[processableGroupMessages.length - 1]; + if (lastIgnored?.seq != null) { + advanceLastAgentCursor( + deps.getLastAgentTimestamps(), + deps.saveState, + chatJid, + lastIgnored.seq, + ); + } + logger.info( + { chatJid, group: group.name, groupFolder: group.folder }, + 'Bot-collaboration timeout: no recent human message, skipping', + ); + return; + } + + const loopCmdMsg = groupMessages.find( + (msg) => extractSessionCommand(msg.content, deps.triggerPattern) !== null, + ); + + if (loopCmdMsg) { + if ( + isSessionCommandAllowed( + isMainGroup, + loopCmdMsg.is_from_me === true, + isSessionCommandSenderAllowed(loopCmdMsg.sender), + ) + ) { + deps.queue.closeStdin(chatJid, { + reason: 'session-command-detected', + }); + } + deps.queue.enqueueMessageCheck(chatJid); + return; + } + + if ( + !hasAllowedTrigger({ + chatJid, + messages: processableGroupMessages, + group, + triggerPattern: deps.triggerPattern, + hasImplicitContinuationWindow: continuationTracker.has, + }) + ) { + return; + } + + await processQueuedGroupDispatch({ + chatJid, + group, + channel, + processableGroupMessages, + }); + }; + const startMessageLoop = async (): Promise => { if (messageLoopRunning) { logger.debug('Message loop already running, skipping duplicate start'); @@ -1120,182 +1301,12 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { ); continue; } - - const isMainGroup = group.isMain === true; - const processableGroupMessages = getProcessableMessages( + await processLoopGroupMessages({ chatJid, + group, groupMessages, channel, - ); - - if (processableGroupMessages.length === 0) { - const lastIgnored = groupMessages[groupMessages.length - 1]; - if (lastIgnored?.seq != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - lastIgnored.seq, - ); - } - continue; - } - - if ( - shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages) - ) { - const lastIgnored = - processableGroupMessages[processableGroupMessages.length - 1]; - if (lastIgnored?.seq != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - lastIgnored.seq, - ); - } - logger.info( - { chatJid, group: group.name, groupFolder: group.folder }, - 'Bot-collaboration timeout: no recent human message, skipping', - ); - continue; - } - - const loopCmdMsg = groupMessages.find( - (msg) => - extractSessionCommand(msg.content, deps.triggerPattern) !== - null, - ); - - if (loopCmdMsg) { - if ( - isSessionCommandAllowed( - isMainGroup, - loopCmdMsg.is_from_me === true, - isSessionCommandSenderAllowed(loopCmdMsg.sender), - ) - ) { - deps.queue.closeStdin(chatJid, { - reason: 'session-command-detected', - }); - } - deps.queue.enqueueMessageCheck(chatJid); - continue; - } - - if ( - !hasAllowedTrigger({ - chatJid, - messages: processableGroupMessages, - group, - triggerPattern: deps.triggerPattern, - hasImplicitContinuationWindow: continuationTracker.has, - }) - ) { - continue; - } - - // Use role-aware cursor for paired rooms so the reviewer - // always sees the owner's last messages and vice versa. - const loopPendingTask = hasReviewerLease(chatJid) - ? getLatestOpenPairedTaskForChat(chatJid) - : null; - const loopCursorKey = resolveCursorKey( - chatJid, - loopPendingTask?.status, - ); - - const rawPendingMessages = getMessagesSinceSeq( - chatJid, - deps.getLastAgentTimestamps()[loopCursorKey] || '0', - deps.assistantName, - ); - const pendingMessages = filterLoopingPairedBotMessages( - chatJid, - getProcessableMessages(chatJid, rawPendingMessages, channel), - FAILURE_FINAL_TEXT, - ); - const messagesToSend = - pendingMessages.length > 0 - ? pendingMessages - : processableGroupMessages; - const labeledMessagesToSend = labelPairedSenders( - chatJid, - messagesToSend, - ); - const { - formatted, - botOnlyFollowUpAction, - isBotOnlyPairedFollowUp, - loopCursorKey: dispatchCursorKey, - endSeq, - } = buildQueuedTurnDispatch({ - chatJid, - timezone: deps.timezone, - loopPendingTask, - rawPendingMessages, - messagesToSend, - labeledMessagesToSend, - formatMessages, }); - if ( - await executeBotOnlyPairedFollowUpAction({ - action: botOnlyFollowUpAction, - chatJid, - group, - runId: `loop-merge-ready-${Date.now().toString(36)}`, - channel, - log: logger, - saveState: deps.saveState, - lastAgentTimestamps: deps.getLastAgentTimestamps(), - executeTurn, - enqueueGroupMessageCheck: () => - enqueueScopedGroupMessageCheck(chatJid, group.folder), - closeStdin: () => - deps.queue.closeStdin(chatJid, { - reason: 'paired-pending-turn-follow-up', - }), - }) - ) { - continue; - } - - if (deps.queue.sendMessage(chatJid, formatted)) { - if (endSeq != null) { - advanceLastAgentCursor( - deps.getLastAgentTimestamps(), - deps.saveState, - chatJid, - endSeq, - dispatchCursorKey, - ); - } - logger.debug( - { - transition: 'typing:on', - source: 'follow-up-queued', - chatJid, - group: group.name, - groupFolder: group.folder, - endSeq: endSeq ?? null, - suppressed: isBotOnlyPairedFollowUp, - }, - 'Typing indicator transition', - ); - if (!isBotOnlyPairedFollowUp) { - await channel - .setTyping?.(chatJid, true) - ?.catch((err) => - logger.warn( - { chatJid, err }, - 'Failed to set typing indicator', - ), - ); - } - continue; - } - - enqueueScopedGroupMessageCheck(chatJid, group.folder); } } } catch (err) { diff --git a/src/paired-execution-context-owner.ts b/src/paired-execution-context-owner.ts index a7dca12..80899f6 100644 --- a/src/paired-execution-context-owner.ts +++ b/src/paired-execution-context-owner.ts @@ -5,11 +5,11 @@ import { import { getPairedWorkspace, hasActiveCiWatcherForChat, - updatePairedTask, } from './db.js'; import { logger } from './logger.js'; import { markPairedTaskReviewReady } from './paired-workspace-manager.js'; import { + applyPairedTaskPatch, classifyVerdict, hasCodeChangesSinceRef, requestArbiterOrEscalate, @@ -207,10 +207,12 @@ export function handleOwnerCompletion(args: { const result = markPairedTaskReviewReady(taskId); if (result) { - updatePairedTask(taskId, { - round_trip_count: task.round_trip_count + 1, - review_requested_at: now, - updated_at: now, + applyPairedTaskPatch({ + taskId, + updatedAt: now, + patch: { + round_trip_count: task.round_trip_count + 1, + }, }); logger.info( { taskId, roundTrip: task.round_trip_count + 1 }, diff --git a/src/paired-execution-context-shared.ts b/src/paired-execution-context-shared.ts index 6c1266b..fa8328c 100644 --- a/src/paired-execution-context-shared.ts +++ b/src/paired-execution-context-shared.ts @@ -155,6 +155,17 @@ export function transitionPairedTaskStatus(args: { }); } +export function applyPairedTaskPatch(args: { + taskId: string; + updatedAt: string; + patch: Omit[1], 'updated_at'>; +}): void { + updatePairedTask(args.taskId, { + ...args.patch, + updated_at: args.updatedAt, + }); +} + export function requestArbiterOrEscalate(args: { taskId: string; currentStatus: PairedTaskStatus; diff --git a/src/paired-execution-context.test.ts b/src/paired-execution-context.test.ts index 74cfc96..1f57b57 100644 --- a/src/paired-execution-context.test.ts +++ b/src/paired-execution-context.test.ts @@ -565,7 +565,6 @@ describe('paired execution context', () => { 'task-1', expect.objectContaining({ round_trip_count: 2, - review_requested_at: expect.any(String), }), ); }); diff --git a/src/paired-execution-context.ts b/src/paired-execution-context.ts index ff07c62..3f2a3be 100644 --- a/src/paired-execution-context.ts +++ b/src/paired-execution-context.ts @@ -19,7 +19,6 @@ import { getPairedTaskById, getPairedWorkspace, hasActiveCiWatcherForChat, - updatePairedTask, upsertPairedProject, } from './db.js'; import { logger } from './logger.js'; @@ -36,6 +35,7 @@ import { handleReviewerCompletion, } from './paired-execution-context-reviewer.js'; import { + applyPairedTaskPatch, resolveCanonicalSourceRef, requestArbiterOrEscalate, transitionPairedTaskStatus, @@ -238,9 +238,12 @@ export function preparePairedExecutionContext(args: { }, }); } else { - updatePairedTask(latestTask.id, { - ...(hasHuman ? { round_trip_count: 0 } : {}), - updated_at: now, + applyPairedTaskPatch({ + taskId: latestTask.id, + updatedAt: now, + patch: { + ...(hasHuman ? { round_trip_count: 0 } : {}), + }, }); } } @@ -253,9 +256,12 @@ export function preparePairedExecutionContext(args: { if (workspace?.workspace_dir && latestTask.status === 'active') { const wsRef = resolveCanonicalSourceRef(workspace.workspace_dir); if (wsRef !== latestTask.source_ref) { - updatePairedTask(latestTask.id, { - source_ref: wsRef, - updated_at: now, + applyPairedTaskPatch({ + taskId: latestTask.id, + updatedAt: now, + patch: { + source_ref: wsRef, + }, }); } } diff --git a/src/paired-workspace-manager.test.ts b/src/paired-workspace-manager.test.ts index 929547f..1d20003 100644 --- a/src/paired-workspace-manager.test.ts +++ b/src/paired-workspace-manager.test.ts @@ -258,6 +258,31 @@ describe('paired workspace manager', () => { ); }); + it('leaves review_requested_at untouched when review handoff aborts before an owner workspace exists', async () => { + const { db, manager } = await loadModules(); + db._initTestDatabase(); + + const canonicalDir = path.join(tempRoot, 'canonical'); + initCanonicalRepo(canonicalDir); + seedPairedTask(db, canonicalDir, { + taskId: 'paired-task-no-owner-workspace', + groupFolder: 'no-owner-workspace-room', + }); + + const result = manager.markPairedTaskReviewReady( + 'paired-task-no-owner-workspace', + ); + + expect(result).toBeNull(); + expect( + db.getPairedTaskById('paired-task-no-owner-workspace') + ?.review_requested_at, + ).toBeNull(); + expect( + db.getPairedTaskById('paired-task-no-owner-workspace')?.status, + ).toBe('active'); + }); + it('uses the shared DB owner workspace across service-local data dirs', async () => { const canonicalDir = path.join(tempRoot, 'canonical'); fs.mkdirSync(canonicalDir, { recursive: true }); diff --git a/src/paired-workspace-manager.ts b/src/paired-workspace-manager.ts index 327cfe8..7019ac9 100644 --- a/src/paired-workspace-manager.ts +++ b/src/paired-workspace-manager.ts @@ -8,7 +8,6 @@ import { getPairedProject, getPairedTaskById, getPairedWorkspace, - updatePairedTask, upsertPairedWorkspace, } from './db.js'; import { resolvePairedTaskWorkspacePath } from './group-folder.js'; @@ -758,10 +757,6 @@ export function markPairedTaskReviewReady(taskId: string): { reviewerWorkspace: PairedWorkspace; } | null { const requestedAt = new Date().toISOString(); - updatePairedTask(taskId, { - review_requested_at: requestedAt, - updated_at: requestedAt, - }); const ownerWorkspace = getPairedWorkspace(taskId, 'owner'); if (!ownerWorkspace) { @@ -798,7 +793,6 @@ export function markPairedTaskReviewReady(taskId: string): { 'Reviewer will mount owner workspace directly', ); - const now = new Date().toISOString(); const task = getPairedTaskById(taskId); if (!task) { return null; @@ -807,7 +801,10 @@ export function markPairedTaskReviewReady(taskId: string): { taskId, currentStatus: task.status, nextStatus: 'review_ready', - updatedAt: now, + updatedAt: requestedAt, + patch: { + review_requested_at: requestedAt, + }, }); return { ownerWorkspace, reviewerWorkspace }; diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index ff44b0a..14a2134 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -3,7 +3,11 @@ import { CronExpressionParser } from 'cron-parser'; import fs from 'fs'; import { getAgentOutputText } from './agent-output.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js'; -import { resolveAttemptRetryAction } from './agent-attempt-retry.js'; +import { + executeAttemptRetryAction, + runClaudeAttemptWithRotation, + runCodexAttemptWithRotation, +} from './agent-attempt-orchestration.js'; import { getErrorMessage } from './utils.js'; import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js'; @@ -29,10 +33,6 @@ import { } from './group-folder.js'; import { createScopedLogger, logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; -import { - runClaudeRotationLoop, - runCodexRotationLoop, -} from './provider-retry.js'; import { detectCodexRotationTrigger, rotateCodexToken, @@ -447,69 +447,53 @@ async function runTask( retryAfterMs?: number; }, rotationMessage?: string, - ): Promise => { - const logCtx = { + ): Promise<'success' | 'error'> => { + const logContext = { taskId: task.id, group: context.group.name, groupFolder: task.group_folder, }; - const outcome = await runClaudeRotationLoop( + const outcome = await runClaudeAttemptWithRotation({ initialTrigger, - async () => { - const attempt = await runTaskAttempt('claude'); + runAttempt: () => runTaskAttempt('claude'), + logContext, + rotationMessage, + afterAttempt: (attempt) => { result = attempt.attemptResult; error = attempt.attemptError; - return { - output: attempt.output, - sawOutput: attempt.sawOutput, - streamedTriggerReason: attempt.streamedTriggerReason, - }; }, - logCtx, - rotationMessage, - ); + }); - switch (outcome.type) { - case 'success': - error = null; - return; - case 'error': - if (outcome.trigger) { - error = `Claude ${outcome.trigger.reason}`; - } - return; + if (outcome === 'success') { + error = null; } + return outcome; }; const retryCodexTaskWithRotation = async ( initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, - ): Promise => { - const outcome = await runCodexRotationLoop( + ): Promise<'success' | 'error'> => { + const outcome = await runCodexAttemptWithRotation({ initialTrigger, - async () => { - const retryAttempt = await runTaskAttempt('codex'); - result = retryAttempt.attemptResult; - error = retryAttempt.attemptError; - return { - output: retryAttempt.output, - thrownError: null, - sawOutput: retryAttempt.sawOutput, - streamedTriggerReason: retryAttempt.streamedTriggerReason, - }; - }, - { + runAttempt: () => runTaskAttempt('codex'), + logContext: { taskId: task.id, group: context.group.name, groupFolder: task.group_folder, }, rotationMessage, - ); + afterAttempt: (attempt) => { + result = attempt.attemptResult; + error = attempt.attemptError; + }, + }); - if (outcome.type === 'success') { + if (outcome === 'success') { error = null; } + return outcome; }; const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude'; @@ -519,25 +503,17 @@ async function runTask( result = attempt.attemptResult; error = attempt.attemptError; - const retryAction = resolveAttemptRetryAction({ + const retryAction = await executeAttemptRetryAction({ provider, canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0, canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1, attempt, rotationMessage: error, + runClaude: retryClaudeTaskWithRotation, + runCodex: retryCodexTaskWithRotation, }); - if (retryAction.kind === 'claude') { - await retryClaudeTaskWithRotation( - retryAction.trigger, - retryAction.rotationMessage, - ); - } else if (retryAction.kind === 'codex') { - await retryCodexTaskWithRotation( - retryAction.trigger, - retryAction.rotationMessage, - ); - } else if (attempt.output.status === 'error') { + if (retryAction.kind === 'none' && attempt.output.status === 'error') { error = attempt.attemptError || 'Unknown error'; } } // end else (non-exhausted path) diff --git a/src/token-rotation.test.ts b/src/token-rotation.test.ts index e659508..8da8b61 100644 --- a/src/token-rotation.test.ts +++ b/src/token-rotation.test.ts @@ -65,4 +65,44 @@ describe('token-rotation runtime reselection', () => { rateLimited: 0, }); }); + + it('reloads Claude rotation state from disk written by another service', async () => { + const mod = await import('./token-rotation.js'); + const utils = await import('./utils.js'); + const readJsonFile = vi.mocked(utils.readJsonFile); + + readJsonFile.mockReturnValueOnce(null); + mod.initTokenRotation(); + expect(mod.getCurrentToken()).toBe('token-1'); + + readJsonFile.mockReturnValueOnce({ + currentIndex: 1, + rateLimits: [Date.now() + 60_000, null], + }); + mod.reloadTokenRotationStateFromDisk(); + + expect(mod.getCurrentTokenIndex()).toBe(1); + expect(mod.getCurrentToken()).toBe('token-2'); + }); + + it('warns when Claude rotation state cannot be persisted', async () => { + const mod = await import('./token-rotation.js'); + const utils = await import('./utils.js'); + const { logger } = await import('./logger.js'); + + vi.mocked(utils.writeJsonFile).mockImplementation(() => { + throw new Error('disk full'); + }); + + mod.initTokenRotation(); + expect(mod.rotateToken('rate limit')).toBe(true); + + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + stateFile: '/tmp/ejclaw-claude-rot-data/token-rotation-state.json', + err: expect.any(Error), + }), + 'Failed to persist Claude token rotation state', + ); + }); }); diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 4bc88e0..9cf6d1d 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -69,12 +69,15 @@ function saveState(): void { rateLimits: tokens.map((t) => t.rateLimitedUntil), }; writeJsonFile(STATE_FILE, state); - } catch { - /* best effort */ + } catch (err) { + logger.warn( + { stateFile: STATE_FILE, err }, + 'Failed to persist Claude token rotation state', + ); } } -function loadState(): void { +function loadState(quiet = false): void { const state = readJsonFile<{ currentIndex?: number; rateLimits?: (number | null)[]; @@ -93,13 +96,27 @@ function loadState(): void { const until = state.rateLimits[i]; if (typeof until === 'number' && until > now) { tokens[i].rateLimitedUntil = until; + } else { + tokens[i].rateLimitedUntil = null; } } } - logger.info( - { currentIndex, tokenCount: tokens.length }, - 'Token rotation state restored', - ); + if (!quiet) { + logger.info( + { currentIndex, tokenCount: tokens.length }, + 'Token rotation state restored', + ); + } +} + +/** + * Re-read the on-disk rotation state (written by any service). + * Call before dashboard renders so the renderer picks up rotations + * performed by another Claude service process. + */ +export function reloadTokenRotationStateFromDisk(): void { + if (tokens.length <= 1) return; + loadState(true); } function refreshRuntimeTokenSelection(): void {