diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts index e4e2af9..159a100 100644 --- a/src/agent-error-detection.ts +++ b/src/agent-error-detection.ts @@ -126,6 +126,18 @@ export function shouldRotateClaudeToken( ); } +export function isCodexRotationReason( + reason: AgentTriggerReason, +): reason is CodexRotationReason { + return ( + reason === '429' || + reason === 'auth-expired' || + reason === 'org-access-denied' || + reason === 'overloaded' || + reason === 'network-error' + ); +} + // ── Rotation trigger classification ───────────────────────────── export type RotationTriggerResult = diff --git a/src/message-agent-executor-rules.test.ts b/src/message-agent-executor-rules.test.ts index 57471f3..e10a6fd 100644 --- a/src/message-agent-executor-rules.test.ts +++ b/src/message-agent-executor-rules.test.ts @@ -1,11 +1,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -vi.mock('./agent-error-detection.js', () => ({ - classifyRotationTrigger: vi.fn(() => ({ - shouldRetry: false, - reason: '', - })), -})); +vi.mock('./agent-error-detection.js', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + classifyRotationTrigger: vi.fn(() => ({ + shouldRetry: false, + reason: '', + })), + }; +}); vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn(() => ({ @@ -71,6 +76,18 @@ describe('message-agent-executor-rules', () => { ).toEqual({ reason: 'auth-expired' }); }); + it('ignores streamed reasons that are not valid Codex rotation triggers', () => { + expect( + resolveCodexRetryTrigger({ + canRetryCodex: true, + attempt: { + streamedTriggerReason: { reason: 'usage-exhausted' }, + }, + }), + ).toBeNull(); + expect(detectCodexRotationTrigger).not.toHaveBeenCalled(); + }); + it('detects retryable Claude session failures from either flag or classifier', () => { expect( isRetryableClaudeSessionFailureAttempt({ diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index 9215300..c03c33a 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -2,6 +2,7 @@ import { classifyRotationTrigger, type AgentTriggerReason, type CodexRotationReason, + isCodexRotationReason, } from './agent-error-detection.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js'; @@ -94,8 +95,11 @@ export function resolveCodexRetryTrigger(args: { } if (args.attempt.streamedTriggerReason) { + if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) { + return null; + } return { - reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason, + reason: args.attempt.streamedTriggerReason.reason, }; } diff --git a/src/provider-retry.ts b/src/provider-retry.ts index 52dfeea..a5ac85c 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -10,6 +10,7 @@ import { getAgentOutputText } from './agent-output.js'; import { classifyRotationTrigger, type CodexRotationReason, + isCodexRotationReason, shouldRotateClaudeToken, type AgentTriggerReason, } from './agent-error-detection.js'; @@ -210,22 +211,35 @@ function evaluateCodexAttempt( attempt.streamedTriggerReason && output.status !== 'error' ) { + if (!isCodexRotationReason(attempt.streamedTriggerReason.reason)) { + logger.error( + { + ...logContext, + provider: 'codex', + reason: attempt.streamedTriggerReason.reason, + }, + 'Unexpected non-Codex streamed trigger while rotating Codex account', + ); + return { type: 'error' }; + } return { type: 'continue', trigger: { - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + reason: attempt.streamedTriggerReason.reason, }, rotationMessage: getAgentOutputText(output) ?? undefined, }; } if (output.status === 'error') { - const retryTrigger = attempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, - } - : detectCodexRotationTrigger(output.error); + const retryTrigger = + attempt.streamedTriggerReason && + isCodexRotationReason(attempt.streamedTriggerReason.reason) + ? { + shouldRotate: true as const, + reason: attempt.streamedTriggerReason.reason, + } + : detectCodexRotationTrigger(output.error); if (retryTrigger.shouldRotate) { return { type: 'continue', diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index ce13440..99aaec4 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -27,7 +27,7 @@ import { } from './group-folder.js'; import { createScopedLogger, logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; -import { runClaudeRotationLoop } from './provider-retry.js'; +import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js'; import { detectCodexRotationTrigger, rotateCodexToken, @@ -38,6 +38,7 @@ import { classifyRotationTrigger, type AgentTriggerReason, type CodexRotationReason, + isCodexRotationReason, } from './agent-error-detection.js'; import { getTokenCount, @@ -489,65 +490,29 @@ async function runTask( initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, ): Promise => { - let trigger = initialTrigger; - let lastRotationMessage = rotationMessage; - - while ( - getCodexAccountCount() > 1 && - rotateCodexToken(lastRotationMessage) - ) { - log.info( - { - reason: trigger.reason, - }, - 'Codex account unhealthy, retrying scheduled task with rotated account', - ); - - const retryAttempt = await runTaskAttempt('codex'); - result = retryAttempt.attemptResult; - error = retryAttempt.attemptError; - - if ( - !retryAttempt.sawOutput && - retryAttempt.streamedTriggerReason && - retryAttempt.output.status !== 'error' - ) { - trigger = { - reason: retryAttempt.streamedTriggerReason - .reason as CodexRotationReason, + const outcome = await runCodexRotationLoop( + initialTrigger, + async () => { + const retryAttempt = await runTaskAttempt('codex'); + result = retryAttempt.attemptResult; + error = retryAttempt.attemptError; + return { + output: retryAttempt.output, + thrownError: null, + sawOutput: retryAttempt.sawOutput, + streamedTriggerReason: retryAttempt.streamedTriggerReason, }; - lastRotationMessage = - typeof retryAttempt.output.result === 'string' - ? retryAttempt.output.result - : undefined; - continue; - } + }, + { + taskId: task.id, + group: context.group.name, + groupFolder: task.group_folder, + }, + rotationMessage, + ); - if (retryAttempt.output.status === 'error') { - const retryTrigger = retryAttempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: retryAttempt.streamedTriggerReason - .reason as CodexRotationReason, - } - : detectCodexRotationTrigger( - retryAttempt.attemptError || retryAttempt.output.error, - ); - - if (retryTrigger.shouldRotate) { - trigger = { reason: retryTrigger.reason }; - lastRotationMessage = - retryAttempt.attemptError || - retryAttempt.output.error || - undefined; - continue; - } - return; - } - - markCodexTokenHealthy(); + if (outcome.type === 'success') { error = null; - return; } }; @@ -567,11 +532,12 @@ async function runTask( } else if ( provider === 'codex' && attempt.streamedTriggerReason && - !attempt.sawOutput + !attempt.sawOutput && + isCodexRotationReason(attempt.streamedTriggerReason.reason) ) { await retryCodexTaskWithRotation( { - reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + reason: attempt.streamedTriggerReason.reason, }, typeof attempt.output.error === 'string' ? attempt.output.error @@ -592,13 +558,14 @@ async function runTask( }); } } else if (attempt.output.status === 'error' && provider === 'codex') { - const trigger = attempt.streamedTriggerReason - ? { - shouldRotate: true, - reason: attempt.streamedTriggerReason - .reason as CodexRotationReason, - } - : detectCodexRotationTrigger(error); + const trigger = + attempt.streamedTriggerReason && + isCodexRotationReason(attempt.streamedTriggerReason.reason) + ? { + shouldRotate: true as const, + reason: attempt.streamedTriggerReason.reason, + } + : detectCodexRotationTrigger(error); if (trigger.shouldRotate) { await retryCodexTaskWithRotation( { reason: trigger.reason },