Unify codex rotation retry guards

This commit is contained in:
ejclaw
2026-04-06 20:53:48 +09:00
parent 7612b2c4cc
commit fdc27836a3
5 changed files with 94 additions and 80 deletions

View File

@@ -126,6 +126,18 @@ export function shouldRotateClaudeToken(
); );
} }
export function isCodexRotationReason(
reason: AgentTriggerReason,
): reason is CodexRotationReason {
return (
reason === '429' ||
reason === 'auth-expired' ||
reason === 'org-access-denied' ||
reason === 'overloaded' ||
reason === 'network-error'
);
}
// ── Rotation trigger classification ───────────────────────────── // ── Rotation trigger classification ─────────────────────────────
export type RotationTriggerResult = export type RotationTriggerResult =

View File

@@ -1,11 +1,16 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('./agent-error-detection.js', () => ({ vi.mock('./agent-error-detection.js', async (importOriginal) => {
classifyRotationTrigger: vi.fn(() => ({ const actual =
shouldRetry: false, await importOriginal<typeof import('./agent-error-detection.js')>();
reason: '', return {
})), ...actual,
})); classifyRotationTrigger: vi.fn(() => ({
shouldRetry: false,
reason: '',
})),
};
});
vi.mock('./codex-token-rotation.js', () => ({ vi.mock('./codex-token-rotation.js', () => ({
detectCodexRotationTrigger: vi.fn(() => ({ detectCodexRotationTrigger: vi.fn(() => ({
@@ -71,6 +76,18 @@ describe('message-agent-executor-rules', () => {
).toEqual({ reason: 'auth-expired' }); ).toEqual({ reason: 'auth-expired' });
}); });
it('ignores streamed reasons that are not valid Codex rotation triggers', () => {
expect(
resolveCodexRetryTrigger({
canRetryCodex: true,
attempt: {
streamedTriggerReason: { reason: 'usage-exhausted' },
},
}),
).toBeNull();
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
});
it('detects retryable Claude session failures from either flag or classifier', () => { it('detects retryable Claude session failures from either flag or classifier', () => {
expect( expect(
isRetryableClaudeSessionFailureAttempt({ isRetryableClaudeSessionFailureAttempt({

View File

@@ -2,6 +2,7 @@ import {
classifyRotationTrigger, classifyRotationTrigger,
type AgentTriggerReason, type AgentTriggerReason,
type CodexRotationReason, type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js'; } from './agent-error-detection.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import type { PairedRoomRole, PairedTaskStatus } from './types.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js';
@@ -94,8 +95,11 @@ export function resolveCodexRetryTrigger(args: {
} }
if (args.attempt.streamedTriggerReason) { if (args.attempt.streamedTriggerReason) {
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
return null;
}
return { return {
reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason, reason: args.attempt.streamedTriggerReason.reason,
}; };
} }

View File

@@ -10,6 +10,7 @@ import { getAgentOutputText } from './agent-output.js';
import { import {
classifyRotationTrigger, classifyRotationTrigger,
type CodexRotationReason, type CodexRotationReason,
isCodexRotationReason,
shouldRotateClaudeToken, shouldRotateClaudeToken,
type AgentTriggerReason, type AgentTriggerReason,
} from './agent-error-detection.js'; } from './agent-error-detection.js';
@@ -210,22 +211,35 @@ function evaluateCodexAttempt(
attempt.streamedTriggerReason && attempt.streamedTriggerReason &&
output.status !== 'error' output.status !== 'error'
) { ) {
if (!isCodexRotationReason(attempt.streamedTriggerReason.reason)) {
logger.error(
{
...logContext,
provider: 'codex',
reason: attempt.streamedTriggerReason.reason,
},
'Unexpected non-Codex streamed trigger while rotating Codex account',
);
return { type: 'error' };
}
return { return {
type: 'continue', type: 'continue',
trigger: { trigger: {
reason: attempt.streamedTriggerReason.reason as CodexRotationReason, reason: attempt.streamedTriggerReason.reason,
}, },
rotationMessage: getAgentOutputText(output) ?? undefined, rotationMessage: getAgentOutputText(output) ?? undefined,
}; };
} }
if (output.status === 'error') { if (output.status === 'error') {
const retryTrigger = attempt.streamedTriggerReason const retryTrigger =
? { attempt.streamedTriggerReason &&
shouldRotate: true, isCodexRotationReason(attempt.streamedTriggerReason.reason)
reason: attempt.streamedTriggerReason.reason as CodexRotationReason, ? {
} shouldRotate: true as const,
: detectCodexRotationTrigger(output.error); reason: attempt.streamedTriggerReason.reason,
}
: detectCodexRotationTrigger(output.error);
if (retryTrigger.shouldRotate) { if (retryTrigger.shouldRotate) {
return { return {
type: 'continue', type: 'continue',

View File

@@ -27,7 +27,7 @@ import {
} from './group-folder.js'; } from './group-folder.js';
import { createScopedLogger, logger } from './logger.js'; import { createScopedLogger, logger } from './logger.js';
import { createTaskStatusTracker } from './task-status-tracker.js'; import { createTaskStatusTracker } from './task-status-tracker.js';
import { runClaudeRotationLoop } from './provider-retry.js'; import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js';
import { import {
detectCodexRotationTrigger, detectCodexRotationTrigger,
rotateCodexToken, rotateCodexToken,
@@ -38,6 +38,7 @@ import {
classifyRotationTrigger, classifyRotationTrigger,
type AgentTriggerReason, type AgentTriggerReason,
type CodexRotationReason, type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js'; } from './agent-error-detection.js';
import { import {
getTokenCount, getTokenCount,
@@ -489,65 +490,29 @@ async function runTask(
initialTrigger: { reason: CodexRotationReason }, initialTrigger: { reason: CodexRotationReason },
rotationMessage?: string, rotationMessage?: string,
): Promise<void> => { ): Promise<void> => {
let trigger = initialTrigger; const outcome = await runCodexRotationLoop(
let lastRotationMessage = rotationMessage; initialTrigger,
async () => {
while ( const retryAttempt = await runTaskAttempt('codex');
getCodexAccountCount() > 1 && result = retryAttempt.attemptResult;
rotateCodexToken(lastRotationMessage) error = retryAttempt.attemptError;
) { return {
log.info( output: retryAttempt.output,
{ thrownError: null,
reason: trigger.reason, sawOutput: retryAttempt.sawOutput,
}, streamedTriggerReason: retryAttempt.streamedTriggerReason,
'Codex account unhealthy, retrying scheduled task with rotated account',
);
const retryAttempt = await runTaskAttempt('codex');
result = retryAttempt.attemptResult;
error = retryAttempt.attemptError;
if (
!retryAttempt.sawOutput &&
retryAttempt.streamedTriggerReason &&
retryAttempt.output.status !== 'error'
) {
trigger = {
reason: retryAttempt.streamedTriggerReason
.reason as CodexRotationReason,
}; };
lastRotationMessage = },
typeof retryAttempt.output.result === 'string' {
? retryAttempt.output.result taskId: task.id,
: undefined; group: context.group.name,
continue; groupFolder: task.group_folder,
} },
rotationMessage,
);
if (retryAttempt.output.status === 'error') { if (outcome.type === 'success') {
const retryTrigger = retryAttempt.streamedTriggerReason
? {
shouldRotate: true,
reason: retryAttempt.streamedTriggerReason
.reason as CodexRotationReason,
}
: detectCodexRotationTrigger(
retryAttempt.attemptError || retryAttempt.output.error,
);
if (retryTrigger.shouldRotate) {
trigger = { reason: retryTrigger.reason };
lastRotationMessage =
retryAttempt.attemptError ||
retryAttempt.output.error ||
undefined;
continue;
}
return;
}
markCodexTokenHealthy();
error = null; error = null;
return;
} }
}; };
@@ -567,11 +532,12 @@ async function runTask(
} else if ( } else if (
provider === 'codex' && provider === 'codex' &&
attempt.streamedTriggerReason && attempt.streamedTriggerReason &&
!attempt.sawOutput !attempt.sawOutput &&
isCodexRotationReason(attempt.streamedTriggerReason.reason)
) { ) {
await retryCodexTaskWithRotation( await retryCodexTaskWithRotation(
{ {
reason: attempt.streamedTriggerReason.reason as CodexRotationReason, reason: attempt.streamedTriggerReason.reason,
}, },
typeof attempt.output.error === 'string' typeof attempt.output.error === 'string'
? attempt.output.error ? attempt.output.error
@@ -592,13 +558,14 @@ async function runTask(
}); });
} }
} else if (attempt.output.status === 'error' && provider === 'codex') { } else if (attempt.output.status === 'error' && provider === 'codex') {
const trigger = attempt.streamedTriggerReason const trigger =
? { attempt.streamedTriggerReason &&
shouldRotate: true, isCodexRotationReason(attempt.streamedTriggerReason.reason)
reason: attempt.streamedTriggerReason ? {
.reason as CodexRotationReason, shouldRotate: true as const,
} reason: attempt.streamedTriggerReason.reason,
: detectCodexRotationTrigger(error); }
: detectCodexRotationTrigger(error);
if (trigger.shouldRotate) { if (trigger.shouldRotate) {
await retryCodexTaskWithRotation( await retryCodexTaskWithRotation(
{ reason: trigger.reason }, { reason: trigger.reason },