Unify codex rotation retry guards

This commit is contained in:
ejclaw
2026-04-06 20:53:48 +09:00
parent 7612b2c4cc
commit fdc27836a3
5 changed files with 94 additions and 80 deletions

View File

@@ -126,6 +126,18 @@ export function shouldRotateClaudeToken(
);
}
export function isCodexRotationReason(
reason: AgentTriggerReason,
): reason is CodexRotationReason {
return (
reason === '429' ||
reason === 'auth-expired' ||
reason === 'org-access-denied' ||
reason === 'overloaded' ||
reason === 'network-error'
);
}
// ── Rotation trigger classification ─────────────────────────────
export type RotationTriggerResult =

View File

@@ -1,11 +1,16 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('./agent-error-detection.js', () => ({
classifyRotationTrigger: vi.fn(() => ({
shouldRetry: false,
reason: '',
})),
}));
vi.mock('./agent-error-detection.js', async (importOriginal) => {
const actual =
await importOriginal<typeof import('./agent-error-detection.js')>();
return {
...actual,
classifyRotationTrigger: vi.fn(() => ({
shouldRetry: false,
reason: '',
})),
};
});
vi.mock('./codex-token-rotation.js', () => ({
detectCodexRotationTrigger: vi.fn(() => ({
@@ -71,6 +76,18 @@ describe('message-agent-executor-rules', () => {
).toEqual({ reason: 'auth-expired' });
});
it('ignores streamed reasons that are not valid Codex rotation triggers', () => {
expect(
resolveCodexRetryTrigger({
canRetryCodex: true,
attempt: {
streamedTriggerReason: { reason: 'usage-exhausted' },
},
}),
).toBeNull();
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
});
it('detects retryable Claude session failures from either flag or classifier', () => {
expect(
isRetryableClaudeSessionFailureAttempt({

View File

@@ -2,6 +2,7 @@ import {
classifyRotationTrigger,
type AgentTriggerReason,
type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
@@ -94,8 +95,11 @@ export function resolveCodexRetryTrigger(args: {
}
if (args.attempt.streamedTriggerReason) {
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
return null;
}
return {
reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason,
reason: args.attempt.streamedTriggerReason.reason,
};
}

View File

@@ -10,6 +10,7 @@ import { getAgentOutputText } from './agent-output.js';
import {
classifyRotationTrigger,
type CodexRotationReason,
isCodexRotationReason,
shouldRotateClaudeToken,
type AgentTriggerReason,
} from './agent-error-detection.js';
@@ -210,22 +211,35 @@ function evaluateCodexAttempt(
attempt.streamedTriggerReason &&
output.status !== 'error'
) {
if (!isCodexRotationReason(attempt.streamedTriggerReason.reason)) {
logger.error(
{
...logContext,
provider: 'codex',
reason: attempt.streamedTriggerReason.reason,
},
'Unexpected non-Codex streamed trigger while rotating Codex account',
);
return { type: 'error' };
}
return {
type: 'continue',
trigger: {
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
reason: attempt.streamedTriggerReason.reason,
},
rotationMessage: getAgentOutputText(output) ?? undefined,
};
}
if (output.status === 'error') {
const retryTrigger = attempt.streamedTriggerReason
? {
shouldRotate: true,
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
}
: detectCodexRotationTrigger(output.error);
const retryTrigger =
attempt.streamedTriggerReason &&
isCodexRotationReason(attempt.streamedTriggerReason.reason)
? {
shouldRotate: true as const,
reason: attempt.streamedTriggerReason.reason,
}
: detectCodexRotationTrigger(output.error);
if (retryTrigger.shouldRotate) {
return {
type: 'continue',

View File

@@ -27,7 +27,7 @@ import {
} from './group-folder.js';
import { createScopedLogger, logger } from './logger.js';
import { createTaskStatusTracker } from './task-status-tracker.js';
import { runClaudeRotationLoop } from './provider-retry.js';
import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js';
import {
detectCodexRotationTrigger,
rotateCodexToken,
@@ -38,6 +38,7 @@ import {
classifyRotationTrigger,
type AgentTriggerReason,
type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js';
import {
getTokenCount,
@@ -489,65 +490,29 @@ async function runTask(
initialTrigger: { reason: CodexRotationReason },
rotationMessage?: string,
): Promise<void> => {
let trigger = initialTrigger;
let lastRotationMessage = rotationMessage;
while (
getCodexAccountCount() > 1 &&
rotateCodexToken(lastRotationMessage)
) {
log.info(
{
reason: trigger.reason,
},
'Codex account unhealthy, retrying scheduled task with rotated account',
);
const retryAttempt = await runTaskAttempt('codex');
result = retryAttempt.attemptResult;
error = retryAttempt.attemptError;
if (
!retryAttempt.sawOutput &&
retryAttempt.streamedTriggerReason &&
retryAttempt.output.status !== 'error'
) {
trigger = {
reason: retryAttempt.streamedTriggerReason
.reason as CodexRotationReason,
const outcome = await runCodexRotationLoop(
initialTrigger,
async () => {
const retryAttempt = await runTaskAttempt('codex');
result = retryAttempt.attemptResult;
error = retryAttempt.attemptError;
return {
output: retryAttempt.output,
thrownError: null,
sawOutput: retryAttempt.sawOutput,
streamedTriggerReason: retryAttempt.streamedTriggerReason,
};
lastRotationMessage =
typeof retryAttempt.output.result === 'string'
? retryAttempt.output.result
: undefined;
continue;
}
},
{
taskId: task.id,
group: context.group.name,
groupFolder: task.group_folder,
},
rotationMessage,
);
if (retryAttempt.output.status === 'error') {
const retryTrigger = retryAttempt.streamedTriggerReason
? {
shouldRotate: true,
reason: retryAttempt.streamedTriggerReason
.reason as CodexRotationReason,
}
: detectCodexRotationTrigger(
retryAttempt.attemptError || retryAttempt.output.error,
);
if (retryTrigger.shouldRotate) {
trigger = { reason: retryTrigger.reason };
lastRotationMessage =
retryAttempt.attemptError ||
retryAttempt.output.error ||
undefined;
continue;
}
return;
}
markCodexTokenHealthy();
if (outcome.type === 'success') {
error = null;
return;
}
};
@@ -567,11 +532,12 @@ async function runTask(
} else if (
provider === 'codex' &&
attempt.streamedTriggerReason &&
!attempt.sawOutput
!attempt.sawOutput &&
isCodexRotationReason(attempt.streamedTriggerReason.reason)
) {
await retryCodexTaskWithRotation(
{
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
reason: attempt.streamedTriggerReason.reason,
},
typeof attempt.output.error === 'string'
? attempt.output.error
@@ -592,13 +558,14 @@ async function runTask(
});
}
} else if (attempt.output.status === 'error' && provider === 'codex') {
const trigger = attempt.streamedTriggerReason
? {
shouldRotate: true,
reason: attempt.streamedTriggerReason
.reason as CodexRotationReason,
}
: detectCodexRotationTrigger(error);
const trigger =
attempt.streamedTriggerReason &&
isCodexRotationReason(attempt.streamedTriggerReason.reason)
? {
shouldRotate: true as const,
reason: attempt.streamedTriggerReason.reason,
}
: detectCodexRotationTrigger(error);
if (trigger.shouldRotate) {
await retryCodexTaskWithRotation(
{ reason: trigger.reason },