Unify codex rotation retry guards
This commit is contained in:
@@ -126,6 +126,18 @@ export function shouldRotateClaudeToken(
|
||||
);
|
||||
}
|
||||
|
||||
export function isCodexRotationReason(
|
||||
reason: AgentTriggerReason,
|
||||
): reason is CodexRotationReason {
|
||||
return (
|
||||
reason === '429' ||
|
||||
reason === 'auth-expired' ||
|
||||
reason === 'org-access-denied' ||
|
||||
reason === 'overloaded' ||
|
||||
reason === 'network-error'
|
||||
);
|
||||
}
|
||||
|
||||
// ── Rotation trigger classification ─────────────────────────────
|
||||
|
||||
export type RotationTriggerResult =
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./agent-error-detection.js', () => ({
|
||||
classifyRotationTrigger: vi.fn(() => ({
|
||||
shouldRetry: false,
|
||||
reason: '',
|
||||
})),
|
||||
}));
|
||||
vi.mock('./agent-error-detection.js', async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import('./agent-error-detection.js')>();
|
||||
return {
|
||||
...actual,
|
||||
classifyRotationTrigger: vi.fn(() => ({
|
||||
shouldRetry: false,
|
||||
reason: '',
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
detectCodexRotationTrigger: vi.fn(() => ({
|
||||
@@ -71,6 +76,18 @@ describe('message-agent-executor-rules', () => {
|
||||
).toEqual({ reason: 'auth-expired' });
|
||||
});
|
||||
|
||||
it('ignores streamed reasons that are not valid Codex rotation triggers', () => {
|
||||
expect(
|
||||
resolveCodexRetryTrigger({
|
||||
canRetryCodex: true,
|
||||
attempt: {
|
||||
streamedTriggerReason: { reason: 'usage-exhausted' },
|
||||
},
|
||||
}),
|
||||
).toBeNull();
|
||||
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('detects retryable Claude session failures from either flag or classifier', () => {
|
||||
expect(
|
||||
isRetryableClaudeSessionFailureAttempt({
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
classifyRotationTrigger,
|
||||
type AgentTriggerReason,
|
||||
type CodexRotationReason,
|
||||
isCodexRotationReason,
|
||||
} from './agent-error-detection.js';
|
||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||
@@ -94,8 +95,11 @@ export function resolveCodexRetryTrigger(args: {
|
||||
}
|
||||
|
||||
if (args.attempt.streamedTriggerReason) {
|
||||
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||
reason: args.attempt.streamedTriggerReason.reason,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getAgentOutputText } from './agent-output.js';
|
||||
import {
|
||||
classifyRotationTrigger,
|
||||
type CodexRotationReason,
|
||||
isCodexRotationReason,
|
||||
shouldRotateClaudeToken,
|
||||
type AgentTriggerReason,
|
||||
} from './agent-error-detection.js';
|
||||
@@ -210,22 +211,35 @@ function evaluateCodexAttempt(
|
||||
attempt.streamedTriggerReason &&
|
||||
output.status !== 'error'
|
||||
) {
|
||||
if (!isCodexRotationReason(attempt.streamedTriggerReason.reason)) {
|
||||
logger.error(
|
||||
{
|
||||
...logContext,
|
||||
provider: 'codex',
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
},
|
||||
'Unexpected non-Codex streamed trigger while rotating Codex account',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
return {
|
||||
type: 'continue',
|
||||
trigger: {
|
||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
},
|
||||
rotationMessage: getAgentOutputText(output) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (output.status === 'error') {
|
||||
const retryTrigger = attempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRotate: true,
|
||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||
}
|
||||
: detectCodexRotationTrigger(output.error);
|
||||
const retryTrigger =
|
||||
attempt.streamedTriggerReason &&
|
||||
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||
? {
|
||||
shouldRotate: true as const,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
}
|
||||
: detectCodexRotationTrigger(output.error);
|
||||
if (retryTrigger.shouldRotate) {
|
||||
return {
|
||||
type: 'continue',
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
} from './group-folder.js';
|
||||
import { createScopedLogger, logger } from './logger.js';
|
||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||
import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js';
|
||||
import {
|
||||
detectCodexRotationTrigger,
|
||||
rotateCodexToken,
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
classifyRotationTrigger,
|
||||
type AgentTriggerReason,
|
||||
type CodexRotationReason,
|
||||
isCodexRotationReason,
|
||||
} from './agent-error-detection.js';
|
||||
import {
|
||||
getTokenCount,
|
||||
@@ -489,65 +490,29 @@ async function runTask(
|
||||
initialTrigger: { reason: CodexRotationReason },
|
||||
rotationMessage?: string,
|
||||
): Promise<void> => {
|
||||
let trigger = initialTrigger;
|
||||
let lastRotationMessage = rotationMessage;
|
||||
|
||||
while (
|
||||
getCodexAccountCount() > 1 &&
|
||||
rotateCodexToken(lastRotationMessage)
|
||||
) {
|
||||
log.info(
|
||||
{
|
||||
reason: trigger.reason,
|
||||
},
|
||||
'Codex account unhealthy, retrying scheduled task with rotated account',
|
||||
);
|
||||
|
||||
const retryAttempt = await runTaskAttempt('codex');
|
||||
result = retryAttempt.attemptResult;
|
||||
error = retryAttempt.attemptError;
|
||||
|
||||
if (
|
||||
!retryAttempt.sawOutput &&
|
||||
retryAttempt.streamedTriggerReason &&
|
||||
retryAttempt.output.status !== 'error'
|
||||
) {
|
||||
trigger = {
|
||||
reason: retryAttempt.streamedTriggerReason
|
||||
.reason as CodexRotationReason,
|
||||
const outcome = await runCodexRotationLoop(
|
||||
initialTrigger,
|
||||
async () => {
|
||||
const retryAttempt = await runTaskAttempt('codex');
|
||||
result = retryAttempt.attemptResult;
|
||||
error = retryAttempt.attemptError;
|
||||
return {
|
||||
output: retryAttempt.output,
|
||||
thrownError: null,
|
||||
sawOutput: retryAttempt.sawOutput,
|
||||
streamedTriggerReason: retryAttempt.streamedTriggerReason,
|
||||
};
|
||||
lastRotationMessage =
|
||||
typeof retryAttempt.output.result === 'string'
|
||||
? retryAttempt.output.result
|
||||
: undefined;
|
||||
continue;
|
||||
}
|
||||
},
|
||||
{
|
||||
taskId: task.id,
|
||||
group: context.group.name,
|
||||
groupFolder: task.group_folder,
|
||||
},
|
||||
rotationMessage,
|
||||
);
|
||||
|
||||
if (retryAttempt.output.status === 'error') {
|
||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRotate: true,
|
||||
reason: retryAttempt.streamedTriggerReason
|
||||
.reason as CodexRotationReason,
|
||||
}
|
||||
: detectCodexRotationTrigger(
|
||||
retryAttempt.attemptError || retryAttempt.output.error,
|
||||
);
|
||||
|
||||
if (retryTrigger.shouldRotate) {
|
||||
trigger = { reason: retryTrigger.reason };
|
||||
lastRotationMessage =
|
||||
retryAttempt.attemptError ||
|
||||
retryAttempt.output.error ||
|
||||
undefined;
|
||||
continue;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
markCodexTokenHealthy();
|
||||
if (outcome.type === 'success') {
|
||||
error = null;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -567,11 +532,12 @@ async function runTask(
|
||||
} else if (
|
||||
provider === 'codex' &&
|
||||
attempt.streamedTriggerReason &&
|
||||
!attempt.sawOutput
|
||||
!attempt.sawOutput &&
|
||||
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||
) {
|
||||
await retryCodexTaskWithRotation(
|
||||
{
|
||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
},
|
||||
typeof attempt.output.error === 'string'
|
||||
? attempt.output.error
|
||||
@@ -592,13 +558,14 @@ async function runTask(
|
||||
});
|
||||
}
|
||||
} else if (attempt.output.status === 'error' && provider === 'codex') {
|
||||
const trigger = attempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRotate: true,
|
||||
reason: attempt.streamedTriggerReason
|
||||
.reason as CodexRotationReason,
|
||||
}
|
||||
: detectCodexRotationTrigger(error);
|
||||
const trigger =
|
||||
attempt.streamedTriggerReason &&
|
||||
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||
? {
|
||||
shouldRotate: true as const,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
}
|
||||
: detectCodexRotationTrigger(error);
|
||||
if (trigger.shouldRotate) {
|
||||
await retryCodexTaskWithRotation(
|
||||
{ reason: trigger.reason },
|
||||
|
||||
Reference in New Issue
Block a user