Unify codex rotation retry guards
This commit is contained in:
@@ -126,6 +126,18 @@ export function shouldRotateClaudeToken(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isCodexRotationReason(
|
||||||
|
reason: AgentTriggerReason,
|
||||||
|
): reason is CodexRotationReason {
|
||||||
|
return (
|
||||||
|
reason === '429' ||
|
||||||
|
reason === 'auth-expired' ||
|
||||||
|
reason === 'org-access-denied' ||
|
||||||
|
reason === 'overloaded' ||
|
||||||
|
reason === 'network-error'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ── Rotation trigger classification ─────────────────────────────
|
// ── Rotation trigger classification ─────────────────────────────
|
||||||
|
|
||||||
export type RotationTriggerResult =
|
export type RotationTriggerResult =
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
vi.mock('./agent-error-detection.js', () => ({
|
vi.mock('./agent-error-detection.js', async (importOriginal) => {
|
||||||
classifyRotationTrigger: vi.fn(() => ({
|
const actual =
|
||||||
shouldRetry: false,
|
await importOriginal<typeof import('./agent-error-detection.js')>();
|
||||||
reason: '',
|
return {
|
||||||
})),
|
...actual,
|
||||||
}));
|
classifyRotationTrigger: vi.fn(() => ({
|
||||||
|
shouldRetry: false,
|
||||||
|
reason: '',
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock('./codex-token-rotation.js', () => ({
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
detectCodexRotationTrigger: vi.fn(() => ({
|
detectCodexRotationTrigger: vi.fn(() => ({
|
||||||
@@ -71,6 +76,18 @@ describe('message-agent-executor-rules', () => {
|
|||||||
).toEqual({ reason: 'auth-expired' });
|
).toEqual({ reason: 'auth-expired' });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('ignores streamed reasons that are not valid Codex rotation triggers', () => {
|
||||||
|
expect(
|
||||||
|
resolveCodexRetryTrigger({
|
||||||
|
canRetryCodex: true,
|
||||||
|
attempt: {
|
||||||
|
streamedTriggerReason: { reason: 'usage-exhausted' },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).toBeNull();
|
||||||
|
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it('detects retryable Claude session failures from either flag or classifier', () => {
|
it('detects retryable Claude session failures from either flag or classifier', () => {
|
||||||
expect(
|
expect(
|
||||||
isRetryableClaudeSessionFailureAttempt({
|
isRetryableClaudeSessionFailureAttempt({
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import {
|
|||||||
classifyRotationTrigger,
|
classifyRotationTrigger,
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
type CodexRotationReason,
|
type CodexRotationReason,
|
||||||
|
isCodexRotationReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||||
@@ -94,8 +95,11 @@ export function resolveCodexRetryTrigger(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (args.attempt.streamedTriggerReason) {
|
if (args.attempt.streamedTriggerReason) {
|
||||||
|
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason,
|
reason: args.attempt.streamedTriggerReason.reason,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { getAgentOutputText } from './agent-output.js';
|
|||||||
import {
|
import {
|
||||||
classifyRotationTrigger,
|
classifyRotationTrigger,
|
||||||
type CodexRotationReason,
|
type CodexRotationReason,
|
||||||
|
isCodexRotationReason,
|
||||||
shouldRotateClaudeToken,
|
shouldRotateClaudeToken,
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
@@ -210,22 +211,35 @@ function evaluateCodexAttempt(
|
|||||||
attempt.streamedTriggerReason &&
|
attempt.streamedTriggerReason &&
|
||||||
output.status !== 'error'
|
output.status !== 'error'
|
||||||
) {
|
) {
|
||||||
|
if (!isCodexRotationReason(attempt.streamedTriggerReason.reason)) {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
...logContext,
|
||||||
|
provider: 'codex',
|
||||||
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
},
|
||||||
|
'Unexpected non-Codex streamed trigger while rotating Codex account',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
type: 'continue',
|
type: 'continue',
|
||||||
trigger: {
|
trigger: {
|
||||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
},
|
},
|
||||||
rotationMessage: getAgentOutputText(output) ?? undefined,
|
rotationMessage: getAgentOutputText(output) ?? undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (output.status === 'error') {
|
if (output.status === 'error') {
|
||||||
const retryTrigger = attempt.streamedTriggerReason
|
const retryTrigger =
|
||||||
? {
|
attempt.streamedTriggerReason &&
|
||||||
shouldRotate: true,
|
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
? {
|
||||||
}
|
shouldRotate: true as const,
|
||||||
: detectCodexRotationTrigger(output.error);
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
}
|
||||||
|
: detectCodexRotationTrigger(output.error);
|
||||||
if (retryTrigger.shouldRotate) {
|
if (retryTrigger.shouldRotate) {
|
||||||
return {
|
return {
|
||||||
type: 'continue',
|
type: 'continue',
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import {
|
|||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import { createScopedLogger, logger } from './logger.js';
|
import { createScopedLogger, logger } from './logger.js';
|
||||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop, runCodexRotationLoop } from './provider-retry.js';
|
||||||
import {
|
import {
|
||||||
detectCodexRotationTrigger,
|
detectCodexRotationTrigger,
|
||||||
rotateCodexToken,
|
rotateCodexToken,
|
||||||
@@ -38,6 +38,7 @@ import {
|
|||||||
classifyRotationTrigger,
|
classifyRotationTrigger,
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
type CodexRotationReason,
|
type CodexRotationReason,
|
||||||
|
isCodexRotationReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
@@ -489,65 +490,29 @@ async function runTask(
|
|||||||
initialTrigger: { reason: CodexRotationReason },
|
initialTrigger: { reason: CodexRotationReason },
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
let trigger = initialTrigger;
|
const outcome = await runCodexRotationLoop(
|
||||||
let lastRotationMessage = rotationMessage;
|
initialTrigger,
|
||||||
|
async () => {
|
||||||
while (
|
const retryAttempt = await runTaskAttempt('codex');
|
||||||
getCodexAccountCount() > 1 &&
|
result = retryAttempt.attemptResult;
|
||||||
rotateCodexToken(lastRotationMessage)
|
error = retryAttempt.attemptError;
|
||||||
) {
|
return {
|
||||||
log.info(
|
output: retryAttempt.output,
|
||||||
{
|
thrownError: null,
|
||||||
reason: trigger.reason,
|
sawOutput: retryAttempt.sawOutput,
|
||||||
},
|
streamedTriggerReason: retryAttempt.streamedTriggerReason,
|
||||||
'Codex account unhealthy, retrying scheduled task with rotated account',
|
|
||||||
);
|
|
||||||
|
|
||||||
const retryAttempt = await runTaskAttempt('codex');
|
|
||||||
result = retryAttempt.attemptResult;
|
|
||||||
error = retryAttempt.attemptError;
|
|
||||||
|
|
||||||
if (
|
|
||||||
!retryAttempt.sawOutput &&
|
|
||||||
retryAttempt.streamedTriggerReason &&
|
|
||||||
retryAttempt.output.status !== 'error'
|
|
||||||
) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryAttempt.streamedTriggerReason
|
|
||||||
.reason as CodexRotationReason,
|
|
||||||
};
|
};
|
||||||
lastRotationMessage =
|
},
|
||||||
typeof retryAttempt.output.result === 'string'
|
{
|
||||||
? retryAttempt.output.result
|
taskId: task.id,
|
||||||
: undefined;
|
group: context.group.name,
|
||||||
continue;
|
groupFolder: task.group_folder,
|
||||||
}
|
},
|
||||||
|
rotationMessage,
|
||||||
|
);
|
||||||
|
|
||||||
if (retryAttempt.output.status === 'error') {
|
if (outcome.type === 'success') {
|
||||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
|
||||||
? {
|
|
||||||
shouldRotate: true,
|
|
||||||
reason: retryAttempt.streamedTriggerReason
|
|
||||||
.reason as CodexRotationReason,
|
|
||||||
}
|
|
||||||
: detectCodexRotationTrigger(
|
|
||||||
retryAttempt.attemptError || retryAttempt.output.error,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (retryTrigger.shouldRotate) {
|
|
||||||
trigger = { reason: retryTrigger.reason };
|
|
||||||
lastRotationMessage =
|
|
||||||
retryAttempt.attemptError ||
|
|
||||||
retryAttempt.output.error ||
|
|
||||||
undefined;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
markCodexTokenHealthy();
|
|
||||||
error = null;
|
error = null;
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -567,11 +532,12 @@ async function runTask(
|
|||||||
} else if (
|
} else if (
|
||||||
provider === 'codex' &&
|
provider === 'codex' &&
|
||||||
attempt.streamedTriggerReason &&
|
attempt.streamedTriggerReason &&
|
||||||
!attempt.sawOutput
|
!attempt.sawOutput &&
|
||||||
|
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||||
) {
|
) {
|
||||||
await retryCodexTaskWithRotation(
|
await retryCodexTaskWithRotation(
|
||||||
{
|
{
|
||||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
},
|
},
|
||||||
typeof attempt.output.error === 'string'
|
typeof attempt.output.error === 'string'
|
||||||
? attempt.output.error
|
? attempt.output.error
|
||||||
@@ -592,13 +558,14 @@ async function runTask(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if (attempt.output.status === 'error' && provider === 'codex') {
|
} else if (attempt.output.status === 'error' && provider === 'codex') {
|
||||||
const trigger = attempt.streamedTriggerReason
|
const trigger =
|
||||||
? {
|
attempt.streamedTriggerReason &&
|
||||||
shouldRotate: true,
|
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||||
reason: attempt.streamedTriggerReason
|
? {
|
||||||
.reason as CodexRotationReason,
|
shouldRotate: true as const,
|
||||||
}
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
: detectCodexRotationTrigger(error);
|
}
|
||||||
|
: detectCodexRotationTrigger(error);
|
||||||
if (trigger.shouldRotate) {
|
if (trigger.shouldRotate) {
|
||||||
await retryCodexTaskWithRotation(
|
await retryCodexTaskWithRotation(
|
||||||
{ reason: trigger.reason },
|
{ reason: trigger.reason },
|
||||||
|
|||||||
Reference in New Issue
Block a user