Extract executor retry decision rules
This commit is contained in:
115
src/message-agent-executor-rules.test.ts
Normal file
115
src/message-agent-executor-rules.test.ts
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./agent-error-detection.js', () => ({
|
||||||
|
classifyRotationTrigger: vi.fn(() => ({
|
||||||
|
shouldRetry: false,
|
||||||
|
reason: '',
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
|
detectCodexRotationTrigger: vi.fn(() => ({
|
||||||
|
shouldRotate: false,
|
||||||
|
reason: '',
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { classifyRotationTrigger } from './agent-error-detection.js';
|
||||||
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
|
import {
|
||||||
|
isRetryableClaudeSessionFailureAttempt,
|
||||||
|
resolveClaudeRetryTrigger,
|
||||||
|
resolveCodexRetryTrigger,
|
||||||
|
resolvePairedFollowUpQueueAction,
|
||||||
|
} from './message-agent-executor-rules.js';
|
||||||
|
|
||||||
|
describe('message-agent-executor-rules', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves a Claude retry trigger from streamed trigger state', () => {
|
||||||
|
expect(
|
||||||
|
resolveClaudeRetryTrigger({
|
||||||
|
canRetryClaudeCredentials: true,
|
||||||
|
provider: 'claude',
|
||||||
|
attempt: {
|
||||||
|
sawOutput: false,
|
||||||
|
streamedTriggerReason: { reason: 'auth-expired', retryAfterMs: 5000 },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
reason: 'auth-expired',
|
||||||
|
retryAfterMs: 5000,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null for Claude retry when output was already visible', () => {
|
||||||
|
expect(
|
||||||
|
resolveClaudeRetryTrigger({
|
||||||
|
canRetryClaudeCredentials: true,
|
||||||
|
provider: 'claude',
|
||||||
|
attempt: { sawOutput: true },
|
||||||
|
fallbackMessage: '429 rate limit',
|
||||||
|
}),
|
||||||
|
).toBeNull();
|
||||||
|
expect(classifyRotationTrigger).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves a Codex retry trigger from the codex detector', () => {
|
||||||
|
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
||||||
|
shouldRotate: true,
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(
|
||||||
|
resolveCodexRetryTrigger({
|
||||||
|
canRetryCodex: true,
|
||||||
|
attempt: {},
|
||||||
|
rotationMessage: '401 oauth token has expired',
|
||||||
|
}),
|
||||||
|
).toEqual({ reason: 'auth-expired' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('detects retryable Claude session failures from either flag or classifier', () => {
|
||||||
|
expect(
|
||||||
|
isRetryableClaudeSessionFailureAttempt({
|
||||||
|
attempt: {
|
||||||
|
sawOutput: false,
|
||||||
|
retryableSessionFailureDetected: true,
|
||||||
|
},
|
||||||
|
isClaudeCodeAgent: true,
|
||||||
|
provider: 'claude',
|
||||||
|
shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false),
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
|
||||||
|
const shouldRetryFreshSessionOnAgentFailure = vi.fn(() => true);
|
||||||
|
expect(
|
||||||
|
isRetryableClaudeSessionFailureAttempt({
|
||||||
|
attempt: {
|
||||||
|
sawOutput: false,
|
||||||
|
error: new Error('stale session'),
|
||||||
|
},
|
||||||
|
isClaudeCodeAgent: true,
|
||||||
|
provider: 'claude',
|
||||||
|
shouldRetryFreshSessionOnAgentFailure,
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
expect(shouldRetryFreshSessionOnAgentFailure).toHaveBeenCalledWith({
|
||||||
|
result: null,
|
||||||
|
error: 'stale session',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves pending reviewer follow-up requeue from task state', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'reviewer',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'review_ready',
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
});
|
||||||
139
src/message-agent-executor-rules.ts
Normal file
139
src/message-agent-executor-rules.ts
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
import {
|
||||||
|
classifyRotationTrigger,
|
||||||
|
type AgentTriggerReason,
|
||||||
|
type CodexRotationReason,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
|
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
|
export interface ExecutorStreamedTrigger {
|
||||||
|
reason: AgentTriggerReason;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ExecutorAttemptState {
|
||||||
|
sawOutput: boolean;
|
||||||
|
retryableSessionFailureDetected?: boolean;
|
||||||
|
streamedTriggerReason?: ExecutorStreamedTrigger;
|
||||||
|
error?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRetryableClaudeSessionFailureAttempt(args: {
|
||||||
|
attempt: ExecutorAttemptState;
|
||||||
|
isClaudeCodeAgent: boolean;
|
||||||
|
provider: 'claude' | 'codex';
|
||||||
|
shouldRetryFreshSessionOnAgentFailure: (args: {
|
||||||
|
result: null;
|
||||||
|
error: string;
|
||||||
|
}) => boolean;
|
||||||
|
}): boolean {
|
||||||
|
if (
|
||||||
|
!args.isClaudeCodeAgent ||
|
||||||
|
args.provider !== 'claude' ||
|
||||||
|
args.attempt.sawOutput
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.retryableSessionFailureDetected === true) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.error == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return args.shouldRetryFreshSessionOnAgentFailure({
|
||||||
|
result: null,
|
||||||
|
error: getErrorMessage(args.attempt.error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveClaudeRetryTrigger(args: {
|
||||||
|
canRetryClaudeCredentials: boolean;
|
||||||
|
provider: 'claude' | 'codex';
|
||||||
|
attempt: Pick<ExecutorAttemptState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||||
|
fallbackMessage?: string | null;
|
||||||
|
}): { reason: AgentTriggerReason; retryAfterMs?: number } | null {
|
||||||
|
if (
|
||||||
|
!(
|
||||||
|
args.canRetryClaudeCredentials &&
|
||||||
|
args.provider === 'claude' &&
|
||||||
|
!args.attempt.sawOutput
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.streamedTriggerReason) {
|
||||||
|
return {
|
||||||
|
reason: args.attempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const trigger = classifyRotationTrigger(args.fallbackMessage);
|
||||||
|
if (!trigger.shouldRetry) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
reason: trigger.reason,
|
||||||
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveCodexRetryTrigger(args: {
|
||||||
|
canRetryCodex: boolean;
|
||||||
|
attempt: Pick<ExecutorAttemptState, 'streamedTriggerReason'>;
|
||||||
|
rotationMessage?: string | null;
|
||||||
|
}): { reason: CodexRotationReason } | null {
|
||||||
|
if (!args.canRetryCodex) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.streamedTriggerReason) {
|
||||||
|
return {
|
||||||
|
reason: args.attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const trigger = detectCodexRotationTrigger(args.rotationMessage);
|
||||||
|
if (!trigger.shouldRotate) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { reason: trigger.reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PairedFollowUpQueueAction =
|
||||||
|
| 'generic'
|
||||||
|
| 'pending'
|
||||||
|
| 'skip-inline-finalize'
|
||||||
|
| 'none';
|
||||||
|
|
||||||
|
export function resolvePairedFollowUpQueueAction(args: {
|
||||||
|
completedRole: PairedRoomRole;
|
||||||
|
executionStatus: 'succeeded' | 'failed';
|
||||||
|
sawOutput: boolean;
|
||||||
|
taskStatus: PairedTaskStatus | null;
|
||||||
|
}): PairedFollowUpQueueAction {
|
||||||
|
if (args.executionStatus === 'succeeded' && args.sawOutput) {
|
||||||
|
return args.completedRole === 'reviewer' &&
|
||||||
|
args.taskStatus === 'merge_ready'
|
||||||
|
? 'skip-inline-finalize'
|
||||||
|
: args.taskStatus !== 'completed'
|
||||||
|
? 'generic'
|
||||||
|
: 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
const shouldRequeuePendingPairedTurn =
|
||||||
|
(args.completedRole === 'reviewer' || args.completedRole === 'arbiter') &&
|
||||||
|
(args.taskStatus === 'review_ready' ||
|
||||||
|
args.taskStatus === 'in_review' ||
|
||||||
|
args.taskStatus === 'arbiter_requested' ||
|
||||||
|
args.taskStatus === 'in_arbitration' ||
|
||||||
|
args.taskStatus === 'merge_ready');
|
||||||
|
return shouldRequeuePendingPairedTurn ? 'pending' : 'none';
|
||||||
|
}
|
||||||
@@ -37,12 +37,20 @@ import {
|
|||||||
resolveEffectiveAgentType,
|
resolveEffectiveAgentType,
|
||||||
resolveSessionFolder,
|
resolveSessionFolder,
|
||||||
} from './message-runtime-rules.js';
|
} from './message-runtime-rules.js';
|
||||||
|
import {
|
||||||
|
isRetryableClaudeSessionFailureAttempt,
|
||||||
|
resolveClaudeRetryTrigger,
|
||||||
|
resolveCodexRetryTrigger,
|
||||||
|
resolvePairedFollowUpQueueAction,
|
||||||
|
} from './message-agent-executor-rules.js';
|
||||||
import { buildRoomRoleContext } from './room-role-context.js';
|
import { buildRoomRoleContext } from './room-role-context.js';
|
||||||
import {
|
import {
|
||||||
classifyRotationTrigger,
|
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import {
|
||||||
|
runClaudeRotationLoop,
|
||||||
|
runCodexRotationLoop,
|
||||||
|
} from './provider-retry.js';
|
||||||
import {
|
import {
|
||||||
shouldResetSessionOnAgentFailure,
|
shouldResetSessionOnAgentFailure,
|
||||||
shouldRetryFreshSessionOnAgentFailure,
|
shouldRetryFreshSessionOnAgentFailure,
|
||||||
@@ -69,9 +77,7 @@ import {
|
|||||||
} from './streamed-output-evaluator.js';
|
} from './streamed-output-evaluator.js';
|
||||||
import {
|
import {
|
||||||
detectCodexRotationTrigger,
|
detectCodexRotationTrigger,
|
||||||
rotateCodexToken,
|
|
||||||
getCodexAccountCount,
|
getCodexAccountCount,
|
||||||
markCodexTokenHealthy,
|
|
||||||
} from './codex-token-rotation.js';
|
} from './codex-token-rotation.js';
|
||||||
import type { CodexRotationReason } from './agent-error-detection.js';
|
import type { CodexRotationReason } from './agent-error-detection.js';
|
||||||
import { getTokenCount } from './token-rotation.js';
|
import { getTokenCount } from './token-rotation.js';
|
||||||
@@ -169,6 +175,7 @@ export async function runAgentForGroup(
|
|||||||
forceFreshClaudeReviewerSession
|
forceFreshClaudeReviewerSession
|
||||||
? undefined
|
? undefined
|
||||||
: storedSessionId;
|
: storedSessionId;
|
||||||
|
const provider = isClaudeCodeAgent ? 'claude' : 'codex';
|
||||||
const memoryBriefing = currentSessionId
|
const memoryBriefing = currentSessionId
|
||||||
? undefined
|
? undefined
|
||||||
: await buildRoomMemoryBriefing({
|
: await buildRoomMemoryBriefing({
|
||||||
@@ -626,91 +633,27 @@ export async function runAgentForGroup(
|
|||||||
initialTrigger: { reason: CodexRotationReason },
|
initialTrigger: { reason: CodexRotationReason },
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<'success' | 'error'> => {
|
): Promise<'success' | 'error'> => {
|
||||||
let trigger = initialTrigger;
|
const outcome = await runCodexRotationLoop(
|
||||||
let lastRotationMessage = rotationMessage;
|
initialTrigger,
|
||||||
|
async () => {
|
||||||
while (
|
const attempt = await runAttempt('codex');
|
||||||
getCodexAccountCount() > 1 &&
|
return {
|
||||||
rotateCodexToken(lastRotationMessage)
|
output: attempt.output,
|
||||||
) {
|
thrownError: attempt.error,
|
||||||
log.info(
|
sawOutput: attempt.sawOutput,
|
||||||
{ reason: trigger.reason },
|
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||||
'Codex account unhealthy, retrying with rotated account',
|
|
||||||
);
|
|
||||||
|
|
||||||
const retryAttempt = await runAttempt('codex');
|
|
||||||
|
|
||||||
if (retryAttempt.error) {
|
|
||||||
const errMsg = getErrorMessage(retryAttempt.error);
|
|
||||||
const retryTrigger = detectCodexRotationTrigger(errMsg);
|
|
||||||
if (retryTrigger.shouldRotate) {
|
|
||||||
trigger = { reason: retryTrigger.reason };
|
|
||||||
lastRotationMessage = errMsg;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.error(
|
|
||||||
{ provider: 'codex', err: retryAttempt.error },
|
|
||||||
'Rotated Codex account also threw',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
const retryOutput = retryAttempt.output;
|
|
||||||
if (!retryOutput) {
|
|
||||||
log.error(
|
|
||||||
{ provider: 'codex' },
|
|
||||||
'Rotated Codex account produced no output object',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
!retryAttempt.sawOutput &&
|
|
||||||
retryAttempt.streamedTriggerReason &&
|
|
||||||
retryOutput.status !== 'error'
|
|
||||||
) {
|
|
||||||
trigger = {
|
|
||||||
reason: retryAttempt.streamedTriggerReason
|
|
||||||
.reason as CodexRotationReason,
|
|
||||||
};
|
};
|
||||||
lastRotationMessage =
|
|
||||||
typeof retryOutput.result === 'string'
|
|
||||||
? retryOutput.result
|
|
||||||
: undefined;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (retryOutput.status === 'error') {
|
|
||||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
|
||||||
? {
|
|
||||||
shouldRotate: true,
|
|
||||||
reason: retryAttempt.streamedTriggerReason
|
|
||||||
.reason as CodexRotationReason,
|
|
||||||
}
|
|
||||||
: detectCodexRotationTrigger(retryOutput.error);
|
|
||||||
|
|
||||||
if (retryTrigger.shouldRotate) {
|
|
||||||
trigger = { reason: retryTrigger.reason };
|
|
||||||
lastRotationMessage = retryOutput.error ?? undefined;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.error(
|
|
||||||
{
|
|
||||||
provider: 'codex',
|
|
||||||
error: retryOutput.error,
|
|
||||||
},
|
},
|
||||||
'Rotated Codex account failed',
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
},
|
||||||
|
rotationMessage,
|
||||||
);
|
);
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
markCodexTokenHealthy();
|
return outcome.type === 'success' ? 'success' : 'error';
|
||||||
return 'success';
|
|
||||||
}
|
|
||||||
|
|
||||||
return 'error';
|
|
||||||
};
|
};
|
||||||
|
|
||||||
type AgentAttempt = Awaited<ReturnType<typeof runAttempt>>;
|
type AgentAttempt = Awaited<ReturnType<typeof runAttempt>>;
|
||||||
@@ -754,43 +697,16 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const resolveClaudeRetryTrigger = (
|
|
||||||
attempt: AgentAttempt,
|
|
||||||
fallbackMessage?: string | null,
|
|
||||||
): { reason: AgentTriggerReason; retryAfterMs?: number } | null => {
|
|
||||||
if (
|
|
||||||
!(
|
|
||||||
canRetryClaudeCredentials &&
|
|
||||||
provider === 'claude' &&
|
|
||||||
!attempt.sawOutput
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (attempt.streamedTriggerReason) {
|
|
||||||
return {
|
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const trigger = classifyRotationTrigger(fallbackMessage);
|
|
||||||
if (!trigger.shouldRetry) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
reason: trigger.reason,
|
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
const retryClaudeAttemptIfNeeded = async (
|
const retryClaudeAttemptIfNeeded = async (
|
||||||
attempt: AgentAttempt,
|
attempt: AgentAttempt,
|
||||||
rotationMessage?: string | null,
|
rotationMessage?: string | null,
|
||||||
): Promise<'success' | 'error' | null> => {
|
): Promise<'success' | 'error' | null> => {
|
||||||
const trigger = resolveClaudeRetryTrigger(attempt, rotationMessage);
|
const trigger = resolveClaudeRetryTrigger({
|
||||||
|
canRetryClaudeCredentials,
|
||||||
|
provider,
|
||||||
|
attempt,
|
||||||
|
fallbackMessage: rotationMessage,
|
||||||
|
});
|
||||||
if (!trigger) {
|
if (!trigger) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -811,17 +727,12 @@ export async function runAgentForGroup(
|
|||||||
attempt: AgentAttempt,
|
attempt: AgentAttempt,
|
||||||
rotationMessage?: string | null,
|
rotationMessage?: string | null,
|
||||||
): Promise<'success' | 'error' | null> => {
|
): Promise<'success' | 'error' | null> => {
|
||||||
if (isClaudeCodeAgent || getCodexAccountCount() <= 1) {
|
const trigger = resolveCodexRetryTrigger({
|
||||||
return null;
|
canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1,
|
||||||
}
|
attempt,
|
||||||
|
rotationMessage,
|
||||||
const trigger = attempt.streamedTriggerReason
|
});
|
||||||
? {
|
if (!trigger) {
|
||||||
shouldRotate: true,
|
|
||||||
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
|
||||||
}
|
|
||||||
: detectCodexRotationTrigger(rotationMessage);
|
|
||||||
if (!trigger.shouldRotate) {
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -845,58 +756,26 @@ export async function runAgentForGroup(
|
|||||||
return 'error';
|
return 'error';
|
||||||
};
|
};
|
||||||
|
|
||||||
type PairedFollowUpQueueAction =
|
const getPairedExecutionStatus = (): 'succeeded' | 'failed' =>
|
||||||
| 'generic'
|
pairedExecutionStatus;
|
||||||
| 'pending'
|
|
||||||
| 'skip-inline-finalize'
|
|
||||||
| 'none';
|
|
||||||
type PairedTaskStatus = NonNullable<
|
|
||||||
ReturnType<typeof getPairedTaskById>
|
|
||||||
>['status'];
|
|
||||||
|
|
||||||
const resolvePairedFollowUpQueueAction = (
|
|
||||||
completedRole: PairedRoomRole,
|
|
||||||
executionStatus: 'succeeded' | 'failed',
|
|
||||||
sawOutput: boolean,
|
|
||||||
taskStatus: PairedTaskStatus | null,
|
|
||||||
): PairedFollowUpQueueAction => {
|
|
||||||
if (executionStatus === 'succeeded' && sawOutput) {
|
|
||||||
return completedRole === 'reviewer' && taskStatus === 'merge_ready'
|
|
||||||
? 'skip-inline-finalize'
|
|
||||||
: taskStatus !== 'completed'
|
|
||||||
? 'generic'
|
|
||||||
: 'none';
|
|
||||||
}
|
|
||||||
|
|
||||||
const shouldRequeuePendingPairedTurn =
|
|
||||||
(completedRole === 'reviewer' || completedRole === 'arbiter') &&
|
|
||||||
(taskStatus === 'review_ready' ||
|
|
||||||
taskStatus === 'in_review' ||
|
|
||||||
taskStatus === 'arbiter_requested' ||
|
|
||||||
taskStatus === 'in_arbitration' ||
|
|
||||||
taskStatus === 'merge_ready');
|
|
||||||
return shouldRequeuePendingPairedTurn ? 'pending' : 'none';
|
|
||||||
};
|
|
||||||
|
|
||||||
const provider = isClaudeCodeAgent ? 'claude' : 'codex';
|
|
||||||
|
|
||||||
try {
|
|
||||||
let primaryAttempt = await runAttempt(provider);
|
|
||||||
|
|
||||||
const isRetryableClaudeSessionFailure = (
|
const isRetryableClaudeSessionFailure = (
|
||||||
attempt: Awaited<ReturnType<typeof runAttempt>>,
|
attempt: AgentAttempt,
|
||||||
): boolean =>
|
): boolean =>
|
||||||
isClaudeCodeAgent &&
|
isRetryableClaudeSessionFailureAttempt({
|
||||||
provider === 'claude' &&
|
attempt,
|
||||||
!attempt.sawOutput &&
|
isClaudeCodeAgent,
|
||||||
(attempt.retryableSessionFailureDetected === true ||
|
provider,
|
||||||
(attempt.error != null &&
|
shouldRetryFreshSessionOnAgentFailure,
|
||||||
shouldRetryFreshSessionOnAgentFailure({
|
});
|
||||||
result: null,
|
|
||||||
error: getErrorMessage(attempt.error),
|
const recoverRetryableClaudeSessionFailure = async (
|
||||||
})));
|
attempt: AgentAttempt,
|
||||||
|
): Promise<{ attempt: AgentAttempt; resolved: 'success' | 'error' | null }> => {
|
||||||
|
if (!isRetryableClaudeSessionFailure(attempt)) {
|
||||||
|
return { attempt, resolved: null };
|
||||||
|
}
|
||||||
|
|
||||||
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
|
|
||||||
currentSessionId = undefined;
|
currentSessionId = undefined;
|
||||||
deps.clearSession(sessionFolder);
|
deps.clearSession(sessionFolder);
|
||||||
clearRoleSdkSessions();
|
clearRoleSdkSessions();
|
||||||
@@ -904,49 +783,66 @@ export async function runAgentForGroup(
|
|||||||
'Cleared poisoned Claude session before visible output, retrying fresh session',
|
'Cleared poisoned Claude session before visible output, retrying fresh session',
|
||||||
);
|
);
|
||||||
|
|
||||||
primaryAttempt = await runAttempt('claude');
|
const freshAttempt = await runAttempt('claude');
|
||||||
|
if (!isRetryableClaudeSessionFailure(freshAttempt)) {
|
||||||
|
return { attempt: freshAttempt, resolved: null };
|
||||||
|
}
|
||||||
|
|
||||||
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
|
|
||||||
currentSessionId = undefined;
|
currentSessionId = undefined;
|
||||||
deps.clearSession(sessionFolder);
|
deps.clearSession(sessionFolder);
|
||||||
log.warn('Fresh Claude retry also hit a retryable session failure');
|
log.warn('Fresh Claude retry also hit a retryable session failure');
|
||||||
|
log.error('Retryable Claude session failure persisted after fresh retry');
|
||||||
|
return {
|
||||||
|
attempt: freshAttempt,
|
||||||
|
resolved: maybeHandoffAfterError('session-failure', freshAttempt),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
log.error(
|
const handlePrimaryAttemptFailure = async (
|
||||||
'Retryable Claude session failure persisted after fresh retry',
|
attempt: AgentAttempt,
|
||||||
);
|
rotationMessage: string,
|
||||||
return maybeHandoffAfterError('session-failure', primaryAttempt);
|
): Promise<'success' | 'error'> => {
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (primaryAttempt.error) {
|
|
||||||
const errMsg = getErrorMessage(primaryAttempt.error);
|
|
||||||
const claudeRetryResult = await retryClaudeAttemptIfNeeded(
|
const claudeRetryResult = await retryClaudeAttemptIfNeeded(
|
||||||
primaryAttempt,
|
attempt,
|
||||||
errMsg,
|
rotationMessage,
|
||||||
);
|
);
|
||||||
if (claudeRetryResult) {
|
if (claudeRetryResult) {
|
||||||
return claudeRetryResult;
|
return claudeRetryResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
||||||
primaryAttempt,
|
attempt,
|
||||||
errMsg,
|
rotationMessage,
|
||||||
);
|
);
|
||||||
if (codexRetryResult) {
|
if (codexRetryResult) {
|
||||||
return codexRetryResult;
|
return codexRetryResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (attempt.error) {
|
||||||
log.error(
|
log.error(
|
||||||
{
|
{
|
||||||
provider,
|
provider,
|
||||||
err: primaryAttempt.error,
|
err: attempt.error,
|
||||||
},
|
},
|
||||||
'Agent error',
|
'Agent error',
|
||||||
);
|
);
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
const output = primaryAttempt.output;
|
log.error(
|
||||||
|
{
|
||||||
|
provider,
|
||||||
|
error: attempt.output?.error,
|
||||||
|
},
|
||||||
|
'Agent process error',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
};
|
||||||
|
|
||||||
|
const finalizePrimaryAttempt = async (
|
||||||
|
attempt: AgentAttempt,
|
||||||
|
): Promise<'success' | 'error'> => {
|
||||||
|
const output = attempt.output;
|
||||||
if (!output) {
|
if (!output) {
|
||||||
log.error({ provider }, 'Agent produced no output object');
|
log.error({ provider }, 'Agent produced no output object');
|
||||||
return 'error';
|
return 'error';
|
||||||
@@ -963,9 +859,8 @@ export async function runAgentForGroup(
|
|||||||
: null);
|
: null);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!primaryAttempt.sawOutput && output.status !== 'error') {
|
if (!attempt.sawOutput && output.status !== 'error') {
|
||||||
const claudeRetryResult =
|
const claudeRetryResult = await retryClaudeAttemptIfNeeded(attempt);
|
||||||
await retryClaudeAttemptIfNeeded(primaryAttempt);
|
|
||||||
if (claudeRetryResult) {
|
if (claudeRetryResult) {
|
||||||
return claudeRetryResult;
|
return claudeRetryResult;
|
||||||
}
|
}
|
||||||
@@ -983,34 +878,14 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (output.status === 'error') {
|
if (output.status === 'error') {
|
||||||
const claudeRetryResult = await retryClaudeAttemptIfNeeded(
|
return handlePrimaryAttemptFailure(
|
||||||
primaryAttempt,
|
attempt,
|
||||||
output.error,
|
output.error ?? 'Agent process error',
|
||||||
);
|
);
|
||||||
if (claudeRetryResult) {
|
|
||||||
return claudeRetryResult;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
||||||
primaryAttempt,
|
attempt,
|
||||||
output.error,
|
|
||||||
);
|
|
||||||
if (codexRetryResult) {
|
|
||||||
return codexRetryResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.error(
|
|
||||||
{
|
|
||||||
provider,
|
|
||||||
error: output.error,
|
|
||||||
},
|
|
||||||
'Agent process error',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
|
||||||
primaryAttempt,
|
|
||||||
output.error ?? output.result,
|
output.error ?? output.result,
|
||||||
);
|
);
|
||||||
if (codexRetryResult) {
|
if (codexRetryResult) {
|
||||||
@@ -1019,19 +894,19 @@ export async function runAgentForGroup(
|
|||||||
|
|
||||||
// Unresolved streamed trigger — rotation was unavailable or output was
|
// Unresolved streamed trigger — rotation was unavailable or output was
|
||||||
// already forwarded. Surfaces as an error since there is no alternative provider.
|
// already forwarded. Surfaces as an error since there is no alternative provider.
|
||||||
if (primaryAttempt.streamedTriggerReason) {
|
if (attempt.streamedTriggerReason) {
|
||||||
if (
|
if (
|
||||||
isClaudeCodeAgent &&
|
isClaudeCodeAgent &&
|
||||||
maybeHandoffToCodex(
|
maybeHandoffToCodex(
|
||||||
primaryAttempt.streamedTriggerReason.reason,
|
attempt.streamedTriggerReason.reason,
|
||||||
primaryAttempt.sawVisibleOutput,
|
attempt.sawVisibleOutput,
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
return 'success';
|
return 'success';
|
||||||
}
|
}
|
||||||
log.error(
|
log.error(
|
||||||
{
|
{
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
},
|
},
|
||||||
'Agent trigger detected but could not be resolved',
|
'Agent trigger detected but could not be resolved',
|
||||||
);
|
);
|
||||||
@@ -1040,30 +915,45 @@ export async function runAgentForGroup(
|
|||||||
|
|
||||||
// success-null-result with no visible output — agent returned nothing useful.
|
// success-null-result with no visible output — agent returned nothing useful.
|
||||||
// But if output was already delivered to Discord (sawOutput), treat as success.
|
// But if output was already delivered to Discord (sawOutput), treat as success.
|
||||||
if (
|
if (attempt.sawSuccessNullResultWithoutOutput && !attempt.sawOutput) {
|
||||||
primaryAttempt.sawSuccessNullResultWithoutOutput &&
|
log.error('Agent returned success with null result and no visible output');
|
||||||
!primaryAttempt.sawOutput
|
|
||||||
) {
|
|
||||||
log.error(
|
|
||||||
'Agent returned success with null result and no visible output',
|
|
||||||
);
|
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
pairedExecutionStatus = 'succeeded';
|
||||||
pairedSawOutput = primaryAttempt.sawOutput;
|
pairedSawOutput = attempt.sawOutput;
|
||||||
return 'success';
|
return 'success';
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
let primaryAttempt = await runAttempt(provider);
|
||||||
|
const recoveredSessionAttempt =
|
||||||
|
await recoverRetryableClaudeSessionFailure(primaryAttempt);
|
||||||
|
if (recoveredSessionAttempt.resolved) {
|
||||||
|
return recoveredSessionAttempt.resolved;
|
||||||
|
}
|
||||||
|
primaryAttempt = recoveredSessionAttempt.attempt;
|
||||||
|
|
||||||
|
if (primaryAttempt.error) {
|
||||||
|
return await handlePrimaryAttemptFailure(
|
||||||
|
primaryAttempt,
|
||||||
|
getErrorMessage(primaryAttempt.error),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return await finalizePrimaryAttempt(primaryAttempt);
|
||||||
} finally {
|
} finally {
|
||||||
if (pairedExecutionContext && !pairedExecutionCompleted) {
|
if (pairedExecutionContext && !pairedExecutionCompleted) {
|
||||||
const completedRole = roomRoleContext?.role ?? 'owner';
|
const completedRole = roomRoleContext?.role ?? 'owner';
|
||||||
|
const completionStatus = getPairedExecutionStatus();
|
||||||
// Owner was interrupted without producing output (e.g. /stop) —
|
// Owner was interrupted without producing output (e.g. /stop) —
|
||||||
// treat as failed so reviewer is not auto-triggered.
|
// treat as failed so reviewer is not auto-triggered.
|
||||||
const effectiveStatus =
|
const effectiveStatus =
|
||||||
completedRole === 'owner' &&
|
completedRole === 'owner' &&
|
||||||
pairedExecutionStatus === 'succeeded' &&
|
completionStatus === 'succeeded' &&
|
||||||
!pairedSawOutput
|
!pairedSawOutput
|
||||||
? 'failed'
|
? 'failed'
|
||||||
: pairedExecutionStatus;
|
: completionStatus;
|
||||||
completePairedExecutionContext({
|
completePairedExecutionContext({
|
||||||
taskId: pairedExecutionContext.task.id,
|
taskId: pairedExecutionContext.task.id,
|
||||||
role: completedRole,
|
role: completedRole,
|
||||||
@@ -1123,12 +1013,12 @@ export async function runAgentForGroup(
|
|||||||
if (pairedExecutionContext) {
|
if (pairedExecutionContext) {
|
||||||
const completedRole = roomRoleContext?.role ?? 'owner';
|
const completedRole = roomRoleContext?.role ?? 'owner';
|
||||||
const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id);
|
const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id);
|
||||||
const queueAction = resolvePairedFollowUpQueueAction(
|
const queueAction = resolvePairedFollowUpQueueAction({
|
||||||
completedRole,
|
completedRole,
|
||||||
pairedExecutionStatus,
|
executionStatus: pairedExecutionStatus,
|
||||||
pairedSawOutput,
|
sawOutput: pairedSawOutput,
|
||||||
finishedCheck?.status ?? null,
|
taskStatus: finishedCheck?.status ?? null,
|
||||||
);
|
});
|
||||||
if (queueAction === 'generic') {
|
if (queueAction === 'generic') {
|
||||||
deps.queue.enqueueMessageCheck(chatJid);
|
deps.queue.enqueueMessageCheck(chatJid);
|
||||||
} else if (queueAction === 'skip-inline-finalize') {
|
} else if (queueAction === 'skip-inline-finalize') {
|
||||||
|
|||||||
@@ -115,4 +115,32 @@ describe('resolveCodexFallbackHandoff', () => {
|
|||||||
logMessage: 'Fallback disabled for role, skipping handoff',
|
logMessage: 'Fallback disabled for role, skipping handoff',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('returns none when visible output was already emitted', () => {
|
||||||
|
const result = resolveCodexFallbackHandoff({
|
||||||
|
activeRole: 'reviewer',
|
||||||
|
effectiveAgentType: 'claude-code',
|
||||||
|
hasReviewer: true,
|
||||||
|
fallbackEnabled: true,
|
||||||
|
reason: 'auth-expired',
|
||||||
|
sawVisibleOutput: true,
|
||||||
|
prompt: 'please review',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({ type: 'none' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns none when no reviewer is configured for the room', () => {
|
||||||
|
const result = resolveCodexFallbackHandoff({
|
||||||
|
activeRole: 'owner',
|
||||||
|
effectiveAgentType: 'claude-code',
|
||||||
|
hasReviewer: false,
|
||||||
|
fallbackEnabled: true,
|
||||||
|
reason: '429',
|
||||||
|
sawVisibleOutput: false,
|
||||||
|
prompt: 'please continue',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({ type: 'none' });
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -9,6 +9,16 @@ vi.mock('./logger.js', () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
|
detectCodexRotationTrigger: vi.fn(() => ({
|
||||||
|
shouldRotate: false,
|
||||||
|
reason: '',
|
||||||
|
})),
|
||||||
|
getCodexAccountCount: vi.fn(() => 1),
|
||||||
|
markCodexTokenHealthy: vi.fn(),
|
||||||
|
rotateCodexToken: vi.fn(() => false),
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock('./token-rotation.js', () => ({
|
vi.mock('./token-rotation.js', () => ({
|
||||||
rotateToken: vi.fn(() => false),
|
rotateToken: vi.fn(() => false),
|
||||||
getTokenCount: vi.fn(() => 1),
|
getTokenCount: vi.fn(() => 1),
|
||||||
@@ -21,6 +31,13 @@ vi.mock('./token-refresh.js', () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
|
import { runCodexRotationLoop } from './provider-retry.js';
|
||||||
|
import {
|
||||||
|
detectCodexRotationTrigger,
|
||||||
|
getCodexAccountCount,
|
||||||
|
markCodexTokenHealthy,
|
||||||
|
rotateCodexToken,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
import {
|
import {
|
||||||
getCurrentTokenIndex,
|
getCurrentTokenIndex,
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
@@ -184,3 +201,81 @@ describe('runClaudeRotationLoop', () => {
|
|||||||
expect(rotateToken).toHaveBeenNthCalledWith(2, 'retry with next token');
|
expect(rotateToken).toHaveBeenNthCalledWith(2, 'retry with next token');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('runCodexRotationLoop', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(getCodexAccountCount).mockReturnValue(1);
|
||||||
|
vi.mocked(rotateCodexToken).mockReturnValue(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rotates and succeeds after a Codex auth trigger', async () => {
|
||||||
|
vi.mocked(getCodexAccountCount).mockReturnValue(2);
|
||||||
|
vi.mocked(rotateCodexToken).mockReturnValueOnce(true);
|
||||||
|
|
||||||
|
const outcome = await runCodexRotationLoop(
|
||||||
|
{ reason: 'auth-expired' },
|
||||||
|
async () => ({
|
||||||
|
output: { status: 'success', result: 'ok' },
|
||||||
|
sawOutput: true,
|
||||||
|
}),
|
||||||
|
{ runId: 'rotate-codex-auth' },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(outcome).toEqual({ type: 'success' });
|
||||||
|
expect(rotateCodexToken).toHaveBeenCalledTimes(1);
|
||||||
|
expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('continues rotation when streamed trigger arrives before visible output', async () => {
|
||||||
|
vi.mocked(getCodexAccountCount).mockReturnValue(2);
|
||||||
|
vi.mocked(rotateCodexToken).mockReturnValueOnce(true).mockReturnValueOnce(false);
|
||||||
|
|
||||||
|
let attempts = 0;
|
||||||
|
const outcome = await runCodexRotationLoop(
|
||||||
|
{ reason: 'auth-expired' },
|
||||||
|
async () => {
|
||||||
|
attempts += 1;
|
||||||
|
if (attempts === 1) {
|
||||||
|
return {
|
||||||
|
output: { status: 'success', result: 'rotate next' },
|
||||||
|
sawOutput: false,
|
||||||
|
streamedTriggerReason: { reason: 'auth-expired' },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
output: { status: 'success', result: 'ok' },
|
||||||
|
sawOutput: true,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
{ runId: 'rotate-codex-streamed' },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(outcome).toEqual({ type: 'error' });
|
||||||
|
expect(rotateCodexToken).toHaveBeenNthCalledWith(2, 'rotate next');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the Codex detector for thrown errors', async () => {
|
||||||
|
vi.mocked(getCodexAccountCount).mockReturnValue(2);
|
||||||
|
vi.mocked(rotateCodexToken).mockReturnValueOnce(true);
|
||||||
|
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
||||||
|
shouldRotate: true,
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
const outcome = await runCodexRotationLoop(
|
||||||
|
{ reason: 'auth-expired' },
|
||||||
|
async () => ({
|
||||||
|
thrownError: new Error('OAuth token expired'),
|
||||||
|
sawOutput: false,
|
||||||
|
}),
|
||||||
|
{ runId: 'rotate-codex-error' },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(outcome).toEqual({ type: 'error' });
|
||||||
|
expect(detectCodexRotationTrigger).toHaveBeenCalledWith(
|
||||||
|
'OAuth token expired',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -9,9 +9,16 @@ import type { AgentOutput } from './agent-runner.js';
|
|||||||
import { getAgentOutputText } from './agent-output.js';
|
import { getAgentOutputText } from './agent-output.js';
|
||||||
import {
|
import {
|
||||||
classifyRotationTrigger,
|
classifyRotationTrigger,
|
||||||
|
type CodexRotationReason,
|
||||||
shouldRotateClaudeToken,
|
shouldRotateClaudeToken,
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
|
import {
|
||||||
|
detectCodexRotationTrigger,
|
||||||
|
getCodexAccountCount,
|
||||||
|
markCodexTokenHealthy,
|
||||||
|
rotateCodexToken,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { getErrorMessage } from './utils.js';
|
import { getErrorMessage } from './utils.js';
|
||||||
import {
|
import {
|
||||||
@@ -42,6 +49,15 @@ export type RotationOutcome =
|
|||||||
| { type: 'success'; sawOutput: boolean }
|
| { type: 'success'; sawOutput: boolean }
|
||||||
| { type: 'error'; trigger?: TriggerInfo };
|
| { type: 'error'; trigger?: TriggerInfo };
|
||||||
|
|
||||||
|
export interface CodexRotationAttemptResult {
|
||||||
|
output?: Pick<AgentOutput, 'status' | 'result' | 'output' | 'error'>;
|
||||||
|
thrownError?: unknown;
|
||||||
|
sawOutput: boolean;
|
||||||
|
streamedTriggerReason?: TriggerInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CodexRotationOutcome = { type: 'success' } | { type: 'error' };
|
||||||
|
|
||||||
type AttemptOutcome =
|
type AttemptOutcome =
|
||||||
| { type: 'success'; sawOutput: boolean }
|
| { type: 'success'; sawOutput: boolean }
|
||||||
| { type: 'error'; trigger?: TriggerInfo }
|
| { type: 'error'; trigger?: TriggerInfo }
|
||||||
@@ -51,6 +67,15 @@ type AttemptOutcome =
|
|||||||
rotationMessage?: string;
|
rotationMessage?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type CodexAttemptOutcome =
|
||||||
|
| { type: 'success' }
|
||||||
|
| { type: 'error' }
|
||||||
|
| {
|
||||||
|
type: 'continue';
|
||||||
|
trigger: { reason: CodexRotationReason };
|
||||||
|
rotationMessage?: string;
|
||||||
|
};
|
||||||
|
|
||||||
function evaluateClaudeAttempt(
|
function evaluateClaudeAttempt(
|
||||||
attempt: RotationAttemptResult,
|
attempt: RotationAttemptResult,
|
||||||
logContext: Record<string, unknown>,
|
logContext: Record<string, unknown>,
|
||||||
@@ -149,6 +174,77 @@ function evaluateClaudeAttempt(
|
|||||||
return { type: 'success', sawOutput: attempt.sawOutput };
|
return { type: 'success', sawOutput: attempt.sawOutput };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function evaluateCodexAttempt(
|
||||||
|
attempt: CodexRotationAttemptResult,
|
||||||
|
logContext: Record<string, unknown>,
|
||||||
|
): CodexAttemptOutcome {
|
||||||
|
if (attempt.thrownError) {
|
||||||
|
const errMsg = getErrorMessage(attempt.thrownError);
|
||||||
|
const retryTrigger = detectCodexRotationTrigger(errMsg);
|
||||||
|
if (retryTrigger.shouldRotate) {
|
||||||
|
return {
|
||||||
|
type: 'continue',
|
||||||
|
trigger: { reason: retryTrigger.reason },
|
||||||
|
rotationMessage: errMsg,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'codex', err: attempt.thrownError },
|
||||||
|
'Rotated Codex account also threw',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const output = attempt.output;
|
||||||
|
if (!output) {
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'codex' },
|
||||||
|
'Rotated Codex account produced no output object',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!attempt.sawOutput &&
|
||||||
|
attempt.streamedTriggerReason &&
|
||||||
|
output.status !== 'error'
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
type: 'continue',
|
||||||
|
trigger: {
|
||||||
|
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||||
|
},
|
||||||
|
rotationMessage: getAgentOutputText(output) ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (output.status === 'error') {
|
||||||
|
const retryTrigger = attempt.streamedTriggerReason
|
||||||
|
? {
|
||||||
|
shouldRotate: true,
|
||||||
|
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||||
|
}
|
||||||
|
: detectCodexRotationTrigger(output.error);
|
||||||
|
if (retryTrigger.shouldRotate) {
|
||||||
|
return {
|
||||||
|
type: 'continue',
|
||||||
|
trigger: { reason: retryTrigger.reason },
|
||||||
|
rotationMessage: output.error ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'codex', error: output.error },
|
||||||
|
'Rotated Codex account failed',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
markCodexTokenHealthy();
|
||||||
|
return { type: 'success' };
|
||||||
|
}
|
||||||
|
|
||||||
// ── Shared rotation loop ─────────────────────────────────────────
|
// ── Shared rotation loop ─────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -238,3 +334,39 @@ export async function runClaudeRotationLoop(
|
|||||||
);
|
);
|
||||||
return { type: 'error', trigger };
|
return { type: 'error', trigger };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function runCodexRotationLoop(
|
||||||
|
initialTrigger: { reason: CodexRotationReason },
|
||||||
|
runAttempt: () => Promise<CodexRotationAttemptResult>,
|
||||||
|
logContext: Record<string, unknown>,
|
||||||
|
rotationMessage?: string,
|
||||||
|
): Promise<CodexRotationOutcome> {
|
||||||
|
let trigger = initialTrigger;
|
||||||
|
let lastRotationMessage = rotationMessage;
|
||||||
|
|
||||||
|
while (
|
||||||
|
getCodexAccountCount() > 1 &&
|
||||||
|
rotateCodexToken(lastRotationMessage)
|
||||||
|
) {
|
||||||
|
logger.info(
|
||||||
|
{ ...logContext, reason: trigger.reason },
|
||||||
|
'Codex account unhealthy, retrying with rotated account',
|
||||||
|
);
|
||||||
|
|
||||||
|
const rotatedAttemptOutcome = evaluateCodexAttempt(
|
||||||
|
await runAttempt(),
|
||||||
|
logContext,
|
||||||
|
);
|
||||||
|
if (rotatedAttemptOutcome.type === 'success') {
|
||||||
|
return rotatedAttemptOutcome;
|
||||||
|
}
|
||||||
|
if (rotatedAttemptOutcome.type === 'error') {
|
||||||
|
return rotatedAttemptOutcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
trigger = rotatedAttemptOutcome.trigger;
|
||||||
|
lastRotationMessage = rotatedAttemptOutcome.rotationMessage;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user