diff --git a/src/agent-error-detection.test.ts b/src/agent-error-detection.test.ts new file mode 100644 index 0000000..1454c73 --- /dev/null +++ b/src/agent-error-detection.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; + +import { + classifyClaudeAuthError, + isClaudeOrgAccessDeniedMessage, + isNoFallbackCooldownReason, + shouldRotateClaudeToken, +} from './agent-error-detection.js'; + +describe('agent-error-detection', () => { + it('detects Claude org access denied banners', () => { + expect( + isClaudeOrgAccessDeniedMessage( + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + ), + ).toBe(true); + }); + + it('classifies org access denied banners as org-access-denied', () => { + expect( + classifyClaudeAuthError( + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + ), + ).toEqual({ + category: 'org-access-denied', + reason: 'org-access-denied', + }); + }); + + it('classifies terminated 403 auth failures as org-access-denied', () => { + expect( + classifyClaudeAuthError( + 'Failed to authenticate. API Error: 403 terminated', + ), + ).toEqual({ + category: 'org-access-denied', + reason: 'org-access-denied', + }); + }); + + it('marks only Claude quota/auth reasons as Claude rotation reasons', () => { + expect(shouldRotateClaudeToken('429')).toBe(true); + expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true); + expect(shouldRotateClaudeToken('auth-expired')).toBe(true); + expect(shouldRotateClaudeToken('org-access-denied')).toBe(true); + expect(shouldRotateClaudeToken('overloaded')).toBe(false); + expect(shouldRotateClaudeToken('success-null-result')).toBe(false); + }); + + it('marks only no-fallback cooldown reasons as skip-worthy', () => { + expect(isNoFallbackCooldownReason('usage-exhausted')).toBe(true); + expect(isNoFallbackCooldownReason('auth-expired')).toBe(true); + expect(isNoFallbackCooldownReason('org-access-denied')).toBe(true); + expect(isNoFallbackCooldownReason('429')).toBe(false); + expect(isNoFallbackCooldownReason('success-null-result')).toBe(false); + }); +}); diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts index d323dcc..0d6192d 100644 --- a/src/agent-error-detection.ts +++ b/src/agent-error-detection.ts @@ -68,7 +68,35 @@ export function isClaudeOrgAccessDeniedMessage(text: string): boolean { // ── Rotation decision ─────────────────────────────────────────── -export function shouldRotateClaudeToken(reason: string): boolean { +export type AgentTriggerReason = + | '429' + | 'usage-exhausted' + | 'auth-expired' + | 'org-access-denied' + | 'overloaded' + | 'network-error' + | 'success-null-result'; + +export type FallbackTriggerReason = Exclude< + AgentTriggerReason, + 'usage-exhausted' | 'success-null-result' +>; + +export type ClaudeRotationReason = Extract< + AgentTriggerReason, + '429' | 'usage-exhausted' | 'auth-expired' | 'org-access-denied' +>; + +export type CodexRotationReason = FallbackTriggerReason; + +export type NoFallbackCooldownReason = Extract< + AgentTriggerReason, + 'usage-exhausted' | 'auth-expired' | 'org-access-denied' +>; + +export function shouldRotateClaudeToken( + reason: AgentTriggerReason, +): reason is ClaudeRotationReason { return ( reason === '429' || reason === 'usage-exhausted' || @@ -77,6 +105,16 @@ export function shouldRotateClaudeToken(reason: string): boolean { ); } +export function isNoFallbackCooldownReason( + reason: AgentTriggerReason, +): reason is NoFallbackCooldownReason { + return ( + reason === 'usage-exhausted' || + reason === 'auth-expired' || + reason === 'org-access-denied' + ); +} + // ── Unified error classification ──────────────────────────────── export type ErrorCategory = @@ -87,11 +125,37 @@ export type ErrorCategory = | 'network-error' | 'none'; -export interface AgentErrorClassification { - category: ErrorCategory; - reason: string; // '429' | 'auth-expired' | 'org-access-denied' | 'overloaded' | 'network-error' | '' - retryAfterMs?: number; -} +export type AgentErrorClassification = + | { + category: 'none'; + reason: ''; + retryAfterMs?: undefined; + } + | { + category: 'rate-limit'; + reason: '429'; + retryAfterMs?: number; + } + | { + category: 'auth-expired'; + reason: 'auth-expired'; + retryAfterMs?: undefined; + } + | { + category: 'org-access-denied'; + reason: 'org-access-denied'; + retryAfterMs?: undefined; + } + | { + category: 'overloaded'; + reason: 'overloaded'; + retryAfterMs?: undefined; + } + | { + category: 'network-error'; + reason: 'network-error'; + retryAfterMs?: undefined; + }; const NONE: AgentErrorClassification = { category: 'none', diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 71fad44..b0badc1 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -17,6 +17,7 @@ import path from 'path'; import { classifyAgentError, classifyCodexAuthError, + type CodexRotationReason, } from './agent-error-detection.js'; import { DATA_DIR } from './config.js'; import { logger } from './logger.js'; @@ -42,10 +43,15 @@ interface CodexAccount { resetD7At?: string; } -export interface CodexRotationTriggerResult { - shouldRotate: boolean; - reason: string; -} +export type CodexRotationTriggerResult = + | { + shouldRotate: false; + reason: ''; + } + | { + shouldRotate: true; + reason: CodexRotationReason; + }; function parseJwtAuth(idToken: string): { planType: string; diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 92a0ab7..6ca28e3 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -57,7 +57,6 @@ vi.mock('./provider-fallback.js', () => ({ hasGroupProviderOverride: vi.fn(() => false), isFallbackEnabled: vi.fn(() => true), isPrimaryNoFallbackCooldownActive: vi.fn(() => false), - isUsageExhausted: vi.fn(() => false), markPrimaryCooldown: vi.fn(), })); diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 95543b4..bb80d66 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -35,6 +35,7 @@ import { getCodexAccountCount, markCodexTokenHealthy, } from './codex-token-rotation.js'; +import type { AgentTriggerReason, CodexRotationReason } from './agent-error-detection.js'; import { getTokenCount } from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; @@ -124,7 +125,7 @@ export async function runAgentForGroup( sawOutput: boolean; sawSuccessNullResultWithoutOutput: boolean; streamedTriggerReason?: { - reason: string; + reason: AgentTriggerReason; retryAfterMs?: number; }; }> => { @@ -293,7 +294,7 @@ export async function runAgentForGroup( }; const runFallbackAttempt = async ( - reason: string, + reason: AgentTriggerReason, retryAfterMs?: number, ): Promise<'success' | 'error'> => { const fallbackName = getFallbackProviderName(); @@ -347,7 +348,7 @@ export async function runAgentForGroup( }; const retryCodexWithRotation = async ( - initialTrigger: { reason: string }, + initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, ): Promise<'success' | 'error'> => { let trigger = initialTrigger; @@ -407,7 +408,9 @@ export async function runAgentForGroup( retryAttempt.streamedTriggerReason && retryOutput.status !== 'error' ) { - trigger = { reason: retryAttempt.streamedTriggerReason.reason }; + trigger = { + reason: retryAttempt.streamedTriggerReason.reason as CodexRotationReason, + }; lastRotationMessage = typeof retryOutput.result === 'string' ? retryOutput.result @@ -419,7 +422,8 @@ export async function runAgentForGroup( const retryTrigger = retryAttempt.streamedTriggerReason ? { shouldRotate: true, - reason: retryAttempt.streamedTriggerReason.reason, + reason: + retryAttempt.streamedTriggerReason.reason as CodexRotationReason, } : detectCodexRotationTrigger(retryOutput.error); @@ -451,7 +455,7 @@ export async function runAgentForGroup( const retryClaudeWithRotation = async ( initialTrigger: { - reason: string; + reason: AgentTriggerReason; retryAfterMs?: number; }, rotationMessage?: string, @@ -667,7 +671,9 @@ export async function runAgentForGroup( getCodexAccountCount() > 1 ) { return retryCodexWithRotation( - { reason: primaryAttempt.streamedTriggerReason.reason }, + { + reason: primaryAttempt.streamedTriggerReason.reason as CodexRotationReason, + }, output.error ?? output.result ?? undefined, ); } diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index b1a2936..dc62486 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -17,6 +17,9 @@ import fs from 'fs'; import { classifyAgentError, classifyClaudeAuthError, + isNoFallbackCooldownReason, + type AgentTriggerReason, + type FallbackTriggerReason, } from './agent-error-detection.js'; import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js'; import { getEnv } from './env.js'; @@ -27,16 +30,22 @@ import { rotateToken, getTokenCount } from './token-rotation.js'; export type ProviderName = 'claude' | string; // fallback name is configurable -export interface FallbackTriggerResult { - shouldFallback: boolean; - reason: string; - retryAfterMs?: number; -} +export type FallbackTriggerResult = + | { + shouldFallback: false; + reason: ''; + retryAfterMs?: undefined; + } + | { + shouldFallback: true; + reason: FallbackTriggerReason; + retryAfterMs?: number; + }; interface CooldownState { startedAt: number; expiresAt: number; - reason: string; + reason: AgentTriggerReason; } interface FallbackConfig { @@ -59,11 +68,6 @@ let lastUsageAvailabilityCheck: { let usageAvailabilityCheckPromise: Promise< 'available' | 'exhausted' | 'unknown' > | null = null; -const NO_FALLBACK_COOLDOWN_REASONS = new Set([ - 'usage-exhausted', - 'auth-expired', - 'org-access-denied', -]); const USAGE_RECOVERY_RECHECK_MS = 30_000; @@ -290,7 +294,7 @@ export async function getActiveProvider(): Promise { * the fallback provider until the cooldown expires. */ export function markPrimaryCooldown( - reason: string, + reason: AgentTriggerReason, retryAfterMs?: number, ): void { const config = loadConfig(); @@ -331,14 +335,9 @@ export function clearPrimaryCooldown(): void { } } -/** Check if Claude is currently in usage-exhausted cooldown. */ -export function isUsageExhausted(): boolean { - return cooldown?.reason === 'usage-exhausted'; -} - /** Check whether the active primary cooldown should suppress fallback entirely. */ export function isPrimaryNoFallbackCooldownActive(): boolean { - return cooldown ? NO_FALLBACK_COOLDOWN_REASONS.has(cooldown.reason) : false; + return cooldown ? isNoFallbackCooldownReason(cooldown.reason) : false; } /** Get current cooldown info (for diagnostics / status dashboard). */ diff --git a/src/provider-retry.test.ts b/src/provider-retry.test.ts new file mode 100644 index 0000000..ea6f970 --- /dev/null +++ b/src/provider-retry.test.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./logger.js', () => ({ + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +vi.mock('./provider-fallback.js', () => ({ + detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })), + markPrimaryCooldown: vi.fn(), +})); + +vi.mock('./token-rotation.js', () => ({ + rotateToken: vi.fn(() => false), + getTokenCount: vi.fn(() => 1), + markTokenHealthy: vi.fn(), +})); + +import { markPrimaryCooldown } from './provider-fallback.js'; +import { runClaudeRotationLoop } from './provider-retry.js'; +import { + getTokenCount, + markTokenHealthy, + rotateToken, +} from './token-rotation.js'; + +describe('runClaudeRotationLoop', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getTokenCount).mockReturnValue(1); + vi.mocked(rotateToken).mockReturnValue(false); + }); + + it('rotates and succeeds after an org-access-denied trigger', async () => { + vi.mocked(getTokenCount).mockReturnValue(2); + vi.mocked(rotateToken).mockReturnValueOnce(true); + + const outcome = await runClaudeRotationLoop( + { reason: 'org-access-denied' }, + async () => ({ + output: { status: 'success', result: 'ok' }, + sawOutput: true, + }), + { runId: 'rotate-org-access' }, + ); + + expect(outcome).toEqual({ type: 'success' }); + expect(rotateToken).toHaveBeenCalledTimes(1); + expect(markTokenHealthy).toHaveBeenCalledTimes(1); + expect(markPrimaryCooldown).not.toHaveBeenCalled(); + }); + + it('marks no-fallback cooldown when all Claude tokens are org-access-denied', async () => { + vi.mocked(getTokenCount).mockReturnValue(2); + + const outcome = await runClaudeRotationLoop( + { reason: 'org-access-denied' }, + async () => ({ + output: { status: 'success', result: 'should not run' }, + sawOutput: true, + }), + { runId: 'no-fallback-org-access' }, + ); + + expect(outcome).toEqual({ + type: 'no-fallback', + trigger: { reason: 'org-access-denied' }, + }); + expect(markPrimaryCooldown).toHaveBeenCalledWith( + 'org-access-denied', + undefined, + ); + }); + + it('returns success-null-result as a fallback trigger after rotation', async () => { + vi.mocked(getTokenCount).mockReturnValue(2); + vi.mocked(rotateToken).mockReturnValueOnce(true); + + const outcome = await runClaudeRotationLoop( + { reason: '429' }, + async () => ({ + output: { status: 'success', result: null }, + sawOutput: false, + sawSuccessNullResult: true, + }), + { runId: 'success-null-result' }, + ); + + expect(outcome).toEqual({ + type: 'needs-fallback', + trigger: { reason: 'success-null-result' }, + }); + expect(markPrimaryCooldown).not.toHaveBeenCalled(); + }); +}); diff --git a/src/provider-retry.ts b/src/provider-retry.ts index a9e4472..8ca94ba 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -5,7 +5,11 @@ * to eliminate the ~255-line structural duplication. */ -import { shouldRotateClaudeToken } from './agent-error-detection.js'; +import { + isNoFallbackCooldownReason, + shouldRotateClaudeToken, + type AgentTriggerReason, +} from './agent-error-detection.js'; import { logger } from './logger.js'; import { getErrorMessage } from './utils.js'; import { @@ -21,7 +25,7 @@ import { // ── Types ──────────────────────────────────────────────────────── export interface TriggerInfo { - reason: string; + reason: AgentTriggerReason; retryAfterMs?: number; } @@ -163,11 +167,7 @@ export async function runClaudeRotationLoop( // ── All tokens exhausted ── // Usage/auth/org access failures: don't fall back to Kimi - if ( - trigger.reason === 'usage-exhausted' || - trigger.reason === 'auth-expired' || - trigger.reason === 'org-access-denied' - ) { + if (isNoFallbackCooldownReason(trigger.reason)) { markPrimaryCooldown(trigger.reason, trigger.retryAfterMs); logger.info( { ...logContext, reason: trigger.reason }, diff --git a/src/streamed-output-evaluator.ts b/src/streamed-output-evaluator.ts index 218d35d..5db3d4b 100644 --- a/src/streamed-output-evaluator.ts +++ b/src/streamed-output-evaluator.ts @@ -3,13 +3,14 @@ import { isClaudeAuthExpiredMessage, isClaudeOrgAccessDeniedMessage, isClaudeUsageExhaustedMessage, + type AgentTriggerReason, } from './agent-error-detection.js'; import type { AgentOutput } from './agent-runner.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import { detectFallbackTrigger } from './provider-fallback.js'; export interface StreamedTriggerReason { - reason: string; + reason: AgentTriggerReason; retryAfterMs?: number; } @@ -51,7 +52,8 @@ export function evaluateStreamedOutput( !state.sawOutput && typeof output.result === 'string' ) { - const triggerReason = isClaudeUsageExhaustedMessage(output.result) + const triggerReason: AgentTriggerReason | undefined = + isClaudeUsageExhaustedMessage(output.result) ? 'usage-exhausted' : isClaudeOrgAccessDeniedMessage(output.result) ? 'org-access-denied' diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 3094b7d..0fbd528 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -49,6 +49,7 @@ import { getCodexAccountCount, markCodexTokenHealthy, } from './codex-token-rotation.js'; +import type { AgentTriggerReason, CodexRotationReason } from './agent-error-detection.js'; import { getTokenCount, markTokenHealthy, @@ -290,7 +291,7 @@ async function runTask( output: AgentOutput; sawOutput: boolean; streamedTriggerReason?: { - reason: string; + reason: AgentTriggerReason; retryAfterMs?: number; }; attemptResult: string | null; @@ -410,7 +411,7 @@ async function runTask( }; const runFallbackTaskAttempt = async ( - reason: string, + reason: AgentTriggerReason, retryAfterMs?: number, ): Promise => { if (!canFallback) { @@ -443,7 +444,7 @@ async function runTask( const retryClaudeTaskWithRotation = async ( initialTrigger: { - reason: string; + reason: AgentTriggerReason; retryAfterMs?: number; }, rotationMessage?: string, @@ -489,7 +490,7 @@ async function runTask( }; const retryCodexTaskWithRotation = async ( - initialTrigger: { reason: string }, + initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, ): Promise => { let trigger = initialTrigger; @@ -518,7 +519,9 @@ async function runTask( retryAttempt.streamedTriggerReason && retryAttempt.output.status !== 'error' ) { - trigger = { reason: retryAttempt.streamedTriggerReason.reason }; + trigger = { + reason: retryAttempt.streamedTriggerReason.reason as CodexRotationReason, + }; lastRotationMessage = typeof retryAttempt.output.result === 'string' ? retryAttempt.output.result @@ -530,7 +533,8 @@ async function runTask( const retryTrigger = retryAttempt.streamedTriggerReason ? { shouldRotate: true, - reason: retryAttempt.streamedTriggerReason.reason, + reason: + retryAttempt.streamedTriggerReason.reason as CodexRotationReason, } : detectCodexRotationTrigger( retryAttempt.attemptError || retryAttempt.output.error, @@ -589,7 +593,9 @@ async function runTask( !attempt.sawOutput ) { await retryCodexTaskWithRotation( - { reason: attempt.streamedTriggerReason.reason }, + { + reason: attempt.streamedTriggerReason.reason as CodexRotationReason, + }, typeof attempt.output.error === 'string' ? attempt.output.error : undefined, @@ -612,7 +618,7 @@ async function runTask( const trigger = attempt.streamedTriggerReason ? { shouldRotate: true, - reason: attempt.streamedTriggerReason.reason, + reason: attempt.streamedTriggerReason.reason as CodexRotationReason, } : detectCodexRotationTrigger(error); if (trigger.shouldRotate) {