diff --git a/src/streamed-output-evaluator.test.ts b/src/streamed-output-evaluator.test.ts new file mode 100644 index 0000000..9f3d29a --- /dev/null +++ b/src/streamed-output-evaluator.test.ts @@ -0,0 +1,381 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +import type { AgentOutput } from './agent-runner.js'; + +// ── Mocks ────────────────────────────────────────────────────── +vi.mock('./agent-error-detection.js', () => ({ + isClaudeUsageExhaustedMessage: vi.fn(() => false), + isClaudeAuthExpiredMessage: vi.fn(() => false), + isClaudeAuthError: vi.fn(() => false), +})); + +vi.mock('./provider-fallback.js', () => ({ + detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })), +})); + +vi.mock('./codex-token-rotation.js', () => ({ + detectCodexRotationTrigger: vi.fn(() => ({ + shouldRotate: false, + reason: '', + })), +})); + +import { + isClaudeUsageExhaustedMessage, + isClaudeAuthExpiredMessage, + isClaudeAuthError, +} from './agent-error-detection.js'; +import { detectFallbackTrigger } from './provider-fallback.js'; +import { detectCodexRotationTrigger } from './codex-token-rotation.js'; + +import { + evaluateStreamedOutput, + type StreamedOutputState, + type EvaluateStreamedOutputOptions, +} from './streamed-output-evaluator.js'; + +// ── Helpers ──────────────────────────────────────────────────── +function freshState(): StreamedOutputState { + return { sawOutput: false, sawSuccessNullResultWithoutOutput: false }; +} + +const claudeOpts: EvaluateStreamedOutputOptions = { + agentType: 'claude', + provider: 'claude', +}; + +const codexOpts: EvaluateStreamedOutputOptions = { + agentType: 'codex', + provider: 'codex', +}; + +function successOutput(result: string | null): AgentOutput { + return { status: 'success', result }; +} + +function errorOutput(error: string): AgentOutput { + return { status: 'error', result: null, error }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false); + vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false); + vi.mocked(isClaudeAuthError).mockReturnValue(false); + vi.mocked(detectFallbackTrigger).mockReturnValue({ + shouldFallback: false, + reason: '', + }); + vi.mocked(detectCodexRotationTrigger).mockReturnValue({ + shouldRotate: false, + reason: '', + }); +}); + +// ── Tests ────────────────────────────────────────────────────── + +describe('evaluateStreamedOutput', () => { + describe('normal output forwarding', () => { + it('forwards success output and sets sawOutput', () => { + const result = evaluateStreamedOutput( + successOutput('Hello'), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(true); + expect(result.state.sawOutput).toBe(true); + expect(result.newTrigger).toBeUndefined(); + }); + + it('forwards output for non-primary agent type', () => { + const result = evaluateStreamedOutput( + successOutput('Hello'), + freshState(), + { agentType: 'claude', provider: 'fallback' }, + ); + expect(result.shouldForwardOutput).toBe(true); + expect(result.state.sawOutput).toBe(true); + }); + + it('does not set sawOutput for null result', () => { + const result = evaluateStreamedOutput( + successOutput(null), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(true); + expect(result.state.sawOutput).toBe(false); + }); + }); + + describe('Claude usage-exhausted banner', () => { + it('suppresses output and returns newTrigger', () => { + vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput("You're out of extra usage. Resets at 5pm."), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(false); + expect(result.newTrigger).toEqual({ reason: 'usage-exhausted' }); + expect(result.state.streamedTriggerReason).toEqual({ + reason: 'usage-exhausted', + }); + }); + + it('does not fire on non-primary Claude', () => { + vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput('banner text'), + freshState(), + { agentType: 'claude', provider: 'fallback' }, + ); + // Non-primary: skip banner check, forward normally + expect(result.shouldForwardOutput).toBe(true); + expect(result.newTrigger).toBeUndefined(); + }); + + it('does not fire when output was already seen', () => { + vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput('banner text'), + { ...freshState(), sawOutput: true }, + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(true); + expect(result.newTrigger).toBeUndefined(); + }); + }); + + describe('Claude auth-expired banner', () => { + it('suppresses output and returns newTrigger', () => { + vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput('Failed to authenticate...'), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(false); + expect(result.newTrigger).toEqual({ reason: 'auth-expired' }); + }); + }); + + describe('duplicate trigger suppression', () => { + it('suppresses output but returns no newTrigger when already triggered', () => { + vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true); + + const stateWithTrigger: StreamedOutputState = { + ...freshState(), + streamedTriggerReason: { reason: 'usage-exhausted' }, + }; + const result = evaluateStreamedOutput( + successOutput('second banner'), + stateWithTrigger, + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(false); + expect(result.newTrigger).toBeUndefined(); + // Existing trigger preserved + expect(result.state.streamedTriggerReason).toEqual({ + reason: 'usage-exhausted', + }); + }); + }); + + describe('Claude auth error suppression', () => { + it('suppresses auth error output when option enabled', () => { + vi.mocked(isClaudeAuthError).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput('Failed to authenticate 401 authentication_error'), + freshState(), + { ...claudeOpts, suppressClaudeAuthErrorOutput: true }, + ); + expect(result.shouldForwardOutput).toBe(false); + expect(result.suppressedAuthError).toBe(true); + expect(result.newTrigger).toBeUndefined(); + }); + + it('forwards auth error output when option not enabled', () => { + vi.mocked(isClaudeAuthError).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput('Failed to authenticate 401 authentication_error'), + freshState(), + claudeOpts, + ); + // No suppressClaudeAuthErrorOutput — forwarded normally + expect(result.shouldForwardOutput).toBe(true); + expect(result.suppressedAuthError).toBeUndefined(); + }); + }); + + describe('success null result tracking', () => { + it('tracks success-null-result on primary Claude when option enabled', () => { + const result = evaluateStreamedOutput(successOutput(null), freshState(), { + ...claudeOpts, + trackSuccessNullResult: true, + }); + expect(result.state.sawSuccessNullResultWithoutOutput).toBe(true); + expect(result.shouldForwardOutput).toBe(true); + }); + + it('does not track when sawOutput is already true', () => { + const result = evaluateStreamedOutput( + successOutput(null), + { ...freshState(), sawOutput: true }, + { ...claudeOpts, trackSuccessNullResult: true }, + ); + expect(result.state.sawSuccessNullResultWithoutOutput).toBe(false); + }); + + it('does not track when option is disabled', () => { + const result = evaluateStreamedOutput( + successOutput(null), + freshState(), + claudeOpts, + ); + expect(result.state.sawSuccessNullResultWithoutOutput).toBe(false); + }); + + it('does not track for non-primary Claude', () => { + const result = evaluateStreamedOutput(successOutput(null), freshState(), { + agentType: 'claude', + provider: 'fallback', + trackSuccessNullResult: true, + }); + expect(result.state.sawSuccessNullResultWithoutOutput).toBe(false); + }); + }); + + describe('error → Claude fallback trigger', () => { + it('returns fallback trigger with retryAfterMs', () => { + vi.mocked(detectFallbackTrigger).mockReturnValue({ + shouldFallback: true, + reason: '429', + retryAfterMs: 30000, + }); + + const result = evaluateStreamedOutput( + errorOutput('429 Too Many Requests'), + freshState(), + claudeOpts, + ); + expect(result.newTrigger).toEqual({ + reason: '429', + retryAfterMs: 30000, + }); + expect(result.state.streamedTriggerReason).toEqual({ + reason: '429', + retryAfterMs: 30000, + }); + // Without shortCircuit, error is still forwarded + expect(result.shouldForwardOutput).toBe(true); + }); + + it('suppresses error forward when shortCircuitTriggeredErrors is set', () => { + vi.mocked(detectFallbackTrigger).mockReturnValue({ + shouldFallback: true, + reason: '429', + }); + + const result = evaluateStreamedOutput(errorOutput('429'), freshState(), { + ...claudeOpts, + shortCircuitTriggeredErrors: true, + }); + expect(result.shouldForwardOutput).toBe(false); + expect(result.newTrigger).toEqual({ reason: '429' }); + }); + + it('skips fallback check when output was already seen', () => { + vi.mocked(detectFallbackTrigger).mockReturnValue({ + shouldFallback: true, + reason: '429', + }); + + const result = evaluateStreamedOutput( + errorOutput('429'), + { ...freshState(), sawOutput: true }, + claudeOpts, + ); + expect(result.newTrigger).toBeUndefined(); + expect(result.shouldForwardOutput).toBe(true); + }); + + it('skips fallback check when already triggered', () => { + vi.mocked(detectFallbackTrigger).mockReturnValue({ + shouldFallback: true, + reason: '429', + }); + + const result = evaluateStreamedOutput( + errorOutput('429'), + { + ...freshState(), + streamedTriggerReason: { reason: 'usage-exhausted' }, + }, + claudeOpts, + ); + expect(result.newTrigger).toBeUndefined(); + }); + }); + + describe('error → Codex rotation trigger', () => { + it('returns rotation trigger for Codex primary', () => { + vi.mocked(detectCodexRotationTrigger).mockReturnValue({ + shouldRotate: true, + reason: '429', + }); + + const result = evaluateStreamedOutput( + errorOutput('429 rate limit'), + freshState(), + codexOpts, + ); + expect(result.newTrigger).toEqual({ reason: '429' }); + expect(result.state.streamedTriggerReason).toEqual({ reason: '429' }); + }); + + it('does not check Codex rotation for Claude agent type', () => { + vi.mocked(detectCodexRotationTrigger).mockReturnValue({ + shouldRotate: true, + reason: '429', + }); + + const result = evaluateStreamedOutput( + errorOutput('429'), + freshState(), + claudeOpts, + ); + // Claude uses detectFallbackTrigger, not detectCodexRotationTrigger + expect(detectCodexRotationTrigger).not.toHaveBeenCalled(); + }); + }); + + describe('error without trigger match', () => { + it('forwards error normally when no trigger matches', () => { + const result = evaluateStreamedOutput( + errorOutput('some random error'), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(true); + expect(result.newTrigger).toBeUndefined(); + }); + }); + + describe('state immutability', () => { + it('does not mutate the input state object', () => { + const inputState = freshState(); + const frozen = { ...inputState }; + + evaluateStreamedOutput(successOutput('Hello'), inputState, claudeOpts); + + expect(inputState).toEqual(frozen); + }); + }); +}); diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 7bce7ae..384ca34 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -535,7 +535,10 @@ export async function startUnifiedDashboard( // Codex service: fetch own usage and expose via status snapshot. // Active account every usageUpdateInterval; full scan on startup + hourly. // Collector returns data; we own the cache state. - const applyRefresh = (result: { rows: UsageRow[]; fetchedAt: string | null }) => { + const applyRefresh = (result: { + rows: UsageRow[]; + fetchedAt: string | null; + }) => { cachedCodexUsageRows = result.rows; if (result.fetchedAt) codexUsageFetchedAt = result.fetchedAt; }; @@ -543,7 +546,10 @@ export async function startUnifiedDashboard( applyRefresh(r); return refreshActiveCodexUsage().then(applyRefresh); }); - setInterval(() => void refreshActiveCodexUsage().then(applyRefresh), opts.usageUpdateInterval); + setInterval( + () => void refreshActiveCodexUsage().then(applyRefresh), + opts.usageUpdateInterval, + ); setInterval( () => void refreshAllCodexAccountUsage().then(applyRefresh), CODEX_FULL_SCAN_INTERVAL,