From 60a9fe86ecc1b1db607911164f943dfbcf66c0f7 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Wed, 25 Mar 2026 17:24:10 +0900 Subject: [PATCH] fix: detect org-access-denied errors, suppress chat output, and rotate Claude tokens When a Claude account is suspended, the API returns "Your organization does not have access to Claude" on the success path or "Failed to authenticate. API Error: 403 terminated" on the error path. Both are now classified as 'org-access-denied', suppressed from Discord output, and trigger automatic token rotation. If all tokens are exhausted, enters cooldown without Kimi fallback (same as usage-exhausted and auth-expired). Changes: - agent-error-detection: add isClaudeOrgAccessDeniedMessage(), expand classifyClaudeAuthError() and shouldRotateClaudeToken() - streamed-output-evaluator: detect org-access-denied in success-path chain - provider-fallback: add NO_FALLBACK_COOLDOWN_REASONS set and isPrimaryNoFallbackCooldownActive() helper - provider-retry: handle org-access-denied in rotation loop - message-agent-executor / task-scheduler: use generalized no-fallback cooldown check - Tests: +8 test cases across 5 test files (415 total passing) Co-Authored-By: Claude Opus 4.6 --- src/agent-error-detection.ts | 31 +++++- src/message-agent-executor.test.ts | 146 ++++++++++++++++++++++++++ src/message-agent-executor.ts | 8 +- src/message-runtime.test.ts | 1 + src/provider-fallback.test.ts | 33 ++++++ src/provider-fallback.ts | 13 ++- src/provider-retry.ts | 9 +- src/streamed-output-evaluator.test.ts | 22 ++++ src/streamed-output-evaluator.ts | 3 + src/task-scheduler.test.ts | 108 +++++++++++++++++++ src/task-scheduler.ts | 14 ++- 11 files changed, 372 insertions(+), 16 deletions(-) diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts index c1c171a..d323dcc 100644 --- a/src/agent-error-detection.ts +++ b/src/agent-error-detection.ts @@ -54,13 +54,26 @@ export function isClaudeAuthExpiredMessage(text: string): boolean { ); } +export function isClaudeOrgAccessDeniedMessage(text: string): boolean { + const normalized = text.trim().toLowerCase().replace(/\s+/g, ' '); + const hasOrgAccessDeniedMarker = normalized.includes( + 'does not have access to claude', + ); + const hasRecoveryHint = + normalized.includes('please login again') || + normalized.includes('contact your administrator'); + + return hasOrgAccessDeniedMarker && hasRecoveryHint; +} + // ── Rotation decision ─────────────────────────────────────────── export function shouldRotateClaudeToken(reason: string): boolean { return ( reason === '429' || reason === 'usage-exhausted' || - reason === 'auth-expired' + reason === 'auth-expired' || + reason === 'org-access-denied' ); } @@ -69,13 +82,14 @@ export function shouldRotateClaudeToken(reason: string): boolean { export type ErrorCategory = | 'rate-limit' | 'auth-expired' + | 'org-access-denied' | 'overloaded' | 'network-error' | 'none'; export interface AgentErrorClassification { category: ErrorCategory; - reason: string; // '429' | 'auth-expired' | 'overloaded' | 'network-error' | '' + reason: string; // '429' | 'auth-expired' | 'org-access-denied' | 'overloaded' | 'network-error' | '' retryAfterMs?: number; } @@ -142,6 +156,19 @@ export function classifyClaudeAuthError( if (!error) return NONE; const lower = error.toLowerCase(); + const hasOrgAccessDeniedMarker = + lower.includes('your organization does not have access to claude') || + (lower.includes('does not have access to claude') && + lower.includes('contact your administrator')); + const hasTerminated403AuthFailure = + lower.includes('failed to authenticate') && + lower.includes('403') && + lower.includes('terminated'); + + if (hasOrgAccessDeniedMarker || hasTerminated403AuthFailure) { + return { category: 'org-access-denied', reason: 'org-access-denied' }; + } + if ( (lower.includes('failed to authenticate') || lower.includes('authentication_error')) && diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 8991fc7..92a0ab7 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -30,6 +30,14 @@ vi.mock('./logger.js', () => ({ vi.mock('./provider-fallback.js', () => ({ detectFallbackTrigger: vi.fn((error?: string | null) => { const lower = (error || '').toLowerCase(); + if ( + lower.includes('does not have access to claude') || + (lower.includes('failed to authenticate') && + lower.includes('403') && + lower.includes('terminated')) + ) { + return { shouldFallback: true, reason: 'org-access-denied' }; + } if ( lower.includes('429') || lower.includes('rate limit') || @@ -48,6 +56,7 @@ vi.mock('./provider-fallback.js', () => ({ getFallbackProviderName: vi.fn(() => 'kimi'), hasGroupProviderOverride: vi.fn(() => false), isFallbackEnabled: vi.fn(() => true), + isPrimaryNoFallbackCooldownActive: vi.fn(() => false), isUsageExhausted: vi.fn(() => false), markPrimaryCooldown: vi.fn(), })); @@ -370,6 +379,143 @@ describe('runAgentForGroup Claude rotation', () => { expect(outputs).toEqual([]); }); + it('rotates to another Claude account when Claude streams an org access denied banner', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + }); + await onOutput?.({ + status: 'success', + phase: 'final', + result: + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'org access denied 회전 성공 응답', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-org-access-denied-claude', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); + expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); + expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + expect(outputs).toEqual(['org access denied 회전 성공 응답']); + }); + + it('stops after all Claude accounts are org-access-denied without falling back to Kimi', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'error', + result: null, + error: 'Failed to authenticate. API Error: 403 terminated', + }); + return { + status: 'error', + result: null, + error: 'Failed to authenticate. API Error: 403 terminated', + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'Kimi 폴백 응답입니다.', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-org-access-denied-no-fallback', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('error'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2); + expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( + 'org-access-denied', + undefined, + ); + expect(outputs).toEqual([]); + }); + + it('skips execution entirely when Claude no-fallback cooldown is already active', async () => { + vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('kimi'); + vi.mocked( + providerFallback.isPrimaryNoFallbackCooldownActive, + ).mockReturnValue(true); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-skip-primary-cooldown', + }); + + expect(result).toBe('error'); + expect(agentRunner.runAgentProcess).not.toHaveBeenCalled(); + }); + it('does not mistake a normal response quoting the banner text for a usage error', async () => { const outputs: string[] = []; diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 5f2f31f..95543b4 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -20,7 +20,7 @@ import { getFallbackProviderName, hasGroupProviderOverride, isFallbackEnabled, - isUsageExhausted, + isPrimaryNoFallbackCooldownActive, markPrimaryCooldown, } from './provider-fallback.js'; import { runClaudeRotationLoop } from './provider-retry.js'; @@ -508,11 +508,11 @@ export async function runAgentForGroup( const provider = canFallback ? await getActiveProvider() : 'claude'; - // Already in usage-exhausted cooldown — log only, no response - if (provider !== 'claude' && isUsageExhausted()) { + // Already in no-fallback Claude cooldown — log only, no response + if (provider !== 'claude' && isPrimaryNoFallbackCooldownActive()) { logger.info( { chatJid, group: group.name, runId, provider }, - 'Claude usage exhausted (cooldown active), silently skipping', + 'Claude primary cooldown active, silently skipping', ); return 'error'; } diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 814c77b..cfe46ec 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -131,6 +131,7 @@ vi.mock('./provider-fallback.js', () => ({ getFallbackProviderName: vi.fn(() => 'kimi'), hasGroupProviderOverride: vi.fn(() => false), isFallbackEnabled: vi.fn(() => true), + isPrimaryNoFallbackCooldownActive: vi.fn(() => false), markPrimaryCooldown: vi.fn(), })); diff --git a/src/provider-fallback.test.ts b/src/provider-fallback.test.ts index 393ec1a..445ef6c 100644 --- a/src/provider-fallback.test.ts +++ b/src/provider-fallback.test.ts @@ -40,6 +40,7 @@ import { detectFallbackTrigger, getActiveProvider, getCooldownInfo, + isPrimaryNoFallbackCooldownActive, markPrimaryCooldown, resetFallbackConfig, } from './provider-fallback.js'; @@ -133,4 +134,36 @@ describe('provider fallback usage recovery', () => { reason: 'auth-expired', }); }); + + it('treats Claude org access denied banners as an org-access-denied fallback trigger', () => { + expect( + detectFallbackTrigger( + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + ), + ).toEqual({ + shouldFallback: true, + reason: 'org-access-denied', + }); + }); + + it('treats terminated 403 auth failures as an org-access-denied fallback trigger', () => { + expect( + detectFallbackTrigger( + 'Failed to authenticate. API Error: 403 terminated', + ), + ).toEqual({ + shouldFallback: true, + reason: 'org-access-denied', + }); + }); + + it('marks org-access-denied as a no-fallback cooldown reason', () => { + markPrimaryCooldown('org-access-denied', 60_000); + + expect(isPrimaryNoFallbackCooldownActive()).toBe(true); + expect(getCooldownInfo()).toMatchObject({ + active: true, + reason: 'org-access-denied', + }); + }); }); diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index 902ad33..b1a2936 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -59,6 +59,11 @@ let lastUsageAvailabilityCheck: { let usageAvailabilityCheckPromise: Promise< 'available' | 'exhausted' | 'unknown' > | null = null; +const NO_FALLBACK_COOLDOWN_REASONS = new Set([ + 'usage-exhausted', + 'auth-expired', + 'org-access-denied', +]); const USAGE_RECOVERY_RECHECK_MS = 30_000; @@ -331,6 +336,11 @@ export function isUsageExhausted(): boolean { return cooldown?.reason === 'usage-exhausted'; } +/** Check whether the active primary cooldown should suppress fallback entirely. */ +export function isPrimaryNoFallbackCooldownActive(): boolean { + return cooldown ? NO_FALLBACK_COOLDOWN_REASONS.has(cooldown.reason) : false; +} + /** Get current cooldown info (for diagnostics / status dashboard). */ export function getCooldownInfo(): { active: boolean; @@ -379,6 +389,7 @@ export function getFallbackEnvOverrides(): Record { * * Triggers: * - 429 / rate limit / too many requests + * - Claude auth/org access failures * - 503 / overloaded (transient provider issue) * - Network / connection errors * @@ -393,7 +404,7 @@ export function detectFallbackTrigger( if (!error) return { shouldFallback: false, reason: '' }; // Delegated to shared SSOT — original priority preserved: - // 429 first, then auth-expired, then 503/network + // 429 first, then Claude auth/org errors, then 503/network const common = classifyAgentError(error); // 429 rate-limit (highest priority) diff --git a/src/provider-retry.ts b/src/provider-retry.ts index 227531c..a9e4472 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -37,7 +37,7 @@ export type RotationOutcome = | { type: 'success' } | { type: 'error'; message?: string } | { type: 'needs-fallback'; trigger: TriggerInfo } - | { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired + | { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired/org-access-denied // ── Shared rotation loop ───────────────────────────────────────── @@ -63,7 +63,7 @@ export async function runClaudeRotationLoop( ) { logger.info( { ...logContext, reason: trigger.reason }, - 'Claude rate-limited, retrying with rotated account', + 'Claude account unavailable, retrying with rotated account', ); const attempt = await runAttempt(); @@ -162,10 +162,11 @@ export async function runClaudeRotationLoop( // ── All tokens exhausted ── - // Usage exhausted or auth-expired: don't fall back to Kimi + // Usage/auth/org access failures: don't fall back to Kimi if ( trigger.reason === 'usage-exhausted' || - trigger.reason === 'auth-expired' + trigger.reason === 'auth-expired' || + trigger.reason === 'org-access-denied' ) { markPrimaryCooldown(trigger.reason, trigger.retryAfterMs); logger.info( diff --git a/src/streamed-output-evaluator.test.ts b/src/streamed-output-evaluator.test.ts index 90fd8b9..bd7ebf1 100644 --- a/src/streamed-output-evaluator.test.ts +++ b/src/streamed-output-evaluator.test.ts @@ -5,6 +5,7 @@ import type { AgentOutput } from './agent-runner.js'; // ── Mocks ────────────────────────────────────────────────────── vi.mock('./agent-error-detection.js', () => ({ isClaudeUsageExhaustedMessage: vi.fn(() => false), + isClaudeOrgAccessDeniedMessage: vi.fn(() => false), isClaudeAuthExpiredMessage: vi.fn(() => false), isClaudeAuthError: vi.fn(() => false), })); @@ -22,6 +23,7 @@ vi.mock('./codex-token-rotation.js', () => ({ import { isClaudeUsageExhaustedMessage, + isClaudeOrgAccessDeniedMessage, isClaudeAuthExpiredMessage, isClaudeAuthError, } from './agent-error-detection.js'; @@ -60,6 +62,7 @@ function errorOutput(error: string): AgentOutput { beforeEach(() => { vi.clearAllMocks(); vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false); + vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(false); vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false); vi.mocked(isClaudeAuthError).mockReturnValue(false); vi.mocked(detectFallbackTrigger).mockReturnValue({ @@ -164,6 +167,25 @@ describe('evaluateStreamedOutput', () => { }); }); + describe('Claude org-access-denied banner', () => { + it('suppresses output and returns newTrigger', () => { + vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput( + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + ), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(false); + expect(result.newTrigger).toEqual({ reason: 'org-access-denied' }); + expect(result.state.streamedTriggerReason).toEqual({ + reason: 'org-access-denied', + }); + }); + }); + describe('duplicate trigger suppression', () => { it('suppresses output but returns no newTrigger when already triggered', () => { vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true); diff --git a/src/streamed-output-evaluator.ts b/src/streamed-output-evaluator.ts index da6877d..ca38e24 100644 --- a/src/streamed-output-evaluator.ts +++ b/src/streamed-output-evaluator.ts @@ -1,6 +1,7 @@ import { isClaudeAuthError, isClaudeAuthExpiredMessage, + isClaudeOrgAccessDeniedMessage, isClaudeUsageExhaustedMessage, } from './agent-error-detection.js'; import type { AgentOutput } from './agent-runner.js'; @@ -52,6 +53,8 @@ export function evaluateStreamedOutput( ) { const triggerReason = isClaudeUsageExhaustedMessage(output.result) ? 'usage-exhausted' + : isClaudeOrgAccessDeniedMessage(output.result) + ? 'org-access-denied' : isClaudeAuthExpiredMessage(output.result) ? 'auth-expired' : undefined; diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index d8f772d..9b7bdb3 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -11,6 +11,14 @@ const { runAgentProcessMock, writeTasksSnapshotMock } = vi.hoisted(() => ({ vi.mock('./provider-fallback.js', () => ({ detectFallbackTrigger: vi.fn((error?: string | null) => { const lower = (error || '').toLowerCase(); + if ( + lower.includes('does not have access to claude') || + (lower.includes('failed to authenticate') && + lower.includes('403') && + lower.includes('terminated')) + ) { + return { shouldFallback: true, reason: 'org-access-denied' }; + } if ( lower.includes('429') || lower.includes('rate limit') || @@ -29,6 +37,7 @@ vi.mock('./provider-fallback.js', () => ({ getFallbackProviderName: vi.fn(() => 'kimi'), hasGroupProviderOverride: vi.fn(() => false), isFallbackEnabled: vi.fn(() => true), + isPrimaryNoFallbackCooldownActive: vi.fn(() => false), markPrimaryCooldown: vi.fn(), })); @@ -493,6 +502,105 @@ Check the run. ); }); + it('suppresses Claude org access denied banners for scheduled tasks and retries with a rotated account', async () => { + const dueAt = new Date(Date.now() - 60_000).toISOString(); + createTask({ + id: 'task-org-access-denied-banner', + group_folder: 'shared-group', + chat_jid: 'shared@g.us', + agent_type: 'claude-code', + prompt: 'claude task', + schedule_type: 'once', + schedule_value: dueAt, + context_mode: 'isolated', + next_run: dueAt, + status: 'active', + created_at: '2026-02-22T00:00:00.000Z', + }); + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true); + + (runAgentProcessMock as any) + .mockImplementationOnce( + async ( + _group: unknown, + _input: unknown, + _onProcess: unknown, + onOutput?: (output: Record) => Promise, + ) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + }); + await onOutput?.({ + status: 'success', + result: + 'Your organization does not have access to Claude. Please login again or contact your administrator.', + }); + return { + status: 'success', + result: null, + }; + }, + ) + .mockImplementationOnce( + async ( + _group: unknown, + _input: unknown, + _onProcess: unknown, + onOutput?: (output: Record) => Promise, + ) => { + await onOutput?.({ + status: 'success', + result: 'rotated scheduled task org-access response', + }); + return { + status: 'success', + result: null, + }; + }, + ); + + const enqueueTask = vi.fn( + (_groupJid: string, _taskId: string, fn: () => Promise) => { + void fn(); + }, + ); + const sendMessage = vi.fn(async () => {}); + + startSchedulerLoop({ + serviceAgentType: 'claude-code', + registeredGroups: () => ({ + 'shared@g.us': { + name: 'Shared', + folder: 'shared-group', + trigger: '@Claude', + added_at: '2026-02-22T00:00:00.000Z', + agentType: 'claude-code', + }, + }), + getSessions: () => ({}), + queue: { enqueueTask } as any, + onProcess: () => {}, + sendMessage, + }); + + await vi.advanceTimersByTimeAsync(10); + + expect(runAgentProcessMock).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); + expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); + expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage).toHaveBeenCalledWith( + 'shared@g.us', + 'rotated scheduled task org-access response', + ); + }); + it('retries Codex scheduled tasks with a rotated account on streamed auth expiry', async () => { const dueAt = new Date(Date.now() - 60_000).toISOString(); createTask({ diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 7762c80..3094b7d 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -39,7 +39,7 @@ import { getFallbackProviderName, hasGroupProviderOverride, isFallbackEnabled, - isUsageExhausted, + isPrimaryNoFallbackCooldownActive, markPrimaryCooldown, } from './provider-fallback.js'; import { runClaudeRotationLoop } from './provider-retry.js'; @@ -560,13 +560,17 @@ async function runTask( ? await getActiveProvider() : 'claude'; - // Already in usage-exhausted cooldown — skip task instead of running on Kimi - if (isClaudeAgent && provider !== 'claude' && isUsageExhausted()) { + // Already in no-fallback Claude cooldown — skip task instead of running on Kimi + if ( + isClaudeAgent && + provider !== 'claude' && + isPrimaryNoFallbackCooldownActive() + ) { logger.info( { taskId: task.id, group: context.group.name, provider }, - 'Claude usage exhausted (cooldown active), skipping scheduled task', + 'Claude primary cooldown active, skipping scheduled task', ); - error = 'Claude usage exhausted'; + error = 'Claude primary cooldown active'; // Fall through to task completion handling below } else { const attempt = await runTaskAttempt(provider);