fix: detect org-access-denied errors, suppress chat output, and rotate Claude tokens
When a Claude account is suspended, the API returns "Your organization does not have access to Claude" on the success path or "Failed to authenticate. API Error: 403 terminated" on the error path. Both are now classified as 'org-access-denied', suppressed from Discord output, and trigger automatic token rotation. If all tokens are exhausted, enters cooldown without Kimi fallback (same as usage-exhausted and auth-expired). Changes: - agent-error-detection: add isClaudeOrgAccessDeniedMessage(), expand classifyClaudeAuthError() and shouldRotateClaudeToken() - streamed-output-evaluator: detect org-access-denied in success-path chain - provider-fallback: add NO_FALLBACK_COOLDOWN_REASONS set and isPrimaryNoFallbackCooldownActive() helper - provider-retry: handle org-access-denied in rotation loop - message-agent-executor / task-scheduler: use generalized no-fallback cooldown check - Tests: +8 test cases across 5 test files (415 total passing) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -54,13 +54,26 @@ export function isClaudeAuthExpiredMessage(text: string): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isClaudeOrgAccessDeniedMessage(text: string): boolean {
|
||||||
|
const normalized = text.trim().toLowerCase().replace(/\s+/g, ' ');
|
||||||
|
const hasOrgAccessDeniedMarker = normalized.includes(
|
||||||
|
'does not have access to claude',
|
||||||
|
);
|
||||||
|
const hasRecoveryHint =
|
||||||
|
normalized.includes('please login again') ||
|
||||||
|
normalized.includes('contact your administrator');
|
||||||
|
|
||||||
|
return hasOrgAccessDeniedMarker && hasRecoveryHint;
|
||||||
|
}
|
||||||
|
|
||||||
// ── Rotation decision ───────────────────────────────────────────
|
// ── Rotation decision ───────────────────────────────────────────
|
||||||
|
|
||||||
export function shouldRotateClaudeToken(reason: string): boolean {
|
export function shouldRotateClaudeToken(reason: string): boolean {
|
||||||
return (
|
return (
|
||||||
reason === '429' ||
|
reason === '429' ||
|
||||||
reason === 'usage-exhausted' ||
|
reason === 'usage-exhausted' ||
|
||||||
reason === 'auth-expired'
|
reason === 'auth-expired' ||
|
||||||
|
reason === 'org-access-denied'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,13 +82,14 @@ export function shouldRotateClaudeToken(reason: string): boolean {
|
|||||||
export type ErrorCategory =
|
export type ErrorCategory =
|
||||||
| 'rate-limit'
|
| 'rate-limit'
|
||||||
| 'auth-expired'
|
| 'auth-expired'
|
||||||
|
| 'org-access-denied'
|
||||||
| 'overloaded'
|
| 'overloaded'
|
||||||
| 'network-error'
|
| 'network-error'
|
||||||
| 'none';
|
| 'none';
|
||||||
|
|
||||||
export interface AgentErrorClassification {
|
export interface AgentErrorClassification {
|
||||||
category: ErrorCategory;
|
category: ErrorCategory;
|
||||||
reason: string; // '429' | 'auth-expired' | 'overloaded' | 'network-error' | ''
|
reason: string; // '429' | 'auth-expired' | 'org-access-denied' | 'overloaded' | 'network-error' | ''
|
||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,6 +156,19 @@ export function classifyClaudeAuthError(
|
|||||||
if (!error) return NONE;
|
if (!error) return NONE;
|
||||||
const lower = error.toLowerCase();
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
|
const hasOrgAccessDeniedMarker =
|
||||||
|
lower.includes('your organization does not have access to claude') ||
|
||||||
|
(lower.includes('does not have access to claude') &&
|
||||||
|
lower.includes('contact your administrator'));
|
||||||
|
const hasTerminated403AuthFailure =
|
||||||
|
lower.includes('failed to authenticate') &&
|
||||||
|
lower.includes('403') &&
|
||||||
|
lower.includes('terminated');
|
||||||
|
|
||||||
|
if (hasOrgAccessDeniedMarker || hasTerminated403AuthFailure) {
|
||||||
|
return { category: 'org-access-denied', reason: 'org-access-denied' };
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
(lower.includes('failed to authenticate') ||
|
(lower.includes('failed to authenticate') ||
|
||||||
lower.includes('authentication_error')) &&
|
lower.includes('authentication_error')) &&
|
||||||
|
|||||||
@@ -30,6 +30,14 @@ vi.mock('./logger.js', () => ({
|
|||||||
vi.mock('./provider-fallback.js', () => ({
|
vi.mock('./provider-fallback.js', () => ({
|
||||||
detectFallbackTrigger: vi.fn((error?: string | null) => {
|
detectFallbackTrigger: vi.fn((error?: string | null) => {
|
||||||
const lower = (error || '').toLowerCase();
|
const lower = (error || '').toLowerCase();
|
||||||
|
if (
|
||||||
|
lower.includes('does not have access to claude') ||
|
||||||
|
(lower.includes('failed to authenticate') &&
|
||||||
|
lower.includes('403') &&
|
||||||
|
lower.includes('terminated'))
|
||||||
|
) {
|
||||||
|
return { shouldFallback: true, reason: 'org-access-denied' };
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
lower.includes('429') ||
|
lower.includes('429') ||
|
||||||
lower.includes('rate limit') ||
|
lower.includes('rate limit') ||
|
||||||
@@ -48,6 +56,7 @@ vi.mock('./provider-fallback.js', () => ({
|
|||||||
getFallbackProviderName: vi.fn(() => 'kimi'),
|
getFallbackProviderName: vi.fn(() => 'kimi'),
|
||||||
hasGroupProviderOverride: vi.fn(() => false),
|
hasGroupProviderOverride: vi.fn(() => false),
|
||||||
isFallbackEnabled: vi.fn(() => true),
|
isFallbackEnabled: vi.fn(() => true),
|
||||||
|
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
|
||||||
isUsageExhausted: vi.fn(() => false),
|
isUsageExhausted: vi.fn(() => false),
|
||||||
markPrimaryCooldown: vi.fn(),
|
markPrimaryCooldown: vi.fn(),
|
||||||
}));
|
}));
|
||||||
@@ -370,6 +379,143 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
expect(outputs).toEqual([]);
|
expect(outputs).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rotates to another Claude account when Claude streams an org access denied banner', async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
|
||||||
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
|
vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'intermediate',
|
||||||
|
result:
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
});
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result:
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'org access denied 회전 성공 응답',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-org-access-denied-claude',
|
||||||
|
onOutput: async (output) => {
|
||||||
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('success');
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
||||||
|
expect(outputs).toEqual(['org access denied 회전 성공 응답']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stops after all Claude accounts are org-access-denied without falling back to Kimi', async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
|
||||||
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
|
vi.mocked(tokenRotation.rotateToken)
|
||||||
|
.mockReturnValueOnce(true)
|
||||||
|
.mockReturnValueOnce(false);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result:
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'error',
|
||||||
|
result: null,
|
||||||
|
error: 'Failed to authenticate. API Error: 403 terminated',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
result: null,
|
||||||
|
error: 'Failed to authenticate. API Error: 403 terminated',
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'Kimi 폴백 응답입니다.',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-org-access-denied-no-fallback',
|
||||||
|
onOutput: async (output) => {
|
||||||
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('error');
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
|
||||||
|
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
||||||
|
'org-access-denied',
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
expect(outputs).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips execution entirely when Claude no-fallback cooldown is already active', async () => {
|
||||||
|
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('kimi');
|
||||||
|
vi.mocked(
|
||||||
|
providerFallback.isPrimaryNoFallbackCooldownActive,
|
||||||
|
).mockReturnValue(true);
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-skip-primary-cooldown',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('error');
|
||||||
|
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it('does not mistake a normal response quoting the banner text for a usage error', async () => {
|
it('does not mistake a normal response quoting the banner text for a usage error', async () => {
|
||||||
const outputs: string[] = [];
|
const outputs: string[] = [];
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import {
|
|||||||
getFallbackProviderName,
|
getFallbackProviderName,
|
||||||
hasGroupProviderOverride,
|
hasGroupProviderOverride,
|
||||||
isFallbackEnabled,
|
isFallbackEnabled,
|
||||||
isUsageExhausted,
|
isPrimaryNoFallbackCooldownActive,
|
||||||
markPrimaryCooldown,
|
markPrimaryCooldown,
|
||||||
} from './provider-fallback.js';
|
} from './provider-fallback.js';
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
@@ -508,11 +508,11 @@ export async function runAgentForGroup(
|
|||||||
|
|
||||||
const provider = canFallback ? await getActiveProvider() : 'claude';
|
const provider = canFallback ? await getActiveProvider() : 'claude';
|
||||||
|
|
||||||
// Already in usage-exhausted cooldown — log only, no response
|
// Already in no-fallback Claude cooldown — log only, no response
|
||||||
if (provider !== 'claude' && isUsageExhausted()) {
|
if (provider !== 'claude' && isPrimaryNoFallbackCooldownActive()) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid, group: group.name, runId, provider },
|
{ chatJid, group: group.name, runId, provider },
|
||||||
'Claude usage exhausted (cooldown active), silently skipping',
|
'Claude primary cooldown active, silently skipping',
|
||||||
);
|
);
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -131,6 +131,7 @@ vi.mock('./provider-fallback.js', () => ({
|
|||||||
getFallbackProviderName: vi.fn(() => 'kimi'),
|
getFallbackProviderName: vi.fn(() => 'kimi'),
|
||||||
hasGroupProviderOverride: vi.fn(() => false),
|
hasGroupProviderOverride: vi.fn(() => false),
|
||||||
isFallbackEnabled: vi.fn(() => true),
|
isFallbackEnabled: vi.fn(() => true),
|
||||||
|
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
|
||||||
markPrimaryCooldown: vi.fn(),
|
markPrimaryCooldown: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ import {
|
|||||||
detectFallbackTrigger,
|
detectFallbackTrigger,
|
||||||
getActiveProvider,
|
getActiveProvider,
|
||||||
getCooldownInfo,
|
getCooldownInfo,
|
||||||
|
isPrimaryNoFallbackCooldownActive,
|
||||||
markPrimaryCooldown,
|
markPrimaryCooldown,
|
||||||
resetFallbackConfig,
|
resetFallbackConfig,
|
||||||
} from './provider-fallback.js';
|
} from './provider-fallback.js';
|
||||||
@@ -133,4 +134,36 @@ describe('provider fallback usage recovery', () => {
|
|||||||
reason: 'auth-expired',
|
reason: 'auth-expired',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('treats Claude org access denied banners as an org-access-denied fallback trigger', () => {
|
||||||
|
expect(
|
||||||
|
detectFallbackTrigger(
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: 'org-access-denied',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('treats terminated 403 auth failures as an org-access-denied fallback trigger', () => {
|
||||||
|
expect(
|
||||||
|
detectFallbackTrigger(
|
||||||
|
'Failed to authenticate. API Error: 403 terminated',
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: 'org-access-denied',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('marks org-access-denied as a no-fallback cooldown reason', () => {
|
||||||
|
markPrimaryCooldown('org-access-denied', 60_000);
|
||||||
|
|
||||||
|
expect(isPrimaryNoFallbackCooldownActive()).toBe(true);
|
||||||
|
expect(getCooldownInfo()).toMatchObject({
|
||||||
|
active: true,
|
||||||
|
reason: 'org-access-denied',
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -59,6 +59,11 @@ let lastUsageAvailabilityCheck: {
|
|||||||
let usageAvailabilityCheckPromise: Promise<
|
let usageAvailabilityCheckPromise: Promise<
|
||||||
'available' | 'exhausted' | 'unknown'
|
'available' | 'exhausted' | 'unknown'
|
||||||
> | null = null;
|
> | null = null;
|
||||||
|
const NO_FALLBACK_COOLDOWN_REASONS = new Set([
|
||||||
|
'usage-exhausted',
|
||||||
|
'auth-expired',
|
||||||
|
'org-access-denied',
|
||||||
|
]);
|
||||||
|
|
||||||
const USAGE_RECOVERY_RECHECK_MS = 30_000;
|
const USAGE_RECOVERY_RECHECK_MS = 30_000;
|
||||||
|
|
||||||
@@ -331,6 +336,11 @@ export function isUsageExhausted(): boolean {
|
|||||||
return cooldown?.reason === 'usage-exhausted';
|
return cooldown?.reason === 'usage-exhausted';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Check whether the active primary cooldown should suppress fallback entirely. */
|
||||||
|
export function isPrimaryNoFallbackCooldownActive(): boolean {
|
||||||
|
return cooldown ? NO_FALLBACK_COOLDOWN_REASONS.has(cooldown.reason) : false;
|
||||||
|
}
|
||||||
|
|
||||||
/** Get current cooldown info (for diagnostics / status dashboard). */
|
/** Get current cooldown info (for diagnostics / status dashboard). */
|
||||||
export function getCooldownInfo(): {
|
export function getCooldownInfo(): {
|
||||||
active: boolean;
|
active: boolean;
|
||||||
@@ -379,6 +389,7 @@ export function getFallbackEnvOverrides(): Record<string, string> {
|
|||||||
*
|
*
|
||||||
* Triggers:
|
* Triggers:
|
||||||
* - 429 / rate limit / too many requests
|
* - 429 / rate limit / too many requests
|
||||||
|
* - Claude auth/org access failures
|
||||||
* - 503 / overloaded (transient provider issue)
|
* - 503 / overloaded (transient provider issue)
|
||||||
* - Network / connection errors
|
* - Network / connection errors
|
||||||
*
|
*
|
||||||
@@ -393,7 +404,7 @@ export function detectFallbackTrigger(
|
|||||||
if (!error) return { shouldFallback: false, reason: '' };
|
if (!error) return { shouldFallback: false, reason: '' };
|
||||||
|
|
||||||
// Delegated to shared SSOT — original priority preserved:
|
// Delegated to shared SSOT — original priority preserved:
|
||||||
// 429 first, then auth-expired, then 503/network
|
// 429 first, then Claude auth/org errors, then 503/network
|
||||||
const common = classifyAgentError(error);
|
const common = classifyAgentError(error);
|
||||||
|
|
||||||
// 429 rate-limit (highest priority)
|
// 429 rate-limit (highest priority)
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ export type RotationOutcome =
|
|||||||
| { type: 'success' }
|
| { type: 'success' }
|
||||||
| { type: 'error'; message?: string }
|
| { type: 'error'; message?: string }
|
||||||
| { type: 'needs-fallback'; trigger: TriggerInfo }
|
| { type: 'needs-fallback'; trigger: TriggerInfo }
|
||||||
| { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired
|
| { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired/org-access-denied
|
||||||
|
|
||||||
// ── Shared rotation loop ─────────────────────────────────────────
|
// ── Shared rotation loop ─────────────────────────────────────────
|
||||||
|
|
||||||
@@ -63,7 +63,7 @@ export async function runClaudeRotationLoop(
|
|||||||
) {
|
) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ ...logContext, reason: trigger.reason },
|
{ ...logContext, reason: trigger.reason },
|
||||||
'Claude rate-limited, retrying with rotated account',
|
'Claude account unavailable, retrying with rotated account',
|
||||||
);
|
);
|
||||||
|
|
||||||
const attempt = await runAttempt();
|
const attempt = await runAttempt();
|
||||||
@@ -162,10 +162,11 @@ export async function runClaudeRotationLoop(
|
|||||||
|
|
||||||
// ── All tokens exhausted ──
|
// ── All tokens exhausted ──
|
||||||
|
|
||||||
// Usage exhausted or auth-expired: don't fall back to Kimi
|
// Usage/auth/org access failures: don't fall back to Kimi
|
||||||
if (
|
if (
|
||||||
trigger.reason === 'usage-exhausted' ||
|
trigger.reason === 'usage-exhausted' ||
|
||||||
trigger.reason === 'auth-expired'
|
trigger.reason === 'auth-expired' ||
|
||||||
|
trigger.reason === 'org-access-denied'
|
||||||
) {
|
) {
|
||||||
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import type { AgentOutput } from './agent-runner.js';
|
|||||||
// ── Mocks ──────────────────────────────────────────────────────
|
// ── Mocks ──────────────────────────────────────────────────────
|
||||||
vi.mock('./agent-error-detection.js', () => ({
|
vi.mock('./agent-error-detection.js', () => ({
|
||||||
isClaudeUsageExhaustedMessage: vi.fn(() => false),
|
isClaudeUsageExhaustedMessage: vi.fn(() => false),
|
||||||
|
isClaudeOrgAccessDeniedMessage: vi.fn(() => false),
|
||||||
isClaudeAuthExpiredMessage: vi.fn(() => false),
|
isClaudeAuthExpiredMessage: vi.fn(() => false),
|
||||||
isClaudeAuthError: vi.fn(() => false),
|
isClaudeAuthError: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
@@ -22,6 +23,7 @@ vi.mock('./codex-token-rotation.js', () => ({
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
isClaudeUsageExhaustedMessage,
|
isClaudeUsageExhaustedMessage,
|
||||||
|
isClaudeOrgAccessDeniedMessage,
|
||||||
isClaudeAuthExpiredMessage,
|
isClaudeAuthExpiredMessage,
|
||||||
isClaudeAuthError,
|
isClaudeAuthError,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
@@ -60,6 +62,7 @@ function errorOutput(error: string): AgentOutput {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false);
|
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false);
|
||||||
|
vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(false);
|
||||||
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false);
|
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false);
|
||||||
vi.mocked(isClaudeAuthError).mockReturnValue(false);
|
vi.mocked(isClaudeAuthError).mockReturnValue(false);
|
||||||
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
||||||
@@ -164,6 +167,25 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Claude org-access-denied banner', () => {
|
||||||
|
it('suppresses output and returns newTrigger', () => {
|
||||||
|
vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(true);
|
||||||
|
|
||||||
|
const result = evaluateStreamedOutput(
|
||||||
|
successOutput(
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
),
|
||||||
|
freshState(),
|
||||||
|
claudeOpts,
|
||||||
|
);
|
||||||
|
expect(result.shouldForwardOutput).toBe(false);
|
||||||
|
expect(result.newTrigger).toEqual({ reason: 'org-access-denied' });
|
||||||
|
expect(result.state.streamedTriggerReason).toEqual({
|
||||||
|
reason: 'org-access-denied',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('duplicate trigger suppression', () => {
|
describe('duplicate trigger suppression', () => {
|
||||||
it('suppresses output but returns no newTrigger when already triggered', () => {
|
it('suppresses output but returns no newTrigger when already triggered', () => {
|
||||||
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true);
|
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(true);
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import {
|
import {
|
||||||
isClaudeAuthError,
|
isClaudeAuthError,
|
||||||
isClaudeAuthExpiredMessage,
|
isClaudeAuthExpiredMessage,
|
||||||
|
isClaudeOrgAccessDeniedMessage,
|
||||||
isClaudeUsageExhaustedMessage,
|
isClaudeUsageExhaustedMessage,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import type { AgentOutput } from './agent-runner.js';
|
import type { AgentOutput } from './agent-runner.js';
|
||||||
@@ -52,6 +53,8 @@ export function evaluateStreamedOutput(
|
|||||||
) {
|
) {
|
||||||
const triggerReason = isClaudeUsageExhaustedMessage(output.result)
|
const triggerReason = isClaudeUsageExhaustedMessage(output.result)
|
||||||
? 'usage-exhausted'
|
? 'usage-exhausted'
|
||||||
|
: isClaudeOrgAccessDeniedMessage(output.result)
|
||||||
|
? 'org-access-denied'
|
||||||
: isClaudeAuthExpiredMessage(output.result)
|
: isClaudeAuthExpiredMessage(output.result)
|
||||||
? 'auth-expired'
|
? 'auth-expired'
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ const { runAgentProcessMock, writeTasksSnapshotMock } = vi.hoisted(() => ({
|
|||||||
vi.mock('./provider-fallback.js', () => ({
|
vi.mock('./provider-fallback.js', () => ({
|
||||||
detectFallbackTrigger: vi.fn((error?: string | null) => {
|
detectFallbackTrigger: vi.fn((error?: string | null) => {
|
||||||
const lower = (error || '').toLowerCase();
|
const lower = (error || '').toLowerCase();
|
||||||
|
if (
|
||||||
|
lower.includes('does not have access to claude') ||
|
||||||
|
(lower.includes('failed to authenticate') &&
|
||||||
|
lower.includes('403') &&
|
||||||
|
lower.includes('terminated'))
|
||||||
|
) {
|
||||||
|
return { shouldFallback: true, reason: 'org-access-denied' };
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
lower.includes('429') ||
|
lower.includes('429') ||
|
||||||
lower.includes('rate limit') ||
|
lower.includes('rate limit') ||
|
||||||
@@ -29,6 +37,7 @@ vi.mock('./provider-fallback.js', () => ({
|
|||||||
getFallbackProviderName: vi.fn(() => 'kimi'),
|
getFallbackProviderName: vi.fn(() => 'kimi'),
|
||||||
hasGroupProviderOverride: vi.fn(() => false),
|
hasGroupProviderOverride: vi.fn(() => false),
|
||||||
isFallbackEnabled: vi.fn(() => true),
|
isFallbackEnabled: vi.fn(() => true),
|
||||||
|
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
|
||||||
markPrimaryCooldown: vi.fn(),
|
markPrimaryCooldown: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -493,6 +502,105 @@ Check the run.
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('suppresses Claude org access denied banners for scheduled tasks and retries with a rotated account', async () => {
|
||||||
|
const dueAt = new Date(Date.now() - 60_000).toISOString();
|
||||||
|
createTask({
|
||||||
|
id: 'task-org-access-denied-banner',
|
||||||
|
group_folder: 'shared-group',
|
||||||
|
chat_jid: 'shared@g.us',
|
||||||
|
agent_type: 'claude-code',
|
||||||
|
prompt: 'claude task',
|
||||||
|
schedule_type: 'once',
|
||||||
|
schedule_value: dueAt,
|
||||||
|
context_mode: 'isolated',
|
||||||
|
next_run: dueAt,
|
||||||
|
status: 'active',
|
||||||
|
created_at: '2026-02-22T00:00:00.000Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
|
vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true);
|
||||||
|
|
||||||
|
(runAgentProcessMock as any)
|
||||||
|
.mockImplementationOnce(
|
||||||
|
async (
|
||||||
|
_group: unknown,
|
||||||
|
_input: unknown,
|
||||||
|
_onProcess: unknown,
|
||||||
|
onOutput?: (output: Record<string, unknown>) => Promise<void>,
|
||||||
|
) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'intermediate',
|
||||||
|
result:
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
});
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
result:
|
||||||
|
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.mockImplementationOnce(
|
||||||
|
async (
|
||||||
|
_group: unknown,
|
||||||
|
_input: unknown,
|
||||||
|
_onProcess: unknown,
|
||||||
|
onOutput?: (output: Record<string, unknown>) => Promise<void>,
|
||||||
|
) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
result: 'rotated scheduled task org-access response',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const enqueueTask = vi.fn(
|
||||||
|
(_groupJid: string, _taskId: string, fn: () => Promise<void>) => {
|
||||||
|
void fn();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const sendMessage = vi.fn(async () => {});
|
||||||
|
|
||||||
|
startSchedulerLoop({
|
||||||
|
serviceAgentType: 'claude-code',
|
||||||
|
registeredGroups: () => ({
|
||||||
|
'shared@g.us': {
|
||||||
|
name: 'Shared',
|
||||||
|
folder: 'shared-group',
|
||||||
|
trigger: '@Claude',
|
||||||
|
added_at: '2026-02-22T00:00:00.000Z',
|
||||||
|
agentType: 'claude-code',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
queue: { enqueueTask } as any,
|
||||||
|
onProcess: () => {},
|
||||||
|
sendMessage,
|
||||||
|
});
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
|
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
||||||
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
||||||
|
expect(sendMessage).toHaveBeenCalledTimes(1);
|
||||||
|
expect(sendMessage).toHaveBeenCalledWith(
|
||||||
|
'shared@g.us',
|
||||||
|
'rotated scheduled task org-access response',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('retries Codex scheduled tasks with a rotated account on streamed auth expiry', async () => {
|
it('retries Codex scheduled tasks with a rotated account on streamed auth expiry', async () => {
|
||||||
const dueAt = new Date(Date.now() - 60_000).toISOString();
|
const dueAt = new Date(Date.now() - 60_000).toISOString();
|
||||||
createTask({
|
createTask({
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ import {
|
|||||||
getFallbackProviderName,
|
getFallbackProviderName,
|
||||||
hasGroupProviderOverride,
|
hasGroupProviderOverride,
|
||||||
isFallbackEnabled,
|
isFallbackEnabled,
|
||||||
isUsageExhausted,
|
isPrimaryNoFallbackCooldownActive,
|
||||||
markPrimaryCooldown,
|
markPrimaryCooldown,
|
||||||
} from './provider-fallback.js';
|
} from './provider-fallback.js';
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
@@ -560,13 +560,17 @@ async function runTask(
|
|||||||
? await getActiveProvider()
|
? await getActiveProvider()
|
||||||
: 'claude';
|
: 'claude';
|
||||||
|
|
||||||
// Already in usage-exhausted cooldown — skip task instead of running on Kimi
|
// Already in no-fallback Claude cooldown — skip task instead of running on Kimi
|
||||||
if (isClaudeAgent && provider !== 'claude' && isUsageExhausted()) {
|
if (
|
||||||
|
isClaudeAgent &&
|
||||||
|
provider !== 'claude' &&
|
||||||
|
isPrimaryNoFallbackCooldownActive()
|
||||||
|
) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId: task.id, group: context.group.name, provider },
|
{ taskId: task.id, group: context.group.name, provider },
|
||||||
'Claude usage exhausted (cooldown active), skipping scheduled task',
|
'Claude primary cooldown active, skipping scheduled task',
|
||||||
);
|
);
|
||||||
error = 'Claude usage exhausted';
|
error = 'Claude primary cooldown active';
|
||||||
// Fall through to task completion handling below
|
// Fall through to task completion handling below
|
||||||
} else {
|
} else {
|
||||||
const attempt = await runTaskAttempt(provider);
|
const attempt = await runTaskAttempt(provider);
|
||||||
|
|||||||
Reference in New Issue
Block a user