fix: Claude token rotation, usage-exhausted fallback, and usage API caching
- Fix token rotation not taking effect: getCurrentToken() was shadowed by static .env CLAUDE_CODE_OAUTH_TOKEN value in agent environment - Don't fall back to Kimi on usage-exhausted (only on transient 429/network) - Add disk cache for Claude usage API data (survives restarts, 429s) - Rate-limit usage API calls to 1 per token per 5 minutes - Add ignoreRateLimits option to rotateToken() for exhausted recovery - Rotate to next token in getActiveProvider() when current is exhausted - Add isUsageExhausted() helper to provider-fallback
This commit is contained in:
Eyejoker
@@ -18,6 +18,7 @@ import {
|
||||
getFallbackProviderName,
|
||||
hasGroupProviderOverride,
|
||||
isFallbackEnabled,
|
||||
isUsageExhausted,
|
||||
markPrimaryCooldown,
|
||||
} from './provider-fallback.js';
|
||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
||||
@@ -42,6 +43,25 @@ export interface MessageAgentExecutorDeps {
|
||||
clearSession: (groupFolder: string) => void;
|
||||
}
|
||||
|
||||
function isClaudeUsageExhaustedMessage(text: string): boolean {
|
||||
const normalized = text
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[’‘`]/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/^error:\s*/i, '');
|
||||
const looksLikeBanner =
|
||||
normalized.startsWith("you're out of extra usage") ||
|
||||
normalized.startsWith('you are out of extra usage') ||
|
||||
normalized.startsWith("you've hit your limit") ||
|
||||
normalized.startsWith('you have hit your limit');
|
||||
const hasResetHint =
|
||||
normalized.includes('resets ') ||
|
||||
normalized.includes('reset at ') ||
|
||||
normalized.includes('try again');
|
||||
return looksLikeBanner && hasResetHint && normalized.length <= 160;
|
||||
}
|
||||
|
||||
export async function runAgentForGroup(
|
||||
deps: MessageAgentExecutorDeps,
|
||||
args: {
|
||||
@@ -137,6 +157,30 @@ export async function runAgentForGroup(
|
||||
) {
|
||||
resetSessionRequested = true;
|
||||
}
|
||||
if (
|
||||
canFallback &&
|
||||
provider === 'claude' &&
|
||||
output.status === 'success' &&
|
||||
!sawOutput &&
|
||||
typeof output.result === 'string' &&
|
||||
isClaudeUsageExhaustedMessage(output.result)
|
||||
) {
|
||||
if (!streamedTriggerReason) {
|
||||
logger.warn(
|
||||
{
|
||||
chatJid,
|
||||
group: group.name,
|
||||
runId,
|
||||
resultPreview: output.result.slice(0, 120),
|
||||
},
|
||||
'Detected Claude usage exhaustion banner in successful output',
|
||||
);
|
||||
}
|
||||
streamedTriggerReason = {
|
||||
reason: 'usage-exhausted',
|
||||
};
|
||||
return;
|
||||
}
|
||||
if (output.result !== null && output.result !== undefined) {
|
||||
sawOutput = true;
|
||||
} else if (
|
||||
@@ -291,7 +335,164 @@ export async function runAgentForGroup(
|
||||
return 'success';
|
||||
};
|
||||
|
||||
const provider = canFallback ? getActiveProvider() : 'claude';
|
||||
const shouldRotateClaudeToken = (reason: string): boolean =>
|
||||
reason === '429' || reason === 'usage-exhausted';
|
||||
|
||||
const retryClaudeWithRotation = async (
|
||||
initialTrigger: {
|
||||
reason: string;
|
||||
retryAfterMs?: number;
|
||||
},
|
||||
rotationMessage?: string,
|
||||
): Promise<'success' | 'error'> => {
|
||||
let trigger = initialTrigger;
|
||||
let lastRotationMessage = rotationMessage;
|
||||
|
||||
while (
|
||||
shouldRotateClaudeToken(trigger.reason) &&
|
||||
getTokenCount() > 1 &&
|
||||
rotateToken(lastRotationMessage)
|
||||
) {
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||
'Claude rate-limited, retrying with rotated account',
|
||||
);
|
||||
|
||||
const retryAttempt = await runAttempt('claude');
|
||||
|
||||
if (retryAttempt.error) {
|
||||
if (!retryAttempt.sawOutput) {
|
||||
const errMsg =
|
||||
retryAttempt.error instanceof Error
|
||||
? retryAttempt.error.message
|
||||
: String(retryAttempt.error);
|
||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
||||
? {
|
||||
shouldFallback: true,
|
||||
reason: retryAttempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
||||
}
|
||||
: detectFallbackTrigger(errMsg);
|
||||
if (retryTrigger.shouldFallback) {
|
||||
trigger = {
|
||||
reason: retryTrigger.reason,
|
||||
retryAfterMs: retryTrigger.retryAfterMs,
|
||||
};
|
||||
lastRotationMessage = errMsg;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{
|
||||
chatJid,
|
||||
group: group.name,
|
||||
groupFolder: group.folder,
|
||||
runId,
|
||||
provider: 'claude',
|
||||
err: retryAttempt.error,
|
||||
},
|
||||
'Rotated Claude account also threw',
|
||||
);
|
||||
return 'error';
|
||||
}
|
||||
|
||||
const retryOutput = retryAttempt.output;
|
||||
if (!retryOutput) {
|
||||
logger.error(
|
||||
{
|
||||
chatJid,
|
||||
group: group.name,
|
||||
groupFolder: group.folder,
|
||||
runId,
|
||||
provider: 'claude',
|
||||
},
|
||||
'Rotated Claude account produced no output object',
|
||||
);
|
||||
return 'error';
|
||||
}
|
||||
|
||||
if (
|
||||
!retryAttempt.sawOutput &&
|
||||
retryAttempt.streamedTriggerReason &&
|
||||
retryOutput.status !== 'error'
|
||||
) {
|
||||
trigger = {
|
||||
reason: retryAttempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
||||
};
|
||||
lastRotationMessage =
|
||||
typeof retryOutput.result === 'string' ? retryOutput.result : undefined;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
!retryAttempt.sawOutput &&
|
||||
retryAttempt.sawSuccessNullResultWithoutOutput
|
||||
) {
|
||||
return runFallbackAttempt('success-null-result');
|
||||
}
|
||||
|
||||
if (retryOutput.status === 'error') {
|
||||
if (!retryAttempt.sawOutput) {
|
||||
const retryTrigger = retryAttempt.streamedTriggerReason
|
||||
? {
|
||||
shouldFallback: true,
|
||||
reason: retryAttempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs,
|
||||
}
|
||||
: detectFallbackTrigger(retryOutput.error);
|
||||
if (retryTrigger.shouldFallback) {
|
||||
trigger = {
|
||||
reason: retryTrigger.reason,
|
||||
retryAfterMs: retryTrigger.retryAfterMs,
|
||||
};
|
||||
lastRotationMessage = retryOutput.error ?? undefined;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{
|
||||
group: group.name,
|
||||
chatJid,
|
||||
runId,
|
||||
provider: 'claude',
|
||||
error: retryOutput.error,
|
||||
},
|
||||
'Rotated Claude account failed',
|
||||
);
|
||||
return 'error';
|
||||
}
|
||||
|
||||
markTokenHealthy();
|
||||
return 'success';
|
||||
}
|
||||
|
||||
// Usage exhausted: don't fall back to Kimi — log only, no response
|
||||
if (trigger.reason === 'usage-exhausted') {
|
||||
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, runId },
|
||||
'All Claude tokens usage-exhausted, silently skipping (no Kimi fallback)',
|
||||
);
|
||||
return 'error';
|
||||
}
|
||||
|
||||
return runFallbackAttempt(trigger.reason, trigger.retryAfterMs);
|
||||
};
|
||||
|
||||
const provider = canFallback ? await getActiveProvider() : 'claude';
|
||||
|
||||
// Already in usage-exhausted cooldown — log only, no response
|
||||
if (provider !== 'claude' && isUsageExhausted()) {
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, runId, provider },
|
||||
'Claude usage exhausted (cooldown active), silently skipping',
|
||||
);
|
||||
return 'error';
|
||||
}
|
||||
|
||||
const primaryAttempt = await runAttempt(provider);
|
||||
|
||||
if (primaryAttempt.error) {
|
||||
@@ -308,19 +509,13 @@ export async function runAgentForGroup(
|
||||
}
|
||||
: detectFallbackTrigger(errMsg);
|
||||
if (trigger.shouldFallback) {
|
||||
// Try rotating token before falling back to another provider
|
||||
if (getTokenCount() > 1 && rotateToken(errMsg)) {
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||
'Rate-limited, retrying with rotated token',
|
||||
);
|
||||
const retryAttempt = await runAttempt('claude');
|
||||
if (!retryAttempt.error) {
|
||||
markTokenHealthy();
|
||||
return 'success';
|
||||
}
|
||||
}
|
||||
return runFallbackAttempt(trigger.reason, trigger.retryAfterMs);
|
||||
return retryClaudeWithRotation(
|
||||
{
|
||||
reason: trigger.reason,
|
||||
retryAfterMs: trigger.retryAfterMs,
|
||||
},
|
||||
errMsg,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,6 +548,19 @@ export async function runAgentForGroup(
|
||||
return 'error';
|
||||
}
|
||||
|
||||
if (
|
||||
canFallback &&
|
||||
provider === 'claude' &&
|
||||
!primaryAttempt.sawOutput &&
|
||||
primaryAttempt.streamedTriggerReason &&
|
||||
output.status !== 'error'
|
||||
) {
|
||||
return retryClaudeWithRotation({
|
||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
canFallback &&
|
||||
provider === 'claude' &&
|
||||
@@ -383,18 +591,13 @@ export async function runAgentForGroup(
|
||||
}
|
||||
: detectFallbackTrigger(output.error);
|
||||
if (trigger.shouldFallback) {
|
||||
if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) {
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||
'Rate-limited (output error), retrying with rotated token',
|
||||
);
|
||||
const retryAttempt = await runAttempt('claude');
|
||||
if (!retryAttempt.error) {
|
||||
markTokenHealthy();
|
||||
return 'success';
|
||||
}
|
||||
}
|
||||
return runFallbackAttempt(trigger.reason, trigger.retryAfterMs);
|
||||
return retryClaudeWithRotation(
|
||||
{
|
||||
reason: trigger.reason,
|
||||
retryAfterMs: trigger.retryAfterMs,
|
||||
},
|
||||
output.error ?? undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user