claude-bot/EJClaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: parse retry-after from errors, cache codex usage per account

Browse Source 
- Parse "try again at" time from rate-limit errors instead of
  fixed 1-hour cooldown, with 3-min buffer after reset
- Pass error message to rotateToken/rotateCodexToken for parsing
- Cache usage % and reset time per Codex account for dashboard
- Persist cached usage in rotation state file across restarts
- Show rate-limited Codex accounts with cached 100% + reset time
- Fix dashboard text indicators (*!/space) for monospace alignment
This commit is contained in:
Eyejoker
2026-03-24 01:16:54 +09:00
parent 2e6fc4a08e
commit 60dab43a3f
5 changed files with 139 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/codex-token-rotation.ts
View File

@@ -25,6 +25,8 @@ interface CodexAccount {
accountId: string;
planType: string;
rateLimitedUntil: number | null;
lastUsagePct?: number;
resetAt?: string;
}
function parsePlanFromJwt(idToken: string): string {
@@ -95,9 +97,13 @@ function saveCodexState(): void {
const state = {
currentIndex,
rateLimits: accounts.map((a) => a.rateLimitedUntil),
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
resetAts: accounts.map((a) => a.resetAt ?? null),
};
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
} catch { /* best effort */ }
} catch {
/* best effort */
}
}
function loadCodexState(): void {
@@ -105,19 +111,71 @@ function loadCodexState(): void {
if (!fs.existsSync(STATE_FILE)) return;
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
const now = Date.now();
if (typeof state.currentIndex === 'number' && state.currentIndex < accounts.length) {
if (
typeof state.currentIndex === 'number' &&
state.currentIndex < accounts.length
) {
currentIndex = state.currentIndex;
}
if (Array.isArray(state.rateLimits)) {
for (let i = 0; i < Math.min(state.rateLimits.length, accounts.length); i++) {
for (
let i = 0;
i < Math.min(state.rateLimits.length, accounts.length);
i++
) {
const until = state.rateLimits[i];
if (typeof until === 'number' && until > now) {
accounts[i].rateLimitedUntil = until;
}
}
}
logger.info({ currentIndex, accountCount: accounts.length }, 'Codex rotation state restored');
} catch { /* start fresh */ }
if (Array.isArray(state.usagePcts)) {
for (let i = 0; i < Math.min(state.usagePcts.length, accounts.length); i++) {
if (typeof state.usagePcts[i] === 'number') accounts[i].lastUsagePct = state.usagePcts[i];
}
}
if (Array.isArray(state.resetAts)) {
for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) {
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i];
}
}
logger.info(
{ currentIndex, accountCount: accounts.length },
'Codex rotation state restored',
);
} catch {
/* start fresh */
}
}
const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time
const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback
/**
* Parse "try again at Mar 26th, 2026 9:00 AM" from error message.
* Returns timestamp in ms, or null if not found.
*/
function parseRetryAfterFromError(error?: string): number | null {
if (!error) return null;
const match = error.match(
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
);
if (!match) return null;
try {
// Remove ordinal suffixes (1st, 2nd, 3rd, 4th)
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
const ts = new Date(cleaned).getTime();
if (Number.isNaN(ts)) return null;
return ts;
} catch {
return null;
}
}
function computeCooldownUntil(error?: string): number {
const retryAt = parseRetryAfterFromError(error);
if (retryAt) return retryAt + BUFFER_MS;
return Date.now() + DEFAULT_COOLDOWN_MS;
}
/** Get the auth.json path for the current active account. */
@@ -130,11 +188,18 @@ export function getActiveCodexAuthPath(): string | null {
* Try to rotate to the next available Codex account.
* Returns true if a fresh account was found.
*/
export function rotateCodexToken(): boolean {
export function rotateCodexToken(errorMessage?: string): boolean {
if (accounts.length <= 1) return false;
const now = Date.now();
accounts[currentIndex].rateLimitedUntil = now + 3_600_000;
const acct = accounts[currentIndex];
acct.rateLimitedUntil = computeCooldownUntil(errorMessage);
acct.lastUsagePct = 100;
// Extract reset time string from error for display
const resetMatch = errorMessage?.match(
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
);
if (resetMatch) acct.resetAt = resetMatch[1];
for (let i = 1; i < accounts.length; i++) {
const idx = (currentIndex + i) % accounts.length;
@@ -178,6 +243,8 @@ export function getAllCodexAccounts(): {
planType: string;
isActive: boolean;
isRateLimited: boolean;
cachedUsagePct?: number;
resetAt?: string;
}[] {
const now = Date.now();
return accounts.map((a, i) => ({
@@ -186,5 +253,7 @@ export function getAllCodexAccounts(): {
planType: a.planType,
isActive: i === currentIndex,
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
cachedUsagePct: a.lastUsagePct,
resetAt: a.resetAt,
}));
}

8
src/message-agent-executor.ts
View File

@@ -309,7 +309,7 @@ export async function runAgentForGroup(
: detectFallbackTrigger(errMsg);
if (trigger.shouldFallback) {
// Try rotating token before falling back to another provider
if (getTokenCount() > 1 && rotateToken()) {
if (getTokenCount() > 1 && rotateToken(errMsg)) {
logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason },
'Rate-limited, retrying with rotated token',
@@ -383,7 +383,7 @@ export async function runAgentForGroup(
}
: detectFallbackTrigger(output.error);
if (trigger.shouldFallback) {
if (getTokenCount() > 1 && rotateToken()) {
if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) {
logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason },
'Rate-limited (output error), retrying with rotated token',
@@ -401,7 +401,7 @@ export async function runAgentForGroup(
// Codex rate-limit rotation (non-Claude agents)
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
const trigger = detectFallbackTrigger(output.error);
if (trigger.shouldFallback && rotateCodexToken()) {
if (trigger.shouldFallback && rotateCodexToken(output.error ?? undefined)) {
logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason },
'Codex rate-limited, retrying with rotated account',
@@ -433,7 +433,7 @@ export async function runAgentForGroup(
!isClaudeCodeAgent &&
primaryAttempt.streamedTriggerReason &&
getCodexAccountCount() > 1 &&
rotateCodexToken()
rotateCodexToken(output.error ?? undefined)
) {
logger.info(
{

10
src/task-scheduler.ts
View File

@@ -332,11 +332,15 @@ async function runTask(
if (trigger.shouldFallback) {
const isCodex = SERVICE_AGENT_TYPE === 'codex';
const rotated = isCodex
? getCodexAccountCount() > 1 && rotateCodexToken()
: getTokenCount() > 1 && rotateToken();
? getCodexAccountCount() > 1 && rotateCodexToken(error)
: getTokenCount() > 1 && rotateToken(error);
if (rotated) {
logger.info(
{ taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason },
{
taskId: task.id,
agent: SERVICE_AGENT_TYPE,
reason: trigger.reason,
},
'Task rate-limited, rotated token — will retry on next schedule',
);
if (isCodex) markCodexTokenHealthy();

54
src/token-rotation.ts
View File

@@ -70,7 +70,9 @@ function saveState(): void {
rateLimits: tokens.map((t) => t.rateLimitedUntil),
};
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
} catch { /* best effort */ }
} catch {
/* best effort */
}
}
function loadState(): void {
@@ -78,19 +80,56 @@ function loadState(): void {
if (!fs.existsSync(STATE_FILE)) return;
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
const now = Date.now();
if (typeof state.currentIndex === 'number' && state.currentIndex < tokens.length) {
if (
typeof state.currentIndex === 'number' &&
state.currentIndex < tokens.length
) {
currentIndex = state.currentIndex;
}
if (Array.isArray(state.rateLimits)) {
for (let i = 0; i < Math.min(state.rateLimits.length, tokens.length); i++) {
for (
let i = 0;
i < Math.min(state.rateLimits.length, tokens.length);
i++
) {
const until = state.rateLimits[i];
if (typeof until === 'number' && until > now) {
tokens[i].rateLimitedUntil = until;
}
}
}
logger.info({ currentIndex, tokenCount: tokens.length }, 'Token rotation state restored');
} catch { /* start fresh */ }
logger.info(
{ currentIndex, tokenCount: tokens.length },
'Token rotation state restored',
);
} catch {
/* start fresh */
}
}
const BUFFER_MS = 3 * 60_000;
const DEFAULT_COOLDOWN_MS = 3_600_000;
function parseRetryAfterFromError(error?: string): number | null {
if (!error) return null;
const match = error.match(
/(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
);
if (!match) return null;
try {
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
const ts = new Date(cleaned).getTime();
if (Number.isNaN(ts)) return null;
return ts;
} catch {
return null;
}
}
function computeCooldownUntil(error?: string): number {
const retryAt = parseRetryAfterFromError(error);
if (retryAt) return retryAt + BUFFER_MS;
return Date.now() + DEFAULT_COOLDOWN_MS;
}
/** Get the current active token. */
@@ -103,12 +142,11 @@ export function getCurrentToken(): string | undefined {
* Try to rotate to the next available (non-rate-limited) token.
* Returns true if a fresh token was found, false if all are exhausted.
*/
export function rotateToken(): boolean {
export function rotateToken(errorMessage?: string): boolean {
if (tokens.length <= 1) return false;
const now = Date.now();
// Mark current as rate-limited (default 1 hour)
tokens[currentIndex].rateLimitedUntil = now + 3_600_000;
tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage);
// Find next available token
for (let i = 1; i < tokens.length; i++) {

8
src/unified-dashboard.ts
View File

@@ -616,10 +616,14 @@ async function buildUsageContent(): Promise<string> {
});
}
} else {
// Show cached usage for rate-limited accounts
const pct = acct.isRateLimited && acct.cachedUsagePct != null
? acct.cachedUsagePct : -1;
const reset = acct.resetAt || '';
rows.push({
name: `${label} ${acct.planType}`,
h5pct: -1,
h5reset: '',
h5pct: pct,
h5reset: reset,
d7pct: -1,
d7reset: '',
});