From 60dab43a3fa2f9b4608ce21a858c01f8b27dc89b Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:16:54 +0900 Subject: [PATCH] feat: parse retry-after from errors, cache codex usage per account - Parse "try again at" time from rate-limit errors instead of fixed 1-hour cooldown, with 3-min buffer after reset - Pass error message to rotateToken/rotateCodexToken for parsing - Cache usage % and reset time per Codex account for dashboard - Persist cached usage in rotation state file across restarts - Show rate-limited Codex accounts with cached 100% + reset time - Fix dashboard text indicators (*!/space) for monospace alignment --- src/codex-token-rotation.ts | 83 ++++++++++++++++++++++++++++++++--- src/message-agent-executor.ts | 8 ++-- src/task-scheduler.ts | 10 +++-- src/token-rotation.ts | 54 +++++++++++++++++++---- src/unified-dashboard.ts | 8 +++- 5 files changed, 139 insertions(+), 24 deletions(-) diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 12de6e4..d57b90c 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -25,6 +25,8 @@ interface CodexAccount { accountId: string; planType: string; rateLimitedUntil: number | null; + lastUsagePct?: number; + resetAt?: string; } function parsePlanFromJwt(idToken: string): string { @@ -95,9 +97,13 @@ function saveCodexState(): void { const state = { currentIndex, rateLimits: accounts.map((a) => a.rateLimitedUntil), + usagePcts: accounts.map((a) => a.lastUsagePct ?? null), + resetAts: accounts.map((a) => a.resetAt ?? null), }; fs.writeFileSync(STATE_FILE, JSON.stringify(state)); - } catch { /* best effort */ } + } catch { + /* best effort */ + } } function loadCodexState(): void { @@ -105,19 +111,71 @@ function loadCodexState(): void { if (!fs.existsSync(STATE_FILE)) return; const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); const now = Date.now(); - if (typeof state.currentIndex === 'number' && state.currentIndex < accounts.length) { + if ( + typeof state.currentIndex === 'number' && + state.currentIndex < accounts.length + ) { currentIndex = state.currentIndex; } if (Array.isArray(state.rateLimits)) { - for (let i = 0; i < Math.min(state.rateLimits.length, accounts.length); i++) { + for ( + let i = 0; + i < Math.min(state.rateLimits.length, accounts.length); + i++ + ) { const until = state.rateLimits[i]; if (typeof until === 'number' && until > now) { accounts[i].rateLimitedUntil = until; } } } - logger.info({ currentIndex, accountCount: accounts.length }, 'Codex rotation state restored'); - } catch { /* start fresh */ } + if (Array.isArray(state.usagePcts)) { + for (let i = 0; i < Math.min(state.usagePcts.length, accounts.length); i++) { + if (typeof state.usagePcts[i] === 'number') accounts[i].lastUsagePct = state.usagePcts[i]; + } + } + if (Array.isArray(state.resetAts)) { + for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) { + if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]; + } + } + logger.info( + { currentIndex, accountCount: accounts.length }, + 'Codex rotation state restored', + ); + } catch { + /* start fresh */ + } +} + +const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time +const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback + +/** + * Parse "try again at Mar 26th, 2026 9:00 AM" from error message. + * Returns timestamp in ms, or null if not found. + */ +function parseRetryAfterFromError(error?: string): number | null { + if (!error) return null; + const match = error.match( + /try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (!match) return null; + try { + // Remove ordinal suffixes (1st, 2nd, 3rd, 4th) + const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); + const ts = new Date(cleaned).getTime(); + if (Number.isNaN(ts)) return null; + return ts; + } catch { + return null; + } +} + +function computeCooldownUntil(error?: string): number { + const retryAt = parseRetryAfterFromError(error); + if (retryAt) return retryAt + BUFFER_MS; + return Date.now() + DEFAULT_COOLDOWN_MS; } /** Get the auth.json path for the current active account. */ @@ -130,11 +188,18 @@ export function getActiveCodexAuthPath(): string | null { * Try to rotate to the next available Codex account. * Returns true if a fresh account was found. */ -export function rotateCodexToken(): boolean { +export function rotateCodexToken(errorMessage?: string): boolean { if (accounts.length <= 1) return false; const now = Date.now(); - accounts[currentIndex].rateLimitedUntil = now + 3_600_000; + const acct = accounts[currentIndex]; + acct.rateLimitedUntil = computeCooldownUntil(errorMessage); + acct.lastUsagePct = 100; + // Extract reset time string from error for display + const resetMatch = errorMessage?.match( + /try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (resetMatch) acct.resetAt = resetMatch[1]; for (let i = 1; i < accounts.length; i++) { const idx = (currentIndex + i) % accounts.length; @@ -178,6 +243,8 @@ export function getAllCodexAccounts(): { planType: string; isActive: boolean; isRateLimited: boolean; + cachedUsagePct?: number; + resetAt?: string; }[] { const now = Date.now(); return accounts.map((a, i) => ({ @@ -186,5 +253,7 @@ export function getAllCodexAccounts(): { planType: a.planType, isActive: i === currentIndex, isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), + cachedUsagePct: a.lastUsagePct, + resetAt: a.resetAt, })); } diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 07f09a1..17c1c55 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -309,7 +309,7 @@ export async function runAgentForGroup( : detectFallbackTrigger(errMsg); if (trigger.shouldFallback) { // Try rotating token before falling back to another provider - if (getTokenCount() > 1 && rotateToken()) { + if (getTokenCount() > 1 && rotateToken(errMsg)) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Rate-limited, retrying with rotated token', @@ -383,7 +383,7 @@ export async function runAgentForGroup( } : detectFallbackTrigger(output.error); if (trigger.shouldFallback) { - if (getTokenCount() > 1 && rotateToken()) { + if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Rate-limited (output error), retrying with rotated token', @@ -401,7 +401,7 @@ export async function runAgentForGroup( // Codex rate-limit rotation (non-Claude agents) if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { const trigger = detectFallbackTrigger(output.error); - if (trigger.shouldFallback && rotateCodexToken()) { + if (trigger.shouldFallback && rotateCodexToken(output.error ?? undefined)) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Codex rate-limited, retrying with rotated account', @@ -433,7 +433,7 @@ export async function runAgentForGroup( !isClaudeCodeAgent && primaryAttempt.streamedTriggerReason && getCodexAccountCount() > 1 && - rotateCodexToken() + rotateCodexToken(output.error ?? undefined) ) { logger.info( { diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 42f0b30..3542c0a 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -332,11 +332,15 @@ async function runTask( if (trigger.shouldFallback) { const isCodex = SERVICE_AGENT_TYPE === 'codex'; const rotated = isCodex - ? getCodexAccountCount() > 1 && rotateCodexToken() - : getTokenCount() > 1 && rotateToken(); + ? getCodexAccountCount() > 1 && rotateCodexToken(error) + : getTokenCount() > 1 && rotateToken(error); if (rotated) { logger.info( - { taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason }, + { + taskId: task.id, + agent: SERVICE_AGENT_TYPE, + reason: trigger.reason, + }, 'Task rate-limited, rotated token — will retry on next schedule', ); if (isCodex) markCodexTokenHealthy(); diff --git a/src/token-rotation.ts b/src/token-rotation.ts index da9f289..2e73dbc 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -70,7 +70,9 @@ function saveState(): void { rateLimits: tokens.map((t) => t.rateLimitedUntil), }; fs.writeFileSync(STATE_FILE, JSON.stringify(state)); - } catch { /* best effort */ } + } catch { + /* best effort */ + } } function loadState(): void { @@ -78,19 +80,56 @@ function loadState(): void { if (!fs.existsSync(STATE_FILE)) return; const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); const now = Date.now(); - if (typeof state.currentIndex === 'number' && state.currentIndex < tokens.length) { + if ( + typeof state.currentIndex === 'number' && + state.currentIndex < tokens.length + ) { currentIndex = state.currentIndex; } if (Array.isArray(state.rateLimits)) { - for (let i = 0; i < Math.min(state.rateLimits.length, tokens.length); i++) { + for ( + let i = 0; + i < Math.min(state.rateLimits.length, tokens.length); + i++ + ) { const until = state.rateLimits[i]; if (typeof until === 'number' && until > now) { tokens[i].rateLimitedUntil = until; } } } - logger.info({ currentIndex, tokenCount: tokens.length }, 'Token rotation state restored'); - } catch { /* start fresh */ } + logger.info( + { currentIndex, tokenCount: tokens.length }, + 'Token rotation state restored', + ); + } catch { + /* start fresh */ + } +} + +const BUFFER_MS = 3 * 60_000; +const DEFAULT_COOLDOWN_MS = 3_600_000; + +function parseRetryAfterFromError(error?: string): number | null { + if (!error) return null; + const match = error.match( + /(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (!match) return null; + try { + const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); + const ts = new Date(cleaned).getTime(); + if (Number.isNaN(ts)) return null; + return ts; + } catch { + return null; + } +} + +function computeCooldownUntil(error?: string): number { + const retryAt = parseRetryAfterFromError(error); + if (retryAt) return retryAt + BUFFER_MS; + return Date.now() + DEFAULT_COOLDOWN_MS; } /** Get the current active token. */ @@ -103,12 +142,11 @@ export function getCurrentToken(): string | undefined { * Try to rotate to the next available (non-rate-limited) token. * Returns true if a fresh token was found, false if all are exhausted. */ -export function rotateToken(): boolean { +export function rotateToken(errorMessage?: string): boolean { if (tokens.length <= 1) return false; const now = Date.now(); - // Mark current as rate-limited (default 1 hour) - tokens[currentIndex].rateLimitedUntil = now + 3_600_000; + tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage); // Find next available token for (let i = 1; i < tokens.length; i++) { diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index fbd5410..b3146da 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -616,10 +616,14 @@ async function buildUsageContent(): Promise { }); } } else { + // Show cached usage for rate-limited accounts + const pct = acct.isRateLimited && acct.cachedUsagePct != null + ? acct.cachedUsagePct : -1; + const reset = acct.resetAt || ''; rows.push({ name: `${label} ${acct.planType}`, - h5pct: -1, - h5reset: '', + h5pct: pct, + h5reset: reset, d7pct: -1, d7reset: '', });