feat: parse retry-after from errors, cache codex usage per account

- Parse "try again at" time from rate-limit errors instead of
  fixed 1-hour cooldown, with 3-min buffer after reset
- Pass error message to rotateToken/rotateCodexToken for parsing
- Cache usage % and reset time per Codex account for dashboard
- Persist cached usage in rotation state file across restarts
- Show rate-limited Codex accounts with cached 100% + reset time
- Fix dashboard text indicators (*!/space) for monospace alignment
This commit is contained in:
Eyejoker
2026-03-24 01:16:54 +09:00
parent 2e6fc4a08e
commit 60dab43a3f
5 changed files with 139 additions and 24 deletions

View File

@@ -25,6 +25,8 @@ interface CodexAccount {
accountId: string; accountId: string;
planType: string; planType: string;
rateLimitedUntil: number | null; rateLimitedUntil: number | null;
lastUsagePct?: number;
resetAt?: string;
} }
function parsePlanFromJwt(idToken: string): string { function parsePlanFromJwt(idToken: string): string {
@@ -95,9 +97,13 @@ function saveCodexState(): void {
const state = { const state = {
currentIndex, currentIndex,
rateLimits: accounts.map((a) => a.rateLimitedUntil), rateLimits: accounts.map((a) => a.rateLimitedUntil),
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
resetAts: accounts.map((a) => a.resetAt ?? null),
}; };
fs.writeFileSync(STATE_FILE, JSON.stringify(state)); fs.writeFileSync(STATE_FILE, JSON.stringify(state));
} catch { /* best effort */ } } catch {
/* best effort */
}
} }
function loadCodexState(): void { function loadCodexState(): void {
@@ -105,19 +111,71 @@ function loadCodexState(): void {
if (!fs.existsSync(STATE_FILE)) return; if (!fs.existsSync(STATE_FILE)) return;
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
const now = Date.now(); const now = Date.now();
if (typeof state.currentIndex === 'number' && state.currentIndex < accounts.length) { if (
typeof state.currentIndex === 'number' &&
state.currentIndex < accounts.length
) {
currentIndex = state.currentIndex; currentIndex = state.currentIndex;
} }
if (Array.isArray(state.rateLimits)) { if (Array.isArray(state.rateLimits)) {
for (let i = 0; i < Math.min(state.rateLimits.length, accounts.length); i++) { for (
let i = 0;
i < Math.min(state.rateLimits.length, accounts.length);
i++
) {
const until = state.rateLimits[i]; const until = state.rateLimits[i];
if (typeof until === 'number' && until > now) { if (typeof until === 'number' && until > now) {
accounts[i].rateLimitedUntil = until; accounts[i].rateLimitedUntil = until;
} }
} }
} }
logger.info({ currentIndex, accountCount: accounts.length }, 'Codex rotation state restored'); if (Array.isArray(state.usagePcts)) {
} catch { /* start fresh */ } for (let i = 0; i < Math.min(state.usagePcts.length, accounts.length); i++) {
if (typeof state.usagePcts[i] === 'number') accounts[i].lastUsagePct = state.usagePcts[i];
}
}
if (Array.isArray(state.resetAts)) {
for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) {
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i];
}
}
logger.info(
{ currentIndex, accountCount: accounts.length },
'Codex rotation state restored',
);
} catch {
/* start fresh */
}
}
const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time
const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback
/**
* Parse "try again at Mar 26th, 2026 9:00 AM" from error message.
* Returns timestamp in ms, or null if not found.
*/
function parseRetryAfterFromError(error?: string): number | null {
if (!error) return null;
const match = error.match(
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
);
if (!match) return null;
try {
// Remove ordinal suffixes (1st, 2nd, 3rd, 4th)
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
const ts = new Date(cleaned).getTime();
if (Number.isNaN(ts)) return null;
return ts;
} catch {
return null;
}
}
function computeCooldownUntil(error?: string): number {
const retryAt = parseRetryAfterFromError(error);
if (retryAt) return retryAt + BUFFER_MS;
return Date.now() + DEFAULT_COOLDOWN_MS;
} }
/** Get the auth.json path for the current active account. */ /** Get the auth.json path for the current active account. */
@@ -130,11 +188,18 @@ export function getActiveCodexAuthPath(): string | null {
* Try to rotate to the next available Codex account. * Try to rotate to the next available Codex account.
* Returns true if a fresh account was found. * Returns true if a fresh account was found.
*/ */
export function rotateCodexToken(): boolean { export function rotateCodexToken(errorMessage?: string): boolean {
if (accounts.length <= 1) return false; if (accounts.length <= 1) return false;
const now = Date.now(); const now = Date.now();
accounts[currentIndex].rateLimitedUntil = now + 3_600_000; const acct = accounts[currentIndex];
acct.rateLimitedUntil = computeCooldownUntil(errorMessage);
acct.lastUsagePct = 100;
// Extract reset time string from error for display
const resetMatch = errorMessage?.match(
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
);
if (resetMatch) acct.resetAt = resetMatch[1];
for (let i = 1; i < accounts.length; i++) { for (let i = 1; i < accounts.length; i++) {
const idx = (currentIndex + i) % accounts.length; const idx = (currentIndex + i) % accounts.length;
@@ -178,6 +243,8 @@ export function getAllCodexAccounts(): {
planType: string; planType: string;
isActive: boolean; isActive: boolean;
isRateLimited: boolean; isRateLimited: boolean;
cachedUsagePct?: number;
resetAt?: string;
}[] { }[] {
const now = Date.now(); const now = Date.now();
return accounts.map((a, i) => ({ return accounts.map((a, i) => ({
@@ -186,5 +253,7 @@ export function getAllCodexAccounts(): {
planType: a.planType, planType: a.planType,
isActive: i === currentIndex, isActive: i === currentIndex,
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
cachedUsagePct: a.lastUsagePct,
resetAt: a.resetAt,
})); }));
} }

View File

@@ -309,7 +309,7 @@ export async function runAgentForGroup(
: detectFallbackTrigger(errMsg); : detectFallbackTrigger(errMsg);
if (trigger.shouldFallback) { if (trigger.shouldFallback) {
// Try rotating token before falling back to another provider // Try rotating token before falling back to another provider
if (getTokenCount() > 1 && rotateToken()) { if (getTokenCount() > 1 && rotateToken(errMsg)) {
logger.info( logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason }, { chatJid, group: group.name, runId, reason: trigger.reason },
'Rate-limited, retrying with rotated token', 'Rate-limited, retrying with rotated token',
@@ -383,7 +383,7 @@ export async function runAgentForGroup(
} }
: detectFallbackTrigger(output.error); : detectFallbackTrigger(output.error);
if (trigger.shouldFallback) { if (trigger.shouldFallback) {
if (getTokenCount() > 1 && rotateToken()) { if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) {
logger.info( logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason }, { chatJid, group: group.name, runId, reason: trigger.reason },
'Rate-limited (output error), retrying with rotated token', 'Rate-limited (output error), retrying with rotated token',
@@ -401,7 +401,7 @@ export async function runAgentForGroup(
// Codex rate-limit rotation (non-Claude agents) // Codex rate-limit rotation (non-Claude agents)
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
const trigger = detectFallbackTrigger(output.error); const trigger = detectFallbackTrigger(output.error);
if (trigger.shouldFallback && rotateCodexToken()) { if (trigger.shouldFallback && rotateCodexToken(output.error ?? undefined)) {
logger.info( logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason }, { chatJid, group: group.name, runId, reason: trigger.reason },
'Codex rate-limited, retrying with rotated account', 'Codex rate-limited, retrying with rotated account',
@@ -433,7 +433,7 @@ export async function runAgentForGroup(
!isClaudeCodeAgent && !isClaudeCodeAgent &&
primaryAttempt.streamedTriggerReason && primaryAttempt.streamedTriggerReason &&
getCodexAccountCount() > 1 && getCodexAccountCount() > 1 &&
rotateCodexToken() rotateCodexToken(output.error ?? undefined)
) { ) {
logger.info( logger.info(
{ {

View File

@@ -332,11 +332,15 @@ async function runTask(
if (trigger.shouldFallback) { if (trigger.shouldFallback) {
const isCodex = SERVICE_AGENT_TYPE === 'codex'; const isCodex = SERVICE_AGENT_TYPE === 'codex';
const rotated = isCodex const rotated = isCodex
? getCodexAccountCount() > 1 && rotateCodexToken() ? getCodexAccountCount() > 1 && rotateCodexToken(error)
: getTokenCount() > 1 && rotateToken(); : getTokenCount() > 1 && rotateToken(error);
if (rotated) { if (rotated) {
logger.info( logger.info(
{ taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason }, {
taskId: task.id,
agent: SERVICE_AGENT_TYPE,
reason: trigger.reason,
},
'Task rate-limited, rotated token — will retry on next schedule', 'Task rate-limited, rotated token — will retry on next schedule',
); );
if (isCodex) markCodexTokenHealthy(); if (isCodex) markCodexTokenHealthy();

View File

@@ -70,7 +70,9 @@ function saveState(): void {
rateLimits: tokens.map((t) => t.rateLimitedUntil), rateLimits: tokens.map((t) => t.rateLimitedUntil),
}; };
fs.writeFileSync(STATE_FILE, JSON.stringify(state)); fs.writeFileSync(STATE_FILE, JSON.stringify(state));
} catch { /* best effort */ } } catch {
/* best effort */
}
} }
function loadState(): void { function loadState(): void {
@@ -78,19 +80,56 @@ function loadState(): void {
if (!fs.existsSync(STATE_FILE)) return; if (!fs.existsSync(STATE_FILE)) return;
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
const now = Date.now(); const now = Date.now();
if (typeof state.currentIndex === 'number' && state.currentIndex < tokens.length) { if (
typeof state.currentIndex === 'number' &&
state.currentIndex < tokens.length
) {
currentIndex = state.currentIndex; currentIndex = state.currentIndex;
} }
if (Array.isArray(state.rateLimits)) { if (Array.isArray(state.rateLimits)) {
for (let i = 0; i < Math.min(state.rateLimits.length, tokens.length); i++) { for (
let i = 0;
i < Math.min(state.rateLimits.length, tokens.length);
i++
) {
const until = state.rateLimits[i]; const until = state.rateLimits[i];
if (typeof until === 'number' && until > now) { if (typeof until === 'number' && until > now) {
tokens[i].rateLimitedUntil = until; tokens[i].rateLimitedUntil = until;
} }
} }
} }
logger.info({ currentIndex, tokenCount: tokens.length }, 'Token rotation state restored'); logger.info(
} catch { /* start fresh */ } { currentIndex, tokenCount: tokens.length },
'Token rotation state restored',
);
} catch {
/* start fresh */
}
}
const BUFFER_MS = 3 * 60_000;
const DEFAULT_COOLDOWN_MS = 3_600_000;
function parseRetryAfterFromError(error?: string): number | null {
if (!error) return null;
const match = error.match(
/(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
);
if (!match) return null;
try {
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
const ts = new Date(cleaned).getTime();
if (Number.isNaN(ts)) return null;
return ts;
} catch {
return null;
}
}
function computeCooldownUntil(error?: string): number {
const retryAt = parseRetryAfterFromError(error);
if (retryAt) return retryAt + BUFFER_MS;
return Date.now() + DEFAULT_COOLDOWN_MS;
} }
/** Get the current active token. */ /** Get the current active token. */
@@ -103,12 +142,11 @@ export function getCurrentToken(): string | undefined {
* Try to rotate to the next available (non-rate-limited) token. * Try to rotate to the next available (non-rate-limited) token.
* Returns true if a fresh token was found, false if all are exhausted. * Returns true if a fresh token was found, false if all are exhausted.
*/ */
export function rotateToken(): boolean { export function rotateToken(errorMessage?: string): boolean {
if (tokens.length <= 1) return false; if (tokens.length <= 1) return false;
const now = Date.now(); const now = Date.now();
// Mark current as rate-limited (default 1 hour) tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage);
tokens[currentIndex].rateLimitedUntil = now + 3_600_000;
// Find next available token // Find next available token
for (let i = 1; i < tokens.length; i++) { for (let i = 1; i < tokens.length; i++) {

View File

@@ -616,10 +616,14 @@ async function buildUsageContent(): Promise<string> {
}); });
} }
} else { } else {
// Show cached usage for rate-limited accounts
const pct = acct.isRateLimited && acct.cachedUsagePct != null
? acct.cachedUsagePct : -1;
const reset = acct.resetAt || '';
rows.push({ rows.push({
name: `${label} ${acct.planType}`, name: `${label} ${acct.planType}`,
h5pct: -1, h5pct: pct,
h5reset: '', h5reset: reset,
d7pct: -1, d7pct: -1,
d7reset: '', d7reset: '',
}); });