feat: parse retry-after from errors, cache codex usage per account
- Parse "try again at" time from rate-limit errors instead of fixed 1-hour cooldown, with 3-min buffer after reset - Pass error message to rotateToken/rotateCodexToken for parsing - Cache usage % and reset time per Codex account for dashboard - Persist cached usage in rotation state file across restarts - Show rate-limited Codex accounts with cached 100% + reset time - Fix dashboard text indicators (*!/space) for monospace alignment
This commit is contained in:
@@ -25,6 +25,8 @@ interface CodexAccount {
|
|||||||
accountId: string;
|
accountId: string;
|
||||||
planType: string;
|
planType: string;
|
||||||
rateLimitedUntil: number | null;
|
rateLimitedUntil: number | null;
|
||||||
|
lastUsagePct?: number;
|
||||||
|
resetAt?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parsePlanFromJwt(idToken: string): string {
|
function parsePlanFromJwt(idToken: string): string {
|
||||||
@@ -95,9 +97,13 @@ function saveCodexState(): void {
|
|||||||
const state = {
|
const state = {
|
||||||
currentIndex,
|
currentIndex,
|
||||||
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
||||||
|
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
|
||||||
|
resetAts: accounts.map((a) => a.resetAt ?? null),
|
||||||
};
|
};
|
||||||
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
|
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
|
||||||
} catch { /* best effort */ }
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadCodexState(): void {
|
function loadCodexState(): void {
|
||||||
@@ -105,19 +111,71 @@ function loadCodexState(): void {
|
|||||||
if (!fs.existsSync(STATE_FILE)) return;
|
if (!fs.existsSync(STATE_FILE)) return;
|
||||||
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
if (typeof state.currentIndex === 'number' && state.currentIndex < accounts.length) {
|
if (
|
||||||
|
typeof state.currentIndex === 'number' &&
|
||||||
|
state.currentIndex < accounts.length
|
||||||
|
) {
|
||||||
currentIndex = state.currentIndex;
|
currentIndex = state.currentIndex;
|
||||||
}
|
}
|
||||||
if (Array.isArray(state.rateLimits)) {
|
if (Array.isArray(state.rateLimits)) {
|
||||||
for (let i = 0; i < Math.min(state.rateLimits.length, accounts.length); i++) {
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.rateLimits.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
const until = state.rateLimits[i];
|
const until = state.rateLimits[i];
|
||||||
if (typeof until === 'number' && until > now) {
|
if (typeof until === 'number' && until > now) {
|
||||||
accounts[i].rateLimitedUntil = until;
|
accounts[i].rateLimitedUntil = until;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.info({ currentIndex, accountCount: accounts.length }, 'Codex rotation state restored');
|
if (Array.isArray(state.usagePcts)) {
|
||||||
} catch { /* start fresh */ }
|
for (let i = 0; i < Math.min(state.usagePcts.length, accounts.length); i++) {
|
||||||
|
if (typeof state.usagePcts[i] === 'number') accounts[i].lastUsagePct = state.usagePcts[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Array.isArray(state.resetAts)) {
|
||||||
|
for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) {
|
||||||
|
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.info(
|
||||||
|
{ currentIndex, accountCount: accounts.length },
|
||||||
|
'Codex rotation state restored',
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
/* start fresh */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time
|
||||||
|
const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse "try again at Mar 26th, 2026 9:00 AM" from error message.
|
||||||
|
* Returns timestamp in ms, or null if not found.
|
||||||
|
*/
|
||||||
|
function parseRetryAfterFromError(error?: string): number | null {
|
||||||
|
if (!error) return null;
|
||||||
|
const match = error.match(
|
||||||
|
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
||||||
|
);
|
||||||
|
if (!match) return null;
|
||||||
|
try {
|
||||||
|
// Remove ordinal suffixes (1st, 2nd, 3rd, 4th)
|
||||||
|
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
|
||||||
|
const ts = new Date(cleaned).getTime();
|
||||||
|
if (Number.isNaN(ts)) return null;
|
||||||
|
return ts;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeCooldownUntil(error?: string): number {
|
||||||
|
const retryAt = parseRetryAfterFromError(error);
|
||||||
|
if (retryAt) return retryAt + BUFFER_MS;
|
||||||
|
return Date.now() + DEFAULT_COOLDOWN_MS;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Get the auth.json path for the current active account. */
|
/** Get the auth.json path for the current active account. */
|
||||||
@@ -130,11 +188,18 @@ export function getActiveCodexAuthPath(): string | null {
|
|||||||
* Try to rotate to the next available Codex account.
|
* Try to rotate to the next available Codex account.
|
||||||
* Returns true if a fresh account was found.
|
* Returns true if a fresh account was found.
|
||||||
*/
|
*/
|
||||||
export function rotateCodexToken(): boolean {
|
export function rotateCodexToken(errorMessage?: string): boolean {
|
||||||
if (accounts.length <= 1) return false;
|
if (accounts.length <= 1) return false;
|
||||||
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
accounts[currentIndex].rateLimitedUntil = now + 3_600_000;
|
const acct = accounts[currentIndex];
|
||||||
|
acct.rateLimitedUntil = computeCooldownUntil(errorMessage);
|
||||||
|
acct.lastUsagePct = 100;
|
||||||
|
// Extract reset time string from error for display
|
||||||
|
const resetMatch = errorMessage?.match(
|
||||||
|
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
||||||
|
);
|
||||||
|
if (resetMatch) acct.resetAt = resetMatch[1];
|
||||||
|
|
||||||
for (let i = 1; i < accounts.length; i++) {
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
const idx = (currentIndex + i) % accounts.length;
|
const idx = (currentIndex + i) % accounts.length;
|
||||||
@@ -178,6 +243,8 @@ export function getAllCodexAccounts(): {
|
|||||||
planType: string;
|
planType: string;
|
||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
isRateLimited: boolean;
|
isRateLimited: boolean;
|
||||||
|
cachedUsagePct?: number;
|
||||||
|
resetAt?: string;
|
||||||
}[] {
|
}[] {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
return accounts.map((a, i) => ({
|
return accounts.map((a, i) => ({
|
||||||
@@ -186,5 +253,7 @@ export function getAllCodexAccounts(): {
|
|||||||
planType: a.planType,
|
planType: a.planType,
|
||||||
isActive: i === currentIndex,
|
isActive: i === currentIndex,
|
||||||
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
||||||
|
cachedUsagePct: a.lastUsagePct,
|
||||||
|
resetAt: a.resetAt,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -309,7 +309,7 @@ export async function runAgentForGroup(
|
|||||||
: detectFallbackTrigger(errMsg);
|
: detectFallbackTrigger(errMsg);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldFallback) {
|
||||||
// Try rotating token before falling back to another provider
|
// Try rotating token before falling back to another provider
|
||||||
if (getTokenCount() > 1 && rotateToken()) {
|
if (getTokenCount() > 1 && rotateToken(errMsg)) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||||
'Rate-limited, retrying with rotated token',
|
'Rate-limited, retrying with rotated token',
|
||||||
@@ -383,7 +383,7 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
: detectFallbackTrigger(output.error);
|
: detectFallbackTrigger(output.error);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldFallback) {
|
||||||
if (getTokenCount() > 1 && rotateToken()) {
|
if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||||
'Rate-limited (output error), retrying with rotated token',
|
'Rate-limited (output error), retrying with rotated token',
|
||||||
@@ -401,7 +401,7 @@ export async function runAgentForGroup(
|
|||||||
// Codex rate-limit rotation (non-Claude agents)
|
// Codex rate-limit rotation (non-Claude agents)
|
||||||
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
|
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
|
||||||
const trigger = detectFallbackTrigger(output.error);
|
const trigger = detectFallbackTrigger(output.error);
|
||||||
if (trigger.shouldFallback && rotateCodexToken()) {
|
if (trigger.shouldFallback && rotateCodexToken(output.error ?? undefined)) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||||
'Codex rate-limited, retrying with rotated account',
|
'Codex rate-limited, retrying with rotated account',
|
||||||
@@ -433,7 +433,7 @@ export async function runAgentForGroup(
|
|||||||
!isClaudeCodeAgent &&
|
!isClaudeCodeAgent &&
|
||||||
primaryAttempt.streamedTriggerReason &&
|
primaryAttempt.streamedTriggerReason &&
|
||||||
getCodexAccountCount() > 1 &&
|
getCodexAccountCount() > 1 &&
|
||||||
rotateCodexToken()
|
rotateCodexToken(output.error ?? undefined)
|
||||||
) {
|
) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -332,11 +332,15 @@ async function runTask(
|
|||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldFallback) {
|
||||||
const isCodex = SERVICE_AGENT_TYPE === 'codex';
|
const isCodex = SERVICE_AGENT_TYPE === 'codex';
|
||||||
const rotated = isCodex
|
const rotated = isCodex
|
||||||
? getCodexAccountCount() > 1 && rotateCodexToken()
|
? getCodexAccountCount() > 1 && rotateCodexToken(error)
|
||||||
: getTokenCount() > 1 && rotateToken();
|
: getTokenCount() > 1 && rotateToken(error);
|
||||||
if (rotated) {
|
if (rotated) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason },
|
{
|
||||||
|
taskId: task.id,
|
||||||
|
agent: SERVICE_AGENT_TYPE,
|
||||||
|
reason: trigger.reason,
|
||||||
|
},
|
||||||
'Task rate-limited, rotated token — will retry on next schedule',
|
'Task rate-limited, rotated token — will retry on next schedule',
|
||||||
);
|
);
|
||||||
if (isCodex) markCodexTokenHealthy();
|
if (isCodex) markCodexTokenHealthy();
|
||||||
|
|||||||
@@ -70,7 +70,9 @@ function saveState(): void {
|
|||||||
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
||||||
};
|
};
|
||||||
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
|
fs.writeFileSync(STATE_FILE, JSON.stringify(state));
|
||||||
} catch { /* best effort */ }
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadState(): void {
|
function loadState(): void {
|
||||||
@@ -78,19 +80,56 @@ function loadState(): void {
|
|||||||
if (!fs.existsSync(STATE_FILE)) return;
|
if (!fs.existsSync(STATE_FILE)) return;
|
||||||
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
if (typeof state.currentIndex === 'number' && state.currentIndex < tokens.length) {
|
if (
|
||||||
|
typeof state.currentIndex === 'number' &&
|
||||||
|
state.currentIndex < tokens.length
|
||||||
|
) {
|
||||||
currentIndex = state.currentIndex;
|
currentIndex = state.currentIndex;
|
||||||
}
|
}
|
||||||
if (Array.isArray(state.rateLimits)) {
|
if (Array.isArray(state.rateLimits)) {
|
||||||
for (let i = 0; i < Math.min(state.rateLimits.length, tokens.length); i++) {
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.rateLimits.length, tokens.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
const until = state.rateLimits[i];
|
const until = state.rateLimits[i];
|
||||||
if (typeof until === 'number' && until > now) {
|
if (typeof until === 'number' && until > now) {
|
||||||
tokens[i].rateLimitedUntil = until;
|
tokens[i].rateLimitedUntil = until;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.info({ currentIndex, tokenCount: tokens.length }, 'Token rotation state restored');
|
logger.info(
|
||||||
} catch { /* start fresh */ }
|
{ currentIndex, tokenCount: tokens.length },
|
||||||
|
'Token rotation state restored',
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
/* start fresh */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const BUFFER_MS = 3 * 60_000;
|
||||||
|
const DEFAULT_COOLDOWN_MS = 3_600_000;
|
||||||
|
|
||||||
|
function parseRetryAfterFromError(error?: string): number | null {
|
||||||
|
if (!error) return null;
|
||||||
|
const match = error.match(
|
||||||
|
/(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
||||||
|
);
|
||||||
|
if (!match) return null;
|
||||||
|
try {
|
||||||
|
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
|
||||||
|
const ts = new Date(cleaned).getTime();
|
||||||
|
if (Number.isNaN(ts)) return null;
|
||||||
|
return ts;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeCooldownUntil(error?: string): number {
|
||||||
|
const retryAt = parseRetryAfterFromError(error);
|
||||||
|
if (retryAt) return retryAt + BUFFER_MS;
|
||||||
|
return Date.now() + DEFAULT_COOLDOWN_MS;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Get the current active token. */
|
/** Get the current active token. */
|
||||||
@@ -103,12 +142,11 @@ export function getCurrentToken(): string | undefined {
|
|||||||
* Try to rotate to the next available (non-rate-limited) token.
|
* Try to rotate to the next available (non-rate-limited) token.
|
||||||
* Returns true if a fresh token was found, false if all are exhausted.
|
* Returns true if a fresh token was found, false if all are exhausted.
|
||||||
*/
|
*/
|
||||||
export function rotateToken(): boolean {
|
export function rotateToken(errorMessage?: string): boolean {
|
||||||
if (tokens.length <= 1) return false;
|
if (tokens.length <= 1) return false;
|
||||||
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
// Mark current as rate-limited (default 1 hour)
|
tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage);
|
||||||
tokens[currentIndex].rateLimitedUntil = now + 3_600_000;
|
|
||||||
|
|
||||||
// Find next available token
|
// Find next available token
|
||||||
for (let i = 1; i < tokens.length; i++) {
|
for (let i = 1; i < tokens.length; i++) {
|
||||||
|
|||||||
@@ -616,10 +616,14 @@ async function buildUsageContent(): Promise<string> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
// Show cached usage for rate-limited accounts
|
||||||
|
const pct = acct.isRateLimited && acct.cachedUsagePct != null
|
||||||
|
? acct.cachedUsagePct : -1;
|
||||||
|
const reset = acct.resetAt || '';
|
||||||
rows.push({
|
rows.push({
|
||||||
name: `${label} ${acct.planType}`,
|
name: `${label} ${acct.planType}`,
|
||||||
h5pct: -1,
|
h5pct: pct,
|
||||||
h5reset: '',
|
h5reset: reset,
|
||||||
d7pct: -1,
|
d7pct: -1,
|
||||||
d7reset: '',
|
d7reset: '',
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user