feat: Codex account rotation + fix rate-limit detection
- New codex-token-rotation module: rotates between multiple
~/.codex-accounts/{n}/auth.json on rate-limit
- Copies active account auth to session dir before each spawn
- Fix detectFallbackTrigger to match "usage limit" / "hit your limit"
- Fix streamed error detection: remove claude-only guard so codex
rate-limit errors are caught even when output.status is "success"
- Add rotation in both task-scheduler and message-agent-executor
This commit is contained in:
@@ -5,6 +5,7 @@ import path from 'path';
|
||||
import { GROUPS_DIR, TIMEZONE } from './config.js';
|
||||
import { isPairedRoomJid } from './db.js';
|
||||
import { readEnvFile } from './env.js';
|
||||
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
||||
import { getCurrentToken } from './token-rotation.js';
|
||||
import {
|
||||
resolveGroupFolderPath,
|
||||
@@ -182,7 +183,10 @@ function prepareCodexSessionEnvironment(args: {
|
||||
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||
|
||||
const authSrc = path.join(hostCodexDir, 'auth.json');
|
||||
const rotatedAuthSrc = getActiveCodexAuthPath();
|
||||
const authSrc = rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||
? rotatedAuthSrc
|
||||
: path.join(hostCodexDir, 'auth.json');
|
||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||
if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst);
|
||||
for (const file of ['config.toml', 'config.json']) {
|
||||
|
||||
128
src/codex-token-rotation.ts
Normal file
128
src/codex-token-rotation.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Codex OAuth Token Rotation
|
||||
*
|
||||
* Rotates between multiple Codex (ChatGPT) OAuth accounts when
|
||||
* rate-limited. Each account is stored as a separate auth.json in
|
||||
* ~/.codex-accounts/{n}/auth.json.
|
||||
*
|
||||
* The active account's auth.json is copied to the session directory
|
||||
* before each agent spawn (existing behavior in agent-runner-environment).
|
||||
* On rate-limit, we rotate to the next account.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
import { logger } from './logger.js';
|
||||
|
||||
interface CodexAccount {
|
||||
index: number;
|
||||
authPath: string;
|
||||
accountId: string;
|
||||
rateLimitedUntil: number | null;
|
||||
}
|
||||
|
||||
const accounts: CodexAccount[] = [];
|
||||
let currentIndex = 0;
|
||||
let initialized = false;
|
||||
|
||||
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||
|
||||
export function initCodexTokenRotation(): void {
|
||||
if (initialized) return;
|
||||
initialized = true;
|
||||
|
||||
if (!fs.existsSync(ACCOUNTS_DIR)) {
|
||||
logger.info({ dir: ACCOUNTS_DIR }, 'Codex accounts dir not found, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
const dirs = fs.readdirSync(ACCOUNTS_DIR)
|
||||
.filter((d) => /^\d+$/.test(d))
|
||||
.sort((a, b) => parseInt(a) - parseInt(b));
|
||||
|
||||
for (const dir of dirs) {
|
||||
const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json');
|
||||
if (!fs.existsSync(authPath)) continue;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(authPath, 'utf-8'));
|
||||
const accountId = data?.tokens?.account_id || `account-${dir}`;
|
||||
accounts.push({
|
||||
index: accounts.length,
|
||||
authPath,
|
||||
accountId,
|
||||
rateLimitedUntil: null,
|
||||
});
|
||||
} catch {
|
||||
logger.warn({ authPath }, 'Failed to parse codex account auth.json');
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{ count: accounts.length, dir: ACCOUNTS_DIR },
|
||||
`Codex token rotation: ${accounts.length} account(s) found`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Get the auth.json path for the current active account. */
|
||||
export function getActiveCodexAuthPath(): string | null {
|
||||
if (accounts.length === 0) return null;
|
||||
return accounts[currentIndex]?.authPath ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to rotate to the next available Codex account.
|
||||
* Returns true if a fresh account was found.
|
||||
*/
|
||||
export function rotateCodexToken(): boolean {
|
||||
if (accounts.length <= 1) return false;
|
||||
|
||||
const now = Date.now();
|
||||
accounts[currentIndex].rateLimitedUntil = now + 3_600_000;
|
||||
|
||||
for (let i = 1; i < accounts.length; i++) {
|
||||
const idx = (currentIndex + i) % accounts.length;
|
||||
const acct = accounts[idx];
|
||||
if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) {
|
||||
acct.rateLimitedUntil = null;
|
||||
currentIndex = idx;
|
||||
logger.info(
|
||||
{ accountIndex: currentIndex, totalAccounts: accounts.length, accountId: acct.accountId },
|
||||
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn('All Codex accounts are rate-limited');
|
||||
return false;
|
||||
}
|
||||
|
||||
export function markCodexTokenHealthy(): void {
|
||||
if (accounts.length === 0) return;
|
||||
const acct = accounts[currentIndex];
|
||||
if (acct?.rateLimitedUntil) {
|
||||
acct.rateLimitedUntil = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getCodexAccountCount(): number {
|
||||
return accounts.length;
|
||||
}
|
||||
|
||||
export function getAllCodexAccounts(): {
|
||||
index: number;
|
||||
accountId: string;
|
||||
isActive: boolean;
|
||||
isRateLimited: boolean;
|
||||
}[] {
|
||||
const now = Date.now();
|
||||
return accounts.map((a, i) => ({
|
||||
index: i,
|
||||
accountId: a.accountId,
|
||||
isActive: i === currentIndex,
|
||||
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
||||
}));
|
||||
}
|
||||
@@ -63,14 +63,14 @@ import { startUnifiedDashboard } from './unified-dashboard.js';
|
||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||
import { logger } from './logger.js';
|
||||
import { normalizeStoredSeqCursor } from './message-cursor.js';
|
||||
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
||||
import { initTokenRotation } from './token-rotation.js';
|
||||
|
||||
// Re-export for backwards compatibility during refactor
|
||||
export { escapeXml, formatMessages } from './router.js';
|
||||
export { composeDashboardContent } from './dashboard-render.js';
|
||||
|
||||
// Initialize token rotation early (reads CLAUDE_CODE_OAUTH_TOKENS from env)
|
||||
initTokenRotation();
|
||||
// Token rotation is initialized lazily on first use or at startup below
|
||||
|
||||
export async function sendFormattedChannelMessage(
|
||||
channels: Channel[],
|
||||
@@ -297,6 +297,8 @@ async function main(): Promise<void> {
|
||||
const processStartedAtMs = Date.now();
|
||||
initDatabase();
|
||||
logger.info('Database initialized');
|
||||
initTokenRotation();
|
||||
initCodexTokenRotation();
|
||||
loadState();
|
||||
|
||||
// Graceful shutdown handlers
|
||||
|
||||
@@ -21,7 +21,16 @@ import {
|
||||
markPrimaryCooldown,
|
||||
} from './provider-fallback.js';
|
||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
||||
import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js';
|
||||
import {
|
||||
rotateCodexToken,
|
||||
getCodexAccountCount,
|
||||
markCodexTokenHealthy,
|
||||
} from './codex-token-rotation.js';
|
||||
import {
|
||||
rotateToken,
|
||||
getTokenCount,
|
||||
markTokenHealthy,
|
||||
} from './token-rotation.js';
|
||||
import type { RegisteredGroup } from './types.js';
|
||||
|
||||
export interface MessageAgentExecutorDeps {
|
||||
@@ -138,7 +147,6 @@ export async function runAgentForGroup(
|
||||
sawSuccessNullResultWithoutOutput = true;
|
||||
}
|
||||
if (
|
||||
provider === 'claude' &&
|
||||
output.status === 'error' &&
|
||||
!sawOutput &&
|
||||
!streamedTriggerReason
|
||||
@@ -390,6 +398,22 @@ export async function runAgentForGroup(
|
||||
}
|
||||
}
|
||||
|
||||
// Codex rate-limit rotation (non-Claude agents)
|
||||
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
|
||||
const trigger = detectFallbackTrigger(output.error);
|
||||
if (trigger.shouldFallback && rotateCodexToken()) {
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||
'Codex rate-limited, retrying with rotated account',
|
||||
);
|
||||
const retryAttempt = await runAttempt('codex');
|
||||
if (!retryAttempt.error) {
|
||||
markCodexTokenHealthy();
|
||||
return 'success';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{
|
||||
group: group.name,
|
||||
@@ -403,5 +427,25 @@ export async function runAgentForGroup(
|
||||
return 'error';
|
||||
}
|
||||
|
||||
// Codex may report success but have streamed a rate-limit error.
|
||||
// Rotate token so the NEXT request uses a fresh account.
|
||||
if (
|
||||
!isClaudeCodeAgent &&
|
||||
primaryAttempt.streamedTriggerReason &&
|
||||
getCodexAccountCount() > 1
|
||||
) {
|
||||
if (rotateCodexToken()) {
|
||||
logger.info(
|
||||
{
|
||||
chatJid,
|
||||
group: group.name,
|
||||
runId,
|
||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||
},
|
||||
'Codex rate-limited (streamed), rotated account for next request',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return 'success';
|
||||
}
|
||||
|
||||
@@ -247,6 +247,8 @@ export function detectFallbackTrigger(
|
||||
if (
|
||||
lower.includes('429') ||
|
||||
lower.includes('rate limit') ||
|
||||
lower.includes('usage limit') ||
|
||||
lower.includes('hit your limit') ||
|
||||
lower.includes('too many requests') ||
|
||||
lower.includes('rate_limit')
|
||||
) {
|
||||
|
||||
@@ -29,6 +29,17 @@ import {
|
||||
} from './group-folder.js';
|
||||
import { logger } from './logger.js';
|
||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||
import { detectFallbackTrigger } from './provider-fallback.js';
|
||||
import {
|
||||
rotateCodexToken,
|
||||
getCodexAccountCount,
|
||||
markCodexTokenHealthy,
|
||||
} from './codex-token-rotation.js';
|
||||
import {
|
||||
rotateToken,
|
||||
getTokenCount,
|
||||
markTokenHealthy,
|
||||
} from './token-rotation.js';
|
||||
import {
|
||||
evaluateTaskSuspension,
|
||||
formatSuspensionNotice,
|
||||
@@ -315,6 +326,27 @@ async function runTask(
|
||||
updateTask(task.id, { suspended_until: null });
|
||||
}
|
||||
|
||||
// Try token rotation before suspending
|
||||
if (error) {
|
||||
const trigger = detectFallbackTrigger(error);
|
||||
if (trigger.shouldFallback) {
|
||||
const isCodex = SERVICE_AGENT_TYPE === 'codex';
|
||||
const rotated = isCodex
|
||||
? getCodexAccountCount() > 1 && rotateCodexToken()
|
||||
: getTokenCount() > 1 && rotateToken();
|
||||
if (rotated) {
|
||||
logger.info(
|
||||
{ taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason },
|
||||
'Task rate-limited, rotated token — will retry on next schedule',
|
||||
);
|
||||
if (isCodex) markCodexTokenHealthy();
|
||||
else markTokenHealthy();
|
||||
// Clear the error so suspension doesn't trigger
|
||||
error = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for repeated quota/auth errors → auto-suspend
|
||||
let suspended = false;
|
||||
if (error) {
|
||||
|
||||
Reference in New Issue
Block a user