diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index c32587e..43b548e 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -5,6 +5,7 @@ import path from 'path'; import { GROUPS_DIR, TIMEZONE } from './config.js'; import { isPairedRoomJid } from './db.js'; import { readEnvFile } from './env.js'; +import { getActiveCodexAuthPath } from './codex-token-rotation.js'; import { getCurrentToken } from './token-rotation.js'; import { resolveGroupFolderPath, @@ -182,7 +183,10 @@ function prepareCodexSessionEnvironment(args: { const sessionCodexDir = path.join(args.sessionRootDir, '.codex'); fs.mkdirSync(sessionCodexDir, { recursive: true }); - const authSrc = path.join(hostCodexDir, 'auth.json'); + const rotatedAuthSrc = getActiveCodexAuthPath(); + const authSrc = rotatedAuthSrc && fs.existsSync(rotatedAuthSrc) + ? rotatedAuthSrc + : path.join(hostCodexDir, 'auth.json'); const authDst = path.join(sessionCodexDir, 'auth.json'); if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst); for (const file of ['config.toml', 'config.json']) { diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts new file mode 100644 index 0000000..092d642 --- /dev/null +++ b/src/codex-token-rotation.ts @@ -0,0 +1,128 @@ +/** + * Codex OAuth Token Rotation + * + * Rotates between multiple Codex (ChatGPT) OAuth accounts when + * rate-limited. Each account is stored as a separate auth.json in + * ~/.codex-accounts/{n}/auth.json. + * + * The active account's auth.json is copied to the session directory + * before each agent spawn (existing behavior in agent-runner-environment). + * On rate-limit, we rotate to the next account. + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { logger } from './logger.js'; + +interface CodexAccount { + index: number; + authPath: string; + accountId: string; + rateLimitedUntil: number | null; +} + +const accounts: CodexAccount[] = []; +let currentIndex = 0; +let initialized = false; + +const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts'); + +export function initCodexTokenRotation(): void { + if (initialized) return; + initialized = true; + + if (!fs.existsSync(ACCOUNTS_DIR)) { + logger.info({ dir: ACCOUNTS_DIR }, 'Codex accounts dir not found, skipping'); + return; + } + + const dirs = fs.readdirSync(ACCOUNTS_DIR) + .filter((d) => /^\d+$/.test(d)) + .sort((a, b) => parseInt(a) - parseInt(b)); + + for (const dir of dirs) { + const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json'); + if (!fs.existsSync(authPath)) continue; + + try { + const data = JSON.parse(fs.readFileSync(authPath, 'utf-8')); + const accountId = data?.tokens?.account_id || `account-${dir}`; + accounts.push({ + index: accounts.length, + authPath, + accountId, + rateLimitedUntil: null, + }); + } catch { + logger.warn({ authPath }, 'Failed to parse codex account auth.json'); + } + } + + logger.info( + { count: accounts.length, dir: ACCOUNTS_DIR }, + `Codex token rotation: ${accounts.length} account(s) found`, + ); +} + +/** Get the auth.json path for the current active account. */ +export function getActiveCodexAuthPath(): string | null { + if (accounts.length === 0) return null; + return accounts[currentIndex]?.authPath ?? null; +} + +/** + * Try to rotate to the next available Codex account. + * Returns true if a fresh account was found. + */ +export function rotateCodexToken(): boolean { + if (accounts.length <= 1) return false; + + const now = Date.now(); + accounts[currentIndex].rateLimitedUntil = now + 3_600_000; + + for (let i = 1; i < accounts.length; i++) { + const idx = (currentIndex + i) % accounts.length; + const acct = accounts[idx]; + if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) { + acct.rateLimitedUntil = null; + currentIndex = idx; + logger.info( + { accountIndex: currentIndex, totalAccounts: accounts.length, accountId: acct.accountId }, + `Codex rotated to account #${currentIndex + 1}/${accounts.length}`, + ); + return true; + } + } + + logger.warn('All Codex accounts are rate-limited'); + return false; +} + +export function markCodexTokenHealthy(): void { + if (accounts.length === 0) return; + const acct = accounts[currentIndex]; + if (acct?.rateLimitedUntil) { + acct.rateLimitedUntil = null; + } +} + +export function getCodexAccountCount(): number { + return accounts.length; +} + +export function getAllCodexAccounts(): { + index: number; + accountId: string; + isActive: boolean; + isRateLimited: boolean; +}[] { + const now = Date.now(); + return accounts.map((a, i) => ({ + index: i, + accountId: a.accountId, + isActive: i === currentIndex, + isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), + })); +} diff --git a/src/index.ts b/src/index.ts index 8e5c216..45db441 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,14 +63,14 @@ import { startUnifiedDashboard } from './unified-dashboard.js'; import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { normalizeStoredSeqCursor } from './message-cursor.js'; +import { initCodexTokenRotation } from './codex-token-rotation.js'; import { initTokenRotation } from './token-rotation.js'; // Re-export for backwards compatibility during refactor export { escapeXml, formatMessages } from './router.js'; export { composeDashboardContent } from './dashboard-render.js'; -// Initialize token rotation early (reads CLAUDE_CODE_OAUTH_TOKENS from env) -initTokenRotation(); +// Token rotation is initialized lazily on first use or at startup below export async function sendFormattedChannelMessage( channels: Channel[], @@ -297,6 +297,8 @@ async function main(): Promise { const processStartedAtMs = Date.now(); initDatabase(); logger.info('Database initialized'); + initTokenRotation(); + initCodexTokenRotation(); loadState(); // Graceful shutdown handlers diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 968bf1c..5cd7b5e 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -21,7 +21,16 @@ import { markPrimaryCooldown, } from './provider-fallback.js'; import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; -import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js'; +import { + rotateCodexToken, + getCodexAccountCount, + markCodexTokenHealthy, +} from './codex-token-rotation.js'; +import { + rotateToken, + getTokenCount, + markTokenHealthy, +} from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; export interface MessageAgentExecutorDeps { @@ -138,7 +147,6 @@ export async function runAgentForGroup( sawSuccessNullResultWithoutOutput = true; } if ( - provider === 'claude' && output.status === 'error' && !sawOutput && !streamedTriggerReason @@ -390,6 +398,22 @@ export async function runAgentForGroup( } } + // Codex rate-limit rotation (non-Claude agents) + if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { + const trigger = detectFallbackTrigger(output.error); + if (trigger.shouldFallback && rotateCodexToken()) { + logger.info( + { chatJid, group: group.name, runId, reason: trigger.reason }, + 'Codex rate-limited, retrying with rotated account', + ); + const retryAttempt = await runAttempt('codex'); + if (!retryAttempt.error) { + markCodexTokenHealthy(); + return 'success'; + } + } + } + logger.error( { group: group.name, @@ -403,5 +427,25 @@ export async function runAgentForGroup( return 'error'; } + // Codex may report success but have streamed a rate-limit error. + // Rotate token so the NEXT request uses a fresh account. + if ( + !isClaudeCodeAgent && + primaryAttempt.streamedTriggerReason && + getCodexAccountCount() > 1 + ) { + if (rotateCodexToken()) { + logger.info( + { + chatJid, + group: group.name, + runId, + reason: primaryAttempt.streamedTriggerReason.reason, + }, + 'Codex rate-limited (streamed), rotated account for next request', + ); + } + } + return 'success'; } diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index 451f29d..f79d6c0 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -247,6 +247,8 @@ export function detectFallbackTrigger( if ( lower.includes('429') || lower.includes('rate limit') || + lower.includes('usage limit') || + lower.includes('hit your limit') || lower.includes('too many requests') || lower.includes('rate_limit') ) { diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 04895fc..42f0b30 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -29,6 +29,17 @@ import { } from './group-folder.js'; import { logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; +import { detectFallbackTrigger } from './provider-fallback.js'; +import { + rotateCodexToken, + getCodexAccountCount, + markCodexTokenHealthy, +} from './codex-token-rotation.js'; +import { + rotateToken, + getTokenCount, + markTokenHealthy, +} from './token-rotation.js'; import { evaluateTaskSuspension, formatSuspensionNotice, @@ -315,6 +326,27 @@ async function runTask( updateTask(task.id, { suspended_until: null }); } + // Try token rotation before suspending + if (error) { + const trigger = detectFallbackTrigger(error); + if (trigger.shouldFallback) { + const isCodex = SERVICE_AGENT_TYPE === 'codex'; + const rotated = isCodex + ? getCodexAccountCount() > 1 && rotateCodexToken() + : getTokenCount() > 1 && rotateToken(); + if (rotated) { + logger.info( + { taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason }, + 'Task rate-limited, rotated token — will retry on next schedule', + ); + if (isCodex) markCodexTokenHealthy(); + else markTokenHealthy(); + // Clear the error so suspension doesn't trigger + error = null; + } + } + } + // Check for repeated quota/auth errors → auto-suspend let suspended = false; if (error) {