feat: Codex account rotation + fix rate-limit detection

- New codex-token-rotation module: rotates between multiple
  ~/.codex-accounts/{n}/auth.json on rate-limit
- Copies active account auth to session dir before each spawn
- Fix detectFallbackTrigger to match "usage limit" / "hit your limit"
- Fix streamed error detection: remove claude-only guard so codex
  rate-limit errors are caught even when output.status is "success"
- Add rotation in both task-scheduler and message-agent-executor
This commit is contained in:
Eyejoker
2026-03-23 23:06:17 +09:00
parent 0b86f937f4
commit 816b0a39cc
6 changed files with 217 additions and 5 deletions

View File

@@ -5,6 +5,7 @@ import path from 'path';
import { GROUPS_DIR, TIMEZONE } from './config.js';
import { isPairedRoomJid } from './db.js';
import { readEnvFile } from './env.js';
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
import { getCurrentToken } from './token-rotation.js';
import {
resolveGroupFolderPath,
@@ -182,7 +183,10 @@ function prepareCodexSessionEnvironment(args: {
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
fs.mkdirSync(sessionCodexDir, { recursive: true });
const authSrc = path.join(hostCodexDir, 'auth.json');
const rotatedAuthSrc = getActiveCodexAuthPath();
const authSrc = rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
? rotatedAuthSrc
: path.join(hostCodexDir, 'auth.json');
const authDst = path.join(sessionCodexDir, 'auth.json');
if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst);
for (const file of ['config.toml', 'config.json']) {

128
src/codex-token-rotation.ts Normal file
View File

@@ -0,0 +1,128 @@
/**
* Codex OAuth Token Rotation
*
* Rotates between multiple Codex (ChatGPT) OAuth accounts when
* rate-limited. Each account is stored as a separate auth.json in
* ~/.codex-accounts/{n}/auth.json.
*
* The active account's auth.json is copied to the session directory
* before each agent spawn (existing behavior in agent-runner-environment).
* On rate-limit, we rotate to the next account.
*/
import fs from 'fs';
import os from 'os';
import path from 'path';
import { logger } from './logger.js';
interface CodexAccount {
index: number;
authPath: string;
accountId: string;
rateLimitedUntil: number | null;
}
const accounts: CodexAccount[] = [];
let currentIndex = 0;
let initialized = false;
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
export function initCodexTokenRotation(): void {
if (initialized) return;
initialized = true;
if (!fs.existsSync(ACCOUNTS_DIR)) {
logger.info({ dir: ACCOUNTS_DIR }, 'Codex accounts dir not found, skipping');
return;
}
const dirs = fs.readdirSync(ACCOUNTS_DIR)
.filter((d) => /^\d+$/.test(d))
.sort((a, b) => parseInt(a) - parseInt(b));
for (const dir of dirs) {
const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json');
if (!fs.existsSync(authPath)) continue;
try {
const data = JSON.parse(fs.readFileSync(authPath, 'utf-8'));
const accountId = data?.tokens?.account_id || `account-${dir}`;
accounts.push({
index: accounts.length,
authPath,
accountId,
rateLimitedUntil: null,
});
} catch {
logger.warn({ authPath }, 'Failed to parse codex account auth.json');
}
}
logger.info(
{ count: accounts.length, dir: ACCOUNTS_DIR },
`Codex token rotation: ${accounts.length} account(s) found`,
);
}
/** Get the auth.json path for the current active account. */
export function getActiveCodexAuthPath(): string | null {
if (accounts.length === 0) return null;
return accounts[currentIndex]?.authPath ?? null;
}
/**
* Try to rotate to the next available Codex account.
* Returns true if a fresh account was found.
*/
export function rotateCodexToken(): boolean {
if (accounts.length <= 1) return false;
const now = Date.now();
accounts[currentIndex].rateLimitedUntil = now + 3_600_000;
for (let i = 1; i < accounts.length; i++) {
const idx = (currentIndex + i) % accounts.length;
const acct = accounts[idx];
if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) {
acct.rateLimitedUntil = null;
currentIndex = idx;
logger.info(
{ accountIndex: currentIndex, totalAccounts: accounts.length, accountId: acct.accountId },
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
);
return true;
}
}
logger.warn('All Codex accounts are rate-limited');
return false;
}
export function markCodexTokenHealthy(): void {
if (accounts.length === 0) return;
const acct = accounts[currentIndex];
if (acct?.rateLimitedUntil) {
acct.rateLimitedUntil = null;
}
}
export function getCodexAccountCount(): number {
return accounts.length;
}
export function getAllCodexAccounts(): {
index: number;
accountId: string;
isActive: boolean;
isRateLimited: boolean;
}[] {
const now = Date.now();
return accounts.map((a, i) => ({
index: i,
accountId: a.accountId,
isActive: i === currentIndex,
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
}));
}

View File

@@ -63,14 +63,14 @@ import { startUnifiedDashboard } from './unified-dashboard.js';
import { Channel, NewMessage, RegisteredGroup } from './types.js';
import { logger } from './logger.js';
import { normalizeStoredSeqCursor } from './message-cursor.js';
import { initCodexTokenRotation } from './codex-token-rotation.js';
import { initTokenRotation } from './token-rotation.js';
// Re-export for backwards compatibility during refactor
export { escapeXml, formatMessages } from './router.js';
export { composeDashboardContent } from './dashboard-render.js';
// Initialize token rotation early (reads CLAUDE_CODE_OAUTH_TOKENS from env)
initTokenRotation();
// Token rotation is initialized lazily on first use or at startup below
export async function sendFormattedChannelMessage(
channels: Channel[],
@@ -297,6 +297,8 @@ async function main(): Promise<void> {
const processStartedAtMs = Date.now();
initDatabase();
logger.info('Database initialized');
initTokenRotation();
initCodexTokenRotation();
loadState();
// Graceful shutdown handlers

View File

@@ -21,7 +21,16 @@ import {
markPrimaryCooldown,
} from './provider-fallback.js';
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js';
import {
rotateCodexToken,
getCodexAccountCount,
markCodexTokenHealthy,
} from './codex-token-rotation.js';
import {
rotateToken,
getTokenCount,
markTokenHealthy,
} from './token-rotation.js';
import type { RegisteredGroup } from './types.js';
export interface MessageAgentExecutorDeps {
@@ -138,7 +147,6 @@ export async function runAgentForGroup(
sawSuccessNullResultWithoutOutput = true;
}
if (
provider === 'claude' &&
output.status === 'error' &&
!sawOutput &&
!streamedTriggerReason
@@ -390,6 +398,22 @@ export async function runAgentForGroup(
}
}
// Codex rate-limit rotation (non-Claude agents)
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
const trigger = detectFallbackTrigger(output.error);
if (trigger.shouldFallback && rotateCodexToken()) {
logger.info(
{ chatJid, group: group.name, runId, reason: trigger.reason },
'Codex rate-limited, retrying with rotated account',
);
const retryAttempt = await runAttempt('codex');
if (!retryAttempt.error) {
markCodexTokenHealthy();
return 'success';
}
}
}
logger.error(
{
group: group.name,
@@ -403,5 +427,25 @@ export async function runAgentForGroup(
return 'error';
}
// Codex may report success but have streamed a rate-limit error.
// Rotate token so the NEXT request uses a fresh account.
if (
!isClaudeCodeAgent &&
primaryAttempt.streamedTriggerReason &&
getCodexAccountCount() > 1
) {
if (rotateCodexToken()) {
logger.info(
{
chatJid,
group: group.name,
runId,
reason: primaryAttempt.streamedTriggerReason.reason,
},
'Codex rate-limited (streamed), rotated account for next request',
);
}
}
return 'success';
}

View File

@@ -247,6 +247,8 @@ export function detectFallbackTrigger(
if (
lower.includes('429') ||
lower.includes('rate limit') ||
lower.includes('usage limit') ||
lower.includes('hit your limit') ||
lower.includes('too many requests') ||
lower.includes('rate_limit')
) {

View File

@@ -29,6 +29,17 @@ import {
} from './group-folder.js';
import { logger } from './logger.js';
import { createTaskStatusTracker } from './task-status-tracker.js';
import { detectFallbackTrigger } from './provider-fallback.js';
import {
rotateCodexToken,
getCodexAccountCount,
markCodexTokenHealthy,
} from './codex-token-rotation.js';
import {
rotateToken,
getTokenCount,
markTokenHealthy,
} from './token-rotation.js';
import {
evaluateTaskSuspension,
formatSuspensionNotice,
@@ -315,6 +326,27 @@ async function runTask(
updateTask(task.id, { suspended_until: null });
}
// Try token rotation before suspending
if (error) {
const trigger = detectFallbackTrigger(error);
if (trigger.shouldFallback) {
const isCodex = SERVICE_AGENT_TYPE === 'codex';
const rotated = isCodex
? getCodexAccountCount() > 1 && rotateCodexToken()
: getTokenCount() > 1 && rotateToken();
if (rotated) {
logger.info(
{ taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason },
'Task rate-limited, rotated token — will retry on next schedule',
);
if (isCodex) markCodexTokenHealthy();
else markTokenHealthy();
// Clear the error so suspension doesn't trigger
error = null;
}
}
}
// Check for repeated quota/auth errors → auto-suspend
let suspended = false;
if (error) {