feat: Codex account rotation + fix rate-limit detection
- New codex-token-rotation module: rotates between multiple
~/.codex-accounts/{n}/auth.json on rate-limit
- Copies active account auth to session dir before each spawn
- Fix detectFallbackTrigger to match "usage limit" / "hit your limit"
- Fix streamed error detection: remove claude-only guard so codex
rate-limit errors are caught even when output.status is "success"
- Add rotation in both task-scheduler and message-agent-executor
This commit is contained in:
@@ -5,6 +5,7 @@ import path from 'path';
|
|||||||
import { GROUPS_DIR, TIMEZONE } from './config.js';
|
import { GROUPS_DIR, TIMEZONE } from './config.js';
|
||||||
import { isPairedRoomJid } from './db.js';
|
import { isPairedRoomJid } from './db.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { readEnvFile } from './env.js';
|
||||||
|
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
||||||
import { getCurrentToken } from './token-rotation.js';
|
import { getCurrentToken } from './token-rotation.js';
|
||||||
import {
|
import {
|
||||||
resolveGroupFolderPath,
|
resolveGroupFolderPath,
|
||||||
@@ -182,7 +183,10 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
||||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||||
|
|
||||||
const authSrc = path.join(hostCodexDir, 'auth.json');
|
const rotatedAuthSrc = getActiveCodexAuthPath();
|
||||||
|
const authSrc = rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||||
|
? rotatedAuthSrc
|
||||||
|
: path.join(hostCodexDir, 'auth.json');
|
||||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||||
if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst);
|
if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst);
|
||||||
for (const file of ['config.toml', 'config.json']) {
|
for (const file of ['config.toml', 'config.json']) {
|
||||||
|
|||||||
128
src/codex-token-rotation.ts
Normal file
128
src/codex-token-rotation.ts
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
/**
|
||||||
|
* Codex OAuth Token Rotation
|
||||||
|
*
|
||||||
|
* Rotates between multiple Codex (ChatGPT) OAuth accounts when
|
||||||
|
* rate-limited. Each account is stored as a separate auth.json in
|
||||||
|
* ~/.codex-accounts/{n}/auth.json.
|
||||||
|
*
|
||||||
|
* The active account's auth.json is copied to the session directory
|
||||||
|
* before each agent spawn (existing behavior in agent-runner-environment).
|
||||||
|
* On rate-limit, we rotate to the next account.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
import os from 'os';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
|
||||||
|
interface CodexAccount {
|
||||||
|
index: number;
|
||||||
|
authPath: string;
|
||||||
|
accountId: string;
|
||||||
|
rateLimitedUntil: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const accounts: CodexAccount[] = [];
|
||||||
|
let currentIndex = 0;
|
||||||
|
let initialized = false;
|
||||||
|
|
||||||
|
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||||
|
|
||||||
|
export function initCodexTokenRotation(): void {
|
||||||
|
if (initialized) return;
|
||||||
|
initialized = true;
|
||||||
|
|
||||||
|
if (!fs.existsSync(ACCOUNTS_DIR)) {
|
||||||
|
logger.info({ dir: ACCOUNTS_DIR }, 'Codex accounts dir not found, skipping');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dirs = fs.readdirSync(ACCOUNTS_DIR)
|
||||||
|
.filter((d) => /^\d+$/.test(d))
|
||||||
|
.sort((a, b) => parseInt(a) - parseInt(b));
|
||||||
|
|
||||||
|
for (const dir of dirs) {
|
||||||
|
const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json');
|
||||||
|
if (!fs.existsSync(authPath)) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(fs.readFileSync(authPath, 'utf-8'));
|
||||||
|
const accountId = data?.tokens?.account_id || `account-${dir}`;
|
||||||
|
accounts.push({
|
||||||
|
index: accounts.length,
|
||||||
|
authPath,
|
||||||
|
accountId,
|
||||||
|
rateLimitedUntil: null,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
logger.warn({ authPath }, 'Failed to parse codex account auth.json');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{ count: accounts.length, dir: ACCOUNTS_DIR },
|
||||||
|
`Codex token rotation: ${accounts.length} account(s) found`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get the auth.json path for the current active account. */
|
||||||
|
export function getActiveCodexAuthPath(): string | null {
|
||||||
|
if (accounts.length === 0) return null;
|
||||||
|
return accounts[currentIndex]?.authPath ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Try to rotate to the next available Codex account.
|
||||||
|
* Returns true if a fresh account was found.
|
||||||
|
*/
|
||||||
|
export function rotateCodexToken(): boolean {
|
||||||
|
if (accounts.length <= 1) return false;
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
accounts[currentIndex].rateLimitedUntil = now + 3_600_000;
|
||||||
|
|
||||||
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
|
const idx = (currentIndex + i) % accounts.length;
|
||||||
|
const acct = accounts[idx];
|
||||||
|
if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) {
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
currentIndex = idx;
|
||||||
|
logger.info(
|
||||||
|
{ accountIndex: currentIndex, totalAccounts: accounts.length, accountId: acct.accountId },
|
||||||
|
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.warn('All Codex accounts are rate-limited');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markCodexTokenHealthy(): void {
|
||||||
|
if (accounts.length === 0) return;
|
||||||
|
const acct = accounts[currentIndex];
|
||||||
|
if (acct?.rateLimitedUntil) {
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCodexAccountCount(): number {
|
||||||
|
return accounts.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAllCodexAccounts(): {
|
||||||
|
index: number;
|
||||||
|
accountId: string;
|
||||||
|
isActive: boolean;
|
||||||
|
isRateLimited: boolean;
|
||||||
|
}[] {
|
||||||
|
const now = Date.now();
|
||||||
|
return accounts.map((a, i) => ({
|
||||||
|
index: i,
|
||||||
|
accountId: a.accountId,
|
||||||
|
isActive: i === currentIndex,
|
||||||
|
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
||||||
|
}));
|
||||||
|
}
|
||||||
@@ -63,14 +63,14 @@ import { startUnifiedDashboard } from './unified-dashboard.js';
|
|||||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { normalizeStoredSeqCursor } from './message-cursor.js';
|
import { normalizeStoredSeqCursor } from './message-cursor.js';
|
||||||
|
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
||||||
import { initTokenRotation } from './token-rotation.js';
|
import { initTokenRotation } from './token-rotation.js';
|
||||||
|
|
||||||
// Re-export for backwards compatibility during refactor
|
// Re-export for backwards compatibility during refactor
|
||||||
export { escapeXml, formatMessages } from './router.js';
|
export { escapeXml, formatMessages } from './router.js';
|
||||||
export { composeDashboardContent } from './dashboard-render.js';
|
export { composeDashboardContent } from './dashboard-render.js';
|
||||||
|
|
||||||
// Initialize token rotation early (reads CLAUDE_CODE_OAUTH_TOKENS from env)
|
// Token rotation is initialized lazily on first use or at startup below
|
||||||
initTokenRotation();
|
|
||||||
|
|
||||||
export async function sendFormattedChannelMessage(
|
export async function sendFormattedChannelMessage(
|
||||||
channels: Channel[],
|
channels: Channel[],
|
||||||
@@ -297,6 +297,8 @@ async function main(): Promise<void> {
|
|||||||
const processStartedAtMs = Date.now();
|
const processStartedAtMs = Date.now();
|
||||||
initDatabase();
|
initDatabase();
|
||||||
logger.info('Database initialized');
|
logger.info('Database initialized');
|
||||||
|
initTokenRotation();
|
||||||
|
initCodexTokenRotation();
|
||||||
loadState();
|
loadState();
|
||||||
|
|
||||||
// Graceful shutdown handlers
|
// Graceful shutdown handlers
|
||||||
|
|||||||
@@ -21,7 +21,16 @@ import {
|
|||||||
markPrimaryCooldown,
|
markPrimaryCooldown,
|
||||||
} from './provider-fallback.js';
|
} from './provider-fallback.js';
|
||||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
||||||
import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js';
|
import {
|
||||||
|
rotateCodexToken,
|
||||||
|
getCodexAccountCount,
|
||||||
|
markCodexTokenHealthy,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
|
import {
|
||||||
|
rotateToken,
|
||||||
|
getTokenCount,
|
||||||
|
markTokenHealthy,
|
||||||
|
} from './token-rotation.js';
|
||||||
import type { RegisteredGroup } from './types.js';
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
export interface MessageAgentExecutorDeps {
|
export interface MessageAgentExecutorDeps {
|
||||||
@@ -138,7 +147,6 @@ export async function runAgentForGroup(
|
|||||||
sawSuccessNullResultWithoutOutput = true;
|
sawSuccessNullResultWithoutOutput = true;
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
provider === 'claude' &&
|
|
||||||
output.status === 'error' &&
|
output.status === 'error' &&
|
||||||
!sawOutput &&
|
!sawOutput &&
|
||||||
!streamedTriggerReason
|
!streamedTriggerReason
|
||||||
@@ -390,6 +398,22 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Codex rate-limit rotation (non-Claude agents)
|
||||||
|
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
|
||||||
|
const trigger = detectFallbackTrigger(output.error);
|
||||||
|
if (trigger.shouldFallback && rotateCodexToken()) {
|
||||||
|
logger.info(
|
||||||
|
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||||
|
'Codex rate-limited, retrying with rotated account',
|
||||||
|
);
|
||||||
|
const retryAttempt = await runAttempt('codex');
|
||||||
|
if (!retryAttempt.error) {
|
||||||
|
markCodexTokenHealthy();
|
||||||
|
return 'success';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
logger.error(
|
logger.error(
|
||||||
{
|
{
|
||||||
group: group.name,
|
group: group.name,
|
||||||
@@ -403,5 +427,25 @@ export async function runAgentForGroup(
|
|||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Codex may report success but have streamed a rate-limit error.
|
||||||
|
// Rotate token so the NEXT request uses a fresh account.
|
||||||
|
if (
|
||||||
|
!isClaudeCodeAgent &&
|
||||||
|
primaryAttempt.streamedTriggerReason &&
|
||||||
|
getCodexAccountCount() > 1
|
||||||
|
) {
|
||||||
|
if (rotateCodexToken()) {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
runId,
|
||||||
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
|
},
|
||||||
|
'Codex rate-limited (streamed), rotated account for next request',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return 'success';
|
return 'success';
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -247,6 +247,8 @@ export function detectFallbackTrigger(
|
|||||||
if (
|
if (
|
||||||
lower.includes('429') ||
|
lower.includes('429') ||
|
||||||
lower.includes('rate limit') ||
|
lower.includes('rate limit') ||
|
||||||
|
lower.includes('usage limit') ||
|
||||||
|
lower.includes('hit your limit') ||
|
||||||
lower.includes('too many requests') ||
|
lower.includes('too many requests') ||
|
||||||
lower.includes('rate_limit')
|
lower.includes('rate_limit')
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -29,6 +29,17 @@ import {
|
|||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||||
|
import { detectFallbackTrigger } from './provider-fallback.js';
|
||||||
|
import {
|
||||||
|
rotateCodexToken,
|
||||||
|
getCodexAccountCount,
|
||||||
|
markCodexTokenHealthy,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
|
import {
|
||||||
|
rotateToken,
|
||||||
|
getTokenCount,
|
||||||
|
markTokenHealthy,
|
||||||
|
} from './token-rotation.js';
|
||||||
import {
|
import {
|
||||||
evaluateTaskSuspension,
|
evaluateTaskSuspension,
|
||||||
formatSuspensionNotice,
|
formatSuspensionNotice,
|
||||||
@@ -315,6 +326,27 @@ async function runTask(
|
|||||||
updateTask(task.id, { suspended_until: null });
|
updateTask(task.id, { suspended_until: null });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Try token rotation before suspending
|
||||||
|
if (error) {
|
||||||
|
const trigger = detectFallbackTrigger(error);
|
||||||
|
if (trigger.shouldFallback) {
|
||||||
|
const isCodex = SERVICE_AGENT_TYPE === 'codex';
|
||||||
|
const rotated = isCodex
|
||||||
|
? getCodexAccountCount() > 1 && rotateCodexToken()
|
||||||
|
: getTokenCount() > 1 && rotateToken();
|
||||||
|
if (rotated) {
|
||||||
|
logger.info(
|
||||||
|
{ taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason },
|
||||||
|
'Task rate-limited, rotated token — will retry on next schedule',
|
||||||
|
);
|
||||||
|
if (isCodex) markCodexTokenHealthy();
|
||||||
|
else markTokenHealthy();
|
||||||
|
// Clear the error so suspension doesn't trigger
|
||||||
|
error = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check for repeated quota/auth errors → auto-suspend
|
// Check for repeated quota/auth errors → auto-suspend
|
||||||
let suspended = false;
|
let suspended = false;
|
||||||
if (error) {
|
if (error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user