fix: recover Codex rotation auth failures

- mark Codex bearer/refresh failures as terminal auth-expired states

- sync refreshed session auth back to rotation slots and revive refreshed dead_auth slots

- stop paired arbiter retry loops when Codex accounts are unavailable

- add regression coverage for rotation leases, follow-up suppression, and arbiter closure
This commit is contained in:
ejclaw
2026-06-02 00:21:16 +09:00
parent 1ff6b3434f
commit 03d4c81192
21 changed files with 1082 additions and 52 deletions

View File

@@ -0,0 +1,29 @@
import { describe, expect, it } from 'vitest';
import { resolveAttemptRetryAction } from './agent-attempt-retry.js';
describe('resolveAttemptRetryAction', () => {
it('uses the streamed Codex trigger message as the rotation message', () => {
const errorMessage =
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
const action = resolveAttemptRetryAction({
provider: 'codex',
canRetryClaudeCredentials: false,
canRetryCodex: true,
attempt: {
sawOutput: false,
streamedTriggerReason: {
reason: 'auth-expired',
message: errorMessage,
} as any,
},
});
expect(action).toEqual({
kind: 'codex',
trigger: { reason: 'auth-expired' },
rotationMessage: errorMessage,
});
});
});

View File

@@ -10,6 +10,7 @@ import { getErrorMessage } from './utils.js';
export interface AttemptStreamedTrigger { export interface AttemptStreamedTrigger {
reason: AgentTriggerReason; reason: AgentTriggerReason;
retryAfterMs?: number; retryAfterMs?: number;
message?: string;
} }
export interface AttemptRetryState { export interface AttemptRetryState {
@@ -131,7 +132,10 @@ export function resolveAttemptRetryAction(args: {
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>; attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
rotationMessage?: string | null; rotationMessage?: string | null;
}): AttemptRetryAction { }): AttemptRetryAction {
const normalizedRotationMessage = args.rotationMessage ?? undefined; const normalizedRotationMessage =
args.rotationMessage ??
args.attempt.streamedTriggerReason?.message ??
undefined;
const claudeTrigger = resolveClaudeRetryTrigger({ const claudeTrigger = resolveClaudeRetryTrigger({
canRetryClaudeCredentials: args.canRetryClaudeCredentials, canRetryClaudeCredentials: args.canRetryClaudeCredentials,

View File

@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
import { import {
classifyAgentError, classifyAgentError,
classifyClaudeAuthError, classifyClaudeAuthError,
classifyCodexAuthError,
detectClaudeProviderFailureMessage, detectClaudeProviderFailureMessage,
isClaudeOrgAccessDeniedMessage, isClaudeOrgAccessDeniedMessage,
shouldRotateClaudeToken, shouldRotateClaudeToken,
@@ -66,6 +67,32 @@ describe('agent-error-detection', () => {
}); });
}); });
it('classifies Codex reused refresh-token errors as auth-expired', () => {
expect(
classifyCodexAuthError(
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.',
),
).toEqual({
category: 'auth-expired',
reason: 'auth-expired',
});
});
it('classifies a Codex auth-expired trigger reason as auth-expired', () => {
expect(classifyCodexAuthError('auth-expired')).toEqual({
category: 'auth-expired',
reason: 'auth-expired',
});
});
it('classifies Codex workspace credit exhaustion as rate-limit', () => {
expect(classifyAgentError('Workspace out of credits')).toEqual({
category: 'rate-limit',
reason: '429',
retryAfterMs: undefined,
});
});
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => { it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
expect(shouldRotateClaudeToken('429')).toBe(true); expect(shouldRotateClaudeToken('429')).toBe(true);
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true); expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);

View File

@@ -249,6 +249,8 @@ export function classifyAgentError(
lower.includes('429') || lower.includes('429') ||
lower.includes('rate limit') || lower.includes('rate limit') ||
lower.includes('usage limit') || lower.includes('usage limit') ||
lower.includes('out of credits') ||
lower.includes('workspace out of credits') ||
lower.includes('hit your limit') || lower.includes('hit your limit') ||
lower.includes('too many requests') || lower.includes('too many requests') ||
lower.includes('rate_limit') lower.includes('rate_limit')
@@ -335,11 +337,17 @@ export function classifyCodexAuthError(
const lower = error.toLowerCase(); const lower = error.toLowerCase();
if ( if (
lower.includes('auth-expired') ||
lower.includes('auth expired') ||
lower.includes('401') || lower.includes('401') ||
lower.includes('authentication_error') || lower.includes('authentication_error') ||
lower.includes('failed to authenticate') || lower.includes('failed to authenticate') ||
lower.includes('access token could not be refreshed') ||
lower.includes('oauth token has expired') || lower.includes('oauth token has expired') ||
lower.includes('refresh token was already used') ||
lower.includes('refresh your existing token') || lower.includes('refresh your existing token') ||
lower.includes('log out and sign in again') ||
lower.includes('app_session_terminated') ||
lower.includes('unauthorized') lower.includes('unauthorized')
) { ) {
return { category: 'auth-expired', reason: 'auth-expired' }; return { category: 'auth-expired', reason: 'auth-expired' };

View File

@@ -5,9 +5,18 @@ import path from 'path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { EJCLAW_ENV } from 'ejclaw-runners-shared'; import { EJCLAW_ENV } from 'ejclaw-runners-shared';
const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({ const {
mockReadEnvFile,
mockGetActiveCodexAuthPath,
mockGetCodexAccountCount,
mockClaimCodexAuthLease,
mockFindCodexAccountIndexByAuthPath,
} = vi.hoisted(() => ({
mockReadEnvFile: vi.fn<() => Record<string, string>>(), mockReadEnvFile: vi.fn<() => Record<string, string>>(),
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(), mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
mockGetCodexAccountCount: vi.fn<() => number>(),
mockClaimCodexAuthLease: vi.fn<() => null>(),
mockFindCodexAccountIndexByAuthPath: vi.fn<() => number | null>(),
})); }));
vi.mock('./config.js', () => ({ vi.mock('./config.js', () => ({
@@ -29,6 +38,9 @@ vi.mock('./env.js', () => ({
vi.mock('./codex-token-rotation.js', () => ({ vi.mock('./codex-token-rotation.js', () => ({
getActiveCodexAuthPath: mockGetActiveCodexAuthPath, getActiveCodexAuthPath: mockGetActiveCodexAuthPath,
getCodexAccountCount: mockGetCodexAccountCount,
claimCodexAuthLease: mockClaimCodexAuthLease,
findCodexAccountIndexByAuthPath: mockFindCodexAccountIndexByAuthPath,
})); }));
vi.mock('./token-rotation.js', () => ({ vi.mock('./token-rotation.js', () => ({
@@ -159,6 +171,12 @@ describe('prepareGroupEnvironment codex auth handling', () => {
mockReadEnvFile.mockReset(); mockReadEnvFile.mockReset();
mockGetActiveCodexAuthPath.mockReset(); mockGetActiveCodexAuthPath.mockReset();
mockGetCodexAccountCount.mockReset();
mockGetCodexAccountCount.mockReturnValue(0);
mockClaimCodexAuthLease.mockReset();
mockClaimCodexAuthLease.mockReturnValue(null);
mockFindCodexAccountIndexByAuthPath.mockReset();
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
}); });
afterEach(() => { afterEach(() => {
@@ -240,6 +258,36 @@ describe('prepareGroupEnvironment codex auth handling', () => {
expect(auth).toEqual(rotatedAuth); expect(auth).toEqual(rotatedAuth);
}); });
it('fails fast instead of launching Codex without OAuth when rotation accounts exist but no lease is available', () => {
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
fs.writeFileSync(
rotatedAuthPath,
JSON.stringify({
auth_mode: 'chatgpt',
tokens: { access_token: 'locked-token' },
}),
);
mockGetCodexAccountCount.mockReturnValue(1);
mockClaimCodexAuthLease.mockReturnValue(null);
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
mockReadEnvFile.mockReturnValue({});
expect(() => prepareGroupEnvironment(group, false, 'dc:test')).toThrow(
/auth-expired: All Codex rotation accounts unavailable/,
);
const authPath = path.join(
tempRoot,
'sessions',
group.folder,
'services',
'codex-main',
'.codex',
'auth.json',
);
expect(fs.existsSync(authPath)).toBe(false);
});
it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => { it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => {
vi.mocked(config.isReviewService).mockReturnValue(true); vi.mocked(config.isReviewService).mockReturnValue(true);
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true); vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
@@ -411,6 +459,12 @@ describe('prepareGroupEnvironment Codex MCP room role env', () => {
mockReadEnvFile.mockReset(); mockReadEnvFile.mockReset();
mockGetActiveCodexAuthPath.mockReset(); mockGetActiveCodexAuthPath.mockReset();
mockGetCodexAccountCount.mockReset();
mockGetCodexAccountCount.mockReturnValue(0);
mockClaimCodexAuthLease.mockReset();
mockClaimCodexAuthLease.mockReturnValue(null);
mockFindCodexAccountIndexByAuthPath.mockReset();
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
resetRoutingMocks(); resetRoutingMocks();
}); });
@@ -484,6 +538,12 @@ describe('prepareGroupEnvironment room skill overrides', () => {
}); });
mockReadEnvFile.mockReset(); mockReadEnvFile.mockReset();
mockGetActiveCodexAuthPath.mockReset(); mockGetActiveCodexAuthPath.mockReset();
mockGetCodexAccountCount.mockReset();
mockGetCodexAccountCount.mockReturnValue(0);
mockClaimCodexAuthLease.mockReset();
mockClaimCodexAuthLease.mockReturnValue(null);
mockFindCodexAccountIndexByAuthPath.mockReset();
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
}); });
afterEach(() => { afterEach(() => {
@@ -616,6 +676,12 @@ describe('prepareGroupEnvironment Codex goals handling', () => {
mockReadEnvFile.mockReset(); mockReadEnvFile.mockReset();
mockGetActiveCodexAuthPath.mockReset(); mockGetActiveCodexAuthPath.mockReset();
mockGetCodexAccountCount.mockReset();
mockGetCodexAccountCount.mockReturnValue(0);
mockClaimCodexAuthLease.mockReset();
mockClaimCodexAuthLease.mockReturnValue(null);
mockFindCodexAccountIndexByAuthPath.mockReset();
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
vi.mocked(config.isReviewService).mockReturnValue(false); vi.mocked(config.isReviewService).mockReturnValue(false);
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false); vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false);
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
@@ -706,6 +772,12 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
mockReadEnvFile.mockReset(); mockReadEnvFile.mockReset();
mockGetActiveCodexAuthPath.mockReset(); mockGetActiveCodexAuthPath.mockReset();
mockGetCodexAccountCount.mockReset();
mockGetCodexAccountCount.mockReturnValue(0);
mockClaimCodexAuthLease.mockReset();
mockClaimCodexAuthLease.mockReturnValue(null);
mockFindCodexAccountIndexByAuthPath.mockReset();
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
}); });
afterEach(() => { afterEach(() => {

View File

@@ -12,7 +12,13 @@ import {
} from './config.js'; } from './config.js';
import { logger } from './logger.js'; import { logger } from './logger.js';
import { readEnvFile } from './env.js'; import { readEnvFile } from './env.js';
import { getActiveCodexAuthPath } from './codex-token-rotation.js'; import {
claimCodexAuthLease,
findCodexAccountIndexByAuthPath,
getActiveCodexAuthPath,
getCodexAccountCount,
type CodexAuthLease,
} from './codex-token-rotation.js';
import { readCodexFeatureFromFile } from './codex-config-features.js'; import { readCodexFeatureFromFile } from './codex-config-features.js';
import { ensureClaudeSessionSettings } from './claude-session-settings.js'; import { ensureClaudeSessionSettings } from './claude-session-settings.js';
import { import {
@@ -140,17 +146,32 @@ function ensureClaudeGlobalSettingsFile(sessionDir: string): void {
fs.writeFileSync(settingsFile, '{}\n'); fs.writeFileSync(settingsFile, '{}\n');
} }
function syncHostCodexSessionFiles(sessionCodexDir: string): void { export interface PreparedCodexSessionAuth {
canonicalAuthPath: string;
sessionAuthPath: string;
accountIndex: number | null;
lease?: CodexAuthLease;
}
function syncHostCodexSessionFiles(
sessionCodexDir: string,
): PreparedCodexSessionAuth | null {
const hostCodexDir = path.join(os.homedir(), '.codex'); const hostCodexDir = path.join(os.homedir(), '.codex');
fs.mkdirSync(sessionCodexDir, { recursive: true }); fs.mkdirSync(sessionCodexDir, { recursive: true });
const authDst = path.join(sessionCodexDir, 'auth.json'); const authDst = path.join(sessionCodexDir, 'auth.json');
const rotatedAuthSrc = getActiveCodexAuthPath(); const hasRotationAccounts = getCodexAccountCount() > 0;
const lease = claimCodexAuthLease();
const rotatedAuthSrc =
lease?.authPath ?? (!hasRotationAccounts ? getActiveCodexAuthPath() : null);
const fallbackAuthSrc = path.join(hostCodexDir, 'auth.json');
const authSrc = const authSrc =
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc) rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
? rotatedAuthSrc ? rotatedAuthSrc
: path.join(hostCodexDir, 'auth.json'); : !hasRotationAccounts && fs.existsSync(fallbackAuthSrc)
if (fs.existsSync(authSrc)) { ? fallbackAuthSrc
: null;
if (authSrc) {
fs.copyFileSync(authSrc, authDst); fs.copyFileSync(authSrc, authDst);
} else if (fs.existsSync(authDst)) { } else if (fs.existsSync(authDst)) {
fs.unlinkSync(authDst); fs.unlinkSync(authDst);
@@ -163,6 +184,28 @@ function syncHostCodexSessionFiles(sessionCodexDir: string): void {
fs.copyFileSync(src, dst); fs.copyFileSync(src, dst);
} }
} }
if (
!rotatedAuthSrc ||
authSrc !== rotatedAuthSrc ||
!fs.existsSync(authDst)
) {
lease?.release();
if (hasRotationAccounts) {
throw new Error(
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
);
}
return null;
}
return {
canonicalAuthPath: rotatedAuthSrc,
sessionAuthPath: authDst,
accountIndex:
lease?.accountIndex ?? findCodexAccountIndexByAuthPath(rotatedAuthSrc),
...(lease ? { lease } : {}),
};
} }
function upsertEjclawMcpServerSection(args: { function upsertEjclawMcpServerSection(args: {
@@ -357,7 +400,7 @@ function prepareCodexSessionEnvironment(args: {
useFailoverPromptPack: boolean; useFailoverPromptPack: boolean;
memoryBriefing?: string; memoryBriefing?: string;
skillOverrides?: StoredRoomSkillOverride[]; skillOverrides?: StoredRoomSkillOverride[];
}): void { }): PreparedCodexSessionAuth | null {
// API key auth intentionally removed — Codex uses OAuth only. // API key auth intentionally removed — Codex uses OAuth only.
// Never pass any API key to Codex child process to prevent API billing. // Never pass any API key to Codex child process to prevent API billing.
delete args.env.OPENAI_API_KEY; delete args.env.OPENAI_API_KEY;
@@ -376,7 +419,7 @@ function prepareCodexSessionEnvironment(args: {
if (codexEffort) args.env.CODEX_EFFORT = codexEffort; if (codexEffort) args.env.CODEX_EFFORT = codexEffort;
const sessionCodexDir = path.join(args.sessionRootDir, '.codex'); const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
syncHostCodexSessionFiles(sessionCodexDir); const codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
const overlayPath = path.join(args.groupDir, '.codex', 'config.toml'); const overlayPath = path.join(args.groupDir, '.codex', 'config.toml');
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml'); const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
@@ -506,16 +549,19 @@ function prepareCodexSessionEnvironment(args: {
delete args.env.ANTHROPIC_BASE_URL; delete args.env.ANTHROPIC_BASE_URL;
delete args.env.CLAUDE_CODE_OAUTH_TOKEN; delete args.env.CLAUDE_CODE_OAUTH_TOKEN;
args.env.CODEX_HOME = sessionCodexDir; args.env.CODEX_HOME = sessionCodexDir;
return codexSessionAuth;
} }
export interface PreparedGroupEnvironment { export interface PreparedGroupEnvironment {
env: Record<string, string>; env: Record<string, string>;
groupDir: string; groupDir: string;
runnerDir: string; runnerDir: string;
codexSessionAuth?: PreparedCodexSessionAuth | null;
} }
export interface PreparedReadonlySessionEnvironment { export interface PreparedReadonlySessionEnvironment {
codexHomeDir?: string; codexHomeDir?: string;
codexSessionAuth?: PreparedCodexSessionAuth | null;
} }
export function prepareGroupEnvironment( export function prepareGroupEnvironment(
@@ -657,8 +703,9 @@ export function prepareGroupEnvironment(
runtimeTaskId, runtimeTaskId,
}); });
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
if (agentType === 'codex') { if (agentType === 'codex') {
prepareCodexSessionEnvironment({ codexSessionAuth = prepareCodexSessionEnvironment({
env, env,
envVars, envVars,
projectRoot, projectRoot,
@@ -677,7 +724,7 @@ export function prepareGroupEnvironment(
prepareClaudeEnvironment({ env, envVars, group }); prepareClaudeEnvironment({ env, envVars, group });
} }
return { env, groupDir, runnerDir }; return { env, groupDir, runnerDir, codexSessionAuth };
} }
/** /**
@@ -716,6 +763,7 @@ export function prepareReadonlySessionEnvironment(args: {
skillOverrides, skillOverrides,
} = args; } = args;
const projectRoot = process.cwd(); const projectRoot = process.cwd();
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
fs.mkdirSync(sessionDir, { recursive: true }); fs.mkdirSync(sessionDir, { recursive: true });
ensureClaudeSessionSettings(sessionDir); ensureClaudeSessionSettings(sessionDir);
@@ -785,7 +833,7 @@ export function prepareReadonlySessionEnvironment(args: {
if (sessionClaudeMd) { if (sessionClaudeMd) {
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n'); fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
const sessionCodexDir = path.join(sessionDir, '.codex'); const sessionCodexDir = path.join(sessionDir, '.codex');
syncHostCodexSessionFiles(sessionCodexDir); codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
fs.writeFileSync( fs.writeFileSync(
path.join(sessionCodexDir, 'AGENTS.md'), path.join(sessionCodexDir, 'AGENTS.md'),
sessionClaudeMd + '\n', sessionClaudeMd + '\n',
@@ -826,5 +874,5 @@ export function prepareReadonlySessionEnvironment(args: {
} else if (fs.existsSync(sessionClaudeMdPath)) { } else if (fs.existsSync(sessionClaudeMdPath)) {
fs.unlinkSync(sessionClaudeMdPath); fs.unlinkSync(sessionClaudeMdPath);
} }
return { codexHomeDir }; return { codexHomeDir, codexSessionAuth };
} }

View File

@@ -14,7 +14,9 @@ import {
import { import {
prepareReadonlySessionEnvironment, prepareReadonlySessionEnvironment,
prepareGroupEnvironment, prepareGroupEnvironment,
type PreparedCodexSessionAuth,
} from './agent-runner-environment.js'; } from './agent-runner-environment.js';
import { syncCodexSessionAuthBack } from './codex-token-rotation.js';
import { runSpawnedAgentProcess } from './agent-runner-process.js'; import { runSpawnedAgentProcess } from './agent-runner-process.js';
import { getStoredRoomSkillOverrides } from './db.js'; import { getStoredRoomSkillOverrides } from './db.js';
export { export {
@@ -76,6 +78,36 @@ function readRoomSkillOverridesForRunner(
} }
} }
function releaseCodexAuthSession(
auth: PreparedCodexSessionAuth | null | undefined,
): void {
auth?.lease?.release();
}
function finalizeCodexAuthSession(
auth: PreparedCodexSessionAuth | null | undefined,
): void {
if (!auth) return;
try {
syncCodexSessionAuthBack({
canonicalAuthPath: auth.canonicalAuthPath,
sessionAuthPath: auth.sessionAuthPath,
accountIndex: auth.accountIndex,
});
} catch (err) {
logger.warn(
{
err,
canonicalAuthPath: auth.canonicalAuthPath,
accountIndex: auth.accountIndex,
},
'Failed to sync Codex session auth back to canonical slot',
);
} finally {
auth.lease?.release();
}
}
export async function runAgentProcess( export async function runAgentProcess(
group: RegisteredGroup, group: RegisteredGroup,
input: AgentInput, input: AgentInput,
@@ -92,18 +124,15 @@ export async function runAgentProcess(
// ── Host process mode (owner) ─────────────────────────────────── // ── Host process mode (owner) ───────────────────────────────────
const startTime = Date.now(); const startTime = Date.now();
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid); const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
const { env, groupDir, runnerDir } = prepareGroupEnvironment( const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, {
group, memoryBriefing: input.memoryBriefing,
input.isMain, runtimeTaskId: input.runtimeTaskId,
input.chatJid, useTaskScopedSession: input.useTaskScopedSession,
{ skillOverrides,
memoryBriefing: input.memoryBriefing, roomRole: input.roomRoleContext?.role,
runtimeTaskId: input.runtimeTaskId, });
useTaskScopedSession: input.useTaskScopedSession, const { env, groupDir, runnerDir } = prepared;
skillOverrides, let codexSessionAuth = prepared.codexSessionAuth;
roomRole: input.roomRoleContext?.role,
},
);
// Apply env overrides (caller-provided) // Apply env overrides (caller-provided)
if (envOverrides) { if (envOverrides) {
@@ -131,6 +160,8 @@ export async function runAgentProcess(
skillOverrides, skillOverrides,
}); });
if ((group.agentType || 'claude-code') === 'codex') { if ((group.agentType || 'claude-code') === 'codex') {
releaseCodexAuthSession(codexSessionAuth);
codexSessionAuth = readonlySession.codexSessionAuth;
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex'); env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
if (readonlySession.codexHomeDir) { if (readonlySession.codexHomeDir) {
env.HOME = readonlySession.codexHomeDir; env.HOME = readonlySession.codexHomeDir;
@@ -152,6 +183,7 @@ export async function runAgentProcess(
{ runnerDir, chatJid: input.chatJid, runId: input.runId }, { runnerDir, chatJid: input.chatJid, runId: input.runId },
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build', 'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
); );
releaseCodexAuthSession(codexSessionAuth);
return { return {
status: 'error', status: 'error',
result: null, result: null,
@@ -199,6 +231,22 @@ export async function runAgentProcess(
logsDir, logsDir,
startTime, startTime,
onOutput, onOutput,
}).then(resolve); })
.then((output) => {
finalizeCodexAuthSession(codexSessionAuth);
resolve(output);
})
.catch((err: unknown) => {
finalizeCodexAuthSession(codexSessionAuth);
logger.error(
{ err, processName, chatJid: input.chatJid, runId: input.runId },
'Spawned agent process runner failed',
);
resolve({
status: 'error',
result: null,
error: err instanceof Error ? err.message : String(err),
});
});
}); });
} }

View File

@@ -165,6 +165,192 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
expect(mod.getCodexAccountCount()).toBe(4); expect(mod.getCodexAccountCount()).toBe(4);
expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath); expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath);
}); });
it('marks refresh-token reuse as dead auth instead of a recoverable cooldown', async () => {
const agentErrors = await import('./agent-error-detection.js');
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
category: 'auth-expired',
reason: 'auth-expired',
});
const mod = await import('./codex-token-rotation.js');
mod.initCodexTokenRotation();
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
const accounts = mod.getAllCodexAccounts();
expect(accounts[0]).toEqual(
expect.objectContaining({
isAuthDead: true,
authStatus: 'dead_auth',
}),
);
expect(accounts[0].isRateLimited).toBe(false);
expect(accounts[1].isActive).toBe(true);
});
it('recovers a dead_auth account when canonical auth.json is refreshed before lease claim', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
try {
const agentErrors = await import('./agent-error-detection.js');
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
category: 'auth-expired',
reason: 'auth-expired',
});
const mod = await import('./codex-token-rotation.js');
const utils = await import('./utils.js');
mod.initCodexTokenRotation();
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
expect(mod.getAllCodexAccounts()[0].isAuthDead).toBe(true);
const refreshedAt = new Date('2026-01-01T00:00:10.000Z');
const authPath = path.join(tempHome, '.codex-accounts', '0', 'auth.json');
fs.writeFileSync(
authPath,
JSON.stringify({
auth_mode: 'chatgpt',
tokens: { account_id: 'acct-0', access_token: 'refreshed-token' },
}),
);
fs.utimesSync(authPath, refreshedAt, refreshedAt);
mod.setCurrentCodexAccountIndex(0);
const lease = mod.claimCodexAuthLease();
try {
expect(lease).toEqual(
expect.objectContaining({ accountIndex: 0, authPath }),
);
expect(mod.getAllCodexAccounts()[0]).toEqual(
expect.objectContaining({
authStatus: 'healthy',
isAuthDead: false,
isActive: true,
}),
);
expect(vi.mocked(utils.writeJsonFile)).toHaveBeenCalled();
} finally {
lease?.release();
}
} finally {
vi.useRealTimers();
}
});
it('leases different accounts for concurrent Codex runs', async () => {
const mod = await import('./codex-token-rotation.js');
mod.initCodexTokenRotation();
const first = mod.claimCodexAuthLease();
const second = mod.claimCodexAuthLease();
try {
expect(first).toEqual(
expect.objectContaining({
accountIndex: 0,
authPath: expect.any(String),
}),
);
expect(second).toEqual(
expect.objectContaining({
accountIndex: 1,
authPath: expect.any(String),
}),
);
expect(second?.authPath).not.toBe(first?.authPath);
} finally {
second?.release();
first?.release();
}
});
it('syncs a refreshed session auth.json back to the canonical slot atomically', async () => {
const mod = await import('./codex-token-rotation.js');
mod.initCodexTokenRotation();
const canonicalAuthPath = path.join(
tempHome,
'.codex-accounts',
'0',
'auth.json',
);
const sessionAuthPath = path.join(
tempHome,
'session',
'.codex',
'auth.json',
);
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
fs.writeFileSync(
sessionAuthPath,
JSON.stringify({
auth_mode: 'chatgpt',
tokens: {
account_id: 'acct-0',
access_token: 'new-access',
refresh_token: 'new-refresh',
},
}),
);
const synced = mod.syncCodexSessionAuthBack({
canonicalAuthPath,
sessionAuthPath,
accountIndex: 0,
});
expect(synced).toBe(true);
const canonical = JSON.parse(fs.readFileSync(canonicalAuthPath, 'utf-8'));
expect(canonical.tokens).toEqual(
expect.objectContaining({
account_id: 'acct-0',
access_token: 'new-access',
refresh_token: 'new-refresh',
}),
);
});
it('refuses to sync a session auth.json that belongs to another Codex account', async () => {
const mod = await import('./codex-token-rotation.js');
mod.initCodexTokenRotation();
const canonicalAuthPath = path.join(
tempHome,
'.codex-accounts',
'0',
'auth.json',
);
const before = fs.readFileSync(canonicalAuthPath, 'utf-8');
const sessionAuthPath = path.join(
tempHome,
'wrong-account',
'.codex',
'auth.json',
);
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
fs.writeFileSync(
sessionAuthPath,
JSON.stringify({
auth_mode: 'chatgpt',
tokens: {
account_id: 'acct-other',
access_token: 'other-access',
refresh_token: 'other-refresh',
},
}),
);
expect(
mod.syncCodexSessionAuthBack({
canonicalAuthPath,
sessionAuthPath,
accountIndex: 0,
}),
).toBe(false);
expect(fs.readFileSync(canonicalAuthPath, 'utf-8')).toBe(before);
});
}); });
describe('codex-token-rotation single-account fallback', () => { describe('codex-token-rotation single-account fallback', () => {

View File

@@ -23,7 +23,6 @@ import { DATA_DIR } from './config.js';
import { logger } from './logger.js'; import { logger } from './logger.js';
import { import {
computeCooldownUntil, computeCooldownUntil,
findNextAvailable,
parseRetryAfterFromError, parseRetryAfterFromError,
} from './token-rotation-base.js'; } from './token-rotation-base.js';
import { readJsonFile, writeJsonFile } from './utils.js'; import { readJsonFile, writeJsonFile } from './utils.js';
@@ -34,15 +33,27 @@ interface CodexAccount {
index: number; index: number;
authPath: string; authPath: string;
accountId: string; accountId: string;
authFileMtimeMs: number;
planType: string; planType: string;
subscriptionUntil: string | null; subscriptionUntil: string | null;
rateLimitedUntil: number | null; rateLimitedUntil: number | null;
authStatus: 'healthy' | 'dead_auth';
authDeadAt: number | null;
authDeadReason: string | null;
leasedUntil: number | null;
leaseId: string | null;
lastUsagePct?: number; lastUsagePct?: number;
lastUsageD7Pct?: number; lastUsageD7Pct?: number;
resetAt?: string; resetAt?: string;
resetD7At?: string; resetD7At?: string;
} }
export interface CodexAuthLease {
accountIndex: number;
authPath: string;
release: () => void;
}
export type CodexRotationTriggerResult = export type CodexRotationTriggerResult =
| { | {
shouldRotate: false; shouldRotate: false;
@@ -77,10 +88,20 @@ const accounts: CodexAccount[] = [];
let currentIndex = 0; let currentIndex = 0;
let initialized = false; let initialized = false;
const CODEX_AUTH_LEASE_MS = 2 * 60 * 60_000;
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts'); const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex'); const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json'); const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json');
function readAuthFileMtimeMs(authPath: string): number {
try {
return fs.statSync(authPath).mtimeMs;
} catch {
return 0;
}
}
function loadCodexAccount( function loadCodexAccount(
authPath: string, authPath: string,
fallbackAccountId: string, fallbackAccountId: string,
@@ -99,9 +120,15 @@ function loadCodexAccount(
index: accounts.length, index: accounts.length,
authPath, authPath,
accountId, accountId,
authFileMtimeMs: readAuthFileMtimeMs(authPath),
planType: jwt.planType, planType: jwt.planType,
subscriptionUntil: jwt.expiresAt, subscriptionUntil: jwt.expiresAt,
rateLimitedUntil: null, rateLimitedUntil: null,
authStatus: 'healthy',
authDeadAt: null,
authDeadReason: null,
leasedUntil: null,
leaseId: null,
}); });
return true; return true;
} }
@@ -149,6 +176,8 @@ function saveCodexState(): void {
const state = { const state = {
currentIndex, currentIndex,
rateLimits: accounts.map((a) => a.rateLimitedUntil), rateLimits: accounts.map((a) => a.rateLimitedUntil),
authDeadAts: accounts.map((a) => a.authDeadAt),
authDeadReasons: accounts.map((a) => a.authDeadReason),
usagePcts: accounts.map((a) => a.lastUsagePct ?? null), usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null), usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null),
resetAts: accounts.map((a) => a.resetAt ?? null), resetAts: accounts.map((a) => a.resetAt ?? null),
@@ -167,6 +196,8 @@ function loadCodexState(quiet = false): void {
const state = readJsonFile<{ const state = readJsonFile<{
currentIndex?: number; currentIndex?: number;
rateLimits?: (number | null)[]; rateLimits?: (number | null)[];
authDeadAts?: (number | null)[];
authDeadReasons?: (string | null)[];
usagePcts?: (number | null)[]; usagePcts?: (number | null)[];
usageD7Pcts?: (number | null)[]; usageD7Pcts?: (number | null)[];
resetAts?: (string | null)[]; resetAts?: (string | null)[];
@@ -175,6 +206,7 @@ function loadCodexState(quiet = false): void {
if (!state) return; if (!state) return;
const now = Date.now(); const now = Date.now();
let restoredDeadAuth = false;
if ( if (
typeof state.currentIndex === 'number' && typeof state.currentIndex === 'number' &&
state.currentIndex < accounts.length state.currentIndex < accounts.length
@@ -195,6 +227,39 @@ function loadCodexState(quiet = false): void {
} }
} }
} }
if (Array.isArray(state.authDeadAts)) {
for (
let i = 0;
i < Math.min(state.authDeadAts.length, accounts.length);
i++
) {
const deadAt = state.authDeadAts[i];
const acct = accounts[i];
if (typeof deadAt === 'number' && deadAt > 0) {
const currentMtime = readAuthFileMtimeMs(acct.authPath);
acct.authFileMtimeMs = currentMtime;
if (currentMtime > deadAt) {
acct.authStatus = 'dead_auth';
acct.authDeadAt = deadAt;
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
restoredDeadAuth =
restoreDeadAuthIfAuthFileChanged(acct) || restoredDeadAuth;
} else {
acct.authStatus = 'dead_auth';
acct.authDeadAt = deadAt;
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
acct.rateLimitedUntil = null;
}
}
}
}
if (
currentIndex < accounts.length &&
accounts[currentIndex]?.authStatus === 'dead_auth'
) {
const nextIdx = findNextCodexAvailable(currentIndex);
if (nextIdx !== null) currentIndex = nextIdx;
}
if (Array.isArray(state.usagePcts)) { if (Array.isArray(state.usagePcts)) {
for ( for (
let i = 0; let i = 0;
@@ -239,6 +304,7 @@ function loadCodexState(quiet = false): void {
'Codex rotation state restored', 'Codex rotation state restored',
); );
} }
if (restoredDeadAuth) saveCodexState();
} }
/** /**
@@ -309,6 +375,265 @@ export function getCodexAuthPath(
return accounts[accountIndex]?.authPath ?? null; return accounts[accountIndex]?.authPath ?? null;
} }
function codexLockPath(authPath: string): string {
return path.join(path.dirname(authPath), '.ejclaw-auth.lock');
}
function isPidAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function readExistingLease(lockPath: string): {
leaseId?: string;
pid?: number;
expiresAt?: number;
} | null {
const data = readJsonFile<{
leaseId?: string;
pid?: number;
expiresAt?: number;
}>(lockPath);
return data ?? null;
}
function tryAcquireDiskLease(
acct: CodexAccount,
leaseId: string,
now: number,
): boolean {
const lockPath = codexLockPath(acct.authPath);
const payload = JSON.stringify(
{
leaseId,
pid: process.pid,
accountIndex: acct.index,
createdAt: now,
expiresAt: now + CODEX_AUTH_LEASE_MS,
},
null,
2,
);
try {
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
return true;
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') return false;
}
const existing = readExistingLease(lockPath);
const expired = Boolean(existing?.expiresAt && existing.expiresAt <= now);
const deadPid = Boolean(existing?.pid && !isPidAlive(existing.pid));
if (!expired && !deadPid) return false;
try {
fs.unlinkSync(lockPath);
} catch {
return false;
}
try {
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
return true;
} catch {
return false;
}
}
function releaseDiskLease(authPath: string, leaseId: string): void {
const lockPath = codexLockPath(authPath);
const existing = readExistingLease(lockPath);
if (existing?.leaseId && existing.leaseId !== leaseId) return;
try {
fs.unlinkSync(lockPath);
} catch {
// Best effort only. Expired/stale locks are cleared by the next claimant.
}
}
function isCodexAccountUsable(
acct: CodexAccount,
now = Date.now(),
opts?: { ignoreRateLimits?: boolean; ignoreD7?: boolean },
): boolean {
if (restoreDeadAuthIfAuthFileChanged(acct)) saveCodexState();
if (acct.authStatus === 'dead_auth') return false;
if (
!opts?.ignoreRateLimits &&
acct.rateLimitedUntil &&
acct.rateLimitedUntil > now
) {
return false;
}
if (
!opts?.ignoreD7 &&
acct.lastUsageD7Pct != null &&
acct.lastUsageD7Pct >= 100
) {
return false;
}
if (acct.leasedUntil && acct.leasedUntil > now) return false;
const existing = readExistingLease(codexLockPath(acct.authPath));
if (!existing) return true;
if (existing.expiresAt && existing.expiresAt <= now) return true;
if (existing.pid && !isPidAlive(existing.pid)) return true;
return false;
}
function restoreDeadAuthIfAuthFileChanged(acct: CodexAccount): boolean {
if (acct.authStatus !== 'dead_auth' || !acct.authDeadAt) return false;
const currentMtime = readAuthFileMtimeMs(acct.authPath);
acct.authFileMtimeMs = currentMtime;
if (currentMtime <= acct.authDeadAt) return false;
const previousDeadAt = acct.authDeadAt;
acct.authStatus = 'healthy';
acct.authDeadAt = null;
acct.authDeadReason = null;
acct.rateLimitedUntil = null;
logger.info(
{
transition: 'rotation:auth-file-refreshed',
accountIndex: acct.index,
previousDeadAt: new Date(previousDeadAt).toISOString(),
authFileMtime: new Date(currentMtime).toISOString(),
},
`Codex account #${acct.index + 1}/${accounts.length} marked healthy after auth.json refresh`,
);
return true;
}
function markCodexAccountDeadAuth(acct: CodexAccount, reason?: string): void {
acct.authStatus = 'dead_auth';
acct.authDeadAt = Date.now();
acct.authDeadReason = reason || 'auth-expired';
acct.rateLimitedUntil = null;
acct.leasedUntil = null;
acct.leaseId = null;
logger.warn(
{
transition: 'rotation:dead-auth',
accountIndex: acct.index,
accountId: acct.accountId,
reason: acct.authDeadReason,
},
`Codex account #${acct.index + 1}/${accounts.length} marked dead; re-auth required`,
);
}
function updateAccountMetadataFromAuthFile(acct: CodexAccount): void {
const data = readJsonFile<{
tokens?: { account_id?: string; id_token?: string };
}>(acct.authPath);
if (data?.tokens?.account_id) acct.accountId = data.tokens.account_id;
const jwt = parseJwtAuth(data?.tokens?.id_token || '');
acct.planType = jwt.planType;
acct.subscriptionUntil = jwt.expiresAt;
acct.authFileMtimeMs = readAuthFileMtimeMs(acct.authPath);
}
export function claimCodexAuthLease(): CodexAuthLease | null {
if (accounts.length === 0) return null;
const now = Date.now();
const attempts = accounts.length;
for (let offset = 0; offset < attempts; offset += 1) {
const idx = (currentIndex + offset) % accounts.length;
const acct = accounts[idx];
if (!isCodexAccountUsable(acct, now)) continue;
const leaseId = `${process.pid}-${now}-${idx}-${Math.random()
.toString(36)
.slice(2)}`;
if (!tryAcquireDiskLease(acct, leaseId, now)) continue;
currentIndex = idx;
acct.leasedUntil = now + CODEX_AUTH_LEASE_MS;
acct.leaseId = leaseId;
return {
accountIndex: idx,
authPath: acct.authPath,
release: () => {
if (acct.leaseId === leaseId) {
acct.leasedUntil = null;
acct.leaseId = null;
}
releaseDiskLease(acct.authPath, leaseId);
},
};
}
return null;
}
export function syncCodexSessionAuthBack(args: {
canonicalAuthPath: string;
sessionAuthPath: string;
accountIndex?: number | null;
}): boolean {
if (!fs.existsSync(args.sessionAuthPath)) return false;
if (!fs.existsSync(args.canonicalAuthPath)) return false;
const canonical = readJsonFile<{
auth_mode?: string;
tokens?: { account_id?: string };
}>(args.canonicalAuthPath);
const session = readJsonFile<{
auth_mode?: string;
tokens?: { account_id?: string };
}>(args.sessionAuthPath);
if (!canonical || !session?.tokens) return false;
const canonicalAccount = canonical.tokens?.account_id;
const sessionAccount = session.tokens.account_id;
if (
canonicalAccount &&
sessionAccount &&
canonicalAccount !== sessionAccount
) {
logger.warn(
{
canonicalAuthPath: args.canonicalAuthPath,
sessionAuthPath: args.sessionAuthPath,
accountIndex: args.accountIndex ?? null,
},
'Refusing to sync Codex session auth back: account mismatch',
);
return false;
}
const sessionRaw = fs.readFileSync(args.sessionAuthPath, 'utf-8');
const canonicalRaw = fs.readFileSync(args.canonicalAuthPath, 'utf-8');
if (sessionRaw === canonicalRaw) return false;
const tmpPath = `${args.canonicalAuthPath}.tmp-${process.pid}-${Date.now()}`;
fs.writeFileSync(tmpPath, sessionRaw, { mode: 0o600 });
fs.renameSync(tmpPath, args.canonicalAuthPath);
const idx =
args.accountIndex ??
findCodexAccountIndexByAuthPath(args.canonicalAuthPath);
if (idx != null && accounts[idx]) {
const acct = accounts[idx];
acct.authStatus = 'healthy';
acct.authDeadAt = null;
acct.authDeadReason = null;
updateAccountMetadataFromAuthFile(acct);
saveCodexState();
}
logger.info(
{
accountIndex: idx,
canonicalAuthPath: args.canonicalAuthPath,
},
'Synced refreshed Codex session auth back to canonical account slot',
);
return true;
}
export function detectCodexRotationTrigger( export function detectCodexRotationTrigger(
error?: string | null, error?: string | null,
): CodexRotationTriggerResult { ): CodexRotationTriggerResult {
@@ -341,16 +666,23 @@ export function rotateCodexToken(
const previousIndex = currentIndex; const previousIndex = currentIndex;
const acct = accounts[currentIndex]; const acct = accounts[currentIndex];
const cooldownUntil = computeCooldownUntil(errorMessage); const authFailure = classifyCodexAuthError(errorMessage);
acct.rateLimitedUntil = cooldownUntil; let cooldownUntil: number | null = null;
acct.lastUsagePct = 100;
// Extract reset time string from error for display if (authFailure.category === 'auth-expired') {
const retryAt = parseRetryAfterFromError(errorMessage); markCodexAccountDeadAuth(acct, errorMessage || authFailure.reason);
if (retryAt) { } else {
acct.resetAt = new Date(retryAt).toISOString(); cooldownUntil = computeCooldownUntil(errorMessage);
acct.rateLimitedUntil = cooldownUntil;
acct.lastUsagePct = 100;
// Extract reset time string from error for display
const retryAt = parseRetryAfterFromError(errorMessage);
if (retryAt) {
acct.resetAt = new Date(retryAt).toISOString();
}
} }
const nextIdx = findNextAvailable(accounts, currentIndex, opts); const nextIdx = findNextCodexAvailable(currentIndex, opts);
if (nextIdx !== null) { if (nextIdx !== null) {
accounts[nextIdx].rateLimitedUntil = null; accounts[nextIdx].rateLimitedUntil = null;
currentIndex = nextIdx; currentIndex = nextIdx;
@@ -364,6 +696,7 @@ export function rotateCodexToken(
ignoreRL: opts?.ignoreRateLimits ?? false, ignoreRL: opts?.ignoreRateLimits ?? false,
cooldownUntil: cooldownUntil:
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null, cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
authDead: authFailure.category === 'auth-expired',
reason: errorMessage ?? null, reason: errorMessage ?? null,
}, },
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`, `Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
@@ -380,28 +713,40 @@ export function rotateCodexToken(
ignoreRL: opts?.ignoreRateLimits ?? false, ignoreRL: opts?.ignoreRateLimits ?? false,
cooldownUntil: cooldownUntil:
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null, cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
authDead: authFailure.category === 'auth-expired',
reason: errorMessage ?? null, reason: errorMessage ?? null,
}, },
'All Codex accounts are rate-limited', authFailure.category === 'auth-expired'
? 'All Codex accounts unavailable after auth failure; re-auth required'
: 'All Codex accounts are rate-limited',
); );
saveCodexState();
return false; return false;
} }
/** /**
* Find the next Codex account that is neither rate-limited nor 7d-exhausted. * Find the next Codex account that is neither rate-limited nor 7d-exhausted.
*/ */
function findNextCodexAvailable(fromIndex?: number): number | null { function findNextCodexAvailable(
fromIndex?: number,
opts?: { ignoreRateLimits?: boolean },
): number | null {
const now = Date.now(); const now = Date.now();
const start = fromIndex ?? currentIndex; const start = fromIndex ?? currentIndex;
for (let i = 1; i < accounts.length; i++) { for (let i = 1; i < accounts.length; i++) {
const idx = (start + i) % accounts.length; const idx = (start + i) % accounts.length;
const acct = accounts[idx]; const acct = accounts[idx];
const rlOk = !acct.rateLimitedUntil || acct.rateLimitedUntil <= now; if (isCodexAccountUsable(acct, now, opts)) return idx;
const usageOk = acct.lastUsageD7Pct == null || acct.lastUsageD7Pct < 100;
if (rlOk && usageOk) return idx;
} }
// All exhausted — fall back to rate-limit-only check // All d7-exhausted — fall back to rate-limit/dead/lease checks only.
return findNextAvailable(accounts, start); for (let i = 1; i < accounts.length; i++) {
const idx = (start + i) % accounts.length;
const acct = accounts[idx];
if (isCodexAccountUsable(acct, now, { ...opts, ignoreD7: true })) {
return idx;
}
}
return null;
} }
/** /**
@@ -469,9 +814,17 @@ export function updateCodexAccountUsage(
export function markCodexTokenHealthy(): void { export function markCodexTokenHealthy(): void {
if (accounts.length === 0) return; if (accounts.length === 0) return;
const acct = accounts[currentIndex]; const acct = accounts[currentIndex];
let changed = false;
if (acct?.authStatus === 'dead_auth') {
acct.authStatus = 'healthy';
acct.authDeadAt = null;
acct.authDeadReason = null;
changed = true;
}
if (acct?.rateLimitedUntil) { if (acct?.rateLimitedUntil) {
const previousCooldownUntil = acct.rateLimitedUntil; const previousCooldownUntil = acct.rateLimitedUntil;
acct.rateLimitedUntil = null; acct.rateLimitedUntil = null;
changed = true;
logger.info( logger.info(
{ {
transition: 'rotation:clear-rate-limit', transition: 'rotation:clear-rate-limit',
@@ -481,8 +834,8 @@ export function markCodexTokenHealthy(): void {
}, },
'Cleared Codex account rate-limit state after successful response', 'Cleared Codex account rate-limit state after successful response',
); );
saveCodexState();
} }
if (changed) saveCodexState();
} }
export function getCodexAccountCount(): number { export function getCodexAccountCount(): number {
@@ -495,6 +848,10 @@ export function getAllCodexAccounts(): {
planType: string; planType: string;
isActive: boolean; isActive: boolean;
isRateLimited: boolean; isRateLimited: boolean;
isAuthDead?: boolean;
authStatus?: 'healthy' | 'dead_auth';
authDeadAt?: string;
isLeased?: boolean;
cachedUsagePct?: number; cachedUsagePct?: number;
cachedUsageD7Pct?: number; cachedUsageD7Pct?: number;
resetAt?: string; resetAt?: string;
@@ -507,6 +864,10 @@ export function getAllCodexAccounts(): {
planType: a.planType, planType: a.planType,
isActive: i === currentIndex, isActive: i === currentIndex,
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
isAuthDead: a.authStatus === 'dead_auth',
authStatus: a.authStatus,
authDeadAt: a.authDeadAt ? new Date(a.authDeadAt).toISOString() : undefined,
isLeased: Boolean(a.leasedUntil && a.leasedUntil > now),
cachedUsagePct: a.lastUsagePct, cachedUsagePct: a.lastUsagePct,
cachedUsageD7Pct: a.lastUsageD7Pct, cachedUsageD7Pct: a.lastUsageD7Pct,
resetAt: a.resetAt, resetAt: a.resetAt,

View File

@@ -0,0 +1,71 @@
import { describe, expect, it, vi } from 'vitest';
const mockRunAgentProcess = vi.hoisted(() => vi.fn());
vi.mock('./agent-runner.js', () => ({
runAgentProcess: mockRunAgentProcess,
}));
vi.mock('./codex-token-rotation.js', () => ({
getCodexAccountCount: vi.fn(() => 2),
}));
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
import type { AgentOutput } from './agent-runner.js';
describe('runMessageAgentAttempt', () => {
it('stores pre-stream Codex launch errors in paired summary', async () => {
const error = new Error(
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
);
mockRunAgentProcess.mockRejectedValueOnce(error);
const updateSummary = vi.fn();
const attempt = await runMessageAgentAttempt({
provider: 'codex',
currentSessionId: undefined,
isClaudeCodeAgent: false,
canRetryClaudeCredentials: false,
shouldPersistSession: false,
effectiveGroup: {
name: 'Test',
folder: 'test',
trigger: '@test',
added_at: new Date().toISOString(),
agentType: 'codex',
},
agentInput: {
prompt: 'arbiter prompt',
groupFolder: 'test',
chatJid: 'dc:test',
runId: 'run-test',
isMain: false,
assistantName: 'Andy',
},
activeRole: 'arbiter',
effectiveServiceId: 'codex-review',
effectiveAgentType: 'codex',
sessionFolder: 'test:arbiter',
onPersistSession: vi.fn(),
registerProcess: vi.fn(),
onOutput: vi.fn(async (_output: AgentOutput) => undefined),
pairedExecutionLifecycle: {
updateSummary,
recordFinalOutputBeforeDelivery: vi.fn(() => false),
},
log: {
child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
} as any,
});
expect(attempt.error).toBe(error);
expect(attempt.sawOutput).toBe(false);
expect(updateSummary).toHaveBeenCalledWith({
errorText:
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
});
});
});

View File

@@ -14,6 +14,7 @@ import {
shouldResetCodexSessionOnAgentFailure, shouldResetCodexSessionOnAgentFailure,
shouldResetSessionOnAgentFailure, shouldResetSessionOnAgentFailure,
} from './session-recovery.js'; } from './session-recovery.js';
import { getErrorMessage } from './utils.js';
import type { import type {
AgentType, AgentType,
OutboundAttachment, OutboundAttachment,
@@ -162,6 +163,9 @@ class MessageAgentAttemptRunner {
); );
return this.buildAttempt({ output }); return this.buildAttempt({ output });
} catch (error) { } catch (error) {
this.args.pairedExecutionLifecycle.updateSummary({
errorText: getErrorMessage(error),
});
return this.buildAttempt({ error }); return this.buildAttempt({ error });
} }
} }

View File

@@ -505,8 +505,11 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
if (!missingVisibleVerdict) { if (!missingVisibleVerdict) {
return false; return false;
} }
this.pairedExecutionSummary = const noVerdictSummary =
'Execution completed without a visible terminal verdict.'; 'Execution completed without a visible terminal verdict.';
this.pairedExecutionSummary = this.pairedExecutionSummary
? `${this.pairedExecutionSummary}\n${noVerdictSummary}`
: noVerdictSummary;
log.warn( log.warn(
{ {
pairedTaskId: pairedExecutionContext?.task.id ?? null, pairedTaskId: pairedExecutionContext?.task.id ?? null,

View File

@@ -172,6 +172,19 @@ describe('message-agent-executor-rules', () => {
).toBe('pending'); ).toBe('pending');
}); });
it('does not requeue a silent arbiter failure caused by Codex account auth expiry', () => {
expect(
resolvePairedFollowUpQueueAction({
completedRole: 'arbiter',
executionStatus: 'failed',
sawOutput: false,
taskStatus: 'arbiter_requested',
outputSummary:
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.\nExecution completed without a visible terminal verdict.',
}),
).toBe('none');
});
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => { it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
expect( expect(
resolvePairedFollowUpQueueAction({ resolvePairedFollowUpQueueAction({

View File

@@ -3,6 +3,7 @@ import {
resolveNextTurnAction, resolveNextTurnAction,
shouldRetrySilentOwnerExecution, shouldRetrySilentOwnerExecution,
} from './message-runtime-rules.js'; } from './message-runtime-rules.js';
import { classifyCodexAuthError } from './agent-error-detection.js';
import { parseVisibleVerdict } from './paired-verdict.js'; import { parseVisibleVerdict } from './paired-verdict.js';
import type { PairedRoomRole, PairedTaskStatus } from './types.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js';
export { export {
@@ -17,6 +18,17 @@ export {
export type PairedFollowUpQueueAction = 'pending' | 'none'; export type PairedFollowUpQueueAction = 'pending' | 'none';
function isSilentCodexAccountFailure(summary?: string | null): boolean {
if (!summary) return false;
if (classifyCodexAuthError(summary).category !== 'none') return true;
const lower = summary.toLowerCase();
return (
lower.includes('workspace out of credits') ||
lower.includes('out of credits') ||
/all\s+codex(?:\s+rotation)?\s+accounts/i.test(summary)
);
}
export function resolvePairedFollowUpQueueAction(args: { export function resolvePairedFollowUpQueueAction(args: {
completedRole: PairedRoomRole; completedRole: PairedRoomRole;
executionStatus: 'succeeded' | 'failed'; executionStatus: 'succeeded' | 'failed';
@@ -24,6 +36,15 @@ export function resolvePairedFollowUpQueueAction(args: {
taskStatus: PairedTaskStatus | null; taskStatus: PairedTaskStatus | null;
outputSummary?: string | null; outputSummary?: string | null;
}): PairedFollowUpQueueAction { }): PairedFollowUpQueueAction {
if (
args.executionStatus === 'failed' &&
args.sawOutput === false &&
(args.completedRole === 'reviewer' || args.completedRole === 'arbiter') &&
isSilentCodexAccountFailure(args.outputSummary)
) {
return 'none';
}
if ( if (
shouldRetrySilentOwnerExecution({ shouldRetrySilentOwnerExecution({
completedRole: args.completedRole, completedRole: args.completedRole,

View File

@@ -1,15 +1,52 @@
import { logger } from './logger.js'; import { logger } from './logger.js';
import {
classifyAgentError,
classifyCodexAuthError,
} from './agent-error-detection.js';
import { transitionPairedTaskStatus } from './paired-task-status.js'; import { transitionPairedTaskStatus } from './paired-task-status.js';
import { classifyArbiterVerdict } from './paired-verdict.js'; import { classifyArbiterVerdict } from './paired-verdict.js';
import type { PairedTask } from './types.js'; import type { PairedTask } from './types.js';
const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0; const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0;
function isTerminalCodexAccountFailure(summary?: string | null): boolean {
if (!summary) return false;
if (classifyCodexAuthError(summary).category !== 'none') return true;
if (classifyAgentError(summary).category === 'rate-limit') return true;
return /all\s+codex(?:\s+rotation)?\s+accounts/i.test(summary);
}
export function handleFailedArbiterExecution(args: { export function handleFailedArbiterExecution(args: {
task: PairedTask; task: PairedTask;
taskId: string; taskId: string;
summary?: string | null;
}): void { }): void {
const { task, taskId } = args; const { task, taskId, summary } = args;
if (isTerminalCodexAccountFailure(summary)) {
const now = new Date().toISOString();
transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: 'completed',
expectedUpdatedAt: task.updated_at,
updatedAt: now,
patch: {
arbiter_verdict: 'escalate',
arbiter_requested_at: null,
completion_reason: 'arbiter_codex_unavailable',
},
});
logger.warn(
{
taskId,
role: 'arbiter',
status: task.status,
summary: summary?.slice(0, 200),
},
'Completed arbiter task after terminal Codex account failure instead of re-arming arbitration loop',
);
return;
}
const now = new Date().toISOString(); const now = new Date().toISOString();
const fallbackStatus = const fallbackStatus =
task.status === 'in_arbitration' || task.status === 'arbiter_requested' task.status === 'in_arbitration' || task.status === 'arbiter_requested'

View File

@@ -197,6 +197,33 @@ describe('paired execution routing loop guards', () => {
); );
}); });
it('does not re-arm arbiter loop after terminal Codex account failure', () => {
vi.mocked(db.getPairedTaskById).mockReturnValue(
buildPairedTask({
status: 'in_arbitration',
updated_at: '2026-03-28T00:00:05.000Z',
}),
);
completePairedExecutionContext({
taskId: 'task-1',
role: 'arbiter',
status: 'failed',
summary:
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
});
expect(db.updatePairedTask).toHaveBeenCalledWith(
'task-1',
expect.objectContaining({
status: 'completed',
arbiter_verdict: 'escalate',
arbiter_requested_at: null,
completion_reason: 'arbiter_codex_unavailable',
}),
);
});
it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => { it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => {
vi.mocked(db.getPairedTaskById).mockReturnValue( vi.mocked(db.getPairedTaskById).mockReturnValue(
buildPairedTask({ buildPairedTask({

View File

@@ -666,7 +666,11 @@ export function completePairedExecutionContext(args: {
return; return;
} }
if (role === 'arbiter') { if (role === 'arbiter') {
handleFailedArbiterExecution({ task, taskId }); handleFailedArbiterExecution({
task,
taskId,
summary: args.summary,
});
return; return;
} }
handleFailedOwnerExecution({ task, taskId, summary: args.summary }); handleFailedOwnerExecution({ task, taskId, summary: args.summary });

View File

@@ -224,6 +224,7 @@ describe('runCodexRotationLoop', () => {
expect(outcome).toEqual({ type: 'success' }); expect(outcome).toEqual({ type: 'success' });
expect(rotateCodexToken).toHaveBeenCalledTimes(1); expect(rotateCodexToken).toHaveBeenCalledTimes(1);
expect(rotateCodexToken).toHaveBeenCalledWith('auth-expired');
expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1); expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1);
}); });

View File

@@ -36,6 +36,7 @@ import { clearGlobalFailover } from './service-routing.js';
export interface TriggerInfo { export interface TriggerInfo {
reason: AgentTriggerReason; reason: AgentTriggerReason;
retryAfterMs?: number; retryAfterMs?: number;
message?: string;
} }
export interface RotationAttemptResult { export interface RotationAttemptResult {
@@ -206,6 +207,34 @@ function evaluateCodexAttempt(
return { type: 'error' }; return { type: 'error' };
} }
if (
!attempt.sawOutput &&
typeof output.error === 'string' &&
output.error.length > 0
) {
const retryTrigger =
attempt.streamedTriggerReason &&
isCodexRotationReason(attempt.streamedTriggerReason.reason)
? {
shouldRotate: true as const,
reason: attempt.streamedTriggerReason.reason,
}
: detectCodexRotationTrigger(output.error);
if (retryTrigger.shouldRotate) {
return {
type: 'continue',
trigger: { reason: retryTrigger.reason },
rotationMessage: output.error,
};
}
logger.error(
{ ...logContext, provider: 'codex', error: output.error },
'Rotated Codex account returned an error field without visible output',
);
return { type: 'error' };
}
if ( if (
!attempt.sawOutput && !attempt.sawOutput &&
attempt.streamedTriggerReason && attempt.streamedTriggerReason &&
@@ -350,7 +379,7 @@ export async function runClaudeRotationLoop(
} }
export async function runCodexRotationLoop( export async function runCodexRotationLoop(
initialTrigger: { reason: CodexRotationReason }, initialTrigger: { reason: CodexRotationReason; message?: string },
runAttempt: () => Promise<CodexRotationAttemptResult>, runAttempt: () => Promise<CodexRotationAttemptResult>,
logContext: Record<string, unknown>, logContext: Record<string, unknown>,
rotationMessage?: string, rotationMessage?: string,
@@ -358,7 +387,13 @@ export async function runCodexRotationLoop(
let trigger = initialTrigger; let trigger = initialTrigger;
let lastRotationMessage = rotationMessage; let lastRotationMessage = rotationMessage;
while (getCodexAccountCount() > 1 && rotateCodexToken(lastRotationMessage)) { while (getCodexAccountCount() > 1) {
const rotationReasonMessage =
lastRotationMessage ?? trigger.message ?? trigger.reason;
if (!rotateCodexToken(rotationReasonMessage)) {
break;
}
logger.info( logger.info(
{ ...logContext, reason: trigger.reason }, { ...logContext, reason: trigger.reason },
'Codex account unhealthy, retrying with rotated account', 'Codex account unhealthy, retrying with rotated account',

View File

@@ -513,18 +513,26 @@ describe('evaluateStreamedOutput', () => {
describe('error → Codex rotation trigger', () => { describe('error → Codex rotation trigger', () => {
it('returns rotation trigger for Codex primary', () => { it('returns rotation trigger for Codex primary', () => {
const errorMessage =
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
vi.mocked(detectCodexRotationTrigger).mockReturnValue({ vi.mocked(detectCodexRotationTrigger).mockReturnValue({
shouldRotate: true, shouldRotate: true,
reason: '429', reason: '429',
}); });
const result = evaluateStreamedOutput( const result = evaluateStreamedOutput(
errorOutput('429 rate limit'), errorOutput(errorMessage),
freshState(), freshState(),
codexOpts, codexOpts,
); );
expect(result.newTrigger).toEqual({ reason: '429' }); expect(result.newTrigger).toEqual({
expect(result.state.streamedTriggerReason).toEqual({ reason: '429' }); reason: '429',
message: errorMessage,
});
expect(result.state.streamedTriggerReason).toEqual({
reason: '429',
message: errorMessage,
});
}); });
it('does not check Codex rotation for Claude agent type', () => { it('does not check Codex rotation for Claude agent type', () => {

View File

@@ -19,6 +19,7 @@ import {
export interface StreamedTriggerReason { export interface StreamedTriggerReason {
reason: AgentTriggerReason; reason: AgentTriggerReason;
retryAfterMs?: number; retryAfterMs?: number;
message?: string;
} }
export interface StreamedOutputState { export interface StreamedOutputState {
@@ -139,6 +140,28 @@ export function evaluateStreamedOutput(
nextState.sawSuccessNullResultWithoutOutput = true; nextState.sawSuccessNullResultWithoutOutput = true;
} }
if (
isPrimaryCodex &&
typeof output.error === 'string' &&
output.error.length > 0 &&
!nextState.sawOutput &&
!nextState.streamedTriggerReason
) {
const trigger = detectCodexRotationTrigger(output.error);
if (trigger.shouldRotate) {
const newTrigger: StreamedTriggerReason = {
reason: trigger.reason,
message: output.error,
};
nextState.streamedTriggerReason = newTrigger;
return {
state: nextState,
shouldForwardOutput: !options.shortCircuitTriggeredErrors,
newTrigger,
};
}
}
if ( if (
output.status === 'error' && output.status === 'error' &&
!nextState.sawOutput && !nextState.sawOutput &&
@@ -157,7 +180,7 @@ export function evaluateStreamedOutput(
} else if (isPrimaryCodex) { } else if (isPrimaryCodex) {
const trigger = detectCodexRotationTrigger(output.error); const trigger = detectCodexRotationTrigger(output.error);
if (trigger.shouldRotate) { if (trigger.shouldRotate) {
newTrigger = { reason: trigger.reason }; newTrigger = { reason: trigger.reason, message: output.error };
} }
} }