fix: recover Codex rotation auth failures
- mark Codex bearer/refresh failures as terminal auth-expired states - sync refreshed session auth back to rotation slots and revive refreshed dead_auth slots - stop paired arbiter retry loops when Codex accounts are unavailable - add regression coverage for rotation leases, follow-up suppression, and arbiter closure
This commit is contained in:
29
src/agent-attempt-retry.test.ts
Normal file
29
src/agent-attempt-retry.test.ts
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { resolveAttemptRetryAction } from './agent-attempt-retry.js';
|
||||||
|
|
||||||
|
describe('resolveAttemptRetryAction', () => {
|
||||||
|
it('uses the streamed Codex trigger message as the rotation message', () => {
|
||||||
|
const errorMessage =
|
||||||
|
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
|
||||||
|
|
||||||
|
const action = resolveAttemptRetryAction({
|
||||||
|
provider: 'codex',
|
||||||
|
canRetryClaudeCredentials: false,
|
||||||
|
canRetryCodex: true,
|
||||||
|
attempt: {
|
||||||
|
sawOutput: false,
|
||||||
|
streamedTriggerReason: {
|
||||||
|
reason: 'auth-expired',
|
||||||
|
message: errorMessage,
|
||||||
|
} as any,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(action).toEqual({
|
||||||
|
kind: 'codex',
|
||||||
|
trigger: { reason: 'auth-expired' },
|
||||||
|
rotationMessage: errorMessage,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -10,6 +10,7 @@ import { getErrorMessage } from './utils.js';
|
|||||||
export interface AttemptStreamedTrigger {
|
export interface AttemptStreamedTrigger {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
|
message?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AttemptRetryState {
|
export interface AttemptRetryState {
|
||||||
@@ -131,7 +132,10 @@ export function resolveAttemptRetryAction(args: {
|
|||||||
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||||
rotationMessage?: string | null;
|
rotationMessage?: string | null;
|
||||||
}): AttemptRetryAction {
|
}): AttemptRetryAction {
|
||||||
const normalizedRotationMessage = args.rotationMessage ?? undefined;
|
const normalizedRotationMessage =
|
||||||
|
args.rotationMessage ??
|
||||||
|
args.attempt.streamedTriggerReason?.message ??
|
||||||
|
undefined;
|
||||||
|
|
||||||
const claudeTrigger = resolveClaudeRetryTrigger({
|
const claudeTrigger = resolveClaudeRetryTrigger({
|
||||||
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
|
|||||||
import {
|
import {
|
||||||
classifyAgentError,
|
classifyAgentError,
|
||||||
classifyClaudeAuthError,
|
classifyClaudeAuthError,
|
||||||
|
classifyCodexAuthError,
|
||||||
detectClaudeProviderFailureMessage,
|
detectClaudeProviderFailureMessage,
|
||||||
isClaudeOrgAccessDeniedMessage,
|
isClaudeOrgAccessDeniedMessage,
|
||||||
shouldRotateClaudeToken,
|
shouldRotateClaudeToken,
|
||||||
@@ -66,6 +67,32 @@ describe('agent-error-detection', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('classifies Codex reused refresh-token errors as auth-expired', () => {
|
||||||
|
expect(
|
||||||
|
classifyCodexAuthError(
|
||||||
|
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.',
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('classifies a Codex auth-expired trigger reason as auth-expired', () => {
|
||||||
|
expect(classifyCodexAuthError('auth-expired')).toEqual({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('classifies Codex workspace credit exhaustion as rate-limit', () => {
|
||||||
|
expect(classifyAgentError('Workspace out of credits')).toEqual({
|
||||||
|
category: 'rate-limit',
|
||||||
|
reason: '429',
|
||||||
|
retryAfterMs: undefined,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
||||||
expect(shouldRotateClaudeToken('429')).toBe(true);
|
expect(shouldRotateClaudeToken('429')).toBe(true);
|
||||||
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
||||||
|
|||||||
@@ -249,6 +249,8 @@ export function classifyAgentError(
|
|||||||
lower.includes('429') ||
|
lower.includes('429') ||
|
||||||
lower.includes('rate limit') ||
|
lower.includes('rate limit') ||
|
||||||
lower.includes('usage limit') ||
|
lower.includes('usage limit') ||
|
||||||
|
lower.includes('out of credits') ||
|
||||||
|
lower.includes('workspace out of credits') ||
|
||||||
lower.includes('hit your limit') ||
|
lower.includes('hit your limit') ||
|
||||||
lower.includes('too many requests') ||
|
lower.includes('too many requests') ||
|
||||||
lower.includes('rate_limit')
|
lower.includes('rate_limit')
|
||||||
@@ -335,11 +337,17 @@ export function classifyCodexAuthError(
|
|||||||
const lower = error.toLowerCase();
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
lower.includes('auth-expired') ||
|
||||||
|
lower.includes('auth expired') ||
|
||||||
lower.includes('401') ||
|
lower.includes('401') ||
|
||||||
lower.includes('authentication_error') ||
|
lower.includes('authentication_error') ||
|
||||||
lower.includes('failed to authenticate') ||
|
lower.includes('failed to authenticate') ||
|
||||||
|
lower.includes('access token could not be refreshed') ||
|
||||||
lower.includes('oauth token has expired') ||
|
lower.includes('oauth token has expired') ||
|
||||||
|
lower.includes('refresh token was already used') ||
|
||||||
lower.includes('refresh your existing token') ||
|
lower.includes('refresh your existing token') ||
|
||||||
|
lower.includes('log out and sign in again') ||
|
||||||
|
lower.includes('app_session_terminated') ||
|
||||||
lower.includes('unauthorized')
|
lower.includes('unauthorized')
|
||||||
) {
|
) {
|
||||||
return { category: 'auth-expired', reason: 'auth-expired' };
|
return { category: 'auth-expired', reason: 'auth-expired' };
|
||||||
|
|||||||
@@ -5,9 +5,18 @@ import path from 'path';
|
|||||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
import { EJCLAW_ENV } from 'ejclaw-runners-shared';
|
import { EJCLAW_ENV } from 'ejclaw-runners-shared';
|
||||||
|
|
||||||
const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({
|
const {
|
||||||
|
mockReadEnvFile,
|
||||||
|
mockGetActiveCodexAuthPath,
|
||||||
|
mockGetCodexAccountCount,
|
||||||
|
mockClaimCodexAuthLease,
|
||||||
|
mockFindCodexAccountIndexByAuthPath,
|
||||||
|
} = vi.hoisted(() => ({
|
||||||
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
||||||
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
||||||
|
mockGetCodexAccountCount: vi.fn<() => number>(),
|
||||||
|
mockClaimCodexAuthLease: vi.fn<() => null>(),
|
||||||
|
mockFindCodexAccountIndexByAuthPath: vi.fn<() => number | null>(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
@@ -29,6 +38,9 @@ vi.mock('./env.js', () => ({
|
|||||||
|
|
||||||
vi.mock('./codex-token-rotation.js', () => ({
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
getActiveCodexAuthPath: mockGetActiveCodexAuthPath,
|
getActiveCodexAuthPath: mockGetActiveCodexAuthPath,
|
||||||
|
getCodexAccountCount: mockGetCodexAccountCount,
|
||||||
|
claimCodexAuthLease: mockClaimCodexAuthLease,
|
||||||
|
findCodexAccountIndexByAuthPath: mockFindCodexAccountIndexByAuthPath,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./token-rotation.js', () => ({
|
vi.mock('./token-rotation.js', () => ({
|
||||||
@@ -159,6 +171,12 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -240,6 +258,36 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
expect(auth).toEqual(rotatedAuth);
|
expect(auth).toEqual(rotatedAuth);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('fails fast instead of launching Codex without OAuth when rotation accounts exist but no lease is available', () => {
|
||||||
|
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||||
|
fs.writeFileSync(
|
||||||
|
rotatedAuthPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: { access_token: 'locked-token' },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(1);
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
|
||||||
|
mockReadEnvFile.mockReturnValue({});
|
||||||
|
|
||||||
|
expect(() => prepareGroupEnvironment(group, false, 'dc:test')).toThrow(
|
||||||
|
/auth-expired: All Codex rotation accounts unavailable/,
|
||||||
|
);
|
||||||
|
|
||||||
|
const authPath = path.join(
|
||||||
|
tempRoot,
|
||||||
|
'sessions',
|
||||||
|
group.folder,
|
||||||
|
'services',
|
||||||
|
'codex-main',
|
||||||
|
'.codex',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
expect(fs.existsSync(authPath)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => {
|
it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => {
|
||||||
vi.mocked(config.isReviewService).mockReturnValue(true);
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||||
@@ -411,6 +459,12 @@ describe('prepareGroupEnvironment Codex MCP room role env', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
resetRoutingMocks();
|
resetRoutingMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -484,6 +538,12 @@ describe('prepareGroupEnvironment room skill overrides', () => {
|
|||||||
});
|
});
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -616,6 +676,12 @@ describe('prepareGroupEnvironment Codex goals handling', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
vi.mocked(config.isReviewService).mockReturnValue(false);
|
vi.mocked(config.isReviewService).mockReturnValue(false);
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false);
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false);
|
||||||
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||||
@@ -706,6 +772,12 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
|||||||
@@ -12,7 +12,13 @@ import {
|
|||||||
} from './config.js';
|
} from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { readEnvFile } from './env.js';
|
||||||
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
import {
|
||||||
|
claimCodexAuthLease,
|
||||||
|
findCodexAccountIndexByAuthPath,
|
||||||
|
getActiveCodexAuthPath,
|
||||||
|
getCodexAccountCount,
|
||||||
|
type CodexAuthLease,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
import { readCodexFeatureFromFile } from './codex-config-features.js';
|
import { readCodexFeatureFromFile } from './codex-config-features.js';
|
||||||
import { ensureClaudeSessionSettings } from './claude-session-settings.js';
|
import { ensureClaudeSessionSettings } from './claude-session-settings.js';
|
||||||
import {
|
import {
|
||||||
@@ -140,17 +146,32 @@ function ensureClaudeGlobalSettingsFile(sessionDir: string): void {
|
|||||||
fs.writeFileSync(settingsFile, '{}\n');
|
fs.writeFileSync(settingsFile, '{}\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
function syncHostCodexSessionFiles(sessionCodexDir: string): void {
|
export interface PreparedCodexSessionAuth {
|
||||||
|
canonicalAuthPath: string;
|
||||||
|
sessionAuthPath: string;
|
||||||
|
accountIndex: number | null;
|
||||||
|
lease?: CodexAuthLease;
|
||||||
|
}
|
||||||
|
|
||||||
|
function syncHostCodexSessionFiles(
|
||||||
|
sessionCodexDir: string,
|
||||||
|
): PreparedCodexSessionAuth | null {
|
||||||
const hostCodexDir = path.join(os.homedir(), '.codex');
|
const hostCodexDir = path.join(os.homedir(), '.codex');
|
||||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||||
|
|
||||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||||
const rotatedAuthSrc = getActiveCodexAuthPath();
|
const hasRotationAccounts = getCodexAccountCount() > 0;
|
||||||
|
const lease = claimCodexAuthLease();
|
||||||
|
const rotatedAuthSrc =
|
||||||
|
lease?.authPath ?? (!hasRotationAccounts ? getActiveCodexAuthPath() : null);
|
||||||
|
const fallbackAuthSrc = path.join(hostCodexDir, 'auth.json');
|
||||||
const authSrc =
|
const authSrc =
|
||||||
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||||
? rotatedAuthSrc
|
? rotatedAuthSrc
|
||||||
: path.join(hostCodexDir, 'auth.json');
|
: !hasRotationAccounts && fs.existsSync(fallbackAuthSrc)
|
||||||
if (fs.existsSync(authSrc)) {
|
? fallbackAuthSrc
|
||||||
|
: null;
|
||||||
|
if (authSrc) {
|
||||||
fs.copyFileSync(authSrc, authDst);
|
fs.copyFileSync(authSrc, authDst);
|
||||||
} else if (fs.existsSync(authDst)) {
|
} else if (fs.existsSync(authDst)) {
|
||||||
fs.unlinkSync(authDst);
|
fs.unlinkSync(authDst);
|
||||||
@@ -163,6 +184,28 @@ function syncHostCodexSessionFiles(sessionCodexDir: string): void {
|
|||||||
fs.copyFileSync(src, dst);
|
fs.copyFileSync(src, dst);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!rotatedAuthSrc ||
|
||||||
|
authSrc !== rotatedAuthSrc ||
|
||||||
|
!fs.existsSync(authDst)
|
||||||
|
) {
|
||||||
|
lease?.release();
|
||||||
|
if (hasRotationAccounts) {
|
||||||
|
throw new Error(
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
canonicalAuthPath: rotatedAuthSrc,
|
||||||
|
sessionAuthPath: authDst,
|
||||||
|
accountIndex:
|
||||||
|
lease?.accountIndex ?? findCodexAccountIndexByAuthPath(rotatedAuthSrc),
|
||||||
|
...(lease ? { lease } : {}),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function upsertEjclawMcpServerSection(args: {
|
function upsertEjclawMcpServerSection(args: {
|
||||||
@@ -357,7 +400,7 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
useFailoverPromptPack: boolean;
|
useFailoverPromptPack: boolean;
|
||||||
memoryBriefing?: string;
|
memoryBriefing?: string;
|
||||||
skillOverrides?: StoredRoomSkillOverride[];
|
skillOverrides?: StoredRoomSkillOverride[];
|
||||||
}): void {
|
}): PreparedCodexSessionAuth | null {
|
||||||
// API key auth intentionally removed — Codex uses OAuth only.
|
// API key auth intentionally removed — Codex uses OAuth only.
|
||||||
// Never pass any API key to Codex child process to prevent API billing.
|
// Never pass any API key to Codex child process to prevent API billing.
|
||||||
delete args.env.OPENAI_API_KEY;
|
delete args.env.OPENAI_API_KEY;
|
||||||
@@ -376,7 +419,7 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
if (codexEffort) args.env.CODEX_EFFORT = codexEffort;
|
if (codexEffort) args.env.CODEX_EFFORT = codexEffort;
|
||||||
|
|
||||||
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
||||||
syncHostCodexSessionFiles(sessionCodexDir);
|
const codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||||
|
|
||||||
const overlayPath = path.join(args.groupDir, '.codex', 'config.toml');
|
const overlayPath = path.join(args.groupDir, '.codex', 'config.toml');
|
||||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||||
@@ -506,16 +549,19 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
delete args.env.ANTHROPIC_BASE_URL;
|
delete args.env.ANTHROPIC_BASE_URL;
|
||||||
delete args.env.CLAUDE_CODE_OAUTH_TOKEN;
|
delete args.env.CLAUDE_CODE_OAUTH_TOKEN;
|
||||||
args.env.CODEX_HOME = sessionCodexDir;
|
args.env.CODEX_HOME = sessionCodexDir;
|
||||||
|
return codexSessionAuth;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PreparedGroupEnvironment {
|
export interface PreparedGroupEnvironment {
|
||||||
env: Record<string, string>;
|
env: Record<string, string>;
|
||||||
groupDir: string;
|
groupDir: string;
|
||||||
runnerDir: string;
|
runnerDir: string;
|
||||||
|
codexSessionAuth?: PreparedCodexSessionAuth | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PreparedReadonlySessionEnvironment {
|
export interface PreparedReadonlySessionEnvironment {
|
||||||
codexHomeDir?: string;
|
codexHomeDir?: string;
|
||||||
|
codexSessionAuth?: PreparedCodexSessionAuth | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function prepareGroupEnvironment(
|
export function prepareGroupEnvironment(
|
||||||
@@ -657,8 +703,9 @@ export function prepareGroupEnvironment(
|
|||||||
runtimeTaskId,
|
runtimeTaskId,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
|
||||||
if (agentType === 'codex') {
|
if (agentType === 'codex') {
|
||||||
prepareCodexSessionEnvironment({
|
codexSessionAuth = prepareCodexSessionEnvironment({
|
||||||
env,
|
env,
|
||||||
envVars,
|
envVars,
|
||||||
projectRoot,
|
projectRoot,
|
||||||
@@ -677,7 +724,7 @@ export function prepareGroupEnvironment(
|
|||||||
prepareClaudeEnvironment({ env, envVars, group });
|
prepareClaudeEnvironment({ env, envVars, group });
|
||||||
}
|
}
|
||||||
|
|
||||||
return { env, groupDir, runnerDir };
|
return { env, groupDir, runnerDir, codexSessionAuth };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -716,6 +763,7 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
skillOverrides,
|
skillOverrides,
|
||||||
} = args;
|
} = args;
|
||||||
const projectRoot = process.cwd();
|
const projectRoot = process.cwd();
|
||||||
|
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
|
||||||
|
|
||||||
fs.mkdirSync(sessionDir, { recursive: true });
|
fs.mkdirSync(sessionDir, { recursive: true });
|
||||||
ensureClaudeSessionSettings(sessionDir);
|
ensureClaudeSessionSettings(sessionDir);
|
||||||
@@ -785,7 +833,7 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
if (sessionClaudeMd) {
|
if (sessionClaudeMd) {
|
||||||
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
||||||
const sessionCodexDir = path.join(sessionDir, '.codex');
|
const sessionCodexDir = path.join(sessionDir, '.codex');
|
||||||
syncHostCodexSessionFiles(sessionCodexDir);
|
codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
path.join(sessionCodexDir, 'AGENTS.md'),
|
path.join(sessionCodexDir, 'AGENTS.md'),
|
||||||
sessionClaudeMd + '\n',
|
sessionClaudeMd + '\n',
|
||||||
@@ -826,5 +874,5 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
} else if (fs.existsSync(sessionClaudeMdPath)) {
|
} else if (fs.existsSync(sessionClaudeMdPath)) {
|
||||||
fs.unlinkSync(sessionClaudeMdPath);
|
fs.unlinkSync(sessionClaudeMdPath);
|
||||||
}
|
}
|
||||||
return { codexHomeDir };
|
return { codexHomeDir, codexSessionAuth };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ import {
|
|||||||
import {
|
import {
|
||||||
prepareReadonlySessionEnvironment,
|
prepareReadonlySessionEnvironment,
|
||||||
prepareGroupEnvironment,
|
prepareGroupEnvironment,
|
||||||
|
type PreparedCodexSessionAuth,
|
||||||
} from './agent-runner-environment.js';
|
} from './agent-runner-environment.js';
|
||||||
|
import { syncCodexSessionAuthBack } from './codex-token-rotation.js';
|
||||||
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
||||||
import { getStoredRoomSkillOverrides } from './db.js';
|
import { getStoredRoomSkillOverrides } from './db.js';
|
||||||
export {
|
export {
|
||||||
@@ -76,6 +78,36 @@ function readRoomSkillOverridesForRunner(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function releaseCodexAuthSession(
|
||||||
|
auth: PreparedCodexSessionAuth | null | undefined,
|
||||||
|
): void {
|
||||||
|
auth?.lease?.release();
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizeCodexAuthSession(
|
||||||
|
auth: PreparedCodexSessionAuth | null | undefined,
|
||||||
|
): void {
|
||||||
|
if (!auth) return;
|
||||||
|
try {
|
||||||
|
syncCodexSessionAuthBack({
|
||||||
|
canonicalAuthPath: auth.canonicalAuthPath,
|
||||||
|
sessionAuthPath: auth.sessionAuthPath,
|
||||||
|
accountIndex: auth.accountIndex,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
err,
|
||||||
|
canonicalAuthPath: auth.canonicalAuthPath,
|
||||||
|
accountIndex: auth.accountIndex,
|
||||||
|
},
|
||||||
|
'Failed to sync Codex session auth back to canonical slot',
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
auth.lease?.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function runAgentProcess(
|
export async function runAgentProcess(
|
||||||
group: RegisteredGroup,
|
group: RegisteredGroup,
|
||||||
input: AgentInput,
|
input: AgentInput,
|
||||||
@@ -92,18 +124,15 @@ export async function runAgentProcess(
|
|||||||
// ── Host process mode (owner) ───────────────────────────────────
|
// ── Host process mode (owner) ───────────────────────────────────
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
||||||
const { env, groupDir, runnerDir } = prepareGroupEnvironment(
|
const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, {
|
||||||
group,
|
memoryBriefing: input.memoryBriefing,
|
||||||
input.isMain,
|
runtimeTaskId: input.runtimeTaskId,
|
||||||
input.chatJid,
|
useTaskScopedSession: input.useTaskScopedSession,
|
||||||
{
|
skillOverrides,
|
||||||
memoryBriefing: input.memoryBriefing,
|
roomRole: input.roomRoleContext?.role,
|
||||||
runtimeTaskId: input.runtimeTaskId,
|
});
|
||||||
useTaskScopedSession: input.useTaskScopedSession,
|
const { env, groupDir, runnerDir } = prepared;
|
||||||
skillOverrides,
|
let codexSessionAuth = prepared.codexSessionAuth;
|
||||||
roomRole: input.roomRoleContext?.role,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// Apply env overrides (caller-provided)
|
// Apply env overrides (caller-provided)
|
||||||
if (envOverrides) {
|
if (envOverrides) {
|
||||||
@@ -131,6 +160,8 @@ export async function runAgentProcess(
|
|||||||
skillOverrides,
|
skillOverrides,
|
||||||
});
|
});
|
||||||
if ((group.agentType || 'claude-code') === 'codex') {
|
if ((group.agentType || 'claude-code') === 'codex') {
|
||||||
|
releaseCodexAuthSession(codexSessionAuth);
|
||||||
|
codexSessionAuth = readonlySession.codexSessionAuth;
|
||||||
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
|
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
|
||||||
if (readonlySession.codexHomeDir) {
|
if (readonlySession.codexHomeDir) {
|
||||||
env.HOME = readonlySession.codexHomeDir;
|
env.HOME = readonlySession.codexHomeDir;
|
||||||
@@ -152,6 +183,7 @@ export async function runAgentProcess(
|
|||||||
{ runnerDir, chatJid: input.chatJid, runId: input.runId },
|
{ runnerDir, chatJid: input.chatJid, runId: input.runId },
|
||||||
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
|
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
|
||||||
);
|
);
|
||||||
|
releaseCodexAuthSession(codexSessionAuth);
|
||||||
return {
|
return {
|
||||||
status: 'error',
|
status: 'error',
|
||||||
result: null,
|
result: null,
|
||||||
@@ -199,6 +231,22 @@ export async function runAgentProcess(
|
|||||||
logsDir,
|
logsDir,
|
||||||
startTime,
|
startTime,
|
||||||
onOutput,
|
onOutput,
|
||||||
}).then(resolve);
|
})
|
||||||
|
.then((output) => {
|
||||||
|
finalizeCodexAuthSession(codexSessionAuth);
|
||||||
|
resolve(output);
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
finalizeCodexAuthSession(codexSessionAuth);
|
||||||
|
logger.error(
|
||||||
|
{ err, processName, chatJid: input.chatJid, runId: input.runId },
|
||||||
|
'Spawned agent process runner failed',
|
||||||
|
);
|
||||||
|
resolve({
|
||||||
|
status: 'error',
|
||||||
|
result: null,
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,6 +165,192 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
|||||||
expect(mod.getCodexAccountCount()).toBe(4);
|
expect(mod.getCodexAccountCount()).toBe(4);
|
||||||
expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath);
|
expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('marks refresh-token reuse as dead auth instead of a recoverable cooldown', async () => {
|
||||||
|
const agentErrors = await import('./agent-error-detection.js');
|
||||||
|
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
|
||||||
|
|
||||||
|
const accounts = mod.getAllCodexAccounts();
|
||||||
|
expect(accounts[0]).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
isAuthDead: true,
|
||||||
|
authStatus: 'dead_auth',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(accounts[0].isRateLimited).toBe(false);
|
||||||
|
expect(accounts[1].isActive).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('recovers a dead_auth account when canonical auth.json is refreshed before lease claim', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
|
||||||
|
try {
|
||||||
|
const agentErrors = await import('./agent-error-detection.js');
|
||||||
|
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
const utils = await import('./utils.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
|
||||||
|
expect(mod.getAllCodexAccounts()[0].isAuthDead).toBe(true);
|
||||||
|
|
||||||
|
const refreshedAt = new Date('2026-01-01T00:00:10.000Z');
|
||||||
|
const authPath = path.join(tempHome, '.codex-accounts', '0', 'auth.json');
|
||||||
|
fs.writeFileSync(
|
||||||
|
authPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: { account_id: 'acct-0', access_token: 'refreshed-token' },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
fs.utimesSync(authPath, refreshedAt, refreshedAt);
|
||||||
|
|
||||||
|
mod.setCurrentCodexAccountIndex(0);
|
||||||
|
const lease = mod.claimCodexAuthLease();
|
||||||
|
try {
|
||||||
|
expect(lease).toEqual(
|
||||||
|
expect.objectContaining({ accountIndex: 0, authPath }),
|
||||||
|
);
|
||||||
|
expect(mod.getAllCodexAccounts()[0]).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
authStatus: 'healthy',
|
||||||
|
isAuthDead: false,
|
||||||
|
isActive: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(vi.mocked(utils.writeJsonFile)).toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
lease?.release();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('leases different accounts for concurrent Codex runs', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
const first = mod.claimCodexAuthLease();
|
||||||
|
const second = mod.claimCodexAuthLease();
|
||||||
|
|
||||||
|
try {
|
||||||
|
expect(first).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
accountIndex: 0,
|
||||||
|
authPath: expect.any(String),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(second).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
accountIndex: 1,
|
||||||
|
authPath: expect.any(String),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(second?.authPath).not.toBe(first?.authPath);
|
||||||
|
} finally {
|
||||||
|
second?.release();
|
||||||
|
first?.release();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('syncs a refreshed session auth.json back to the canonical slot atomically', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
const canonicalAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'.codex-accounts',
|
||||||
|
'0',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
const sessionAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'session',
|
||||||
|
'.codex',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
sessionAuthPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: {
|
||||||
|
account_id: 'acct-0',
|
||||||
|
access_token: 'new-access',
|
||||||
|
refresh_token: 'new-refresh',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const synced = mod.syncCodexSessionAuthBack({
|
||||||
|
canonicalAuthPath,
|
||||||
|
sessionAuthPath,
|
||||||
|
accountIndex: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(synced).toBe(true);
|
||||||
|
const canonical = JSON.parse(fs.readFileSync(canonicalAuthPath, 'utf-8'));
|
||||||
|
expect(canonical.tokens).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
account_id: 'acct-0',
|
||||||
|
access_token: 'new-access',
|
||||||
|
refresh_token: 'new-refresh',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('refuses to sync a session auth.json that belongs to another Codex account', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
const canonicalAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'.codex-accounts',
|
||||||
|
'0',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
const before = fs.readFileSync(canonicalAuthPath, 'utf-8');
|
||||||
|
const sessionAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'wrong-account',
|
||||||
|
'.codex',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
sessionAuthPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: {
|
||||||
|
account_id: 'acct-other',
|
||||||
|
access_token: 'other-access',
|
||||||
|
refresh_token: 'other-refresh',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
mod.syncCodexSessionAuthBack({
|
||||||
|
canonicalAuthPath,
|
||||||
|
sessionAuthPath,
|
||||||
|
accountIndex: 0,
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
|
expect(fs.readFileSync(canonicalAuthPath, 'utf-8')).toBe(before);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('codex-token-rotation single-account fallback', () => {
|
describe('codex-token-rotation single-account fallback', () => {
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ import { DATA_DIR } from './config.js';
|
|||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import {
|
import {
|
||||||
computeCooldownUntil,
|
computeCooldownUntil,
|
||||||
findNextAvailable,
|
|
||||||
parseRetryAfterFromError,
|
parseRetryAfterFromError,
|
||||||
} from './token-rotation-base.js';
|
} from './token-rotation-base.js';
|
||||||
import { readJsonFile, writeJsonFile } from './utils.js';
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
@@ -34,15 +33,27 @@ interface CodexAccount {
|
|||||||
index: number;
|
index: number;
|
||||||
authPath: string;
|
authPath: string;
|
||||||
accountId: string;
|
accountId: string;
|
||||||
|
authFileMtimeMs: number;
|
||||||
planType: string;
|
planType: string;
|
||||||
subscriptionUntil: string | null;
|
subscriptionUntil: string | null;
|
||||||
rateLimitedUntil: number | null;
|
rateLimitedUntil: number | null;
|
||||||
|
authStatus: 'healthy' | 'dead_auth';
|
||||||
|
authDeadAt: number | null;
|
||||||
|
authDeadReason: string | null;
|
||||||
|
leasedUntil: number | null;
|
||||||
|
leaseId: string | null;
|
||||||
lastUsagePct?: number;
|
lastUsagePct?: number;
|
||||||
lastUsageD7Pct?: number;
|
lastUsageD7Pct?: number;
|
||||||
resetAt?: string;
|
resetAt?: string;
|
||||||
resetD7At?: string;
|
resetD7At?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CodexAuthLease {
|
||||||
|
accountIndex: number;
|
||||||
|
authPath: string;
|
||||||
|
release: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
export type CodexRotationTriggerResult =
|
export type CodexRotationTriggerResult =
|
||||||
| {
|
| {
|
||||||
shouldRotate: false;
|
shouldRotate: false;
|
||||||
@@ -77,10 +88,20 @@ const accounts: CodexAccount[] = [];
|
|||||||
let currentIndex = 0;
|
let currentIndex = 0;
|
||||||
let initialized = false;
|
let initialized = false;
|
||||||
|
|
||||||
|
const CODEX_AUTH_LEASE_MS = 2 * 60 * 60_000;
|
||||||
|
|
||||||
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||||
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
||||||
const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json');
|
const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json');
|
||||||
|
|
||||||
|
function readAuthFileMtimeMs(authPath: string): number {
|
||||||
|
try {
|
||||||
|
return fs.statSync(authPath).mtimeMs;
|
||||||
|
} catch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function loadCodexAccount(
|
function loadCodexAccount(
|
||||||
authPath: string,
|
authPath: string,
|
||||||
fallbackAccountId: string,
|
fallbackAccountId: string,
|
||||||
@@ -99,9 +120,15 @@ function loadCodexAccount(
|
|||||||
index: accounts.length,
|
index: accounts.length,
|
||||||
authPath,
|
authPath,
|
||||||
accountId,
|
accountId,
|
||||||
|
authFileMtimeMs: readAuthFileMtimeMs(authPath),
|
||||||
planType: jwt.planType,
|
planType: jwt.planType,
|
||||||
subscriptionUntil: jwt.expiresAt,
|
subscriptionUntil: jwt.expiresAt,
|
||||||
rateLimitedUntil: null,
|
rateLimitedUntil: null,
|
||||||
|
authStatus: 'healthy',
|
||||||
|
authDeadAt: null,
|
||||||
|
authDeadReason: null,
|
||||||
|
leasedUntil: null,
|
||||||
|
leaseId: null,
|
||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -149,6 +176,8 @@ function saveCodexState(): void {
|
|||||||
const state = {
|
const state = {
|
||||||
currentIndex,
|
currentIndex,
|
||||||
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
||||||
|
authDeadAts: accounts.map((a) => a.authDeadAt),
|
||||||
|
authDeadReasons: accounts.map((a) => a.authDeadReason),
|
||||||
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
|
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
|
||||||
usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null),
|
usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null),
|
||||||
resetAts: accounts.map((a) => a.resetAt ?? null),
|
resetAts: accounts.map((a) => a.resetAt ?? null),
|
||||||
@@ -167,6 +196,8 @@ function loadCodexState(quiet = false): void {
|
|||||||
const state = readJsonFile<{
|
const state = readJsonFile<{
|
||||||
currentIndex?: number;
|
currentIndex?: number;
|
||||||
rateLimits?: (number | null)[];
|
rateLimits?: (number | null)[];
|
||||||
|
authDeadAts?: (number | null)[];
|
||||||
|
authDeadReasons?: (string | null)[];
|
||||||
usagePcts?: (number | null)[];
|
usagePcts?: (number | null)[];
|
||||||
usageD7Pcts?: (number | null)[];
|
usageD7Pcts?: (number | null)[];
|
||||||
resetAts?: (string | null)[];
|
resetAts?: (string | null)[];
|
||||||
@@ -175,6 +206,7 @@ function loadCodexState(quiet = false): void {
|
|||||||
if (!state) return;
|
if (!state) return;
|
||||||
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
let restoredDeadAuth = false;
|
||||||
if (
|
if (
|
||||||
typeof state.currentIndex === 'number' &&
|
typeof state.currentIndex === 'number' &&
|
||||||
state.currentIndex < accounts.length
|
state.currentIndex < accounts.length
|
||||||
@@ -195,6 +227,39 @@ function loadCodexState(quiet = false): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (Array.isArray(state.authDeadAts)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.authDeadAts.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
const deadAt = state.authDeadAts[i];
|
||||||
|
const acct = accounts[i];
|
||||||
|
if (typeof deadAt === 'number' && deadAt > 0) {
|
||||||
|
const currentMtime = readAuthFileMtimeMs(acct.authPath);
|
||||||
|
acct.authFileMtimeMs = currentMtime;
|
||||||
|
if (currentMtime > deadAt) {
|
||||||
|
acct.authStatus = 'dead_auth';
|
||||||
|
acct.authDeadAt = deadAt;
|
||||||
|
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
|
||||||
|
restoredDeadAuth =
|
||||||
|
restoreDeadAuthIfAuthFileChanged(acct) || restoredDeadAuth;
|
||||||
|
} else {
|
||||||
|
acct.authStatus = 'dead_auth';
|
||||||
|
acct.authDeadAt = deadAt;
|
||||||
|
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
currentIndex < accounts.length &&
|
||||||
|
accounts[currentIndex]?.authStatus === 'dead_auth'
|
||||||
|
) {
|
||||||
|
const nextIdx = findNextCodexAvailable(currentIndex);
|
||||||
|
if (nextIdx !== null) currentIndex = nextIdx;
|
||||||
|
}
|
||||||
if (Array.isArray(state.usagePcts)) {
|
if (Array.isArray(state.usagePcts)) {
|
||||||
for (
|
for (
|
||||||
let i = 0;
|
let i = 0;
|
||||||
@@ -239,6 +304,7 @@ function loadCodexState(quiet = false): void {
|
|||||||
'Codex rotation state restored',
|
'Codex rotation state restored',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (restoredDeadAuth) saveCodexState();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -309,6 +375,265 @@ export function getCodexAuthPath(
|
|||||||
return accounts[accountIndex]?.authPath ?? null;
|
return accounts[accountIndex]?.authPath ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function codexLockPath(authPath: string): string {
|
||||||
|
return path.join(path.dirname(authPath), '.ejclaw-auth.lock');
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPidAlive(pid: number): boolean {
|
||||||
|
try {
|
||||||
|
process.kill(pid, 0);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readExistingLease(lockPath: string): {
|
||||||
|
leaseId?: string;
|
||||||
|
pid?: number;
|
||||||
|
expiresAt?: number;
|
||||||
|
} | null {
|
||||||
|
const data = readJsonFile<{
|
||||||
|
leaseId?: string;
|
||||||
|
pid?: number;
|
||||||
|
expiresAt?: number;
|
||||||
|
}>(lockPath);
|
||||||
|
return data ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tryAcquireDiskLease(
|
||||||
|
acct: CodexAccount,
|
||||||
|
leaseId: string,
|
||||||
|
now: number,
|
||||||
|
): boolean {
|
||||||
|
const lockPath = codexLockPath(acct.authPath);
|
||||||
|
const payload = JSON.stringify(
|
||||||
|
{
|
||||||
|
leaseId,
|
||||||
|
pid: process.pid,
|
||||||
|
accountIndex: acct.index,
|
||||||
|
createdAt: now,
|
||||||
|
expiresAt: now + CODEX_AUTH_LEASE_MS,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = readExistingLease(lockPath);
|
||||||
|
const expired = Boolean(existing?.expiresAt && existing.expiresAt <= now);
|
||||||
|
const deadPid = Boolean(existing?.pid && !isPidAlive(existing.pid));
|
||||||
|
if (!expired && !deadPid) return false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function releaseDiskLease(authPath: string, leaseId: string): void {
|
||||||
|
const lockPath = codexLockPath(authPath);
|
||||||
|
const existing = readExistingLease(lockPath);
|
||||||
|
if (existing?.leaseId && existing.leaseId !== leaseId) return;
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
} catch {
|
||||||
|
// Best effort only. Expired/stale locks are cleared by the next claimant.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isCodexAccountUsable(
|
||||||
|
acct: CodexAccount,
|
||||||
|
now = Date.now(),
|
||||||
|
opts?: { ignoreRateLimits?: boolean; ignoreD7?: boolean },
|
||||||
|
): boolean {
|
||||||
|
if (restoreDeadAuthIfAuthFileChanged(acct)) saveCodexState();
|
||||||
|
if (acct.authStatus === 'dead_auth') return false;
|
||||||
|
if (
|
||||||
|
!opts?.ignoreRateLimits &&
|
||||||
|
acct.rateLimitedUntil &&
|
||||||
|
acct.rateLimitedUntil > now
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!opts?.ignoreD7 &&
|
||||||
|
acct.lastUsageD7Pct != null &&
|
||||||
|
acct.lastUsageD7Pct >= 100
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (acct.leasedUntil && acct.leasedUntil > now) return false;
|
||||||
|
|
||||||
|
const existing = readExistingLease(codexLockPath(acct.authPath));
|
||||||
|
if (!existing) return true;
|
||||||
|
if (existing.expiresAt && existing.expiresAt <= now) return true;
|
||||||
|
if (existing.pid && !isPidAlive(existing.pid)) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function restoreDeadAuthIfAuthFileChanged(acct: CodexAccount): boolean {
|
||||||
|
if (acct.authStatus !== 'dead_auth' || !acct.authDeadAt) return false;
|
||||||
|
const currentMtime = readAuthFileMtimeMs(acct.authPath);
|
||||||
|
acct.authFileMtimeMs = currentMtime;
|
||||||
|
if (currentMtime <= acct.authDeadAt) return false;
|
||||||
|
|
||||||
|
const previousDeadAt = acct.authDeadAt;
|
||||||
|
acct.authStatus = 'healthy';
|
||||||
|
acct.authDeadAt = null;
|
||||||
|
acct.authDeadReason = null;
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
transition: 'rotation:auth-file-refreshed',
|
||||||
|
accountIndex: acct.index,
|
||||||
|
previousDeadAt: new Date(previousDeadAt).toISOString(),
|
||||||
|
authFileMtime: new Date(currentMtime).toISOString(),
|
||||||
|
},
|
||||||
|
`Codex account #${acct.index + 1}/${accounts.length} marked healthy after auth.json refresh`,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function markCodexAccountDeadAuth(acct: CodexAccount, reason?: string): void {
|
||||||
|
acct.authStatus = 'dead_auth';
|
||||||
|
acct.authDeadAt = Date.now();
|
||||||
|
acct.authDeadReason = reason || 'auth-expired';
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
acct.leasedUntil = null;
|
||||||
|
acct.leaseId = null;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
transition: 'rotation:dead-auth',
|
||||||
|
accountIndex: acct.index,
|
||||||
|
accountId: acct.accountId,
|
||||||
|
reason: acct.authDeadReason,
|
||||||
|
},
|
||||||
|
`Codex account #${acct.index + 1}/${accounts.length} marked dead; re-auth required`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateAccountMetadataFromAuthFile(acct: CodexAccount): void {
|
||||||
|
const data = readJsonFile<{
|
||||||
|
tokens?: { account_id?: string; id_token?: string };
|
||||||
|
}>(acct.authPath);
|
||||||
|
if (data?.tokens?.account_id) acct.accountId = data.tokens.account_id;
|
||||||
|
const jwt = parseJwtAuth(data?.tokens?.id_token || '');
|
||||||
|
acct.planType = jwt.planType;
|
||||||
|
acct.subscriptionUntil = jwt.expiresAt;
|
||||||
|
acct.authFileMtimeMs = readAuthFileMtimeMs(acct.authPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function claimCodexAuthLease(): CodexAuthLease | null {
|
||||||
|
if (accounts.length === 0) return null;
|
||||||
|
const now = Date.now();
|
||||||
|
const attempts = accounts.length;
|
||||||
|
for (let offset = 0; offset < attempts; offset += 1) {
|
||||||
|
const idx = (currentIndex + offset) % accounts.length;
|
||||||
|
const acct = accounts[idx];
|
||||||
|
if (!isCodexAccountUsable(acct, now)) continue;
|
||||||
|
const leaseId = `${process.pid}-${now}-${idx}-${Math.random()
|
||||||
|
.toString(36)
|
||||||
|
.slice(2)}`;
|
||||||
|
if (!tryAcquireDiskLease(acct, leaseId, now)) continue;
|
||||||
|
currentIndex = idx;
|
||||||
|
acct.leasedUntil = now + CODEX_AUTH_LEASE_MS;
|
||||||
|
acct.leaseId = leaseId;
|
||||||
|
return {
|
||||||
|
accountIndex: idx,
|
||||||
|
authPath: acct.authPath,
|
||||||
|
release: () => {
|
||||||
|
if (acct.leaseId === leaseId) {
|
||||||
|
acct.leasedUntil = null;
|
||||||
|
acct.leaseId = null;
|
||||||
|
}
|
||||||
|
releaseDiskLease(acct.authPath, leaseId);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function syncCodexSessionAuthBack(args: {
|
||||||
|
canonicalAuthPath: string;
|
||||||
|
sessionAuthPath: string;
|
||||||
|
accountIndex?: number | null;
|
||||||
|
}): boolean {
|
||||||
|
if (!fs.existsSync(args.sessionAuthPath)) return false;
|
||||||
|
if (!fs.existsSync(args.canonicalAuthPath)) return false;
|
||||||
|
|
||||||
|
const canonical = readJsonFile<{
|
||||||
|
auth_mode?: string;
|
||||||
|
tokens?: { account_id?: string };
|
||||||
|
}>(args.canonicalAuthPath);
|
||||||
|
const session = readJsonFile<{
|
||||||
|
auth_mode?: string;
|
||||||
|
tokens?: { account_id?: string };
|
||||||
|
}>(args.sessionAuthPath);
|
||||||
|
if (!canonical || !session?.tokens) return false;
|
||||||
|
|
||||||
|
const canonicalAccount = canonical.tokens?.account_id;
|
||||||
|
const sessionAccount = session.tokens.account_id;
|
||||||
|
if (
|
||||||
|
canonicalAccount &&
|
||||||
|
sessionAccount &&
|
||||||
|
canonicalAccount !== sessionAccount
|
||||||
|
) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
canonicalAuthPath: args.canonicalAuthPath,
|
||||||
|
sessionAuthPath: args.sessionAuthPath,
|
||||||
|
accountIndex: args.accountIndex ?? null,
|
||||||
|
},
|
||||||
|
'Refusing to sync Codex session auth back: account mismatch',
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionRaw = fs.readFileSync(args.sessionAuthPath, 'utf-8');
|
||||||
|
const canonicalRaw = fs.readFileSync(args.canonicalAuthPath, 'utf-8');
|
||||||
|
if (sessionRaw === canonicalRaw) return false;
|
||||||
|
|
||||||
|
const tmpPath = `${args.canonicalAuthPath}.tmp-${process.pid}-${Date.now()}`;
|
||||||
|
fs.writeFileSync(tmpPath, sessionRaw, { mode: 0o600 });
|
||||||
|
fs.renameSync(tmpPath, args.canonicalAuthPath);
|
||||||
|
|
||||||
|
const idx =
|
||||||
|
args.accountIndex ??
|
||||||
|
findCodexAccountIndexByAuthPath(args.canonicalAuthPath);
|
||||||
|
if (idx != null && accounts[idx]) {
|
||||||
|
const acct = accounts[idx];
|
||||||
|
acct.authStatus = 'healthy';
|
||||||
|
acct.authDeadAt = null;
|
||||||
|
acct.authDeadReason = null;
|
||||||
|
updateAccountMetadataFromAuthFile(acct);
|
||||||
|
saveCodexState();
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
accountIndex: idx,
|
||||||
|
canonicalAuthPath: args.canonicalAuthPath,
|
||||||
|
},
|
||||||
|
'Synced refreshed Codex session auth back to canonical account slot',
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
export function detectCodexRotationTrigger(
|
export function detectCodexRotationTrigger(
|
||||||
error?: string | null,
|
error?: string | null,
|
||||||
): CodexRotationTriggerResult {
|
): CodexRotationTriggerResult {
|
||||||
@@ -341,16 +666,23 @@ export function rotateCodexToken(
|
|||||||
|
|
||||||
const previousIndex = currentIndex;
|
const previousIndex = currentIndex;
|
||||||
const acct = accounts[currentIndex];
|
const acct = accounts[currentIndex];
|
||||||
const cooldownUntil = computeCooldownUntil(errorMessage);
|
const authFailure = classifyCodexAuthError(errorMessage);
|
||||||
acct.rateLimitedUntil = cooldownUntil;
|
let cooldownUntil: number | null = null;
|
||||||
acct.lastUsagePct = 100;
|
|
||||||
// Extract reset time string from error for display
|
if (authFailure.category === 'auth-expired') {
|
||||||
const retryAt = parseRetryAfterFromError(errorMessage);
|
markCodexAccountDeadAuth(acct, errorMessage || authFailure.reason);
|
||||||
if (retryAt) {
|
} else {
|
||||||
acct.resetAt = new Date(retryAt).toISOString();
|
cooldownUntil = computeCooldownUntil(errorMessage);
|
||||||
|
acct.rateLimitedUntil = cooldownUntil;
|
||||||
|
acct.lastUsagePct = 100;
|
||||||
|
// Extract reset time string from error for display
|
||||||
|
const retryAt = parseRetryAfterFromError(errorMessage);
|
||||||
|
if (retryAt) {
|
||||||
|
acct.resetAt = new Date(retryAt).toISOString();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const nextIdx = findNextAvailable(accounts, currentIndex, opts);
|
const nextIdx = findNextCodexAvailable(currentIndex, opts);
|
||||||
if (nextIdx !== null) {
|
if (nextIdx !== null) {
|
||||||
accounts[nextIdx].rateLimitedUntil = null;
|
accounts[nextIdx].rateLimitedUntil = null;
|
||||||
currentIndex = nextIdx;
|
currentIndex = nextIdx;
|
||||||
@@ -364,6 +696,7 @@ export function rotateCodexToken(
|
|||||||
ignoreRL: opts?.ignoreRateLimits ?? false,
|
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||||
cooldownUntil:
|
cooldownUntil:
|
||||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||||
|
authDead: authFailure.category === 'auth-expired',
|
||||||
reason: errorMessage ?? null,
|
reason: errorMessage ?? null,
|
||||||
},
|
},
|
||||||
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
||||||
@@ -380,28 +713,40 @@ export function rotateCodexToken(
|
|||||||
ignoreRL: opts?.ignoreRateLimits ?? false,
|
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||||
cooldownUntil:
|
cooldownUntil:
|
||||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||||
|
authDead: authFailure.category === 'auth-expired',
|
||||||
reason: errorMessage ?? null,
|
reason: errorMessage ?? null,
|
||||||
},
|
},
|
||||||
'All Codex accounts are rate-limited',
|
authFailure.category === 'auth-expired'
|
||||||
|
? 'All Codex accounts unavailable after auth failure; re-auth required'
|
||||||
|
: 'All Codex accounts are rate-limited',
|
||||||
);
|
);
|
||||||
|
saveCodexState();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
||||||
*/
|
*/
|
||||||
function findNextCodexAvailable(fromIndex?: number): number | null {
|
function findNextCodexAvailable(
|
||||||
|
fromIndex?: number,
|
||||||
|
opts?: { ignoreRateLimits?: boolean },
|
||||||
|
): number | null {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const start = fromIndex ?? currentIndex;
|
const start = fromIndex ?? currentIndex;
|
||||||
for (let i = 1; i < accounts.length; i++) {
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
const idx = (start + i) % accounts.length;
|
const idx = (start + i) % accounts.length;
|
||||||
const acct = accounts[idx];
|
const acct = accounts[idx];
|
||||||
const rlOk = !acct.rateLimitedUntil || acct.rateLimitedUntil <= now;
|
if (isCodexAccountUsable(acct, now, opts)) return idx;
|
||||||
const usageOk = acct.lastUsageD7Pct == null || acct.lastUsageD7Pct < 100;
|
|
||||||
if (rlOk && usageOk) return idx;
|
|
||||||
}
|
}
|
||||||
// All exhausted — fall back to rate-limit-only check
|
// All d7-exhausted — fall back to rate-limit/dead/lease checks only.
|
||||||
return findNextAvailable(accounts, start);
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
|
const idx = (start + i) % accounts.length;
|
||||||
|
const acct = accounts[idx];
|
||||||
|
if (isCodexAccountUsable(acct, now, { ...opts, ignoreD7: true })) {
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -469,9 +814,17 @@ export function updateCodexAccountUsage(
|
|||||||
export function markCodexTokenHealthy(): void {
|
export function markCodexTokenHealthy(): void {
|
||||||
if (accounts.length === 0) return;
|
if (accounts.length === 0) return;
|
||||||
const acct = accounts[currentIndex];
|
const acct = accounts[currentIndex];
|
||||||
|
let changed = false;
|
||||||
|
if (acct?.authStatus === 'dead_auth') {
|
||||||
|
acct.authStatus = 'healthy';
|
||||||
|
acct.authDeadAt = null;
|
||||||
|
acct.authDeadReason = null;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
if (acct?.rateLimitedUntil) {
|
if (acct?.rateLimitedUntil) {
|
||||||
const previousCooldownUntil = acct.rateLimitedUntil;
|
const previousCooldownUntil = acct.rateLimitedUntil;
|
||||||
acct.rateLimitedUntil = null;
|
acct.rateLimitedUntil = null;
|
||||||
|
changed = true;
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
transition: 'rotation:clear-rate-limit',
|
transition: 'rotation:clear-rate-limit',
|
||||||
@@ -481,8 +834,8 @@ export function markCodexTokenHealthy(): void {
|
|||||||
},
|
},
|
||||||
'Cleared Codex account rate-limit state after successful response',
|
'Cleared Codex account rate-limit state after successful response',
|
||||||
);
|
);
|
||||||
saveCodexState();
|
|
||||||
}
|
}
|
||||||
|
if (changed) saveCodexState();
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getCodexAccountCount(): number {
|
export function getCodexAccountCount(): number {
|
||||||
@@ -495,6 +848,10 @@ export function getAllCodexAccounts(): {
|
|||||||
planType: string;
|
planType: string;
|
||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
isRateLimited: boolean;
|
isRateLimited: boolean;
|
||||||
|
isAuthDead?: boolean;
|
||||||
|
authStatus?: 'healthy' | 'dead_auth';
|
||||||
|
authDeadAt?: string;
|
||||||
|
isLeased?: boolean;
|
||||||
cachedUsagePct?: number;
|
cachedUsagePct?: number;
|
||||||
cachedUsageD7Pct?: number;
|
cachedUsageD7Pct?: number;
|
||||||
resetAt?: string;
|
resetAt?: string;
|
||||||
@@ -507,6 +864,10 @@ export function getAllCodexAccounts(): {
|
|||||||
planType: a.planType,
|
planType: a.planType,
|
||||||
isActive: i === currentIndex,
|
isActive: i === currentIndex,
|
||||||
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
||||||
|
isAuthDead: a.authStatus === 'dead_auth',
|
||||||
|
authStatus: a.authStatus,
|
||||||
|
authDeadAt: a.authDeadAt ? new Date(a.authDeadAt).toISOString() : undefined,
|
||||||
|
isLeased: Boolean(a.leasedUntil && a.leasedUntil > now),
|
||||||
cachedUsagePct: a.lastUsagePct,
|
cachedUsagePct: a.lastUsagePct,
|
||||||
cachedUsageD7Pct: a.lastUsageD7Pct,
|
cachedUsageD7Pct: a.lastUsageD7Pct,
|
||||||
resetAt: a.resetAt,
|
resetAt: a.resetAt,
|
||||||
|
|||||||
71
src/message-agent-executor-attempt-runner.test.ts
Normal file
71
src/message-agent-executor-attempt-runner.test.ts
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const mockRunAgentProcess = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock('./agent-runner.js', () => ({
|
||||||
|
runAgentProcess: mockRunAgentProcess,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
|
getCodexAccountCount: vi.fn(() => 2),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
|
||||||
|
import type { AgentOutput } from './agent-runner.js';
|
||||||
|
|
||||||
|
describe('runMessageAgentAttempt', () => {
|
||||||
|
it('stores pre-stream Codex launch errors in paired summary', async () => {
|
||||||
|
const error = new Error(
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
);
|
||||||
|
mockRunAgentProcess.mockRejectedValueOnce(error);
|
||||||
|
const updateSummary = vi.fn();
|
||||||
|
|
||||||
|
const attempt = await runMessageAgentAttempt({
|
||||||
|
provider: 'codex',
|
||||||
|
currentSessionId: undefined,
|
||||||
|
isClaudeCodeAgent: false,
|
||||||
|
canRetryClaudeCredentials: false,
|
||||||
|
shouldPersistSession: false,
|
||||||
|
effectiveGroup: {
|
||||||
|
name: 'Test',
|
||||||
|
folder: 'test',
|
||||||
|
trigger: '@test',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
},
|
||||||
|
agentInput: {
|
||||||
|
prompt: 'arbiter prompt',
|
||||||
|
groupFolder: 'test',
|
||||||
|
chatJid: 'dc:test',
|
||||||
|
runId: 'run-test',
|
||||||
|
isMain: false,
|
||||||
|
assistantName: 'Andy',
|
||||||
|
},
|
||||||
|
activeRole: 'arbiter',
|
||||||
|
effectiveServiceId: 'codex-review',
|
||||||
|
effectiveAgentType: 'codex',
|
||||||
|
sessionFolder: 'test:arbiter',
|
||||||
|
onPersistSession: vi.fn(),
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
onOutput: vi.fn(async (_output: AgentOutput) => undefined),
|
||||||
|
pairedExecutionLifecycle: {
|
||||||
|
updateSummary,
|
||||||
|
recordFinalOutputBeforeDelivery: vi.fn(() => false),
|
||||||
|
},
|
||||||
|
log: {
|
||||||
|
child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(attempt.error).toBe(error);
|
||||||
|
expect(attempt.sawOutput).toBe(false);
|
||||||
|
expect(updateSummary).toHaveBeenCalledWith({
|
||||||
|
errorText:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -14,6 +14,7 @@ import {
|
|||||||
shouldResetCodexSessionOnAgentFailure,
|
shouldResetCodexSessionOnAgentFailure,
|
||||||
shouldResetSessionOnAgentFailure,
|
shouldResetSessionOnAgentFailure,
|
||||||
} from './session-recovery.js';
|
} from './session-recovery.js';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
import type {
|
import type {
|
||||||
AgentType,
|
AgentType,
|
||||||
OutboundAttachment,
|
OutboundAttachment,
|
||||||
@@ -162,6 +163,9 @@ class MessageAgentAttemptRunner {
|
|||||||
);
|
);
|
||||||
return this.buildAttempt({ output });
|
return this.buildAttempt({ output });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
this.args.pairedExecutionLifecycle.updateSummary({
|
||||||
|
errorText: getErrorMessage(error),
|
||||||
|
});
|
||||||
return this.buildAttempt({ error });
|
return this.buildAttempt({ error });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -505,8 +505,11 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
|||||||
if (!missingVisibleVerdict) {
|
if (!missingVisibleVerdict) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
this.pairedExecutionSummary =
|
const noVerdictSummary =
|
||||||
'Execution completed without a visible terminal verdict.';
|
'Execution completed without a visible terminal verdict.';
|
||||||
|
this.pairedExecutionSummary = this.pairedExecutionSummary
|
||||||
|
? `${this.pairedExecutionSummary}\n${noVerdictSummary}`
|
||||||
|
: noVerdictSummary;
|
||||||
log.warn(
|
log.warn(
|
||||||
{
|
{
|
||||||
pairedTaskId: pairedExecutionContext?.task.id ?? null,
|
pairedTaskId: pairedExecutionContext?.task.id ?? null,
|
||||||
|
|||||||
@@ -172,6 +172,19 @@ describe('message-agent-executor-rules', () => {
|
|||||||
).toBe('pending');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not requeue a silent arbiter failure caused by Codex account auth expiry', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'arbiter',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'arbiter_requested',
|
||||||
|
outputSummary:
|
||||||
|
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.\nExecution completed without a visible terminal verdict.',
|
||||||
|
}),
|
||||||
|
).toBe('none');
|
||||||
|
});
|
||||||
|
|
||||||
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
||||||
expect(
|
expect(
|
||||||
resolvePairedFollowUpQueueAction({
|
resolvePairedFollowUpQueueAction({
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import {
|
|||||||
resolveNextTurnAction,
|
resolveNextTurnAction,
|
||||||
shouldRetrySilentOwnerExecution,
|
shouldRetrySilentOwnerExecution,
|
||||||
} from './message-runtime-rules.js';
|
} from './message-runtime-rules.js';
|
||||||
|
import { classifyCodexAuthError } from './agent-error-detection.js';
|
||||||
import { parseVisibleVerdict } from './paired-verdict.js';
|
import { parseVisibleVerdict } from './paired-verdict.js';
|
||||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||||
export {
|
export {
|
||||||
@@ -17,6 +18,17 @@ export {
|
|||||||
|
|
||||||
export type PairedFollowUpQueueAction = 'pending' | 'none';
|
export type PairedFollowUpQueueAction = 'pending' | 'none';
|
||||||
|
|
||||||
|
function isSilentCodexAccountFailure(summary?: string | null): boolean {
|
||||||
|
if (!summary) return false;
|
||||||
|
if (classifyCodexAuthError(summary).category !== 'none') return true;
|
||||||
|
const lower = summary.toLowerCase();
|
||||||
|
return (
|
||||||
|
lower.includes('workspace out of credits') ||
|
||||||
|
lower.includes('out of credits') ||
|
||||||
|
/all\s+codex(?:\s+rotation)?\s+accounts/i.test(summary)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function resolvePairedFollowUpQueueAction(args: {
|
export function resolvePairedFollowUpQueueAction(args: {
|
||||||
completedRole: PairedRoomRole;
|
completedRole: PairedRoomRole;
|
||||||
executionStatus: 'succeeded' | 'failed';
|
executionStatus: 'succeeded' | 'failed';
|
||||||
@@ -24,6 +36,15 @@ export function resolvePairedFollowUpQueueAction(args: {
|
|||||||
taskStatus: PairedTaskStatus | null;
|
taskStatus: PairedTaskStatus | null;
|
||||||
outputSummary?: string | null;
|
outputSummary?: string | null;
|
||||||
}): PairedFollowUpQueueAction {
|
}): PairedFollowUpQueueAction {
|
||||||
|
if (
|
||||||
|
args.executionStatus === 'failed' &&
|
||||||
|
args.sawOutput === false &&
|
||||||
|
(args.completedRole === 'reviewer' || args.completedRole === 'arbiter') &&
|
||||||
|
isSilentCodexAccountFailure(args.outputSummary)
|
||||||
|
) {
|
||||||
|
return 'none';
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
shouldRetrySilentOwnerExecution({
|
shouldRetrySilentOwnerExecution({
|
||||||
completedRole: args.completedRole,
|
completedRole: args.completedRole,
|
||||||
|
|||||||
@@ -1,15 +1,52 @@
|
|||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import {
|
||||||
|
classifyAgentError,
|
||||||
|
classifyCodexAuthError,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
||||||
import { classifyArbiterVerdict } from './paired-verdict.js';
|
import { classifyArbiterVerdict } from './paired-verdict.js';
|
||||||
import type { PairedTask } from './types.js';
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0;
|
const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0;
|
||||||
|
|
||||||
|
function isTerminalCodexAccountFailure(summary?: string | null): boolean {
|
||||||
|
if (!summary) return false;
|
||||||
|
if (classifyCodexAuthError(summary).category !== 'none') return true;
|
||||||
|
if (classifyAgentError(summary).category === 'rate-limit') return true;
|
||||||
|
return /all\s+codex(?:\s+rotation)?\s+accounts/i.test(summary);
|
||||||
|
}
|
||||||
|
|
||||||
export function handleFailedArbiterExecution(args: {
|
export function handleFailedArbiterExecution(args: {
|
||||||
task: PairedTask;
|
task: PairedTask;
|
||||||
taskId: string;
|
taskId: string;
|
||||||
|
summary?: string | null;
|
||||||
}): void {
|
}): void {
|
||||||
const { task, taskId } = args;
|
const { task, taskId, summary } = args;
|
||||||
|
if (isTerminalCodexAccountFailure(summary)) {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'completed',
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'arbiter_codex_unavailable',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'arbiter',
|
||||||
|
status: task.status,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
|
'Completed arbiter task after terminal Codex account failure instead of re-arming arbitration loop',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const fallbackStatus =
|
const fallbackStatus =
|
||||||
task.status === 'in_arbitration' || task.status === 'arbiter_requested'
|
task.status === 'in_arbitration' || task.status === 'arbiter_requested'
|
||||||
|
|||||||
@@ -197,6 +197,33 @@ describe('paired execution routing loop guards', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not re-arm arbiter loop after terminal Codex account failure', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'in_arbitration',
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'arbiter',
|
||||||
|
status: 'failed',
|
||||||
|
summary:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'completed',
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'arbiter_codex_unavailable',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => {
|
it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => {
|
||||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
buildPairedTask({
|
buildPairedTask({
|
||||||
|
|||||||
@@ -666,7 +666,11 @@ export function completePairedExecutionContext(args: {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (role === 'arbiter') {
|
if (role === 'arbiter') {
|
||||||
handleFailedArbiterExecution({ task, taskId });
|
handleFailedArbiterExecution({
|
||||||
|
task,
|
||||||
|
taskId,
|
||||||
|
summary: args.summary,
|
||||||
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
handleFailedOwnerExecution({ task, taskId, summary: args.summary });
|
handleFailedOwnerExecution({ task, taskId, summary: args.summary });
|
||||||
|
|||||||
@@ -224,6 +224,7 @@ describe('runCodexRotationLoop', () => {
|
|||||||
|
|
||||||
expect(outcome).toEqual({ type: 'success' });
|
expect(outcome).toEqual({ type: 'success' });
|
||||||
expect(rotateCodexToken).toHaveBeenCalledTimes(1);
|
expect(rotateCodexToken).toHaveBeenCalledTimes(1);
|
||||||
|
expect(rotateCodexToken).toHaveBeenCalledWith('auth-expired');
|
||||||
expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ import { clearGlobalFailover } from './service-routing.js';
|
|||||||
export interface TriggerInfo {
|
export interface TriggerInfo {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
|
message?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RotationAttemptResult {
|
export interface RotationAttemptResult {
|
||||||
@@ -206,6 +207,34 @@ function evaluateCodexAttempt(
|
|||||||
return { type: 'error' };
|
return { type: 'error' };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!attempt.sawOutput &&
|
||||||
|
typeof output.error === 'string' &&
|
||||||
|
output.error.length > 0
|
||||||
|
) {
|
||||||
|
const retryTrigger =
|
||||||
|
attempt.streamedTriggerReason &&
|
||||||
|
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||||
|
? {
|
||||||
|
shouldRotate: true as const,
|
||||||
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
}
|
||||||
|
: detectCodexRotationTrigger(output.error);
|
||||||
|
if (retryTrigger.shouldRotate) {
|
||||||
|
return {
|
||||||
|
type: 'continue',
|
||||||
|
trigger: { reason: retryTrigger.reason },
|
||||||
|
rotationMessage: output.error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{ ...logContext, provider: 'codex', error: output.error },
|
||||||
|
'Rotated Codex account returned an error field without visible output',
|
||||||
|
);
|
||||||
|
return { type: 'error' };
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
!attempt.sawOutput &&
|
!attempt.sawOutput &&
|
||||||
attempt.streamedTriggerReason &&
|
attempt.streamedTriggerReason &&
|
||||||
@@ -350,7 +379,7 @@ export async function runClaudeRotationLoop(
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function runCodexRotationLoop(
|
export async function runCodexRotationLoop(
|
||||||
initialTrigger: { reason: CodexRotationReason },
|
initialTrigger: { reason: CodexRotationReason; message?: string },
|
||||||
runAttempt: () => Promise<CodexRotationAttemptResult>,
|
runAttempt: () => Promise<CodexRotationAttemptResult>,
|
||||||
logContext: Record<string, unknown>,
|
logContext: Record<string, unknown>,
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
@@ -358,7 +387,13 @@ export async function runCodexRotationLoop(
|
|||||||
let trigger = initialTrigger;
|
let trigger = initialTrigger;
|
||||||
let lastRotationMessage = rotationMessage;
|
let lastRotationMessage = rotationMessage;
|
||||||
|
|
||||||
while (getCodexAccountCount() > 1 && rotateCodexToken(lastRotationMessage)) {
|
while (getCodexAccountCount() > 1) {
|
||||||
|
const rotationReasonMessage =
|
||||||
|
lastRotationMessage ?? trigger.message ?? trigger.reason;
|
||||||
|
if (!rotateCodexToken(rotationReasonMessage)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{ ...logContext, reason: trigger.reason },
|
{ ...logContext, reason: trigger.reason },
|
||||||
'Codex account unhealthy, retrying with rotated account',
|
'Codex account unhealthy, retrying with rotated account',
|
||||||
|
|||||||
@@ -513,18 +513,26 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
|
|
||||||
describe('error → Codex rotation trigger', () => {
|
describe('error → Codex rotation trigger', () => {
|
||||||
it('returns rotation trigger for Codex primary', () => {
|
it('returns rotation trigger for Codex primary', () => {
|
||||||
|
const errorMessage =
|
||||||
|
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
|
||||||
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
||||||
shouldRotate: true,
|
shouldRotate: true,
|
||||||
reason: '429',
|
reason: '429',
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = evaluateStreamedOutput(
|
const result = evaluateStreamedOutput(
|
||||||
errorOutput('429 rate limit'),
|
errorOutput(errorMessage),
|
||||||
freshState(),
|
freshState(),
|
||||||
codexOpts,
|
codexOpts,
|
||||||
);
|
);
|
||||||
expect(result.newTrigger).toEqual({ reason: '429' });
|
expect(result.newTrigger).toEqual({
|
||||||
expect(result.state.streamedTriggerReason).toEqual({ reason: '429' });
|
reason: '429',
|
||||||
|
message: errorMessage,
|
||||||
|
});
|
||||||
|
expect(result.state.streamedTriggerReason).toEqual({
|
||||||
|
reason: '429',
|
||||||
|
message: errorMessage,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not check Codex rotation for Claude agent type', () => {
|
it('does not check Codex rotation for Claude agent type', () => {
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import {
|
|||||||
export interface StreamedTriggerReason {
|
export interface StreamedTriggerReason {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
|
message?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StreamedOutputState {
|
export interface StreamedOutputState {
|
||||||
@@ -139,6 +140,28 @@ export function evaluateStreamedOutput(
|
|||||||
nextState.sawSuccessNullResultWithoutOutput = true;
|
nextState.sawSuccessNullResultWithoutOutput = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
isPrimaryCodex &&
|
||||||
|
typeof output.error === 'string' &&
|
||||||
|
output.error.length > 0 &&
|
||||||
|
!nextState.sawOutput &&
|
||||||
|
!nextState.streamedTriggerReason
|
||||||
|
) {
|
||||||
|
const trigger = detectCodexRotationTrigger(output.error);
|
||||||
|
if (trigger.shouldRotate) {
|
||||||
|
const newTrigger: StreamedTriggerReason = {
|
||||||
|
reason: trigger.reason,
|
||||||
|
message: output.error,
|
||||||
|
};
|
||||||
|
nextState.streamedTriggerReason = newTrigger;
|
||||||
|
return {
|
||||||
|
state: nextState,
|
||||||
|
shouldForwardOutput: !options.shortCircuitTriggeredErrors,
|
||||||
|
newTrigger,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
output.status === 'error' &&
|
output.status === 'error' &&
|
||||||
!nextState.sawOutput &&
|
!nextState.sawOutput &&
|
||||||
@@ -157,7 +180,7 @@ export function evaluateStreamedOutput(
|
|||||||
} else if (isPrimaryCodex) {
|
} else if (isPrimaryCodex) {
|
||||||
const trigger = detectCodexRotationTrigger(output.error);
|
const trigger = detectCodexRotationTrigger(output.error);
|
||||||
if (trigger.shouldRotate) {
|
if (trigger.shouldRotate) {
|
||||||
newTrigger = { reason: trigger.reason };
|
newTrigger = { reason: trigger.reason, message: output.error };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user