diff --git a/src/agent-runner.ts b/src/agent-runner.ts index e3fe5bf..42f50bd 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -16,7 +16,11 @@ import { prepareGroupEnvironment, type PreparedCodexSessionAuth, } from './agent-runner-environment.js'; -import { syncCodexSessionAuthBack } from './codex-token-rotation.js'; +import { + isCodexAuthLeaseAvailable, + syncCodexSessionAuthBack, + waitForCodexAuthLease, +} from './codex-token-rotation.js'; import { syncClaudeSessionAuthBack } from './token-refresh.js'; import { runSpawnedAgentProcess } from './agent-runner-process.js'; import { getStoredRoomSkillOverrides } from './db.js'; @@ -161,6 +165,15 @@ export async function runAgentProcess( // ── Host process mode (owner) ─────────────────────────────────── const startTime = Date.now(); const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid); + if ( + (group.agentType || 'claude-code') === 'codex' && + !isCodexAuthLeaseAvailable() + ) { + // A concurrent Codex run in another room may hold the account lease. + // Wait for it instead of failing the turn outright; if the pool is dead or + // rate-limited this returns at once and the prepare step below fails fast. + await waitForCodexAuthLease(); + } const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, { memoryBriefing: input.memoryBriefing, runtimeTaskId: input.runtimeTaskId, diff --git a/src/codex-token-rotation.test.ts b/src/codex-token-rotation.test.ts index 018af22..e6c1e8b 100644 --- a/src/codex-token-rotation.test.ts +++ b/src/codex-token-rotation.test.ts @@ -434,3 +434,90 @@ describe('codex-token-rotation single-account fallback', () => { ]); }); }); + +describe('codex-token-rotation waitForCodexAuthLease (single account)', () => { + let tempHome: string; + + beforeEach(() => { + vi.resetModules(); + tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-rot-')); + process.env.CODEX_ROT_TEST_HOME = tempHome; + createFakeAccounts(tempHome, 1); + }); + + afterEach(() => { + delete process.env.CODEX_ROT_TEST_HOME; + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it('reports lease availability synchronously', async () => { + const mod = await import('./codex-token-rotation.js'); + mod.initCodexTokenRotation(); + expect(mod.isCodexAuthLeaseAvailable()).toBe(true); + const held = mod.claimCodexAuthLease(); + expect(mod.isCodexAuthLeaseAvailable()).toBe(false); + held?.release(); + expect(mod.isCodexAuthLeaseAvailable()).toBe(true); + }); + + it('returns immediately when the account is free', async () => { + const mod = await import('./codex-token-rotation.js'); + mod.initCodexTokenRotation(); + const sleep = vi.fn(async () => {}); + + await expect(mod.waitForCodexAuthLease({ sleep })).resolves.toBe(true); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('waits for a concurrent run to release the lease instead of failing', async () => { + const mod = await import('./codex-token-rotation.js'); + mod.initCodexTokenRotation(); + + // Another room's Codex run holds the only account. Before the wait, a + // second claim fails outright — that was the skipped-review bug. + const held = mod.claimCodexAuthLease(); + expect(held).not.toBeNull(); + expect(mod.claimCodexAuthLease()).toBeNull(); + + let polls = 0; + const sleep = vi.fn(async () => { + polls += 1; + if (polls === 3) held?.release(); + }); + + await expect(mod.waitForCodexAuthLease({ sleep, pollMs: 1 })).resolves.toBe( + true, + ); + expect(polls).toBe(3); + + const next = mod.claimCodexAuthLease(); + expect(next).not.toBeNull(); + next?.release(); + }); + + it('gives up after the timeout while the lease stays held', async () => { + const mod = await import('./codex-token-rotation.js'); + mod.initCodexTokenRotation(); + const held = mod.claimCodexAuthLease(); + try { + await expect( + mod.waitForCodexAuthLease({ + timeoutMs: 0, + sleep: async () => {}, + }), + ).resolves.toBe(false); + } finally { + held?.release(); + } + }); + + it('does not wait when the account is exhausted rather than leased', async () => { + const mod = await import('./codex-token-rotation.js'); + mod.initCodexTokenRotation(); + mod.updateCodexAccountUsage(80, undefined, 0, 100, undefined); + const sleep = vi.fn(async () => {}); + + await expect(mod.waitForCodexAuthLease({ sleep })).resolves.toBe(false); + expect(sleep).not.toHaveBeenCalled(); + }); +}); diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 3b8066c..80891b9 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -539,6 +539,92 @@ function updateAccountMetadataFromAuthFile(acct: CodexAccount): void { acct.authFileMtimeMs = readAuthFileMtimeMs(acct.authPath); } +function isCodexAccountLeasedByLiveRun( + acct: CodexAccount, + now: number, +): boolean { + if (acct.leasedUntil && acct.leasedUntil > now) return true; + const existing = readExistingLease(codexLockPath(acct.authPath)); + if (!existing) return false; + if (existing.expiresAt && existing.expiresAt <= now) return false; + if (existing.pid && !isPidAlive(existing.pid)) return false; + return true; +} + +/** Synchronous check: can a Codex lease be claimed right now? */ +export function isCodexAuthLeaseAvailable(): boolean { + if (accounts.length === 0) return true; + const now = Date.now(); + return accounts.some((acct) => isCodexAccountUsable(acct, now)); +} + +export const CODEX_AUTH_LEASE_WAIT_MAX_MS = 15 * 60_000; +const CODEX_AUTH_LEASE_WAIT_POLL_MS = 2_000; + +/** + * Wait until a Codex account can be leased. + * + * The lease is exclusive per account so two Codex processes never rotate the + * same refresh token at once. With a single account that also serializes every + * Codex run, and claimCodexAuthLease() fails immediately while another room's + * Codex run holds it — which silently skipped reviews (task closed as + * reviewer_codex_unavailable) whenever two rooms needed Codex at the same time. + * + * Only waits when the sole obstacle is a lease held by a live run. A dead or + * rate-limited pool returns false right away so the caller fails fast as before. + */ +export async function waitForCodexAuthLease(opts?: { + timeoutMs?: number; + pollMs?: number; + sleep?: (ms: number) => Promise; +}): Promise { + if (accounts.length === 0) return true; + const timeoutMs = opts?.timeoutMs ?? CODEX_AUTH_LEASE_WAIT_MAX_MS; + const pollMs = opts?.pollMs ?? CODEX_AUTH_LEASE_WAIT_POLL_MS; + const sleep = + opts?.sleep ?? + ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const startedAt = Date.now(); + let logged = false; + + for (;;) { + const now = Date.now(); + if (accounts.some((acct) => isCodexAccountUsable(acct, now))) { + if (logged) { + logger.info( + { waitedMs: now - startedAt }, + 'Codex account lease became available after waiting', + ); + } + return true; + } + const blockedOnlyByLease = accounts.some( + (acct) => + isCodexAccountLeasedByLiveRun(acct, now) && + acct.authStatus !== 'dead_auth' && + !(acct.rateLimitedUntil && acct.rateLimitedUntil > now) && + !(acct.lastUsageD7Pct != null && acct.lastUsageD7Pct >= 100), + ); + if (!blockedOnlyByLease || now - startedAt >= timeoutMs) { + if (logged) { + logger.warn( + { waitedMs: now - startedAt, timeoutMs, blockedOnlyByLease }, + 'Gave up waiting for a Codex account lease', + ); + } + return false; + } + if (!logged) { + logger.info( + { timeoutMs }, + 'Codex account is leased by another run; waiting before launching Codex', + ); + logged = true; + } + await sleep(pollMs); + } +} + export function claimCodexAuthLease(): CodexAuthLease | null { if (accounts.length === 0) return null; const now = Date.now();