fix: wait for a busy Codex lease instead of skipping the review
The Codex auth lease is exclusive per account so two Codex processes never rotate the same refresh token at once. With the single configured account that also serializes every Codex run, and claimCodexAuthLease() failed immediately while another room's Codex run held it: the turn errored with "Codex rotation pool unavailable", the task closed as reviewer_codex_unavailable and the review was silently skipped. All three such failures on 2026-09-28 coincided exactly with another room's Codex run holding the lease (10:25:55, 18:08:25, 18:22:50 KST). runAgentProcess now waits (up to 15 min, polling every 2s) when the only obstacle is a lease held by a live run. A dead or rate-limited pool still fails fast, and the normal path adds no async hop because the wait is only entered when the lease is actually busy. The exclusivity itself is kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -16,7 +16,11 @@ import {
|
||||
prepareGroupEnvironment,
|
||||
type PreparedCodexSessionAuth,
|
||||
} from './agent-runner-environment.js';
|
||||
import { syncCodexSessionAuthBack } from './codex-token-rotation.js';
|
||||
import {
|
||||
isCodexAuthLeaseAvailable,
|
||||
syncCodexSessionAuthBack,
|
||||
waitForCodexAuthLease,
|
||||
} from './codex-token-rotation.js';
|
||||
import { syncClaudeSessionAuthBack } from './token-refresh.js';
|
||||
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
||||
import { getStoredRoomSkillOverrides } from './db.js';
|
||||
@@ -161,6 +165,15 @@ export async function runAgentProcess(
|
||||
// ── Host process mode (owner) ───────────────────────────────────
|
||||
const startTime = Date.now();
|
||||
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
||||
if (
|
||||
(group.agentType || 'claude-code') === 'codex' &&
|
||||
!isCodexAuthLeaseAvailable()
|
||||
) {
|
||||
// A concurrent Codex run in another room may hold the account lease.
|
||||
// Wait for it instead of failing the turn outright; if the pool is dead or
|
||||
// rate-limited this returns at once and the prepare step below fails fast.
|
||||
await waitForCodexAuthLease();
|
||||
}
|
||||
const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, {
|
||||
memoryBriefing: input.memoryBriefing,
|
||||
runtimeTaskId: input.runtimeTaskId,
|
||||
|
||||
@@ -434,3 +434,90 @@ describe('codex-token-rotation single-account fallback', () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('codex-token-rotation waitForCodexAuthLease (single account)', () => {
|
||||
let tempHome: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-rot-'));
|
||||
process.env.CODEX_ROT_TEST_HOME = tempHome;
|
||||
createFakeAccounts(tempHome, 1);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.CODEX_ROT_TEST_HOME;
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reports lease availability synchronously', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
expect(mod.isCodexAuthLeaseAvailable()).toBe(true);
|
||||
const held = mod.claimCodexAuthLease();
|
||||
expect(mod.isCodexAuthLeaseAvailable()).toBe(false);
|
||||
held?.release();
|
||||
expect(mod.isCodexAuthLeaseAvailable()).toBe(true);
|
||||
});
|
||||
|
||||
it('returns immediately when the account is free', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
const sleep = vi.fn(async () => {});
|
||||
|
||||
await expect(mod.waitForCodexAuthLease({ sleep })).resolves.toBe(true);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('waits for a concurrent run to release the lease instead of failing', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
// Another room's Codex run holds the only account. Before the wait, a
|
||||
// second claim fails outright — that was the skipped-review bug.
|
||||
const held = mod.claimCodexAuthLease();
|
||||
expect(held).not.toBeNull();
|
||||
expect(mod.claimCodexAuthLease()).toBeNull();
|
||||
|
||||
let polls = 0;
|
||||
const sleep = vi.fn(async () => {
|
||||
polls += 1;
|
||||
if (polls === 3) held?.release();
|
||||
});
|
||||
|
||||
await expect(mod.waitForCodexAuthLease({ sleep, pollMs: 1 })).resolves.toBe(
|
||||
true,
|
||||
);
|
||||
expect(polls).toBe(3);
|
||||
|
||||
const next = mod.claimCodexAuthLease();
|
||||
expect(next).not.toBeNull();
|
||||
next?.release();
|
||||
});
|
||||
|
||||
it('gives up after the timeout while the lease stays held', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
const held = mod.claimCodexAuthLease();
|
||||
try {
|
||||
await expect(
|
||||
mod.waitForCodexAuthLease({
|
||||
timeoutMs: 0,
|
||||
sleep: async () => {},
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
} finally {
|
||||
held?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not wait when the account is exhausted rather than leased', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
mod.updateCodexAccountUsage(80, undefined, 0, 100, undefined);
|
||||
const sleep = vi.fn(async () => {});
|
||||
|
||||
await expect(mod.waitForCodexAuthLease({ sleep })).resolves.toBe(false);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -539,6 +539,92 @@ function updateAccountMetadataFromAuthFile(acct: CodexAccount): void {
|
||||
acct.authFileMtimeMs = readAuthFileMtimeMs(acct.authPath);
|
||||
}
|
||||
|
||||
function isCodexAccountLeasedByLiveRun(
|
||||
acct: CodexAccount,
|
||||
now: number,
|
||||
): boolean {
|
||||
if (acct.leasedUntil && acct.leasedUntil > now) return true;
|
||||
const existing = readExistingLease(codexLockPath(acct.authPath));
|
||||
if (!existing) return false;
|
||||
if (existing.expiresAt && existing.expiresAt <= now) return false;
|
||||
if (existing.pid && !isPidAlive(existing.pid)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Synchronous check: can a Codex lease be claimed right now? */
|
||||
export function isCodexAuthLeaseAvailable(): boolean {
|
||||
if (accounts.length === 0) return true;
|
||||
const now = Date.now();
|
||||
return accounts.some((acct) => isCodexAccountUsable(acct, now));
|
||||
}
|
||||
|
||||
export const CODEX_AUTH_LEASE_WAIT_MAX_MS = 15 * 60_000;
|
||||
const CODEX_AUTH_LEASE_WAIT_POLL_MS = 2_000;
|
||||
|
||||
/**
|
||||
* Wait until a Codex account can be leased.
|
||||
*
|
||||
* The lease is exclusive per account so two Codex processes never rotate the
|
||||
* same refresh token at once. With a single account that also serializes every
|
||||
* Codex run, and claimCodexAuthLease() fails immediately while another room's
|
||||
* Codex run holds it — which silently skipped reviews (task closed as
|
||||
* reviewer_codex_unavailable) whenever two rooms needed Codex at the same time.
|
||||
*
|
||||
* Only waits when the sole obstacle is a lease held by a live run. A dead or
|
||||
* rate-limited pool returns false right away so the caller fails fast as before.
|
||||
*/
|
||||
export async function waitForCodexAuthLease(opts?: {
|
||||
timeoutMs?: number;
|
||||
pollMs?: number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}): Promise<boolean> {
|
||||
if (accounts.length === 0) return true;
|
||||
const timeoutMs = opts?.timeoutMs ?? CODEX_AUTH_LEASE_WAIT_MAX_MS;
|
||||
const pollMs = opts?.pollMs ?? CODEX_AUTH_LEASE_WAIT_POLL_MS;
|
||||
const sleep =
|
||||
opts?.sleep ??
|
||||
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
||||
const startedAt = Date.now();
|
||||
let logged = false;
|
||||
|
||||
for (;;) {
|
||||
const now = Date.now();
|
||||
if (accounts.some((acct) => isCodexAccountUsable(acct, now))) {
|
||||
if (logged) {
|
||||
logger.info(
|
||||
{ waitedMs: now - startedAt },
|
||||
'Codex account lease became available after waiting',
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
const blockedOnlyByLease = accounts.some(
|
||||
(acct) =>
|
||||
isCodexAccountLeasedByLiveRun(acct, now) &&
|
||||
acct.authStatus !== 'dead_auth' &&
|
||||
!(acct.rateLimitedUntil && acct.rateLimitedUntil > now) &&
|
||||
!(acct.lastUsageD7Pct != null && acct.lastUsageD7Pct >= 100),
|
||||
);
|
||||
if (!blockedOnlyByLease || now - startedAt >= timeoutMs) {
|
||||
if (logged) {
|
||||
logger.warn(
|
||||
{ waitedMs: now - startedAt, timeoutMs, blockedOnlyByLease },
|
||||
'Gave up waiting for a Codex account lease',
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (!logged) {
|
||||
logger.info(
|
||||
{ timeoutMs },
|
||||
'Codex account is leased by another run; waiting before launching Codex',
|
||||
);
|
||||
logged = true;
|
||||
}
|
||||
await sleep(pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
export function claimCodexAuthLease(): CodexAuthLease | null {
|
||||
if (accounts.length === 0) return null;
|
||||
const now = Date.now();
|
||||
|
||||
Reference in New Issue
Block a user