The Codex auth lease is exclusive per account so two Codex processes never rotate the same refresh token at once. With the single configured account that also serializes every Codex run, and claimCodexAuthLease() failed immediately while another room's Codex run held it: the turn errored with "Codex rotation pool unavailable", the task closed as reviewer_codex_unavailable and the review was silently skipped. All three such failures on 2026-09-28 coincided exactly with another room's Codex run holding the lease (10:25:55, 18:08:25, 18:22:50 KST). runAgentProcess now waits (up to 15 min, polling every 2s) when the only obstacle is a lease held by a live run. A dead or rate-limited pool still fails fast, and the normal path adds no async hop because the wait is only entered when the lease is actually busy. The exclusivity itself is kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
326 lines
9.3 KiB
TypeScript
326 lines
9.3 KiB
TypeScript
import { ChildProcess, spawn } from 'child_process';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
|
|
import {
|
|
EJCLAW_ENV,
|
|
isUnsafeHostPairedModeEnabled,
|
|
} from 'ejclaw-runners-shared';
|
|
|
|
/**
|
|
* Agent Process Runner for EJClaw
|
|
* Spawns agent execution as direct host processes and handles IPC.
|
|
*/
|
|
import {
|
|
prepareReadonlySessionEnvironment,
|
|
prepareGroupEnvironment,
|
|
type PreparedCodexSessionAuth,
|
|
} from './agent-runner-environment.js';
|
|
import {
|
|
isCodexAuthLeaseAvailable,
|
|
syncCodexSessionAuthBack,
|
|
waitForCodexAuthLease,
|
|
} from './codex-token-rotation.js';
|
|
import { syncClaudeSessionAuthBack } from './token-refresh.js';
|
|
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
|
import { getStoredRoomSkillOverrides } from './db.js';
|
|
export {
|
|
type AvailableGroup,
|
|
writeGroupsSnapshot,
|
|
writeTasksSnapshot,
|
|
} from './agent-runner-snapshot.js';
|
|
import { logger } from './logger.js';
|
|
import { RegisteredGroup, RoomRoleContext } from './types.js';
|
|
import type { StoredRoomSkillOverride } from './db/rooms.js';
|
|
|
|
export interface AgentInput {
|
|
prompt: string;
|
|
sessionId?: string;
|
|
memoryBriefing?: string;
|
|
groupFolder: string;
|
|
chatJid: string;
|
|
runId?: string;
|
|
isMain: boolean;
|
|
isScheduledTask?: boolean;
|
|
runtimeTaskId?: string;
|
|
useTaskScopedSession?: boolean;
|
|
assistantName?: string;
|
|
agentType?: 'claude-code' | 'codex';
|
|
codexGoals?: boolean;
|
|
roomRoleContext?: RoomRoleContext;
|
|
}
|
|
|
|
export interface AgentOutput {
|
|
status: 'success' | 'error';
|
|
result: string | null;
|
|
output?: import('./types.js').StructuredAgentOutput;
|
|
phase?: import('./types.js').AgentOutputPhase;
|
|
agentId?: string;
|
|
agentLabel?: string;
|
|
agentDone?: boolean;
|
|
newSessionId?: string;
|
|
error?: string;
|
|
compaction?: {
|
|
completed: boolean;
|
|
trigger?: string | null;
|
|
};
|
|
}
|
|
|
|
function readRoomSkillOverridesForRunner(
|
|
chatJid: string,
|
|
): StoredRoomSkillOverride[] {
|
|
try {
|
|
return getStoredRoomSkillOverrides(chatJid);
|
|
} catch (err) {
|
|
logger.warn(
|
|
{
|
|
err,
|
|
chatJid,
|
|
},
|
|
'Failed to read room skill overrides; falling back to default skills',
|
|
);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
function releaseCodexAuthSession(
|
|
auth: PreparedCodexSessionAuth | null | undefined,
|
|
): void {
|
|
auth?.lease?.release();
|
|
}
|
|
|
|
function finalizeCodexAuthSession(
|
|
auth: PreparedCodexSessionAuth | null | undefined,
|
|
): void {
|
|
if (!auth) return;
|
|
try {
|
|
syncCodexSessionAuthBack({
|
|
canonicalAuthPath: auth.canonicalAuthPath,
|
|
sessionAuthPath: auth.sessionAuthPath,
|
|
accountIndex: auth.accountIndex,
|
|
});
|
|
} catch (err) {
|
|
logger.warn(
|
|
{
|
|
err,
|
|
canonicalAuthPath: auth.canonicalAuthPath,
|
|
accountIndex: auth.accountIndex,
|
|
},
|
|
'Failed to sync Codex session auth back to canonical slot',
|
|
);
|
|
} finally {
|
|
auth.lease?.release();
|
|
}
|
|
}
|
|
|
|
function createCodexAuthSessionFinalizer(
|
|
getAuth: () => PreparedCodexSessionAuth | null | undefined,
|
|
): () => void {
|
|
let finalized = false;
|
|
return () => {
|
|
if (finalized) return;
|
|
finalized = true;
|
|
finalizeCodexAuthSession(getAuth());
|
|
};
|
|
}
|
|
|
|
/**
|
|
* After a Claude agent turn, adopt any token the session's Claude CLI refreshed
|
|
* (rotated) back into the canonical credentials, so the main refresh loop never
|
|
* ends up holding an already-rotated token (which Anthropic revokes as a family).
|
|
* No-op for Codex sessions (sessionClaudeDir undefined).
|
|
*/
|
|
function createClaudeAuthSessionFinalizer(
|
|
sessionClaudeDir: string | undefined,
|
|
): () => void {
|
|
let finalized = false;
|
|
return () => {
|
|
if (finalized) return;
|
|
finalized = true;
|
|
if (!sessionClaudeDir) return;
|
|
try {
|
|
syncClaudeSessionAuthBack(sessionClaudeDir);
|
|
} catch (err) {
|
|
logger.warn(
|
|
{ err, sessionClaudeDir },
|
|
'Failed to sync Claude session auth back to canonical credentials',
|
|
);
|
|
}
|
|
};
|
|
}
|
|
|
|
export async function runAgentProcess(
|
|
group: RegisteredGroup,
|
|
input: AgentInput,
|
|
onProcess: (
|
|
proc: ChildProcess,
|
|
processName: string,
|
|
runtimeIpcDir: string,
|
|
) => void,
|
|
onOutput?: (output: AgentOutput) => Promise<void>,
|
|
envOverrides?: Record<string, string>,
|
|
): Promise<AgentOutput> {
|
|
const unsafeHostPairedMode = isUnsafeHostPairedModeEnabled(envOverrides);
|
|
|
|
// ── Host process mode (owner) ───────────────────────────────────
|
|
const startTime = Date.now();
|
|
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
|
if (
|
|
(group.agentType || 'claude-code') === 'codex' &&
|
|
!isCodexAuthLeaseAvailable()
|
|
) {
|
|
// A concurrent Codex run in another room may hold the account lease.
|
|
// Wait for it instead of failing the turn outright; if the pool is dead or
|
|
// rate-limited this returns at once and the prepare step below fails fast.
|
|
await waitForCodexAuthLease();
|
|
}
|
|
const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, {
|
|
memoryBriefing: input.memoryBriefing,
|
|
runtimeTaskId: input.runtimeTaskId,
|
|
useTaskScopedSession: input.useTaskScopedSession,
|
|
skillOverrides,
|
|
roomRole: input.roomRoleContext?.role,
|
|
});
|
|
const { env, groupDir, runnerDir } = prepared;
|
|
let codexSessionAuth = prepared.codexSessionAuth;
|
|
|
|
// Apply env overrides (caller-provided)
|
|
if (envOverrides) {
|
|
for (const [key, value] of Object.entries(envOverrides)) {
|
|
if (value) env[key] = value;
|
|
}
|
|
}
|
|
if (
|
|
unsafeHostPairedMode &&
|
|
envOverrides?.CLAUDE_CONFIG_DIR &&
|
|
(input.roomRoleContext?.role === 'reviewer' ||
|
|
input.roomRoleContext?.role === 'arbiter')
|
|
) {
|
|
const isCodexGroup = (group.agentType || 'claude-code') === 'codex';
|
|
const previousCodexSessionAuth = codexSessionAuth;
|
|
let readonlySession: ReturnType<typeof prepareReadonlySessionEnvironment>;
|
|
try {
|
|
readonlySession = prepareReadonlySessionEnvironment({
|
|
sessionDir: envOverrides.CLAUDE_CONFIG_DIR,
|
|
chatJid: input.chatJid,
|
|
isMain: input.isMain,
|
|
groupFolder: group.folder,
|
|
agentType: group.agentType || 'claude-code',
|
|
memoryBriefing: input.memoryBriefing,
|
|
role: input.roomRoleContext.role,
|
|
ipcDir: env[EJCLAW_ENV.ipcDir],
|
|
hostIpcDir: env[EJCLAW_ENV.hostIpcDir],
|
|
workDir: envOverrides[EJCLAW_ENV.workDir] || env[EJCLAW_ENV.workDir],
|
|
skillOverrides,
|
|
});
|
|
} catch (err) {
|
|
if (isCodexGroup) {
|
|
releaseCodexAuthSession(previousCodexSessionAuth);
|
|
}
|
|
throw err;
|
|
}
|
|
if (isCodexGroup) {
|
|
releaseCodexAuthSession(previousCodexSessionAuth);
|
|
codexSessionAuth = readonlySession.codexSessionAuth;
|
|
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
|
|
if (readonlySession.codexHomeDir) {
|
|
env.HOME = readonlySession.codexHomeDir;
|
|
}
|
|
}
|
|
}
|
|
if (input.runId) {
|
|
env[EJCLAW_ENV.runId] = input.runId;
|
|
}
|
|
|
|
const safeName = group.folder.replace(/[^a-zA-Z0-9-]/g, '-');
|
|
const processSuffix = input.runId || `${Date.now()}`;
|
|
const processName = `ejclaw-${safeName}-${processSuffix}`;
|
|
const finalizeCodexAuthSessionOnce = createCodexAuthSessionFinalizer(
|
|
() => codexSessionAuth,
|
|
);
|
|
const finalizeClaudeAuthSessionOnce = createClaudeAuthSessionFinalizer(
|
|
(group.agentType || 'claude-code') === 'codex'
|
|
? undefined
|
|
: env.CLAUDE_CONFIG_DIR,
|
|
);
|
|
const finalizeAuthSessionOnce = () => {
|
|
finalizeCodexAuthSessionOnce();
|
|
finalizeClaudeAuthSessionOnce();
|
|
};
|
|
|
|
// Check if runner is built
|
|
const distEntry = path.join(runnerDir, 'dist', 'index.js');
|
|
if (!fs.existsSync(distEntry)) {
|
|
logger.error(
|
|
{ runnerDir, chatJid: input.chatJid, runId: input.runId },
|
|
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
|
|
);
|
|
releaseCodexAuthSession(codexSessionAuth);
|
|
return {
|
|
status: 'error',
|
|
result: null,
|
|
error: `Runner not built at ${distEntry}. Run bun run build:runners first.`,
|
|
};
|
|
}
|
|
|
|
logger.info(
|
|
{
|
|
group: group.name,
|
|
chatJid: input.chatJid,
|
|
groupFolder: input.groupFolder,
|
|
runId: input.runId,
|
|
processName,
|
|
agentType: group.agentType || 'claude-code',
|
|
isMain: input.isMain,
|
|
},
|
|
'Spawning agent process',
|
|
);
|
|
|
|
const logsDir = path.join(groupDir, 'logs');
|
|
fs.mkdirSync(logsDir, { recursive: true });
|
|
|
|
return new Promise((resolve) => {
|
|
const proc = spawn('bun', [distEntry], {
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
cwd: runnerDir,
|
|
env,
|
|
});
|
|
|
|
onProcess(proc, processName, env[EJCLAW_ENV.ipcDir]);
|
|
|
|
const runnerInput: AgentInput = {
|
|
...input,
|
|
...(group.agentConfig?.codexGoals === true ? { codexGoals: true } : {}),
|
|
};
|
|
proc.stdin.write(JSON.stringify(runnerInput));
|
|
proc.stdin.end();
|
|
|
|
runSpawnedAgentProcess({
|
|
proc,
|
|
group,
|
|
input: runnerInput,
|
|
processName,
|
|
logsDir,
|
|
startTime,
|
|
onOutput,
|
|
onTerminalStreamedOutputFlushed: finalizeAuthSessionOnce,
|
|
})
|
|
.then((output) => {
|
|
finalizeAuthSessionOnce();
|
|
resolve(output);
|
|
})
|
|
.catch((err: unknown) => {
|
|
finalizeAuthSessionOnce();
|
|
logger.error(
|
|
{ err, processName, chatJid: input.chatJid, runId: input.runId },
|
|
'Spawned agent process runner failed',
|
|
);
|
|
resolve({
|
|
status: 'error',
|
|
result: null,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
});
|
|
});
|
|
}
|