Fix recurring Codex unavailable in paired rooms (#212)
* fix: avoid codex leases for claude readonly sessions * fix: restore codex lease quality budget * fix: preserve owner finalization on codex failure * fix: wake owner after arbiter codex failure * chore: format readonly codex lease fix
This commit is contained in:
@@ -15,7 +15,13 @@ const {
|
|||||||
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
||||||
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
||||||
mockGetCodexAccountCount: vi.fn<() => number>(),
|
mockGetCodexAccountCount: vi.fn<() => number>(),
|
||||||
mockClaimCodexAuthLease: vi.fn<() => null>(),
|
mockClaimCodexAuthLease: vi.fn<
|
||||||
|
() => {
|
||||||
|
authPath: string;
|
||||||
|
accountIndex: number;
|
||||||
|
release: () => void;
|
||||||
|
} | null
|
||||||
|
>(),
|
||||||
mockFindCodexAccountIndexByAuthPath: vi.fn<() => number | null>(),
|
mockFindCodexAccountIndexByAuthPath: vi.fn<() => number | null>(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -835,6 +841,37 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
|||||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not claim a Codex auth lease for Claude read-only sessions', () => {
|
||||||
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(1);
|
||||||
|
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||||
|
fs.writeFileSync(rotatedAuthPath, '{"auth_mode":"chatgpt"}\n');
|
||||||
|
const release = vi.fn();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue({
|
||||||
|
authPath: rotatedAuthPath,
|
||||||
|
accountIndex: 0,
|
||||||
|
release,
|
||||||
|
});
|
||||||
|
|
||||||
|
const sessionDir = path.join(tempRoot, 'readonly-claude-session');
|
||||||
|
const prepared = prepareReadonlySessionEnvironment({
|
||||||
|
sessionDir,
|
||||||
|
chatJid: 'dc:test',
|
||||||
|
isMain: false,
|
||||||
|
groupFolder: 'codex-test-group',
|
||||||
|
agentType: 'claude-code',
|
||||||
|
memoryBriefing: 'memory briefing',
|
||||||
|
role: 'reviewer',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(prepared.codexSessionAuth).toBeNull();
|
||||||
|
expect(mockClaimCodexAuthLease).not.toHaveBeenCalled();
|
||||||
|
expect(release).not.toHaveBeenCalled();
|
||||||
|
expect(fs.existsSync(path.join(sessionDir, '.codex', 'auth.json'))).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('writes matching AGENTS.md and copies host codex auth/config into the role-scoped session', () => {
|
it('writes matching AGENTS.md and copies host codex auth/config into the role-scoped session', () => {
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||||
|
|
||||||
|
|||||||
@@ -832,32 +832,39 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
const sessionClaudeMdPath = path.join(sessionDir, 'CLAUDE.md');
|
const sessionClaudeMdPath = path.join(sessionDir, 'CLAUDE.md');
|
||||||
if (sessionClaudeMd) {
|
if (sessionClaudeMd) {
|
||||||
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
||||||
const sessionCodexDir = path.join(sessionDir, '.codex');
|
if (agentType === 'codex') {
|
||||||
codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
const sessionCodexDir = path.join(sessionDir, '.codex');
|
||||||
fs.writeFileSync(
|
codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||||
path.join(sessionCodexDir, 'AGENTS.md'),
|
fs.writeFileSync(
|
||||||
sessionClaudeMd + '\n',
|
path.join(sessionCodexDir, 'AGENTS.md'),
|
||||||
);
|
sessionClaudeMd + '\n',
|
||||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
);
|
||||||
const mcpServerPath = path.join(
|
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||||
projectRoot,
|
const mcpServerPath = path.join(
|
||||||
'runners',
|
projectRoot,
|
||||||
'agent-runner',
|
'runners',
|
||||||
'dist',
|
'agent-runner',
|
||||||
'ipc-mcp-stdio.js',
|
'dist',
|
||||||
);
|
'ipc-mcp-stdio.js',
|
||||||
if (fs.existsSync(mcpServerPath)) {
|
);
|
||||||
upsertEjclawMcpServerSection({
|
if (fs.existsSync(mcpServerPath)) {
|
||||||
sessionConfigPath,
|
upsertEjclawMcpServerSection({
|
||||||
mcpServerPath,
|
sessionConfigPath,
|
||||||
ipcDir,
|
mcpServerPath,
|
||||||
hostIpcDir,
|
ipcDir,
|
||||||
chatJid,
|
hostIpcDir,
|
||||||
groupFolder,
|
chatJid,
|
||||||
isMain,
|
groupFolder,
|
||||||
agentType,
|
isMain,
|
||||||
roomRole: role,
|
agentType,
|
||||||
workDir,
|
roomRole: role,
|
||||||
|
workDir,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fs.rmSync(path.join(sessionDir, '.codex'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -60,6 +60,44 @@ function logStreamedOutputDeliveryError(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function logStreamedAgentErrorOutput(
|
||||||
|
output: AgentOutput,
|
||||||
|
group: RegisteredGroup,
|
||||||
|
input: AgentInput,
|
||||||
|
): void {
|
||||||
|
if (output.status !== 'error') return;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
group: group.name,
|
||||||
|
chatJid: input.chatJid,
|
||||||
|
runId: input.runId,
|
||||||
|
error: output.error,
|
||||||
|
newSessionId: output.newSessionId,
|
||||||
|
},
|
||||||
|
'Streamed agent error output',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function chainStreamedOutputDelivery(args: {
|
||||||
|
outputChain: Promise<void>;
|
||||||
|
parsed: AgentOutput;
|
||||||
|
onOutput: (output: AgentOutput) => Promise<void>;
|
||||||
|
onTerminalStreamedOutputFlushed?: (output: AgentOutput) => void;
|
||||||
|
group: RegisteredGroup;
|
||||||
|
input: AgentInput;
|
||||||
|
}): Promise<void> {
|
||||||
|
return args.outputChain.then(async () => {
|
||||||
|
try {
|
||||||
|
await args.onOutput(args.parsed);
|
||||||
|
if (isTerminalStreamedOutput(args.parsed)) {
|
||||||
|
args.onTerminalStreamedOutputFlushed?.(args.parsed);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logStreamedOutputDeliveryError(err, args.group, args.input);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function writeTimeoutLog(args: {
|
function writeTimeoutLog(args: {
|
||||||
logsDir: string;
|
logsDir: string;
|
||||||
input: AgentInput;
|
input: AgentInput;
|
||||||
@@ -188,27 +226,15 @@ export function runSpawnedAgentProcess(
|
|||||||
}
|
}
|
||||||
hadStreamingOutput = true;
|
hadStreamingOutput = true;
|
||||||
resetTimeout();
|
resetTimeout();
|
||||||
if (parsed.status === 'error') {
|
logStreamedAgentErrorOutput(parsed, group, input);
|
||||||
logger.warn(
|
outputChain = chainStreamedOutputDelivery({
|
||||||
{
|
outputChain,
|
||||||
group: group.name,
|
parsed,
|
||||||
chatJid: input.chatJid,
|
onOutput,
|
||||||
runId: input.runId,
|
onTerminalStreamedOutputFlushed:
|
||||||
error: parsed.error,
|
args.onTerminalStreamedOutputFlushed,
|
||||||
newSessionId: parsed.newSessionId,
|
group,
|
||||||
},
|
input,
|
||||||
'Streamed agent error output',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
outputChain = outputChain.then(async () => {
|
|
||||||
try {
|
|
||||||
await onOutput(parsed);
|
|
||||||
if (isTerminalStreamedOutput(parsed)) {
|
|
||||||
args.onTerminalStreamedOutputFlushed?.(parsed);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
logStreamedOutputDeliveryError(err, group, input);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
|
|||||||
@@ -137,6 +137,72 @@ function emitOutputMarker(
|
|||||||
proc.stdout.push(`${OUTPUT_START_MARKER}\n${json}\n${OUTPUT_END_MARKER}\n`);
|
proc.stdout.push(`${OUTPUT_START_MARKER}\n${json}\n${OUTPUT_END_MARKER}\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function mockCodexLeaseEnvironment(releaseLease: () => void) {
|
||||||
|
return vi
|
||||||
|
.spyOn(agentRunnerEnvironment, 'prepareGroupEnvironment')
|
||||||
|
.mockReturnValueOnce({
|
||||||
|
env: {
|
||||||
|
EJCLAW_IPC_DIR: '/tmp/ejclaw-test-data/ipc/test-group',
|
||||||
|
},
|
||||||
|
groupDir: '/tmp/ejclaw-test-groups/test-group',
|
||||||
|
runnerDir: '/tmp/ejclaw-test-runners/codex-runner',
|
||||||
|
codexSessionAuth: {
|
||||||
|
canonicalAuthPath: '/tmp/codex-account/auth.json',
|
||||||
|
sessionAuthPath: '/tmp/codex-session/auth.json',
|
||||||
|
accountIndex: 0,
|
||||||
|
lease: {
|
||||||
|
accountIndex: 0,
|
||||||
|
authPath: '/tmp/codex-account/auth.json',
|
||||||
|
release: releaseLease,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function expectCodexLeaseReleasedAfterFinalOutputFlush() {
|
||||||
|
const releaseLease = vi.fn();
|
||||||
|
const prepareGroupEnvironmentSpy = mockCodexLeaseEnvironment(releaseLease);
|
||||||
|
const onOutput = vi.fn(async () => {});
|
||||||
|
const codexGroup: RegisteredGroup = {
|
||||||
|
...testGroup,
|
||||||
|
agentType: 'codex',
|
||||||
|
};
|
||||||
|
let resultPromise: Promise<AgentOutput> | undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
resultPromise = runAgentProcess(
|
||||||
|
codexGroup,
|
||||||
|
{
|
||||||
|
...testInput,
|
||||||
|
runId: 'run-codex-lease-final',
|
||||||
|
},
|
||||||
|
() => {},
|
||||||
|
onOutput,
|
||||||
|
);
|
||||||
|
|
||||||
|
emitOutputMarker(fakeProc, {
|
||||||
|
status: 'success',
|
||||||
|
result: 'TASK_DONE\n작업 완료',
|
||||||
|
phase: 'final',
|
||||||
|
newSessionId: 'codex-session-lease-final',
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
|
expect(onOutput).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
result: 'TASK_DONE\n작업 완료',
|
||||||
|
phase: 'final',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(releaseLease).toHaveBeenCalledTimes(1);
|
||||||
|
} finally {
|
||||||
|
fakeProc.emit('close', 0);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
await resultPromise?.catch(() => undefined);
|
||||||
|
prepareGroupEnvironmentSpy.mockRestore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
describe('agent-runner timeout behavior', () => {
|
describe('agent-runner timeout behavior', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
@@ -157,23 +223,18 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
onOutput,
|
onOutput,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Emit output with a result
|
|
||||||
emitOutputMarker(fakeProc, {
|
emitOutputMarker(fakeProc, {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
result: 'Here is my response',
|
result: 'Here is my response',
|
||||||
newSessionId: 'session-123',
|
newSessionId: 'session-123',
|
||||||
});
|
});
|
||||||
|
|
||||||
// Let output processing settle
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
// Fire the hard timeout (IDLE_TIMEOUT + 30s = 1830000ms)
|
|
||||||
await vi.advanceTimersByTimeAsync(1830000);
|
await vi.advanceTimersByTimeAsync(1830000);
|
||||||
|
|
||||||
// Emit close event (as if agent was stopped by the timeout)
|
|
||||||
fakeProc.emit('close', 137);
|
fakeProc.emit('close', 137);
|
||||||
|
|
||||||
// Let the promise resolve
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
const result = await resultPromise;
|
const result = await resultPromise;
|
||||||
@@ -193,10 +254,8 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
onOutput,
|
onOutput,
|
||||||
);
|
);
|
||||||
|
|
||||||
// No output emitted — fire the hard timeout
|
|
||||||
await vi.advanceTimersByTimeAsync(1830000);
|
await vi.advanceTimersByTimeAsync(1830000);
|
||||||
|
|
||||||
// Emit close event
|
|
||||||
fakeProc.emit('close', 137);
|
fakeProc.emit('close', 137);
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
@@ -216,7 +275,6 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
onOutput,
|
onOutput,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Emit output
|
|
||||||
emitOutputMarker(fakeProc, {
|
emitOutputMarker(fakeProc, {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
result: 'Done',
|
result: 'Done',
|
||||||
@@ -225,7 +283,6 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
// Normal exit (no timeout)
|
|
||||||
fakeProc.emit('close', 0);
|
fakeProc.emit('close', 0);
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
@@ -236,65 +293,7 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('releases Codex auth lease after final streamed output flushes even before process close', async () => {
|
it('releases Codex auth lease after final streamed output flushes even before process close', async () => {
|
||||||
const releaseLease = vi.fn();
|
await expectCodexLeaseReleasedAfterFinalOutputFlush();
|
||||||
const prepareGroupEnvironmentSpy = vi
|
|
||||||
.spyOn(agentRunnerEnvironment, 'prepareGroupEnvironment')
|
|
||||||
.mockReturnValueOnce({
|
|
||||||
env: {
|
|
||||||
EJCLAW_IPC_DIR: '/tmp/ejclaw-test-data/ipc/test-group',
|
|
||||||
},
|
|
||||||
groupDir: '/tmp/ejclaw-test-groups/test-group',
|
|
||||||
runnerDir: '/tmp/ejclaw-test-runners/codex-runner',
|
|
||||||
codexSessionAuth: {
|
|
||||||
canonicalAuthPath: '/tmp/codex-account/auth.json',
|
|
||||||
sessionAuthPath: '/tmp/codex-session/auth.json',
|
|
||||||
accountIndex: 0,
|
|
||||||
lease: {
|
|
||||||
accountIndex: 0,
|
|
||||||
authPath: '/tmp/codex-account/auth.json',
|
|
||||||
release: releaseLease,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const onOutput = vi.fn(async () => {});
|
|
||||||
const codexGroup: RegisteredGroup = {
|
|
||||||
...testGroup,
|
|
||||||
agentType: 'codex',
|
|
||||||
};
|
|
||||||
let resultPromise: Promise<AgentOutput> | undefined;
|
|
||||||
|
|
||||||
try {
|
|
||||||
resultPromise = runAgentProcess(
|
|
||||||
codexGroup,
|
|
||||||
{
|
|
||||||
...testInput,
|
|
||||||
runId: 'run-codex-lease-final',
|
|
||||||
},
|
|
||||||
() => {},
|
|
||||||
onOutput,
|
|
||||||
);
|
|
||||||
|
|
||||||
emitOutputMarker(fakeProc, {
|
|
||||||
status: 'success',
|
|
||||||
result: 'TASK_DONE\n작업 완료',
|
|
||||||
phase: 'final',
|
|
||||||
newSessionId: 'codex-session-lease-final',
|
|
||||||
});
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
|
||||||
|
|
||||||
expect(onOutput).toHaveBeenCalledWith(
|
|
||||||
expect.objectContaining({
|
|
||||||
result: 'TASK_DONE\n작업 완료',
|
|
||||||
phase: 'final',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(releaseLease).toHaveBeenCalledTimes(1);
|
|
||||||
} finally {
|
|
||||||
fakeProc.emit('close', 0);
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
|
||||||
await resultPromise?.catch(() => undefined);
|
|
||||||
prepareGroupEnvironmentSpy.mockRestore();
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('preserves streamed progress phase metadata', async () => {
|
it('preserves streamed progress phase metadata', async () => {
|
||||||
|
|||||||
@@ -623,6 +623,7 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
|||||||
sawOutput: state.sawOutputForFollowUp,
|
sawOutput: state.sawOutputForFollowUp,
|
||||||
taskStatus: finishedTask?.status ?? null,
|
taskStatus: finishedTask?.status ?? null,
|
||||||
outputSummary: this.pairedExecutionSummary,
|
outputSummary: this.pairedExecutionSummary,
|
||||||
|
ownerFailureCount: finishedTask?.owner_failure_count ?? null,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -172,17 +172,17 @@ describe('message-agent-executor-rules', () => {
|
|||||||
).toBe('pending');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not requeue a silent arbiter failure caused by Codex account auth expiry', () => {
|
it('requeues owner follow-up after a silent arbiter Codex account failure returns the task active', () => {
|
||||||
expect(
|
expect(
|
||||||
resolvePairedFollowUpQueueAction({
|
resolvePairedFollowUpQueueAction({
|
||||||
completedRole: 'arbiter',
|
completedRole: 'arbiter',
|
||||||
executionStatus: 'failed',
|
executionStatus: 'failed',
|
||||||
sawOutput: false,
|
sawOutput: false,
|
||||||
taskStatus: 'arbiter_requested',
|
taskStatus: 'active',
|
||||||
outputSummary:
|
outputSummary:
|
||||||
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.\nExecution completed without a visible terminal verdict.',
|
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.\nExecution completed without a visible terminal verdict.',
|
||||||
}),
|
}),
|
||||||
).toBe('none');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
||||||
@@ -207,7 +207,7 @@ describe('message-agent-executor-rules', () => {
|
|||||||
).toBe('pending');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not requeue a silent owner failure caused by Codex pool unavailability', () => {
|
it('requeues a silent owner failure caused by Codex pool unavailability', () => {
|
||||||
expect(
|
expect(
|
||||||
resolvePairedFollowUpQueueAction({
|
resolvePairedFollowUpQueueAction({
|
||||||
completedRole: 'owner',
|
completedRole: 'owner',
|
||||||
@@ -217,7 +217,7 @@ describe('message-agent-executor-rules', () => {
|
|||||||
outputSummary:
|
outputSummary:
|
||||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||||
}),
|
}),
|
||||||
).toBe('none');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
|
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
|
||||||
|
|||||||
@@ -28,13 +28,19 @@ export function resolvePairedFollowUpQueueAction(args: {
|
|||||||
sawOutput: boolean;
|
sawOutput: boolean;
|
||||||
taskStatus: PairedTaskStatus | null;
|
taskStatus: PairedTaskStatus | null;
|
||||||
outputSummary?: string | null;
|
outputSummary?: string | null;
|
||||||
|
ownerFailureCount?: number | null;
|
||||||
}): PairedFollowUpQueueAction {
|
}): PairedFollowUpQueueAction {
|
||||||
if (
|
if (
|
||||||
args.executionStatus === 'failed' &&
|
args.executionStatus === 'failed' &&
|
||||||
args.sawOutput === false &&
|
args.sawOutput === false &&
|
||||||
isSilentCodexAccountFailure(args.outputSummary)
|
isSilentCodexAccountFailure(args.outputSummary)
|
||||||
) {
|
) {
|
||||||
return 'none';
|
if (args.completedRole === 'arbiter' && args.taskStatus === 'active') {
|
||||||
|
return 'pending';
|
||||||
|
}
|
||||||
|
if (args.completedRole !== 'owner') {
|
||||||
|
return 'none';
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
|||||||
@@ -21,6 +21,11 @@ export type PairedFollowUpSource = Parameters<
|
|||||||
typeof resolveFollowUpDispatch
|
typeof resolveFollowUpDispatch
|
||||||
>[0]['source'];
|
>[0]['source'];
|
||||||
|
|
||||||
|
type PairedFollowUpTaskContext = Pick<
|
||||||
|
PairedTask,
|
||||||
|
'id' | 'status' | 'round_trip_count' | 'updated_at' | 'owner_failure_count'
|
||||||
|
>;
|
||||||
|
|
||||||
export interface PairedFollowUpDecision {
|
export interface PairedFollowUpDecision {
|
||||||
taskId: string | null;
|
taskId: string | null;
|
||||||
taskStatus: PairedTaskStatus | null;
|
taskStatus: PairedTaskStatus | null;
|
||||||
@@ -67,7 +72,10 @@ export function resolveLatestPairedTurnOutputContext(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function resolvePairedFollowUpDecision(args: {
|
export function resolvePairedFollowUpDecision(args: {
|
||||||
task: Pick<PairedTask, 'id' | 'status'> | null | undefined;
|
task:
|
||||||
|
| Pick<PairedFollowUpTaskContext, 'id' | 'status' | 'owner_failure_count'>
|
||||||
|
| null
|
||||||
|
| undefined;
|
||||||
source: PairedFollowUpSource;
|
source: PairedFollowUpSource;
|
||||||
completedRole?: PairedRoomRole;
|
completedRole?: PairedRoomRole;
|
||||||
executionStatus?: 'succeeded' | 'failed';
|
executionStatus?: 'succeeded' | 'failed';
|
||||||
@@ -93,6 +101,7 @@ export function resolvePairedFollowUpDecision(args: {
|
|||||||
taskStatus: args.task?.status ?? null,
|
taskStatus: args.task?.status ?? null,
|
||||||
lastTurnOutputRole,
|
lastTurnOutputRole,
|
||||||
lastTurnOutputVerdict,
|
lastTurnOutputVerdict,
|
||||||
|
ownerFailureCount: args.task?.owner_failure_count ?? null,
|
||||||
});
|
});
|
||||||
const dispatch = resolveFollowUpDispatch({
|
const dispatch = resolveFollowUpDispatch({
|
||||||
source: args.source,
|
source: args.source,
|
||||||
@@ -115,10 +124,7 @@ export function resolvePairedFollowUpDecision(args: {
|
|||||||
export function schedulePairedFollowUpIntent(args: {
|
export function schedulePairedFollowUpIntent(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
intentKind: ScheduledPairedFollowUpIntentKind;
|
intentKind: ScheduledPairedFollowUpIntentKind;
|
||||||
enqueue: () => void;
|
enqueue: () => void;
|
||||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||||
@@ -148,6 +154,7 @@ export function schedulePairedFollowUpIntent(args: {
|
|||||||
taskStatus: args.task.status,
|
taskStatus: args.task.status,
|
||||||
lastTurnOutputRole: latestOutputContext.role,
|
lastTurnOutputRole: latestOutputContext.role,
|
||||||
lastTurnOutputVerdict: latestOutputContext.verdict,
|
lastTurnOutputVerdict: latestOutputContext.verdict,
|
||||||
|
ownerFailureCount: args.task.owner_failure_count ?? null,
|
||||||
intentKind: args.intentKind,
|
intentKind: args.intentKind,
|
||||||
}) &&
|
}) &&
|
||||||
!(
|
!(
|
||||||
@@ -171,10 +178,7 @@ export function schedulePairedFollowUpIntent(args: {
|
|||||||
export function schedulePairedFollowUpWithMessageCheck(args: {
|
export function schedulePairedFollowUpWithMessageCheck(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
intentKind: ScheduledPairedFollowUpIntentKind;
|
intentKind: ScheduledPairedFollowUpIntentKind;
|
||||||
enqueueMessageCheck: () => void;
|
enqueueMessageCheck: () => void;
|
||||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||||
@@ -198,10 +202,7 @@ export function schedulePairedFollowUpWithMessageCheck(args: {
|
|||||||
export function dispatchPairedFollowUpForEvent(args: {
|
export function dispatchPairedFollowUpForEvent(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
source: PairedFollowUpSource;
|
source: PairedFollowUpSource;
|
||||||
completedRole?: PairedRoomRole;
|
completedRole?: PairedRoomRole;
|
||||||
executionStatus?: 'succeeded' | 'failed';
|
executionStatus?: 'succeeded' | 'failed';
|
||||||
@@ -270,10 +271,7 @@ export function dispatchPairedFollowUpForEvent(args: {
|
|||||||
export function enqueuePairedFollowUpAfterEvent(args: {
|
export function enqueuePairedFollowUpAfterEvent(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
source: PairedFollowUpSource;
|
source: PairedFollowUpSource;
|
||||||
completedRole?: PairedRoomRole;
|
completedRole?: PairedRoomRole;
|
||||||
executionStatus?: 'succeeded' | 'failed';
|
executionStatus?: 'succeeded' | 'failed';
|
||||||
|
|||||||
179
src/owner-final-codex-unavailable.test.ts
Normal file
179
src/owner-final-codex-unavailable.test.ts
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./db.js', () => {
|
||||||
|
const updatePairedTask = vi.fn();
|
||||||
|
return {
|
||||||
|
getPairedTaskById: vi.fn(),
|
||||||
|
getPairedWorkspace: vi.fn(),
|
||||||
|
updatePairedTask,
|
||||||
|
updatePairedTaskIfUnchanged: vi.fn((id, _expectedUpdatedAt, updates) => {
|
||||||
|
updatePairedTask(id, updates);
|
||||||
|
return true;
|
||||||
|
}),
|
||||||
|
releasePairedTaskExecutionLease: vi.fn(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock('./paired-workspace-manager.js', () => ({
|
||||||
|
isOwnerWorkspaceRepairNeededError: vi.fn(() => false),
|
||||||
|
markPairedTaskReviewReady: vi.fn(),
|
||||||
|
prepareReviewerWorkspaceForExecution: vi.fn(),
|
||||||
|
provisionOwnerWorkspaceForPairedTask: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./logger.js', () => ({
|
||||||
|
logger: {
|
||||||
|
debug: vi.fn(),
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import * as config from './config.js';
|
||||||
|
import * as db from './db.js';
|
||||||
|
import { completePairedExecutionContext } from './paired-execution-context.js';
|
||||||
|
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
||||||
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
|
const CODEX_UNAVAILABLE_SUMMARY =
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.';
|
||||||
|
|
||||||
|
function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||||
|
return {
|
||||||
|
id: 'task-1',
|
||||||
|
chat_jid: 'dc:test',
|
||||||
|
group_folder: 'ejclaw',
|
||||||
|
owner_service_id: config.CODEX_MAIN_SERVICE_ID,
|
||||||
|
reviewer_service_id: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
review_requested_at: null,
|
||||||
|
round_trip_count: 0,
|
||||||
|
owner_failure_count: 0,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
task_done_then_user_reopen_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
status: 'merge_ready',
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
created_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
updated_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('owner final Codex unavailable handling', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(db.getPairedWorkspace).mockReturnValue(undefined);
|
||||||
|
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves active tasks when the first owner turn cannot start Codex', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
const updates = vi.mocked(db.updatePairedTask).mock.calls[0]?.[1];
|
||||||
|
expect(updates).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
owner_failure_count: 1,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(updates?.status).not.toBe('completed');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves merge_ready when owner finalization cannot start Codex', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
owner_failure_count: 1,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requests arbiter after repeated owner Codex account failures', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
owner_failure_count: 1,
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'arbiter_requested',
|
||||||
|
owner_failure_count: 2,
|
||||||
|
arbiter_requested_at: expect.any(String),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requeues owner finalization once after preserving merge_ready', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'owner',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'merge_ready',
|
||||||
|
ownerFailureCount: 1,
|
||||||
|
outputSummary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('queues arbiter after repeated owner finalization failures', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'owner',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'arbiter_requested',
|
||||||
|
ownerFailureCount: 2,
|
||||||
|
outputSummary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
});
|
||||||
81
src/paired-arbiter-codex-owner-wake.test.ts
Normal file
81
src/paired-arbiter-codex-owner-wake.test.ts
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./db.js', () => ({
|
||||||
|
getPairedTurnOutputs: vi.fn(() => []),
|
||||||
|
reservePairedTurnReservation: vi.fn(() => true),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { getPairedTurnOutputs } from './db.js';
|
||||||
|
import { dispatchPairedFollowUpForEvent } from './message-runtime-follow-up.js';
|
||||||
|
import {
|
||||||
|
matchesExpectedPairedFollowUpIntent,
|
||||||
|
resolveNextTurnAction,
|
||||||
|
} from './message-runtime-rules.js';
|
||||||
|
|
||||||
|
describe('arbiter Codex failure owner wake', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(getPairedTurnOutputs).mockReturnValue([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps owner follow-ups schedulable when active task has owner failure count', () => {
|
||||||
|
const followUpContext = {
|
||||||
|
taskStatus: 'active' as const,
|
||||||
|
lastTurnOutputRole: 'owner' as const,
|
||||||
|
lastTurnOutputVerdict: 'step_done' as const,
|
||||||
|
ownerFailureCount: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(resolveNextTurnAction(followUpContext)).toEqual({
|
||||||
|
kind: 'owner-follow-up',
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
matchesExpectedPairedFollowUpIntent({
|
||||||
|
...followUpContext,
|
||||||
|
intentKind: 'owner-follow-up',
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requeues owner follow-up when arbiter Codex failure returns the task active', () => {
|
||||||
|
const enqueue = vi.fn();
|
||||||
|
vi.mocked(getPairedTurnOutputs).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
task_id: 'task-arbiter-codex-unavailable',
|
||||||
|
turn_number: 1,
|
||||||
|
role: 'owner',
|
||||||
|
output_text: 'STEP_DONE\nwaiting for CI',
|
||||||
|
verdict: 'step_done',
|
||||||
|
created_at: '2026-03-30T00:00:01.000Z',
|
||||||
|
},
|
||||||
|
] as any);
|
||||||
|
|
||||||
|
const result = dispatchPairedFollowUpForEvent({
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-arbiter-codex-unavailable',
|
||||||
|
task: {
|
||||||
|
id: 'task-arbiter-codex-unavailable',
|
||||||
|
status: 'active',
|
||||||
|
round_trip_count: 1,
|
||||||
|
owner_failure_count: 1,
|
||||||
|
updated_at: '2026-03-30T00:00:02.000Z',
|
||||||
|
},
|
||||||
|
source: 'executor-recovery',
|
||||||
|
completedRole: 'arbiter',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
enqueue,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
kind: 'paired-follow-up',
|
||||||
|
intentKind: 'owner-follow-up',
|
||||||
|
scheduled: true,
|
||||||
|
taskStatus: 'active',
|
||||||
|
lastTurnOutputRole: 'owner',
|
||||||
|
nextTurnAction: { kind: 'owner-follow-up' },
|
||||||
|
});
|
||||||
|
expect(enqueue).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -14,16 +14,23 @@ export function handleFailedArbiterExecution(args: {
|
|||||||
const { task, taskId, summary } = args;
|
const { task, taskId, summary } = args;
|
||||||
if (isTerminalCodexAccountFailure(summary)) {
|
if (isTerminalCodexAccountFailure(summary)) {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
|
const nextOwnerFailureCount = Math.max(
|
||||||
|
(task.owner_failure_count ?? 0) + 1,
|
||||||
|
1,
|
||||||
|
);
|
||||||
transitionPairedTaskStatus({
|
transitionPairedTaskStatus({
|
||||||
taskId,
|
taskId,
|
||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
nextStatus: 'completed',
|
nextStatus: 'active',
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
patch: {
|
patch: {
|
||||||
arbiter_verdict: 'escalate',
|
owner_failure_count: nextOwnerFailureCount,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_verdict: null,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
completion_reason: 'arbiter_codex_unavailable',
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@@ -31,9 +38,10 @@ export function handleFailedArbiterExecution(args: {
|
|||||||
taskId,
|
taskId,
|
||||||
role: 'arbiter',
|
role: 'arbiter',
|
||||||
status: task.status,
|
status: task.status,
|
||||||
|
nextOwnerFailureCount,
|
||||||
summary: summary?.slice(0, 200),
|
summary: summary?.slice(0, 200),
|
||||||
},
|
},
|
||||||
'Completed arbiter task after terminal Codex account failure instead of re-arming arbitration loop',
|
'Returned arbiter task to owner after terminal Codex account failure',
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,33 +31,81 @@ export function handleFailedOwnerExecution(args: {
|
|||||||
}): void {
|
}): void {
|
||||||
const { task, taskId, summary } = args;
|
const { task, taskId, summary } = args;
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
|
const nextFailureCount = (task.owner_failure_count ?? 0) + 1;
|
||||||
if (isTerminalCodexAccountFailure(summary)) {
|
if (isTerminalCodexAccountFailure(summary)) {
|
||||||
|
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
||||||
|
requestArbiterOrEscalate({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
now,
|
||||||
|
arbiterLogMessage:
|
||||||
|
'Owner Codex unavailable repeatedly — requesting arbiter',
|
||||||
|
escalateLogMessage:
|
||||||
|
'Owner Codex unavailable repeatedly — escalating to user',
|
||||||
|
logContext: {
|
||||||
|
taskId,
|
||||||
|
role: 'owner',
|
||||||
|
previousStatus: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
|
summary: summary?.slice(0, 160),
|
||||||
|
},
|
||||||
|
patch: {
|
||||||
|
owner_failure_count: nextFailureCount,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
completion_reason: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const patch = {
|
||||||
|
owner_failure_count: nextFailureCount,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
};
|
||||||
|
if (task.status === 'active' || task.status === 'merge_ready') {
|
||||||
|
applyPairedTaskPatch({
|
||||||
|
taskId,
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch,
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'owner',
|
||||||
|
status: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
|
'Preserved owner task after terminal Codex account failure',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
transitionPairedTaskStatus({
|
transitionPairedTaskStatus({
|
||||||
taskId,
|
taskId,
|
||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
nextStatus: 'completed',
|
nextStatus: 'active',
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
patch: {
|
patch,
|
||||||
arbiter_verdict: 'escalate',
|
|
||||||
arbiter_requested_at: null,
|
|
||||||
completion_reason: 'owner_codex_unavailable',
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
taskId,
|
taskId,
|
||||||
role: 'owner',
|
role: 'owner',
|
||||||
status: task.status,
|
status: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
summary: summary?.slice(0, 200),
|
summary: summary?.slice(0, 200),
|
||||||
},
|
},
|
||||||
'Completed owner task after terminal Codex account failure instead of retrying owner loop',
|
'Reset owner task to active after terminal Codex account failure',
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const nextFailureCount = (task.owner_failure_count ?? 0) + 1;
|
|
||||||
|
|
||||||
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
|||||||
@@ -197,10 +197,16 @@ describe('paired execution routing loop guards', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not re-arm arbiter loop after terminal Codex account failure', () => {
|
it('returns arbiter terminal Codex account failures to owner without re-arming arbiter', () => {
|
||||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
buildPairedTask({
|
buildPairedTask({
|
||||||
status: 'in_arbitration',
|
status: 'in_arbitration',
|
||||||
|
owner_failure_count: 1,
|
||||||
|
owner_step_done_streak: 3,
|
||||||
|
finalize_step_done_count: 1,
|
||||||
|
empty_step_done_streak: 2,
|
||||||
|
arbiter_verdict: 'revise',
|
||||||
|
arbiter_requested_at: '2026-03-28T00:00:04.000Z',
|
||||||
updated_at: '2026-03-28T00:00:05.000Z',
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@@ -216,10 +222,13 @@ describe('paired execution routing loop guards', () => {
|
|||||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
'task-1',
|
'task-1',
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
status: 'completed',
|
status: 'active',
|
||||||
arbiter_verdict: 'escalate',
|
owner_failure_count: 2,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_verdict: null,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
completion_reason: 'arbiter_codex_unavailable',
|
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -251,7 +260,7 @@ describe('paired execution routing loop guards', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('completes owner task after terminal Codex account failure instead of retrying owner forever', () => {
|
it('preserves owner task after terminal Codex account failure instead of completing silently', () => {
|
||||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
buildPairedTask({
|
buildPairedTask({
|
||||||
status: 'active',
|
status: 'active',
|
||||||
@@ -270,10 +279,10 @@ describe('paired execution routing loop guards', () => {
|
|||||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
'task-1',
|
'task-1',
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
status: 'completed',
|
owner_failure_count: 1,
|
||||||
arbiter_verdict: 'escalate',
|
arbiter_verdict: null,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
completion_reason: 'owner_codex_unavailable',
|
completion_reason: null,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user