- mark Codex bearer/refresh failures as terminal auth-expired states - sync refreshed session auth back to rotation slots and revive refreshed dead_auth slots - stop paired arbiter retry loops when Codex accounts are unavailable - add regression coverage for rotation leases, follow-up suppression, and arbiter closure
168 lines
4.9 KiB
TypeScript
168 lines
4.9 KiB
TypeScript
import { logger } from './logger.js';
|
|
import {
|
|
classifyAgentError,
|
|
classifyCodexAuthError,
|
|
} from './agent-error-detection.js';
|
|
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
|
import { classifyArbiterVerdict } from './paired-verdict.js';
|
|
import type { PairedTask } from './types.js';
|
|
|
|
const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0;
|
|
|
|
function isTerminalCodexAccountFailure(summary?: string | null): boolean {
|
|
if (!summary) return false;
|
|
if (classifyCodexAuthError(summary).category !== 'none') return true;
|
|
if (classifyAgentError(summary).category === 'rate-limit') return true;
|
|
return /all\s+codex(?:\s+rotation)?\s+accounts/i.test(summary);
|
|
}
|
|
|
|
export function handleFailedArbiterExecution(args: {
|
|
task: PairedTask;
|
|
taskId: string;
|
|
summary?: string | null;
|
|
}): void {
|
|
const { task, taskId, summary } = args;
|
|
if (isTerminalCodexAccountFailure(summary)) {
|
|
const now = new Date().toISOString();
|
|
transitionPairedTaskStatus({
|
|
taskId,
|
|
currentStatus: task.status,
|
|
nextStatus: 'completed',
|
|
expectedUpdatedAt: task.updated_at,
|
|
updatedAt: now,
|
|
patch: {
|
|
arbiter_verdict: 'escalate',
|
|
arbiter_requested_at: null,
|
|
completion_reason: 'arbiter_codex_unavailable',
|
|
},
|
|
});
|
|
logger.warn(
|
|
{
|
|
taskId,
|
|
role: 'arbiter',
|
|
status: task.status,
|
|
summary: summary?.slice(0, 200),
|
|
},
|
|
'Completed arbiter task after terminal Codex account failure instead of re-arming arbitration loop',
|
|
);
|
|
return;
|
|
}
|
|
const now = new Date().toISOString();
|
|
const fallbackStatus =
|
|
task.status === 'in_arbitration' || task.status === 'arbiter_requested'
|
|
? 'arbiter_requested'
|
|
: task.status;
|
|
if (fallbackStatus !== task.status) {
|
|
transitionPairedTaskStatus({
|
|
taskId,
|
|
currentStatus: task.status,
|
|
nextStatus: fallbackStatus,
|
|
expectedUpdatedAt: task.updated_at,
|
|
updatedAt: now,
|
|
});
|
|
logger.warn(
|
|
{
|
|
taskId,
|
|
role: 'arbiter',
|
|
previousStatus: task.status,
|
|
nextStatus: fallbackStatus,
|
|
},
|
|
'Preserved arbiter task in arbitration-requested state after failed execution',
|
|
);
|
|
}
|
|
}
|
|
|
|
export function handleArbiterCompletion(args: {
|
|
task: PairedTask;
|
|
taskId: string;
|
|
summary?: string | null;
|
|
}): void {
|
|
const { task, taskId, summary } = args;
|
|
const now = new Date().toISOString();
|
|
const arbiterVerdict = classifyArbiterVerdict(summary);
|
|
|
|
logger.info(
|
|
{ taskId, arbiterVerdict, summary: summary?.slice(0, 200) },
|
|
'Arbiter verdict rendered',
|
|
);
|
|
|
|
switch (arbiterVerdict) {
|
|
case 'proceed':
|
|
transitionPairedTaskStatus({
|
|
taskId,
|
|
currentStatus: 'in_arbitration',
|
|
nextStatus: 'review_ready',
|
|
expectedUpdatedAt: task.updated_at,
|
|
updatedAt: now,
|
|
patch: {
|
|
round_trip_count: ARBITER_RESOLUTION_ROUND_TRIP_COUNT,
|
|
owner_failure_count: 0,
|
|
owner_step_done_streak: 0,
|
|
finalize_step_done_count: 0,
|
|
empty_step_done_streak: 0,
|
|
arbiter_verdict: arbiterVerdict,
|
|
arbiter_requested_at: null,
|
|
},
|
|
});
|
|
logger.info(
|
|
{ taskId, arbiterVerdict },
|
|
'Arbiter proceeded — returning task to reviewer for approval',
|
|
);
|
|
return;
|
|
case 'revise':
|
|
case 'reset':
|
|
transitionPairedTaskStatus({
|
|
taskId,
|
|
currentStatus: 'in_arbitration',
|
|
nextStatus: 'active',
|
|
expectedUpdatedAt: task.updated_at,
|
|
updatedAt: now,
|
|
patch: {
|
|
round_trip_count: ARBITER_RESOLUTION_ROUND_TRIP_COUNT,
|
|
owner_failure_count: 0,
|
|
owner_step_done_streak: 0,
|
|
finalize_step_done_count: 0,
|
|
empty_step_done_streak: 0,
|
|
arbiter_verdict: arbiterVerdict,
|
|
arbiter_requested_at: null,
|
|
},
|
|
});
|
|
logger.info(
|
|
{ taskId, arbiterVerdict },
|
|
'Arbiter requested owner changes — resuming owner flow',
|
|
);
|
|
return;
|
|
case 'escalate':
|
|
transitionPairedTaskStatus({
|
|
taskId,
|
|
currentStatus: 'in_arbitration',
|
|
nextStatus: 'completed',
|
|
expectedUpdatedAt: task.updated_at,
|
|
updatedAt: now,
|
|
patch: {
|
|
arbiter_verdict: 'escalate',
|
|
completion_reason: 'arbiter_escalated',
|
|
},
|
|
});
|
|
logger.info({ taskId }, 'Arbiter escalated to user — task completed');
|
|
return;
|
|
default:
|
|
transitionPairedTaskStatus({
|
|
taskId,
|
|
currentStatus: 'in_arbitration',
|
|
nextStatus: 'completed',
|
|
expectedUpdatedAt: task.updated_at,
|
|
updatedAt: now,
|
|
patch: {
|
|
arbiter_verdict: 'unknown',
|
|
completion_reason: 'arbiter_escalated',
|
|
},
|
|
});
|
|
logger.warn(
|
|
{ taskId, summary: summary?.slice(0, 200) },
|
|
'Arbiter verdict unrecognized — escalating instead of treating it as approval',
|
|
);
|
|
return;
|
|
}
|
|
}
|