Retry silent owner capacity failures

Classify Codex selected-model capacity as overloaded and requeue silent owner failures that leave paired tasks active.
This commit is contained in:
Eyejoker
2026-05-21 03:46:59 +09:00
committed by GitHub
parent f240cf7820
commit fbaca3ab68
10 changed files with 281 additions and 4 deletions

View File

@@ -55,6 +55,17 @@ describe('agent-error-detection', () => {
expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded'); expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded');
}); });
it('classifies Codex model capacity errors as overloaded', () => {
expect(
classifyAgentError(
'Selected model is at capacity. Please try a different model.',
),
).toEqual({
category: 'overloaded',
reason: 'overloaded',
});
});
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => { it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
expect(shouldRotateClaudeToken('429')).toBe(true); expect(shouldRotateClaudeToken('429')).toBe(true);
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true); expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);

View File

@@ -264,6 +264,8 @@ export function classifyAgentError(
if ( if (
lower.includes('503') || lower.includes('503') ||
lower.includes('overloaded') || lower.includes('overloaded') ||
lower.includes('selected model is at capacity') ||
lower.includes('model is at capacity') ||
((lower.includes('502') || lower.includes('bad gateway')) && ((lower.includes('502') || lower.includes('bad gateway')) &&
(lower.includes('cloudflare') || (lower.includes('cloudflare') ||
lower.includes('<html') || lower.includes('<html') ||

View File

@@ -183,6 +183,17 @@ describe('message-agent-executor-rules', () => {
).toBe('pending'); ).toBe('pending');
}); });
it('resolves pending owner retry after a silent owner failure leaves the task active', () => {
expect(
resolvePairedFollowUpQueueAction({
completedRole: 'owner',
executionStatus: 'failed',
sawOutput: false,
taskStatus: 'active',
}),
).toBe('pending');
});
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => { it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
expect( expect(
resolvePairedFollowUpQueueAction({ resolvePairedFollowUpQueueAction({

View File

@@ -1,6 +1,7 @@
import { import {
resolveFollowUpDispatch, resolveFollowUpDispatch,
resolveNextTurnAction, resolveNextTurnAction,
shouldRetrySilentOwnerExecution,
} from './message-runtime-rules.js'; } from './message-runtime-rules.js';
import { parseVisibleVerdict } from './paired-verdict.js'; import { parseVisibleVerdict } from './paired-verdict.js';
import type { PairedRoomRole, PairedTaskStatus } from './types.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js';
@@ -23,6 +24,17 @@ export function resolvePairedFollowUpQueueAction(args: {
taskStatus: PairedTaskStatus | null; taskStatus: PairedTaskStatus | null;
outputSummary?: string | null; outputSummary?: string | null;
}): PairedFollowUpQueueAction { }): PairedFollowUpQueueAction {
if (
shouldRetrySilentOwnerExecution({
completedRole: args.completedRole,
executionStatus: args.executionStatus,
sawOutput: args.sawOutput,
taskStatus: args.taskStatus,
})
) {
return 'pending';
}
const nextTurnAction = resolveNextTurnAction({ const nextTurnAction = resolveNextTurnAction({
taskStatus: args.taskStatus, taskStatus: args.taskStatus,
lastTurnOutputRole: args.sawOutput ? args.completedRole : null, lastTurnOutputRole: args.sawOutput ? args.completedRole : null,

View File

@@ -124,6 +124,7 @@ export function buildPendingPairedTurn(args: {
verdict: lastTurnOutput?.verdict ?? null, verdict: lastTurnOutput?.verdict ?? null,
outputText: lastTurnOutput?.output_text ?? null, outputText: lastTurnOutput?.output_text ?? null,
}), }),
ownerFailureCount: task.owner_failure_count,
}); });
const recentMessages = getRecentChatMessages(chatJid, 20); const recentMessages = getRecentChatMessages(chatJid, 20);
const lastHumanMessage = getLastHumanMessageContent(chatJid); const lastHumanMessage = getLastHumanMessageContent(chatJid);

View File

@@ -176,6 +176,34 @@ describe('message-runtime-follow-up', () => {
expect(enqueueMessageCheck).toHaveBeenCalledTimes(1); expect(enqueueMessageCheck).toHaveBeenCalledTimes(1);
}); });
it('requeues a silent failed owner execution that leaves the task active', () => {
const enqueue = vi.fn();
const result = dispatchPairedFollowUpForEvent({
chatJid: 'group@test',
runId: 'run-silent-owner-retry',
task: {
id: 'task-silent-owner-retry',
status: 'active',
round_trip_count: 1,
updated_at: '2026-03-30T00:00:00.000Z',
},
source: 'executor-recovery',
completedRole: 'owner',
executionStatus: 'failed',
sawOutput: false,
enqueue,
});
expect(result).toMatchObject({
kind: 'paired-follow-up',
intentKind: 'owner-follow-up',
scheduled: true,
taskStatus: 'active',
lastTurnOutputRole: null,
});
expect(enqueue).toHaveBeenCalledTimes(1);
});
it('uses the scoped message-check enqueuer when scheduling a paired follow-up intent', () => { it('uses the scoped message-check enqueuer when scheduling a paired follow-up intent', () => {
const enqueueMessageCheck = vi.fn(); const enqueueMessageCheck = vi.fn();
const scheduled = schedulePairedFollowUpWithMessageCheck({ const scheduled = schedulePairedFollowUpWithMessageCheck({

View File

@@ -7,6 +7,7 @@ import {
matchesExpectedPairedFollowUpIntent, matchesExpectedPairedFollowUpIntent,
resolveFollowUpDispatch, resolveFollowUpDispatch,
resolveNextTurnAction, resolveNextTurnAction,
shouldRetrySilentOwnerExecution,
type FollowUpDispatch, type FollowUpDispatch,
type NextTurnAction, type NextTurnAction,
} from './message-runtime-rules.js'; } from './message-runtime-rules.js';
@@ -80,11 +81,19 @@ export function resolvePairedFollowUpDecision(args: {
fallbackLastTurnOutputRole: args.fallbackLastTurnOutputRole, fallbackLastTurnOutputRole: args.fallbackLastTurnOutputRole,
fallbackLastTurnOutputVerdict: args.fallbackLastTurnOutputVerdict, fallbackLastTurnOutputVerdict: args.fallbackLastTurnOutputVerdict,
}); });
const nextTurnAction = resolveNextTurnAction({ const silentOwnerRetry = shouldRetrySilentOwnerExecution({
completedRole: args.completedRole,
executionStatus: args.executionStatus,
sawOutput: args.sawOutput,
taskStatus: args.task?.status ?? null, taskStatus: args.task?.status ?? null,
lastTurnOutputRole,
lastTurnOutputVerdict,
}); });
const nextTurnAction = silentOwnerRetry
? ({ kind: 'owner-follow-up' } as const)
: resolveNextTurnAction({
taskStatus: args.task?.status ?? null,
lastTurnOutputRole,
lastTurnOutputVerdict,
});
const dispatch = resolveFollowUpDispatch({ const dispatch = resolveFollowUpDispatch({
source: args.source, source: args.source,
nextTurnAction, nextTurnAction,
@@ -116,6 +125,7 @@ export function schedulePairedFollowUpIntent(args: {
fallbackLastTurnOutputVerdict?: VisibleVerdict | null; fallbackLastTurnOutputVerdict?: VisibleVerdict | null;
lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null; lastTurnOutputVerdict?: VisibleVerdict | null;
allowSilentOwnerRetry?: boolean;
}): boolean { }): boolean {
if (!args.task) { if (!args.task) {
return false; return false;
@@ -139,7 +149,12 @@ export function schedulePairedFollowUpIntent(args: {
lastTurnOutputRole: latestOutputContext.role, lastTurnOutputRole: latestOutputContext.role,
lastTurnOutputVerdict: latestOutputContext.verdict, lastTurnOutputVerdict: latestOutputContext.verdict,
intentKind: args.intentKind, intentKind: args.intentKind,
}) }) &&
!(
args.allowSilentOwnerRetry === true &&
args.task.status === 'active' &&
args.intentKind === 'owner-follow-up'
)
) { ) {
return false; return false;
} }
@@ -230,6 +245,12 @@ export function dispatchPairedFollowUpForEvent(args: {
enqueue: args.enqueue, enqueue: args.enqueue,
lastTurnOutputRole: decision.lastTurnOutputRole, lastTurnOutputRole: decision.lastTurnOutputRole,
lastTurnOutputVerdict: decision.lastTurnOutputVerdict, lastTurnOutputVerdict: decision.lastTurnOutputVerdict,
allowSilentOwnerRetry: shouldRetrySilentOwnerExecution({
completedRole: args.completedRole,
executionStatus: args.executionStatus,
sawOutput: args.sawOutput,
taskStatus: args.task?.status ?? null,
}),
}); });
return { return {

View File

@@ -283,6 +283,7 @@ export async function runQueuedGroupTurn(args: {
hasHumanMessage: false, hasHumanMessage: false,
lastTurnOutputRole, lastTurnOutputRole,
lastTurnOutputVerdict, lastTurnOutputVerdict,
ownerFailureCount: currentTask.owner_failure_count,
}) })
: 'owner'; : 'owner';
if (!turnRole) { if (!turnRole) {

View File

@@ -122,6 +122,7 @@ export function resolveQueuedPairedTurnRole(args: {
hasHumanMessage: boolean; hasHumanMessage: boolean;
lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null; lastTurnOutputVerdict?: VisibleVerdict | null;
ownerFailureCount?: number | null;
}): 'owner' | 'reviewer' | 'arbiter' | null { }): 'owner' | 'reviewer' | 'arbiter' | null {
if (args.hasHumanMessage) { if (args.hasHumanMessage) {
return resolveQueuedTurnRole({ return resolveQueuedTurnRole({
@@ -134,6 +135,7 @@ export function resolveQueuedPairedTurnRole(args: {
taskStatus: args.taskStatus, taskStatus: args.taskStatus,
lastTurnOutputRole: args.lastTurnOutputRole, lastTurnOutputRole: args.lastTurnOutputRole,
lastTurnOutputVerdict: args.lastTurnOutputVerdict, lastTurnOutputVerdict: args.lastTurnOutputVerdict,
ownerFailureCount: args.ownerFailureCount,
}); });
switch (nextTurnAction.kind) { switch (nextTurnAction.kind) {
@@ -170,6 +172,7 @@ export function resolveNextTurnAction(args: {
taskStatus?: PairedTaskStatus | null; taskStatus?: PairedTaskStatus | null;
lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null; lastTurnOutputVerdict?: VisibleVerdict | null;
ownerFailureCount?: number | null;
}): NextTurnAction { }): NextTurnAction {
switch (args.taskStatus) { switch (args.taskStatus) {
case 'review_ready': case 'review_ready':
@@ -187,6 +190,9 @@ export function resolveNextTurnAction(args: {
? { kind: 'none' } ? { kind: 'none' }
: { kind: 'finalize-owner-turn' }; : { kind: 'finalize-owner-turn' };
case 'active': case 'active':
if ((args.ownerFailureCount ?? 0) > 0) {
return { kind: 'owner-follow-up' };
}
return args.lastTurnOutputRole === 'reviewer' || return args.lastTurnOutputRole === 'reviewer' ||
args.lastTurnOutputRole === 'arbiter' args.lastTurnOutputRole === 'arbiter'
? { kind: 'owner-follow-up' } ? { kind: 'owner-follow-up' }
@@ -200,6 +206,7 @@ export function matchesExpectedPairedFollowUpIntent(args: {
taskStatus?: PairedTaskStatus | null; taskStatus?: PairedTaskStatus | null;
lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null; lastTurnOutputVerdict?: VisibleVerdict | null;
ownerFailureCount?: number | null;
intentKind: ScheduledNextTurnActionKind; intentKind: ScheduledNextTurnActionKind;
}): boolean { }): boolean {
return ( return (
@@ -207,6 +214,7 @@ export function matchesExpectedPairedFollowUpIntent(args: {
taskStatus: args.taskStatus, taskStatus: args.taskStatus,
lastTurnOutputRole: args.lastTurnOutputRole, lastTurnOutputRole: args.lastTurnOutputRole,
lastTurnOutputVerdict: args.lastTurnOutputVerdict, lastTurnOutputVerdict: args.lastTurnOutputVerdict,
ownerFailureCount: args.ownerFailureCount,
}).kind === args.intentKind }).kind === args.intentKind
); );
} }
@@ -275,6 +283,20 @@ export function resolveFollowUpDispatch(args: {
} }
} }
export function shouldRetrySilentOwnerExecution(args: {
completedRole?: PairedRoomRole;
executionStatus?: 'succeeded' | 'failed';
sawOutput?: boolean;
taskStatus?: PairedTaskStatus | null;
}): boolean {
return (
args.completedRole === 'owner' &&
args.executionStatus === 'failed' &&
args.sawOutput === false &&
args.taskStatus === 'active'
);
}
/** Cursor key for a role. Owner uses chatJid, others use chatJid:role. */ /** Cursor key for a role. Owner uses chatJid, others use chatJid:role. */
export function resolveCursorKey( export function resolveCursorKey(
chatJid: string, chatJid: string,

View File

@@ -0,0 +1,168 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
_initTestDatabase,
createPairedTask,
getPairedTaskById,
} from './db.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { runQueuedGroupTurn } from './message-runtime-queue.js';
import {
resolveNextTurnAction,
resolveQueuedPairedTurnRole,
} from './message-runtime-rules.js';
import {
resetPairedFollowUpScheduleState,
schedulePairedFollowUpOnce,
} from './paired-follow-up-scheduler.js';
import type { Channel, PairedTask, RegisteredGroup } from './types.js';
function makeGroup(): RegisteredGroup {
return {
name: 'Test Group',
folder: 'test-group',
trigger: '@Andy',
added_at: '2026-03-30T00:00:00.000Z',
requiresTrigger: false,
agentType: 'codex',
workDir: '/repo',
};
}
function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
return {
id: 'task-silent-owner-retry',
chat_jid: 'group@test',
group_folder: 'test-group',
owner_service_id: 'codex-main',
reviewer_service_id: 'claude',
owner_agent_type: 'codex',
reviewer_agent_type: 'claude-code',
arbiter_agent_type: null,
title: null,
source_ref: 'HEAD',
plan_notes: null,
review_requested_at: null,
round_trip_count: 1,
status: 'active',
arbiter_verdict: null,
arbiter_requested_at: null,
completion_reason: null,
owner_failure_count: 1,
created_at: '2026-03-30T00:00:00.000Z',
updated_at: '2026-03-30T00:00:00.000Z',
...overrides,
};
}
function makeChannel(): Channel {
return {
name: 'discord',
connect: vi.fn(),
sendMessage: vi.fn(),
isConnected: vi.fn(() => true),
ownsJid: vi.fn(() => true),
disconnect: vi.fn(),
} as unknown as Channel;
}
describe('silent owner retry recovery', () => {
beforeEach(() => {
_initTestDatabase();
resetPairedFollowUpScheduleState();
});
it('treats Codex selected-model capacity as a rotation trigger', () => {
expect(
detectCodexRotationTrigger(
'Selected model is at capacity. Please try a different model.',
),
).toEqual({ shouldRotate: true, reason: 'overloaded' });
});
it('maps active tasks with a previous silent owner failure to an owner retry', () => {
expect(
resolveNextTurnAction({
taskStatus: 'active',
lastTurnOutputRole: null,
ownerFailureCount: 1,
}),
).toEqual({ kind: 'owner-follow-up' });
expect(
resolveQueuedPairedTurnRole({
taskStatus: 'active',
hasHumanMessage: false,
lastTurnOutputRole: null,
ownerFailureCount: 1,
}),
).toBe('owner');
});
it('runs a queued owner retry after a silent owner failure leaves an active task', async () => {
const task = makeTask();
createPairedTask(task);
expect(
schedulePairedFollowUpOnce({
chatJid: task.chat_jid,
runId: 'run-silent-owner-retry-schedule',
task,
intentKind: 'owner-follow-up',
enqueue: vi.fn(),
}),
).toBe(true);
const executeTurn = vi.fn(async () => ({
outputStatus: 'success' as const,
deliverySucceeded: true,
visiblePhase: 'final',
}));
const outcome = await runQueuedGroupTurn({
chatJid: task.chat_jid,
group: makeGroup(),
runId: 'run-silent-owner-retry',
log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as any,
timezone: 'UTC',
missedMessages: [
{
id: 'bot-silent-owner-retry',
chat_jid: task.chat_jid,
sender: 'owner-bot@test',
sender_name: 'owner',
content: 'retry owner',
timestamp: '2026-03-30T00:00:04.000Z',
seq: 48,
is_bot_message: true,
},
],
task: getPairedTaskById(task.id),
roleToChannel: {
owner: null,
reviewer: makeChannel(),
arbiter: null,
},
ownerChannel: makeChannel(),
lastAgentTimestamps: {},
saveState: vi.fn(),
executeTurn,
getFixedRoleChannelName: () => 'discord-review',
labelPairedSenders: (_chatJid, messages) => messages,
formatMessages: () => 'formatted prompt',
});
expect(outcome).toBe(true);
expect(executeTurn).toHaveBeenCalledWith(
expect.objectContaining({
deliveryRole: 'owner',
forcedRole: 'owner',
pairedTurnIdentity: {
turnId:
'task-silent-owner-retry:2026-03-30T00:00:00.000Z:owner-follow-up',
taskId: 'task-silent-owner-retry',
taskUpdatedAt: '2026-03-30T00:00:00.000Z',
intentKind: 'owner-follow-up',
role: 'owner',
},
}),
);
});
});