Retry silent owner capacity failures

Classify Codex selected-model capacity as overloaded and requeue silent owner failures that leave paired tasks active.
This commit is contained in:
Eyejoker
2026-05-21 03:46:59 +09:00
committed by GitHub
parent f240cf7820
commit fbaca3ab68
10 changed files with 281 additions and 4 deletions

View File

@@ -55,6 +55,17 @@ describe('agent-error-detection', () => {
expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded');
});
it('classifies Codex model capacity errors as overloaded', () => {
expect(
classifyAgentError(
'Selected model is at capacity. Please try a different model.',
),
).toEqual({
category: 'overloaded',
reason: 'overloaded',
});
});
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
expect(shouldRotateClaudeToken('429')).toBe(true);
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);

View File

@@ -264,6 +264,8 @@ export function classifyAgentError(
if (
lower.includes('503') ||
lower.includes('overloaded') ||
lower.includes('selected model is at capacity') ||
lower.includes('model is at capacity') ||
((lower.includes('502') || lower.includes('bad gateway')) &&
(lower.includes('cloudflare') ||
lower.includes('<html') ||

View File

@@ -183,6 +183,17 @@ describe('message-agent-executor-rules', () => {
).toBe('pending');
});
it('resolves pending owner retry after a silent owner failure leaves the task active', () => {
expect(
resolvePairedFollowUpQueueAction({
completedRole: 'owner',
executionStatus: 'failed',
sawOutput: false,
taskStatus: 'active',
}),
).toBe('pending');
});
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
expect(
resolvePairedFollowUpQueueAction({

View File

@@ -1,6 +1,7 @@
import {
resolveFollowUpDispatch,
resolveNextTurnAction,
shouldRetrySilentOwnerExecution,
} from './message-runtime-rules.js';
import { parseVisibleVerdict } from './paired-verdict.js';
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
@@ -23,6 +24,17 @@ export function resolvePairedFollowUpQueueAction(args: {
taskStatus: PairedTaskStatus | null;
outputSummary?: string | null;
}): PairedFollowUpQueueAction {
if (
shouldRetrySilentOwnerExecution({
completedRole: args.completedRole,
executionStatus: args.executionStatus,
sawOutput: args.sawOutput,
taskStatus: args.taskStatus,
})
) {
return 'pending';
}
const nextTurnAction = resolveNextTurnAction({
taskStatus: args.taskStatus,
lastTurnOutputRole: args.sawOutput ? args.completedRole : null,

View File

@@ -124,6 +124,7 @@ export function buildPendingPairedTurn(args: {
verdict: lastTurnOutput?.verdict ?? null,
outputText: lastTurnOutput?.output_text ?? null,
}),
ownerFailureCount: task.owner_failure_count,
});
const recentMessages = getRecentChatMessages(chatJid, 20);
const lastHumanMessage = getLastHumanMessageContent(chatJid);

View File

@@ -176,6 +176,34 @@ describe('message-runtime-follow-up', () => {
expect(enqueueMessageCheck).toHaveBeenCalledTimes(1);
});
it('requeues a silent failed owner execution that leaves the task active', () => {
const enqueue = vi.fn();
const result = dispatchPairedFollowUpForEvent({
chatJid: 'group@test',
runId: 'run-silent-owner-retry',
task: {
id: 'task-silent-owner-retry',
status: 'active',
round_trip_count: 1,
updated_at: '2026-03-30T00:00:00.000Z',
},
source: 'executor-recovery',
completedRole: 'owner',
executionStatus: 'failed',
sawOutput: false,
enqueue,
});
expect(result).toMatchObject({
kind: 'paired-follow-up',
intentKind: 'owner-follow-up',
scheduled: true,
taskStatus: 'active',
lastTurnOutputRole: null,
});
expect(enqueue).toHaveBeenCalledTimes(1);
});
it('uses the scoped message-check enqueuer when scheduling a paired follow-up intent', () => {
const enqueueMessageCheck = vi.fn();
const scheduled = schedulePairedFollowUpWithMessageCheck({

View File

@@ -7,6 +7,7 @@ import {
matchesExpectedPairedFollowUpIntent,
resolveFollowUpDispatch,
resolveNextTurnAction,
shouldRetrySilentOwnerExecution,
type FollowUpDispatch,
type NextTurnAction,
} from './message-runtime-rules.js';
@@ -80,11 +81,19 @@ export function resolvePairedFollowUpDecision(args: {
fallbackLastTurnOutputRole: args.fallbackLastTurnOutputRole,
fallbackLastTurnOutputVerdict: args.fallbackLastTurnOutputVerdict,
});
const nextTurnAction = resolveNextTurnAction({
const silentOwnerRetry = shouldRetrySilentOwnerExecution({
completedRole: args.completedRole,
executionStatus: args.executionStatus,
sawOutput: args.sawOutput,
taskStatus: args.task?.status ?? null,
lastTurnOutputRole,
lastTurnOutputVerdict,
});
const nextTurnAction = silentOwnerRetry
? ({ kind: 'owner-follow-up' } as const)
: resolveNextTurnAction({
taskStatus: args.task?.status ?? null,
lastTurnOutputRole,
lastTurnOutputVerdict,
});
const dispatch = resolveFollowUpDispatch({
source: args.source,
nextTurnAction,
@@ -116,6 +125,7 @@ export function schedulePairedFollowUpIntent(args: {
fallbackLastTurnOutputVerdict?: VisibleVerdict | null;
lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null;
allowSilentOwnerRetry?: boolean;
}): boolean {
if (!args.task) {
return false;
@@ -139,7 +149,12 @@ export function schedulePairedFollowUpIntent(args: {
lastTurnOutputRole: latestOutputContext.role,
lastTurnOutputVerdict: latestOutputContext.verdict,
intentKind: args.intentKind,
})
}) &&
!(
args.allowSilentOwnerRetry === true &&
args.task.status === 'active' &&
args.intentKind === 'owner-follow-up'
)
) {
return false;
}
@@ -230,6 +245,12 @@ export function dispatchPairedFollowUpForEvent(args: {
enqueue: args.enqueue,
lastTurnOutputRole: decision.lastTurnOutputRole,
lastTurnOutputVerdict: decision.lastTurnOutputVerdict,
allowSilentOwnerRetry: shouldRetrySilentOwnerExecution({
completedRole: args.completedRole,
executionStatus: args.executionStatus,
sawOutput: args.sawOutput,
taskStatus: args.task?.status ?? null,
}),
});
return {

View File

@@ -283,6 +283,7 @@ export async function runQueuedGroupTurn(args: {
hasHumanMessage: false,
lastTurnOutputRole,
lastTurnOutputVerdict,
ownerFailureCount: currentTask.owner_failure_count,
})
: 'owner';
if (!turnRole) {

View File

@@ -122,6 +122,7 @@ export function resolveQueuedPairedTurnRole(args: {
hasHumanMessage: boolean;
lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null;
ownerFailureCount?: number | null;
}): 'owner' | 'reviewer' | 'arbiter' | null {
if (args.hasHumanMessage) {
return resolveQueuedTurnRole({
@@ -134,6 +135,7 @@ export function resolveQueuedPairedTurnRole(args: {
taskStatus: args.taskStatus,
lastTurnOutputRole: args.lastTurnOutputRole,
lastTurnOutputVerdict: args.lastTurnOutputVerdict,
ownerFailureCount: args.ownerFailureCount,
});
switch (nextTurnAction.kind) {
@@ -170,6 +172,7 @@ export function resolveNextTurnAction(args: {
taskStatus?: PairedTaskStatus | null;
lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null;
ownerFailureCount?: number | null;
}): NextTurnAction {
switch (args.taskStatus) {
case 'review_ready':
@@ -187,6 +190,9 @@ export function resolveNextTurnAction(args: {
? { kind: 'none' }
: { kind: 'finalize-owner-turn' };
case 'active':
if ((args.ownerFailureCount ?? 0) > 0) {
return { kind: 'owner-follow-up' };
}
return args.lastTurnOutputRole === 'reviewer' ||
args.lastTurnOutputRole === 'arbiter'
? { kind: 'owner-follow-up' }
@@ -200,6 +206,7 @@ export function matchesExpectedPairedFollowUpIntent(args: {
taskStatus?: PairedTaskStatus | null;
lastTurnOutputRole?: PairedRoomRole | null;
lastTurnOutputVerdict?: VisibleVerdict | null;
ownerFailureCount?: number | null;
intentKind: ScheduledNextTurnActionKind;
}): boolean {
return (
@@ -207,6 +214,7 @@ export function matchesExpectedPairedFollowUpIntent(args: {
taskStatus: args.taskStatus,
lastTurnOutputRole: args.lastTurnOutputRole,
lastTurnOutputVerdict: args.lastTurnOutputVerdict,
ownerFailureCount: args.ownerFailureCount,
}).kind === args.intentKind
);
}
@@ -275,6 +283,20 @@ export function resolveFollowUpDispatch(args: {
}
}
export function shouldRetrySilentOwnerExecution(args: {
completedRole?: PairedRoomRole;
executionStatus?: 'succeeded' | 'failed';
sawOutput?: boolean;
taskStatus?: PairedTaskStatus | null;
}): boolean {
return (
args.completedRole === 'owner' &&
args.executionStatus === 'failed' &&
args.sawOutput === false &&
args.taskStatus === 'active'
);
}
/** Cursor key for a role. Owner uses chatJid, others use chatJid:role. */
export function resolveCursorKey(
chatJid: string,

View File

@@ -0,0 +1,168 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
_initTestDatabase,
createPairedTask,
getPairedTaskById,
} from './db.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { runQueuedGroupTurn } from './message-runtime-queue.js';
import {
resolveNextTurnAction,
resolveQueuedPairedTurnRole,
} from './message-runtime-rules.js';
import {
resetPairedFollowUpScheduleState,
schedulePairedFollowUpOnce,
} from './paired-follow-up-scheduler.js';
import type { Channel, PairedTask, RegisteredGroup } from './types.js';
function makeGroup(): RegisteredGroup {
return {
name: 'Test Group',
folder: 'test-group',
trigger: '@Andy',
added_at: '2026-03-30T00:00:00.000Z',
requiresTrigger: false,
agentType: 'codex',
workDir: '/repo',
};
}
function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
return {
id: 'task-silent-owner-retry',
chat_jid: 'group@test',
group_folder: 'test-group',
owner_service_id: 'codex-main',
reviewer_service_id: 'claude',
owner_agent_type: 'codex',
reviewer_agent_type: 'claude-code',
arbiter_agent_type: null,
title: null,
source_ref: 'HEAD',
plan_notes: null,
review_requested_at: null,
round_trip_count: 1,
status: 'active',
arbiter_verdict: null,
arbiter_requested_at: null,
completion_reason: null,
owner_failure_count: 1,
created_at: '2026-03-30T00:00:00.000Z',
updated_at: '2026-03-30T00:00:00.000Z',
...overrides,
};
}
function makeChannel(): Channel {
return {
name: 'discord',
connect: vi.fn(),
sendMessage: vi.fn(),
isConnected: vi.fn(() => true),
ownsJid: vi.fn(() => true),
disconnect: vi.fn(),
} as unknown as Channel;
}
describe('silent owner retry recovery', () => {
beforeEach(() => {
_initTestDatabase();
resetPairedFollowUpScheduleState();
});
it('treats Codex selected-model capacity as a rotation trigger', () => {
expect(
detectCodexRotationTrigger(
'Selected model is at capacity. Please try a different model.',
),
).toEqual({ shouldRotate: true, reason: 'overloaded' });
});
it('maps active tasks with a previous silent owner failure to an owner retry', () => {
expect(
resolveNextTurnAction({
taskStatus: 'active',
lastTurnOutputRole: null,
ownerFailureCount: 1,
}),
).toEqual({ kind: 'owner-follow-up' });
expect(
resolveQueuedPairedTurnRole({
taskStatus: 'active',
hasHumanMessage: false,
lastTurnOutputRole: null,
ownerFailureCount: 1,
}),
).toBe('owner');
});
it('runs a queued owner retry after a silent owner failure leaves an active task', async () => {
const task = makeTask();
createPairedTask(task);
expect(
schedulePairedFollowUpOnce({
chatJid: task.chat_jid,
runId: 'run-silent-owner-retry-schedule',
task,
intentKind: 'owner-follow-up',
enqueue: vi.fn(),
}),
).toBe(true);
const executeTurn = vi.fn(async () => ({
outputStatus: 'success' as const,
deliverySucceeded: true,
visiblePhase: 'final',
}));
const outcome = await runQueuedGroupTurn({
chatJid: task.chat_jid,
group: makeGroup(),
runId: 'run-silent-owner-retry',
log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as any,
timezone: 'UTC',
missedMessages: [
{
id: 'bot-silent-owner-retry',
chat_jid: task.chat_jid,
sender: 'owner-bot@test',
sender_name: 'owner',
content: 'retry owner',
timestamp: '2026-03-30T00:00:04.000Z',
seq: 48,
is_bot_message: true,
},
],
task: getPairedTaskById(task.id),
roleToChannel: {
owner: null,
reviewer: makeChannel(),
arbiter: null,
},
ownerChannel: makeChannel(),
lastAgentTimestamps: {},
saveState: vi.fn(),
executeTurn,
getFixedRoleChannelName: () => 'discord-review',
labelPairedSenders: (_chatJid, messages) => messages,
formatMessages: () => 'formatted prompt',
});
expect(outcome).toBe(true);
expect(executeTurn).toHaveBeenCalledWith(
expect.objectContaining({
deliveryRole: 'owner',
forcedRole: 'owner',
pairedTurnIdentity: {
turnId:
'task-silent-owner-retry:2026-03-30T00:00:00.000Z:owner-follow-up',
taskId: 'task-silent-owner-retry',
taskUpdatedAt: '2026-03-30T00:00:00.000Z',
intentKind: 'owner-follow-up',
role: 'owner',
},
}),
);
});
});