From fbaca3ab681c1ee9d7b6718e2484b04f1ace44f5 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Thu, 21 May 2026 03:46:59 +0900 Subject: [PATCH] Retry silent owner capacity failures Classify Codex selected-model capacity as overloaded and requeue silent owner failures that leave paired tasks active. --- src/agent-error-detection.test.ts | 11 ++ src/agent-error-detection.ts | 2 + src/message-agent-executor-rules.test.ts | 11 ++ src/message-agent-executor-rules.ts | 12 ++ src/message-runtime-flow.ts | 1 + src/message-runtime-follow-up.test.ts | 28 ++++ src/message-runtime-follow-up.ts | 29 +++- src/message-runtime-queue.ts | 1 + src/message-runtime-rules.ts | 22 +++ src/silent-owner-retry.test.ts | 168 +++++++++++++++++++++++ 10 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 src/silent-owner-retry.test.ts diff --git a/src/agent-error-detection.test.ts b/src/agent-error-detection.test.ts index 0e33261..cad0f6e 100644 --- a/src/agent-error-detection.test.ts +++ b/src/agent-error-detection.test.ts @@ -55,6 +55,17 @@ describe('agent-error-detection', () => { expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded'); }); + it('classifies Codex model capacity errors as overloaded', () => { + expect( + classifyAgentError( + 'Selected model is at capacity. Please try a different model.', + ), + ).toEqual({ + category: 'overloaded', + reason: 'overloaded', + }); + }); + it('marks only Claude quota/auth reasons as Claude rotation reasons', () => { expect(shouldRotateClaudeToken('429')).toBe(true); expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true); diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts index 159a100..1eff808 100644 --- a/src/agent-error-detection.ts +++ b/src/agent-error-detection.ts @@ -264,6 +264,8 @@ export function classifyAgentError( if ( lower.includes('503') || lower.includes('overloaded') || + lower.includes('selected model is at capacity') || + lower.includes('model is at capacity') || ((lower.includes('502') || lower.includes('bad gateway')) && (lower.includes('cloudflare') || lower.includes(' { ).toBe('pending'); }); + it('resolves pending owner retry after a silent owner failure leaves the task active', () => { + expect( + resolvePairedFollowUpQueueAction({ + completedRole: 'owner', + executionStatus: 'failed', + sawOutput: false, + taskStatus: 'active', + }), + ).toBe('pending'); + }); + it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => { expect( resolvePairedFollowUpQueueAction({ diff --git a/src/message-agent-executor-rules.ts b/src/message-agent-executor-rules.ts index 7d0f8f5..2ec47f8 100644 --- a/src/message-agent-executor-rules.ts +++ b/src/message-agent-executor-rules.ts @@ -1,6 +1,7 @@ import { resolveFollowUpDispatch, resolveNextTurnAction, + shouldRetrySilentOwnerExecution, } from './message-runtime-rules.js'; import { parseVisibleVerdict } from './paired-verdict.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js'; @@ -23,6 +24,17 @@ export function resolvePairedFollowUpQueueAction(args: { taskStatus: PairedTaskStatus | null; outputSummary?: string | null; }): PairedFollowUpQueueAction { + if ( + shouldRetrySilentOwnerExecution({ + completedRole: args.completedRole, + executionStatus: args.executionStatus, + sawOutput: args.sawOutput, + taskStatus: args.taskStatus, + }) + ) { + return 'pending'; + } + const nextTurnAction = resolveNextTurnAction({ taskStatus: args.taskStatus, lastTurnOutputRole: args.sawOutput ? args.completedRole : null, diff --git a/src/message-runtime-flow.ts b/src/message-runtime-flow.ts index cf957de..35480af 100644 --- a/src/message-runtime-flow.ts +++ b/src/message-runtime-flow.ts @@ -124,6 +124,7 @@ export function buildPendingPairedTurn(args: { verdict: lastTurnOutput?.verdict ?? null, outputText: lastTurnOutput?.output_text ?? null, }), + ownerFailureCount: task.owner_failure_count, }); const recentMessages = getRecentChatMessages(chatJid, 20); const lastHumanMessage = getLastHumanMessageContent(chatJid); diff --git a/src/message-runtime-follow-up.test.ts b/src/message-runtime-follow-up.test.ts index f8c3c76..3c93ff2 100644 --- a/src/message-runtime-follow-up.test.ts +++ b/src/message-runtime-follow-up.test.ts @@ -176,6 +176,34 @@ describe('message-runtime-follow-up', () => { expect(enqueueMessageCheck).toHaveBeenCalledTimes(1); }); + it('requeues a silent failed owner execution that leaves the task active', () => { + const enqueue = vi.fn(); + const result = dispatchPairedFollowUpForEvent({ + chatJid: 'group@test', + runId: 'run-silent-owner-retry', + task: { + id: 'task-silent-owner-retry', + status: 'active', + round_trip_count: 1, + updated_at: '2026-03-30T00:00:00.000Z', + }, + source: 'executor-recovery', + completedRole: 'owner', + executionStatus: 'failed', + sawOutput: false, + enqueue, + }); + + expect(result).toMatchObject({ + kind: 'paired-follow-up', + intentKind: 'owner-follow-up', + scheduled: true, + taskStatus: 'active', + lastTurnOutputRole: null, + }); + expect(enqueue).toHaveBeenCalledTimes(1); + }); + it('uses the scoped message-check enqueuer when scheduling a paired follow-up intent', () => { const enqueueMessageCheck = vi.fn(); const scheduled = schedulePairedFollowUpWithMessageCheck({ diff --git a/src/message-runtime-follow-up.ts b/src/message-runtime-follow-up.ts index 5e9784e..8c7c10b 100644 --- a/src/message-runtime-follow-up.ts +++ b/src/message-runtime-follow-up.ts @@ -7,6 +7,7 @@ import { matchesExpectedPairedFollowUpIntent, resolveFollowUpDispatch, resolveNextTurnAction, + shouldRetrySilentOwnerExecution, type FollowUpDispatch, type NextTurnAction, } from './message-runtime-rules.js'; @@ -80,11 +81,19 @@ export function resolvePairedFollowUpDecision(args: { fallbackLastTurnOutputRole: args.fallbackLastTurnOutputRole, fallbackLastTurnOutputVerdict: args.fallbackLastTurnOutputVerdict, }); - const nextTurnAction = resolveNextTurnAction({ + const silentOwnerRetry = shouldRetrySilentOwnerExecution({ + completedRole: args.completedRole, + executionStatus: args.executionStatus, + sawOutput: args.sawOutput, taskStatus: args.task?.status ?? null, - lastTurnOutputRole, - lastTurnOutputVerdict, }); + const nextTurnAction = silentOwnerRetry + ? ({ kind: 'owner-follow-up' } as const) + : resolveNextTurnAction({ + taskStatus: args.task?.status ?? null, + lastTurnOutputRole, + lastTurnOutputVerdict, + }); const dispatch = resolveFollowUpDispatch({ source: args.source, nextTurnAction, @@ -116,6 +125,7 @@ export function schedulePairedFollowUpIntent(args: { fallbackLastTurnOutputVerdict?: VisibleVerdict | null; lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputVerdict?: VisibleVerdict | null; + allowSilentOwnerRetry?: boolean; }): boolean { if (!args.task) { return false; @@ -139,7 +149,12 @@ export function schedulePairedFollowUpIntent(args: { lastTurnOutputRole: latestOutputContext.role, lastTurnOutputVerdict: latestOutputContext.verdict, intentKind: args.intentKind, - }) + }) && + !( + args.allowSilentOwnerRetry === true && + args.task.status === 'active' && + args.intentKind === 'owner-follow-up' + ) ) { return false; } @@ -230,6 +245,12 @@ export function dispatchPairedFollowUpForEvent(args: { enqueue: args.enqueue, lastTurnOutputRole: decision.lastTurnOutputRole, lastTurnOutputVerdict: decision.lastTurnOutputVerdict, + allowSilentOwnerRetry: shouldRetrySilentOwnerExecution({ + completedRole: args.completedRole, + executionStatus: args.executionStatus, + sawOutput: args.sawOutput, + taskStatus: args.task?.status ?? null, + }), }); return { diff --git a/src/message-runtime-queue.ts b/src/message-runtime-queue.ts index 3f83084..2c30182 100644 --- a/src/message-runtime-queue.ts +++ b/src/message-runtime-queue.ts @@ -283,6 +283,7 @@ export async function runQueuedGroupTurn(args: { hasHumanMessage: false, lastTurnOutputRole, lastTurnOutputVerdict, + ownerFailureCount: currentTask.owner_failure_count, }) : 'owner'; if (!turnRole) { diff --git a/src/message-runtime-rules.ts b/src/message-runtime-rules.ts index 49f2312..d955b55 100644 --- a/src/message-runtime-rules.ts +++ b/src/message-runtime-rules.ts @@ -122,6 +122,7 @@ export function resolveQueuedPairedTurnRole(args: { hasHumanMessage: boolean; lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputVerdict?: VisibleVerdict | null; + ownerFailureCount?: number | null; }): 'owner' | 'reviewer' | 'arbiter' | null { if (args.hasHumanMessage) { return resolveQueuedTurnRole({ @@ -134,6 +135,7 @@ export function resolveQueuedPairedTurnRole(args: { taskStatus: args.taskStatus, lastTurnOutputRole: args.lastTurnOutputRole, lastTurnOutputVerdict: args.lastTurnOutputVerdict, + ownerFailureCount: args.ownerFailureCount, }); switch (nextTurnAction.kind) { @@ -170,6 +172,7 @@ export function resolveNextTurnAction(args: { taskStatus?: PairedTaskStatus | null; lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputVerdict?: VisibleVerdict | null; + ownerFailureCount?: number | null; }): NextTurnAction { switch (args.taskStatus) { case 'review_ready': @@ -187,6 +190,9 @@ export function resolveNextTurnAction(args: { ? { kind: 'none' } : { kind: 'finalize-owner-turn' }; case 'active': + if ((args.ownerFailureCount ?? 0) > 0) { + return { kind: 'owner-follow-up' }; + } return args.lastTurnOutputRole === 'reviewer' || args.lastTurnOutputRole === 'arbiter' ? { kind: 'owner-follow-up' } @@ -200,6 +206,7 @@ export function matchesExpectedPairedFollowUpIntent(args: { taskStatus?: PairedTaskStatus | null; lastTurnOutputRole?: PairedRoomRole | null; lastTurnOutputVerdict?: VisibleVerdict | null; + ownerFailureCount?: number | null; intentKind: ScheduledNextTurnActionKind; }): boolean { return ( @@ -207,6 +214,7 @@ export function matchesExpectedPairedFollowUpIntent(args: { taskStatus: args.taskStatus, lastTurnOutputRole: args.lastTurnOutputRole, lastTurnOutputVerdict: args.lastTurnOutputVerdict, + ownerFailureCount: args.ownerFailureCount, }).kind === args.intentKind ); } @@ -275,6 +283,20 @@ export function resolveFollowUpDispatch(args: { } } +export function shouldRetrySilentOwnerExecution(args: { + completedRole?: PairedRoomRole; + executionStatus?: 'succeeded' | 'failed'; + sawOutput?: boolean; + taskStatus?: PairedTaskStatus | null; +}): boolean { + return ( + args.completedRole === 'owner' && + args.executionStatus === 'failed' && + args.sawOutput === false && + args.taskStatus === 'active' + ); +} + /** Cursor key for a role. Owner uses chatJid, others use chatJid:role. */ export function resolveCursorKey( chatJid: string, diff --git a/src/silent-owner-retry.test.ts b/src/silent-owner-retry.test.ts new file mode 100644 index 0000000..273511d --- /dev/null +++ b/src/silent-owner-retry.test.ts @@ -0,0 +1,168 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + _initTestDatabase, + createPairedTask, + getPairedTaskById, +} from './db.js'; +import { detectCodexRotationTrigger } from './codex-token-rotation.js'; +import { runQueuedGroupTurn } from './message-runtime-queue.js'; +import { + resolveNextTurnAction, + resolveQueuedPairedTurnRole, +} from './message-runtime-rules.js'; +import { + resetPairedFollowUpScheduleState, + schedulePairedFollowUpOnce, +} from './paired-follow-up-scheduler.js'; +import type { Channel, PairedTask, RegisteredGroup } from './types.js'; + +function makeGroup(): RegisteredGroup { + return { + name: 'Test Group', + folder: 'test-group', + trigger: '@Andy', + added_at: '2026-03-30T00:00:00.000Z', + requiresTrigger: false, + agentType: 'codex', + workDir: '/repo', + }; +} + +function makeTask(overrides: Partial = {}): PairedTask { + return { + id: 'task-silent-owner-retry', + chat_jid: 'group@test', + group_folder: 'test-group', + owner_service_id: 'codex-main', + reviewer_service_id: 'claude', + owner_agent_type: 'codex', + reviewer_agent_type: 'claude-code', + arbiter_agent_type: null, + title: null, + source_ref: 'HEAD', + plan_notes: null, + review_requested_at: null, + round_trip_count: 1, + status: 'active', + arbiter_verdict: null, + arbiter_requested_at: null, + completion_reason: null, + owner_failure_count: 1, + created_at: '2026-03-30T00:00:00.000Z', + updated_at: '2026-03-30T00:00:00.000Z', + ...overrides, + }; +} + +function makeChannel(): Channel { + return { + name: 'discord', + connect: vi.fn(), + sendMessage: vi.fn(), + isConnected: vi.fn(() => true), + ownsJid: vi.fn(() => true), + disconnect: vi.fn(), + } as unknown as Channel; +} + +describe('silent owner retry recovery', () => { + beforeEach(() => { + _initTestDatabase(); + resetPairedFollowUpScheduleState(); + }); + + it('treats Codex selected-model capacity as a rotation trigger', () => { + expect( + detectCodexRotationTrigger( + 'Selected model is at capacity. Please try a different model.', + ), + ).toEqual({ shouldRotate: true, reason: 'overloaded' }); + }); + + it('maps active tasks with a previous silent owner failure to an owner retry', () => { + expect( + resolveNextTurnAction({ + taskStatus: 'active', + lastTurnOutputRole: null, + ownerFailureCount: 1, + }), + ).toEqual({ kind: 'owner-follow-up' }); + expect( + resolveQueuedPairedTurnRole({ + taskStatus: 'active', + hasHumanMessage: false, + lastTurnOutputRole: null, + ownerFailureCount: 1, + }), + ).toBe('owner'); + }); + + it('runs a queued owner retry after a silent owner failure leaves an active task', async () => { + const task = makeTask(); + createPairedTask(task); + expect( + schedulePairedFollowUpOnce({ + chatJid: task.chat_jid, + runId: 'run-silent-owner-retry-schedule', + task, + intentKind: 'owner-follow-up', + enqueue: vi.fn(), + }), + ).toBe(true); + const executeTurn = vi.fn(async () => ({ + outputStatus: 'success' as const, + deliverySucceeded: true, + visiblePhase: 'final', + })); + + const outcome = await runQueuedGroupTurn({ + chatJid: task.chat_jid, + group: makeGroup(), + runId: 'run-silent-owner-retry', + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as any, + timezone: 'UTC', + missedMessages: [ + { + id: 'bot-silent-owner-retry', + chat_jid: task.chat_jid, + sender: 'owner-bot@test', + sender_name: 'owner', + content: 'retry owner', + timestamp: '2026-03-30T00:00:04.000Z', + seq: 48, + is_bot_message: true, + }, + ], + task: getPairedTaskById(task.id), + roleToChannel: { + owner: null, + reviewer: makeChannel(), + arbiter: null, + }, + ownerChannel: makeChannel(), + lastAgentTimestamps: {}, + saveState: vi.fn(), + executeTurn, + getFixedRoleChannelName: () => 'discord-review', + labelPairedSenders: (_chatJid, messages) => messages, + formatMessages: () => 'formatted prompt', + }); + + expect(outcome).toBe(true); + expect(executeTurn).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryRole: 'owner', + forcedRole: 'owner', + pairedTurnIdentity: { + turnId: + 'task-silent-owner-retry:2026-03-30T00:00:00.000Z:owner-follow-up', + taskId: 'task-silent-owner-retry', + taskUpdatedAt: '2026-03-30T00:00:00.000Z', + intentKind: 'owner-follow-up', + role: 'owner', + }, + }), + ); + }); +});