Retry silent owner capacity failures
Classify Codex selected-model capacity as overloaded and requeue silent owner failures that leave paired tasks active.
This commit is contained in:
@@ -55,6 +55,17 @@ describe('agent-error-detection', () => {
|
||||
expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded');
|
||||
});
|
||||
|
||||
it('classifies Codex model capacity errors as overloaded', () => {
|
||||
expect(
|
||||
classifyAgentError(
|
||||
'Selected model is at capacity. Please try a different model.',
|
||||
),
|
||||
).toEqual({
|
||||
category: 'overloaded',
|
||||
reason: 'overloaded',
|
||||
});
|
||||
});
|
||||
|
||||
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
||||
expect(shouldRotateClaudeToken('429')).toBe(true);
|
||||
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
||||
|
||||
@@ -264,6 +264,8 @@ export function classifyAgentError(
|
||||
if (
|
||||
lower.includes('503') ||
|
||||
lower.includes('overloaded') ||
|
||||
lower.includes('selected model is at capacity') ||
|
||||
lower.includes('model is at capacity') ||
|
||||
((lower.includes('502') || lower.includes('bad gateway')) &&
|
||||
(lower.includes('cloudflare') ||
|
||||
lower.includes('<html') ||
|
||||
|
||||
@@ -183,6 +183,17 @@ describe('message-agent-executor-rules', () => {
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('resolves pending owner retry after a silent owner failure leaves the task active', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
completedRole: 'owner',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
taskStatus: 'active',
|
||||
}),
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
resolveFollowUpDispatch,
|
||||
resolveNextTurnAction,
|
||||
shouldRetrySilentOwnerExecution,
|
||||
} from './message-runtime-rules.js';
|
||||
import { parseVisibleVerdict } from './paired-verdict.js';
|
||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||
@@ -23,6 +24,17 @@ export function resolvePairedFollowUpQueueAction(args: {
|
||||
taskStatus: PairedTaskStatus | null;
|
||||
outputSummary?: string | null;
|
||||
}): PairedFollowUpQueueAction {
|
||||
if (
|
||||
shouldRetrySilentOwnerExecution({
|
||||
completedRole: args.completedRole,
|
||||
executionStatus: args.executionStatus,
|
||||
sawOutput: args.sawOutput,
|
||||
taskStatus: args.taskStatus,
|
||||
})
|
||||
) {
|
||||
return 'pending';
|
||||
}
|
||||
|
||||
const nextTurnAction = resolveNextTurnAction({
|
||||
taskStatus: args.taskStatus,
|
||||
lastTurnOutputRole: args.sawOutput ? args.completedRole : null,
|
||||
|
||||
@@ -124,6 +124,7 @@ export function buildPendingPairedTurn(args: {
|
||||
verdict: lastTurnOutput?.verdict ?? null,
|
||||
outputText: lastTurnOutput?.output_text ?? null,
|
||||
}),
|
||||
ownerFailureCount: task.owner_failure_count,
|
||||
});
|
||||
const recentMessages = getRecentChatMessages(chatJid, 20);
|
||||
const lastHumanMessage = getLastHumanMessageContent(chatJid);
|
||||
|
||||
@@ -176,6 +176,34 @@ describe('message-runtime-follow-up', () => {
|
||||
expect(enqueueMessageCheck).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('requeues a silent failed owner execution that leaves the task active', () => {
|
||||
const enqueue = vi.fn();
|
||||
const result = dispatchPairedFollowUpForEvent({
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-silent-owner-retry',
|
||||
task: {
|
||||
id: 'task-silent-owner-retry',
|
||||
status: 'active',
|
||||
round_trip_count: 1,
|
||||
updated_at: '2026-03-30T00:00:00.000Z',
|
||||
},
|
||||
source: 'executor-recovery',
|
||||
completedRole: 'owner',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
enqueue,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
kind: 'paired-follow-up',
|
||||
intentKind: 'owner-follow-up',
|
||||
scheduled: true,
|
||||
taskStatus: 'active',
|
||||
lastTurnOutputRole: null,
|
||||
});
|
||||
expect(enqueue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('uses the scoped message-check enqueuer when scheduling a paired follow-up intent', () => {
|
||||
const enqueueMessageCheck = vi.fn();
|
||||
const scheduled = schedulePairedFollowUpWithMessageCheck({
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
matchesExpectedPairedFollowUpIntent,
|
||||
resolveFollowUpDispatch,
|
||||
resolveNextTurnAction,
|
||||
shouldRetrySilentOwnerExecution,
|
||||
type FollowUpDispatch,
|
||||
type NextTurnAction,
|
||||
} from './message-runtime-rules.js';
|
||||
@@ -80,11 +81,19 @@ export function resolvePairedFollowUpDecision(args: {
|
||||
fallbackLastTurnOutputRole: args.fallbackLastTurnOutputRole,
|
||||
fallbackLastTurnOutputVerdict: args.fallbackLastTurnOutputVerdict,
|
||||
});
|
||||
const nextTurnAction = resolveNextTurnAction({
|
||||
const silentOwnerRetry = shouldRetrySilentOwnerExecution({
|
||||
completedRole: args.completedRole,
|
||||
executionStatus: args.executionStatus,
|
||||
sawOutput: args.sawOutput,
|
||||
taskStatus: args.task?.status ?? null,
|
||||
lastTurnOutputRole,
|
||||
lastTurnOutputVerdict,
|
||||
});
|
||||
const nextTurnAction = silentOwnerRetry
|
||||
? ({ kind: 'owner-follow-up' } as const)
|
||||
: resolveNextTurnAction({
|
||||
taskStatus: args.task?.status ?? null,
|
||||
lastTurnOutputRole,
|
||||
lastTurnOutputVerdict,
|
||||
});
|
||||
const dispatch = resolveFollowUpDispatch({
|
||||
source: args.source,
|
||||
nextTurnAction,
|
||||
@@ -116,6 +125,7 @@ export function schedulePairedFollowUpIntent(args: {
|
||||
fallbackLastTurnOutputVerdict?: VisibleVerdict | null;
|
||||
lastTurnOutputRole?: PairedRoomRole | null;
|
||||
lastTurnOutputVerdict?: VisibleVerdict | null;
|
||||
allowSilentOwnerRetry?: boolean;
|
||||
}): boolean {
|
||||
if (!args.task) {
|
||||
return false;
|
||||
@@ -139,7 +149,12 @@ export function schedulePairedFollowUpIntent(args: {
|
||||
lastTurnOutputRole: latestOutputContext.role,
|
||||
lastTurnOutputVerdict: latestOutputContext.verdict,
|
||||
intentKind: args.intentKind,
|
||||
})
|
||||
}) &&
|
||||
!(
|
||||
args.allowSilentOwnerRetry === true &&
|
||||
args.task.status === 'active' &&
|
||||
args.intentKind === 'owner-follow-up'
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
@@ -230,6 +245,12 @@ export function dispatchPairedFollowUpForEvent(args: {
|
||||
enqueue: args.enqueue,
|
||||
lastTurnOutputRole: decision.lastTurnOutputRole,
|
||||
lastTurnOutputVerdict: decision.lastTurnOutputVerdict,
|
||||
allowSilentOwnerRetry: shouldRetrySilentOwnerExecution({
|
||||
completedRole: args.completedRole,
|
||||
executionStatus: args.executionStatus,
|
||||
sawOutput: args.sawOutput,
|
||||
taskStatus: args.task?.status ?? null,
|
||||
}),
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
@@ -283,6 +283,7 @@ export async function runQueuedGroupTurn(args: {
|
||||
hasHumanMessage: false,
|
||||
lastTurnOutputRole,
|
||||
lastTurnOutputVerdict,
|
||||
ownerFailureCount: currentTask.owner_failure_count,
|
||||
})
|
||||
: 'owner';
|
||||
if (!turnRole) {
|
||||
|
||||
@@ -122,6 +122,7 @@ export function resolveQueuedPairedTurnRole(args: {
|
||||
hasHumanMessage: boolean;
|
||||
lastTurnOutputRole?: PairedRoomRole | null;
|
||||
lastTurnOutputVerdict?: VisibleVerdict | null;
|
||||
ownerFailureCount?: number | null;
|
||||
}): 'owner' | 'reviewer' | 'arbiter' | null {
|
||||
if (args.hasHumanMessage) {
|
||||
return resolveQueuedTurnRole({
|
||||
@@ -134,6 +135,7 @@ export function resolveQueuedPairedTurnRole(args: {
|
||||
taskStatus: args.taskStatus,
|
||||
lastTurnOutputRole: args.lastTurnOutputRole,
|
||||
lastTurnOutputVerdict: args.lastTurnOutputVerdict,
|
||||
ownerFailureCount: args.ownerFailureCount,
|
||||
});
|
||||
|
||||
switch (nextTurnAction.kind) {
|
||||
@@ -170,6 +172,7 @@ export function resolveNextTurnAction(args: {
|
||||
taskStatus?: PairedTaskStatus | null;
|
||||
lastTurnOutputRole?: PairedRoomRole | null;
|
||||
lastTurnOutputVerdict?: VisibleVerdict | null;
|
||||
ownerFailureCount?: number | null;
|
||||
}): NextTurnAction {
|
||||
switch (args.taskStatus) {
|
||||
case 'review_ready':
|
||||
@@ -187,6 +190,9 @@ export function resolveNextTurnAction(args: {
|
||||
? { kind: 'none' }
|
||||
: { kind: 'finalize-owner-turn' };
|
||||
case 'active':
|
||||
if ((args.ownerFailureCount ?? 0) > 0) {
|
||||
return { kind: 'owner-follow-up' };
|
||||
}
|
||||
return args.lastTurnOutputRole === 'reviewer' ||
|
||||
args.lastTurnOutputRole === 'arbiter'
|
||||
? { kind: 'owner-follow-up' }
|
||||
@@ -200,6 +206,7 @@ export function matchesExpectedPairedFollowUpIntent(args: {
|
||||
taskStatus?: PairedTaskStatus | null;
|
||||
lastTurnOutputRole?: PairedRoomRole | null;
|
||||
lastTurnOutputVerdict?: VisibleVerdict | null;
|
||||
ownerFailureCount?: number | null;
|
||||
intentKind: ScheduledNextTurnActionKind;
|
||||
}): boolean {
|
||||
return (
|
||||
@@ -207,6 +214,7 @@ export function matchesExpectedPairedFollowUpIntent(args: {
|
||||
taskStatus: args.taskStatus,
|
||||
lastTurnOutputRole: args.lastTurnOutputRole,
|
||||
lastTurnOutputVerdict: args.lastTurnOutputVerdict,
|
||||
ownerFailureCount: args.ownerFailureCount,
|
||||
}).kind === args.intentKind
|
||||
);
|
||||
}
|
||||
@@ -275,6 +283,20 @@ export function resolveFollowUpDispatch(args: {
|
||||
}
|
||||
}
|
||||
|
||||
export function shouldRetrySilentOwnerExecution(args: {
|
||||
completedRole?: PairedRoomRole;
|
||||
executionStatus?: 'succeeded' | 'failed';
|
||||
sawOutput?: boolean;
|
||||
taskStatus?: PairedTaskStatus | null;
|
||||
}): boolean {
|
||||
return (
|
||||
args.completedRole === 'owner' &&
|
||||
args.executionStatus === 'failed' &&
|
||||
args.sawOutput === false &&
|
||||
args.taskStatus === 'active'
|
||||
);
|
||||
}
|
||||
|
||||
/** Cursor key for a role. Owner uses chatJid, others use chatJid:role. */
|
||||
export function resolveCursorKey(
|
||||
chatJid: string,
|
||||
|
||||
168
src/silent-owner-retry.test.ts
Normal file
168
src/silent-owner-retry.test.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import {
|
||||
_initTestDatabase,
|
||||
createPairedTask,
|
||||
getPairedTaskById,
|
||||
} from './db.js';
|
||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||
import { runQueuedGroupTurn } from './message-runtime-queue.js';
|
||||
import {
|
||||
resolveNextTurnAction,
|
||||
resolveQueuedPairedTurnRole,
|
||||
} from './message-runtime-rules.js';
|
||||
import {
|
||||
resetPairedFollowUpScheduleState,
|
||||
schedulePairedFollowUpOnce,
|
||||
} from './paired-follow-up-scheduler.js';
|
||||
import type { Channel, PairedTask, RegisteredGroup } from './types.js';
|
||||
|
||||
function makeGroup(): RegisteredGroup {
|
||||
return {
|
||||
name: 'Test Group',
|
||||
folder: 'test-group',
|
||||
trigger: '@Andy',
|
||||
added_at: '2026-03-30T00:00:00.000Z',
|
||||
requiresTrigger: false,
|
||||
agentType: 'codex',
|
||||
workDir: '/repo',
|
||||
};
|
||||
}
|
||||
|
||||
function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||
return {
|
||||
id: 'task-silent-owner-retry',
|
||||
chat_jid: 'group@test',
|
||||
group_folder: 'test-group',
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
owner_agent_type: 'codex',
|
||||
reviewer_agent_type: 'claude-code',
|
||||
arbiter_agent_type: null,
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: null,
|
||||
round_trip_count: 1,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
owner_failure_count: 1,
|
||||
created_at: '2026-03-30T00:00:00.000Z',
|
||||
updated_at: '2026-03-30T00:00:00.000Z',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeChannel(): Channel {
|
||||
return {
|
||||
name: 'discord',
|
||||
connect: vi.fn(),
|
||||
sendMessage: vi.fn(),
|
||||
isConnected: vi.fn(() => true),
|
||||
ownsJid: vi.fn(() => true),
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as Channel;
|
||||
}
|
||||
|
||||
describe('silent owner retry recovery', () => {
|
||||
beforeEach(() => {
|
||||
_initTestDatabase();
|
||||
resetPairedFollowUpScheduleState();
|
||||
});
|
||||
|
||||
it('treats Codex selected-model capacity as a rotation trigger', () => {
|
||||
expect(
|
||||
detectCodexRotationTrigger(
|
||||
'Selected model is at capacity. Please try a different model.',
|
||||
),
|
||||
).toEqual({ shouldRotate: true, reason: 'overloaded' });
|
||||
});
|
||||
|
||||
it('maps active tasks with a previous silent owner failure to an owner retry', () => {
|
||||
expect(
|
||||
resolveNextTurnAction({
|
||||
taskStatus: 'active',
|
||||
lastTurnOutputRole: null,
|
||||
ownerFailureCount: 1,
|
||||
}),
|
||||
).toEqual({ kind: 'owner-follow-up' });
|
||||
expect(
|
||||
resolveQueuedPairedTurnRole({
|
||||
taskStatus: 'active',
|
||||
hasHumanMessage: false,
|
||||
lastTurnOutputRole: null,
|
||||
ownerFailureCount: 1,
|
||||
}),
|
||||
).toBe('owner');
|
||||
});
|
||||
|
||||
it('runs a queued owner retry after a silent owner failure leaves an active task', async () => {
|
||||
const task = makeTask();
|
||||
createPairedTask(task);
|
||||
expect(
|
||||
schedulePairedFollowUpOnce({
|
||||
chatJid: task.chat_jid,
|
||||
runId: 'run-silent-owner-retry-schedule',
|
||||
task,
|
||||
intentKind: 'owner-follow-up',
|
||||
enqueue: vi.fn(),
|
||||
}),
|
||||
).toBe(true);
|
||||
const executeTurn = vi.fn(async () => ({
|
||||
outputStatus: 'success' as const,
|
||||
deliverySucceeded: true,
|
||||
visiblePhase: 'final',
|
||||
}));
|
||||
|
||||
const outcome = await runQueuedGroupTurn({
|
||||
chatJid: task.chat_jid,
|
||||
group: makeGroup(),
|
||||
runId: 'run-silent-owner-retry',
|
||||
log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as any,
|
||||
timezone: 'UTC',
|
||||
missedMessages: [
|
||||
{
|
||||
id: 'bot-silent-owner-retry',
|
||||
chat_jid: task.chat_jid,
|
||||
sender: 'owner-bot@test',
|
||||
sender_name: 'owner',
|
||||
content: 'retry owner',
|
||||
timestamp: '2026-03-30T00:00:04.000Z',
|
||||
seq: 48,
|
||||
is_bot_message: true,
|
||||
},
|
||||
],
|
||||
task: getPairedTaskById(task.id),
|
||||
roleToChannel: {
|
||||
owner: null,
|
||||
reviewer: makeChannel(),
|
||||
arbiter: null,
|
||||
},
|
||||
ownerChannel: makeChannel(),
|
||||
lastAgentTimestamps: {},
|
||||
saveState: vi.fn(),
|
||||
executeTurn,
|
||||
getFixedRoleChannelName: () => 'discord-review',
|
||||
labelPairedSenders: (_chatJid, messages) => messages,
|
||||
formatMessages: () => 'formatted prompt',
|
||||
});
|
||||
|
||||
expect(outcome).toBe(true);
|
||||
expect(executeTurn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
deliveryRole: 'owner',
|
||||
forcedRole: 'owner',
|
||||
pairedTurnIdentity: {
|
||||
turnId:
|
||||
'task-silent-owner-retry:2026-03-30T00:00:00.000Z:owner-follow-up',
|
||||
taskId: 'task-silent-owner-retry',
|
||||
taskUpdatedAt: '2026-03-30T00:00:00.000Z',
|
||||
intentKind: 'owner-follow-up',
|
||||
role: 'owner',
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user