Unify paired status transitions and retry handling

This commit is contained in:
ejclaw
2026-04-07 02:49:02 +09:00
parent 4511ca5692
commit b8e8c9dabf
13 changed files with 574 additions and 280 deletions

164
src/agent-attempt-retry.ts Normal file
View File

@@ -0,0 +1,164 @@
import {
classifyRotationTrigger,
type AgentTriggerReason,
type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { getErrorMessage } from './utils.js';
export interface AttemptStreamedTrigger {
reason: AgentTriggerReason;
retryAfterMs?: number;
}
export interface AttemptRetryState {
sawOutput: boolean;
retryableSessionFailureDetected?: boolean;
streamedTriggerReason?: AttemptStreamedTrigger;
error?: unknown;
outputError?: string | null;
}
export function isRetryableClaudeSessionFailureAttempt(args: {
attempt: AttemptRetryState;
isClaudeCodeAgent: boolean;
provider: 'claude' | 'codex';
shouldRetryFreshSessionOnAgentFailure: (args: {
result: null;
error: string;
}) => boolean;
}): boolean {
if (
!args.isClaudeCodeAgent ||
args.provider !== 'claude' ||
args.attempt.sawOutput
) {
return false;
}
if (args.attempt.retryableSessionFailureDetected === true) {
return true;
}
if (args.attempt.error == null) {
return false;
}
return args.shouldRetryFreshSessionOnAgentFailure({
result: null,
error: getErrorMessage(args.attempt.error),
});
}
export function resolveClaudeRetryTrigger(args: {
canRetryClaudeCredentials: boolean;
provider: 'claude' | 'codex';
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
fallbackMessage?: string | null;
}): { reason: AgentTriggerReason; retryAfterMs?: number } | null {
if (
!(
args.canRetryClaudeCredentials &&
args.provider === 'claude' &&
!args.attempt.sawOutput
)
) {
return null;
}
if (args.attempt.streamedTriggerReason) {
return {
reason: args.attempt.streamedTriggerReason.reason,
retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs,
};
}
const trigger = classifyRotationTrigger(args.fallbackMessage);
if (!trigger.shouldRetry) {
return null;
}
return {
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,
};
}
export function resolveCodexRetryTrigger(args: {
canRetryCodex: boolean;
attempt: Pick<AttemptRetryState, 'streamedTriggerReason'>;
rotationMessage?: string | null;
}): { reason: CodexRotationReason } | null {
if (!args.canRetryCodex) {
return null;
}
if (args.attempt.streamedTriggerReason) {
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
return null;
}
return {
reason: args.attempt.streamedTriggerReason.reason,
};
}
const trigger = detectCodexRotationTrigger(args.rotationMessage);
if (!trigger.shouldRotate) {
return null;
}
return { reason: trigger.reason };
}
export type AttemptRetryAction =
| {
kind: 'claude';
trigger: { reason: AgentTriggerReason; retryAfterMs?: number };
rotationMessage?: string;
}
| {
kind: 'codex';
trigger: { reason: CodexRotationReason };
rotationMessage?: string;
}
| { kind: 'none' };
export function resolveAttemptRetryAction(args: {
provider: 'claude' | 'codex';
canRetryClaudeCredentials: boolean;
canRetryCodex: boolean;
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
rotationMessage?: string | null;
}): AttemptRetryAction {
const normalizedRotationMessage = args.rotationMessage ?? undefined;
const claudeTrigger = resolveClaudeRetryTrigger({
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
provider: args.provider,
attempt: args.attempt,
fallbackMessage: normalizedRotationMessage,
});
if (claudeTrigger) {
return {
kind: 'claude',
trigger: claudeTrigger,
rotationMessage: normalizedRotationMessage,
};
}
const codexTrigger = resolveCodexRetryTrigger({
canRetryCodex: args.canRetryCodex,
attempt: args.attempt,
rotationMessage: normalizedRotationMessage,
});
if (codexTrigger) {
return {
kind: 'codex',
trigger: codexTrigger,
rotationMessage: normalizedRotationMessage,
};
}
return { kind: 'none' };
}

View File

@@ -23,6 +23,7 @@ import { classifyRotationTrigger } from './agent-error-detection.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { import {
isRetryableClaudeSessionFailureAttempt, isRetryableClaudeSessionFailureAttempt,
resolveAttemptRetryAction,
resolveClaudeRetryTrigger, resolveClaudeRetryTrigger,
resolveCodexRetryTrigger, resolveCodexRetryTrigger,
resolvePairedFollowUpQueueAction, resolvePairedFollowUpQueueAction,
@@ -88,6 +89,47 @@ describe('message-agent-executor-rules', () => {
expect(detectCodexRotationTrigger).not.toHaveBeenCalled(); expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
}); });
it('resolves a shared Claude retry action from fallback output text', () => {
vi.mocked(classifyRotationTrigger).mockReturnValue({
shouldRetry: true,
reason: '429',
retryAfterMs: 30000,
});
expect(
resolveAttemptRetryAction({
provider: 'claude',
canRetryClaudeCredentials: true,
canRetryCodex: false,
attempt: { sawOutput: false },
rotationMessage: '429 rate limit',
}),
).toEqual({
kind: 'claude',
trigger: { reason: '429', retryAfterMs: 30000 },
rotationMessage: '429 rate limit',
});
});
it('resolves a shared Codex retry action from streamed state', () => {
expect(
resolveAttemptRetryAction({
provider: 'codex',
canRetryClaudeCredentials: false,
canRetryCodex: true,
attempt: {
sawOutput: false,
streamedTriggerReason: { reason: 'auth-expired' },
},
rotationMessage: 'oauth token expired',
}),
).toEqual({
kind: 'codex',
trigger: { reason: 'auth-expired' },
rotationMessage: 'oauth token expired',
});
});
it('detects retryable Claude session failures from either flag or classifier', () => { it('detects retryable Claude session failures from either flag or classifier', () => {
expect( expect(
isRetryableClaudeSessionFailureAttempt({ isRetryableClaudeSessionFailureAttempt({

View File

@@ -1,116 +1,14 @@
import {
classifyRotationTrigger,
type AgentTriggerReason,
type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { resolveNextTurnAction } from './message-runtime-rules.js'; import { resolveNextTurnAction } from './message-runtime-rules.js';
import type { PairedRoomRole, PairedTaskStatus } from './types.js'; import type { PairedRoomRole, PairedTaskStatus } from './types.js';
import { getErrorMessage } from './utils.js'; export {
isRetryableClaudeSessionFailureAttempt,
export interface ExecutorStreamedTrigger { resolveAttemptRetryAction,
reason: AgentTriggerReason; resolveClaudeRetryTrigger,
retryAfterMs?: number; resolveCodexRetryTrigger,
} type AttemptRetryAction,
type AttemptRetryState as ExecutorAttemptState,
export interface ExecutorAttemptState { type AttemptStreamedTrigger as ExecutorStreamedTrigger,
sawOutput: boolean; } from './agent-attempt-retry.js';
retryableSessionFailureDetected?: boolean;
streamedTriggerReason?: ExecutorStreamedTrigger;
error?: unknown;
}
export function isRetryableClaudeSessionFailureAttempt(args: {
attempt: ExecutorAttemptState;
isClaudeCodeAgent: boolean;
provider: 'claude' | 'codex';
shouldRetryFreshSessionOnAgentFailure: (args: {
result: null;
error: string;
}) => boolean;
}): boolean {
if (
!args.isClaudeCodeAgent ||
args.provider !== 'claude' ||
args.attempt.sawOutput
) {
return false;
}
if (args.attempt.retryableSessionFailureDetected === true) {
return true;
}
if (args.attempt.error == null) {
return false;
}
return args.shouldRetryFreshSessionOnAgentFailure({
result: null,
error: getErrorMessage(args.attempt.error),
});
}
export function resolveClaudeRetryTrigger(args: {
canRetryClaudeCredentials: boolean;
provider: 'claude' | 'codex';
attempt: Pick<ExecutorAttemptState, 'sawOutput' | 'streamedTriggerReason'>;
fallbackMessage?: string | null;
}): { reason: AgentTriggerReason; retryAfterMs?: number } | null {
if (
!(
args.canRetryClaudeCredentials &&
args.provider === 'claude' &&
!args.attempt.sawOutput
)
) {
return null;
}
if (args.attempt.streamedTriggerReason) {
return {
reason: args.attempt.streamedTriggerReason.reason,
retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs,
};
}
const trigger = classifyRotationTrigger(args.fallbackMessage);
if (!trigger.shouldRetry) {
return null;
}
return {
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,
};
}
export function resolveCodexRetryTrigger(args: {
canRetryCodex: boolean;
attempt: Pick<ExecutorAttemptState, 'streamedTriggerReason'>;
rotationMessage?: string | null;
}): { reason: CodexRotationReason } | null {
if (!args.canRetryCodex) {
return null;
}
if (args.attempt.streamedTriggerReason) {
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
return null;
}
return {
reason: args.attempt.streamedTriggerReason.reason,
};
}
const trigger = detectCodexRotationTrigger(args.rotationMessage);
if (!trigger.shouldRotate) {
return null;
}
return { reason: trigger.reason };
}
export type PairedFollowUpQueueAction = export type PairedFollowUpQueueAction =
| 'generic' | 'generic'

View File

@@ -5,6 +5,10 @@ import { getErrorMessage } from './utils.js';
import { getAgentOutputText } from './agent-output.js'; import { getAgentOutputText } from './agent-output.js';
import { createEvaluatedOutputHandler } from './agent-attempt.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js';
import {
isRetryableClaudeSessionFailureAttempt,
resolveAttemptRetryAction,
} from './agent-attempt-retry.js';
import { import {
AgentOutput, AgentOutput,
runAgentProcess, runAgentProcess,
@@ -31,9 +35,6 @@ import {
import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js'; import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js';
import { resolveExecutionTarget } from './message-runtime-rules.js'; import { resolveExecutionTarget } from './message-runtime-rules.js';
import { import {
isRetryableClaudeSessionFailureAttempt,
resolveClaudeRetryTrigger,
resolveCodexRetryTrigger,
resolvePairedFollowUpQueueAction, resolvePairedFollowUpQueueAction,
} from './message-agent-executor-rules.js'; } from './message-agent-executor-rules.js';
import { buildRoomRoleContext } from './room-role-context.js'; import { buildRoomRoleContext } from './room-role-context.js';
@@ -683,22 +684,23 @@ export async function runAgentForGroup(
attempt: AgentAttempt, attempt: AgentAttempt,
rotationMessage?: string | null, rotationMessage?: string | null,
): Promise<'success' | 'error' | null> => { ): Promise<'success' | 'error' | null> => {
const trigger = resolveClaudeRetryTrigger({ const retryAction = resolveAttemptRetryAction({
canRetryClaudeCredentials,
provider, provider,
canRetryClaudeCredentials,
canRetryCodex: false,
attempt, attempt,
fallbackMessage: rotationMessage, rotationMessage,
}); });
if (!trigger) { if (retryAction.kind !== 'claude') {
return null; return null;
} }
const result = await retryClaudeWithRotation( const result = await retryClaudeWithRotation(
trigger, retryAction.trigger,
rotationMessage ?? undefined, retryAction.rotationMessage,
); );
if (result === 'error') { if (result === 'error') {
return maybeHandoffAfterError(trigger.reason, attempt); return maybeHandoffAfterError(retryAction.trigger.reason, attempt);
} }
pairedExecutionStatus = 'succeeded'; pairedExecutionStatus = 'succeeded';
@@ -709,18 +711,20 @@ export async function runAgentForGroup(
attempt: AgentAttempt, attempt: AgentAttempt,
rotationMessage?: string | null, rotationMessage?: string | null,
): Promise<'success' | 'error' | null> => { ): Promise<'success' | 'error' | null> => {
const trigger = resolveCodexRetryTrigger({ const retryAction = resolveAttemptRetryAction({
provider,
canRetryClaudeCredentials: false,
canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1, canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1,
attempt, attempt,
rotationMessage, rotationMessage,
}); });
if (!trigger) { if (retryAction.kind !== 'codex') {
return null; return null;
} }
const result = await retryCodexWithRotation( const result = await retryCodexWithRotation(
{ reason: trigger.reason }, retryAction.trigger,
rotationMessage ?? undefined, retryAction.rotationMessage,
); );
if (result === 'success') { if (result === 'success') {
pairedExecutionStatus = 'succeeded'; pairedExecutionStatus = 'succeeded';

View File

@@ -8,6 +8,7 @@ import {
buildArbiterPromptForTask, buildArbiterPromptForTask,
buildFinalizePendingPrompt, buildFinalizePendingPrompt,
buildOwnerPendingPrompt, buildOwnerPendingPrompt,
buildPairedTurnPrompt,
buildReviewerPendingPrompt, buildReviewerPendingPrompt,
} from './message-runtime-prompts.js'; } from './message-runtime-prompts.js';
import { import {
@@ -47,6 +48,26 @@ export type BotOnlyPairedFollowUpAction =
nextRole: 'owner' | 'reviewer' | 'arbiter'; nextRole: 'owner' | 'reviewer' | 'arbiter';
}; };
export type QueuedTurnDispatch = {
formatted: string;
botOnlyFollowUpAction: BotOnlyPairedFollowUpAction;
isBotOnlyPairedFollowUp: boolean;
loopCursorKey: string;
endSeq: number | null;
};
export function isBotOnlyPairedRoomTurn(
chatJid: string,
messages: NewMessage[],
): boolean {
return (
hasReviewerLease(chatJid) &&
messages.every(
(message) => message.is_from_me === true || !!message.is_bot_message,
)
);
}
export function resolveLastDeliveredBotRole( export function resolveLastDeliveredBotRole(
messages: NewMessage[], messages: NewMessage[],
): PairedRoomRole | null { ): PairedRoomRole | null {
@@ -384,6 +405,54 @@ export async function executeBotOnlyPairedFollowUpAction(args: {
return true; return true;
} }
export function buildQueuedTurnDispatch(args: {
chatJid: string;
timezone: string;
loopPendingTask: PairedTask | null | undefined;
rawPendingMessages: NewMessage[];
messagesToSend: NewMessage[];
labeledMessagesToSend: NewMessage[];
formatMessages: (messages: NewMessage[], timezone: string) => string;
}): QueuedTurnDispatch {
const loopCursorKey = resolveCursorKey(
args.chatJid,
args.loopPendingTask?.status,
);
const formatted = args.loopPendingTask
? buildPairedTurnPrompt({
taskId: args.loopPendingTask.id,
chatJid: args.chatJid,
timezone: args.timezone,
missedMessages: args.messagesToSend,
labeledFallbackMessages: args.labeledMessagesToSend,
turnOutputs: getPairedTurnOutputs(args.loopPendingTask.id),
})
: args.formatMessages(args.labeledMessagesToSend, args.timezone);
const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn(
args.chatJid,
args.messagesToSend,
);
const pendingCursorSource =
args.rawPendingMessages.length > 0
? args.rawPendingMessages[args.rawPendingMessages.length - 1]
: args.messagesToSend[args.messagesToSend.length - 1];
const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({
chatJid: args.chatJid,
task: args.loopPendingTask,
isBotOnlyPairedFollowUp,
lastDeliveredMessages: args.labeledMessagesToSend,
pendingCursorSource,
});
return {
formatted,
botOnlyFollowUpAction,
isBotOnlyPairedFollowUp,
loopCursorKey,
endSeq: args.messagesToSend[args.messagesToSend.length - 1]?.seq ?? null,
};
}
export function shouldSkipGenericFollowUpAfterDeliveryRetry(args: { export function shouldSkipGenericFollowUpAfterDeliveryRetry(args: {
chatJid: string; chatJid: string;
deliveryRole: PairedRoomRole; deliveryRole: PairedRoomRole;

View File

@@ -43,11 +43,11 @@ import {
} from './message-runtime-rules.js'; } from './message-runtime-rules.js';
import { runAgentForGroup } from './message-agent-executor.js'; import { runAgentForGroup } from './message-agent-executor.js';
import { import {
buildQueuedTurnDispatch,
buildPendingPairedTurn, buildPendingPairedTurn,
executeBotOnlyPairedFollowUpAction, executeBotOnlyPairedFollowUpAction,
executePendingPairedTurn, executePendingPairedTurn,
resolveBotOnlyPairedFollowUpAction, isBotOnlyPairedRoomTurn,
resolveLastDeliveredBotRole,
shouldSkipGenericFollowUpAfterDeliveryRetry, shouldSkipGenericFollowUpAfterDeliveryRetry,
} from './message-runtime-flow.js'; } from './message-runtime-flow.js';
import { MessageTurnController } from './message-turn-controller.js'; import { MessageTurnController } from './message-turn-controller.js';
@@ -58,6 +58,7 @@ import {
buildPairedTurnPrompt, buildPairedTurnPrompt,
buildReviewerPendingPrompt, buildReviewerPendingPrompt,
} from './message-runtime-prompts.js'; } from './message-runtime-prompts.js';
import { transitionPairedTaskStatus } from './paired-execution-context-shared.js';
import { import {
extractSessionCommand, extractSessionCommand,
handleSessionCommand, handleSessionCommand,
@@ -207,15 +208,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
}); });
}; };
const isBotOnlyPairedRoomTurn = (
chatJid: string,
messages: NewMessage[],
): boolean =>
hasReviewerLease(chatJid) &&
messages.every(
(message) => message.is_from_me === true || !!message.is_bot_message,
);
const enqueueScopedGroupMessageCheck = ( const enqueueScopedGroupMessageCheck = (
chatJid: string, chatJid: string,
groupFolder: string, groupFolder: string,
@@ -937,10 +929,15 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
if (hasReviewerLease(chatJid)) { if (hasReviewerLease(chatJid)) {
const task = getLatestOpenPairedTaskForChat(chatJid); const task = getLatestOpenPairedTaskForChat(chatJid);
if (task) { if (task) {
updatePairedTask(task.id, { const now = new Date().toISOString();
status: 'completed', transitionPairedTaskStatus({
completion_reason: 'stopped', taskId: task.id,
updated_at: new Date().toISOString(), currentStatus: task.status,
nextStatus: 'completed',
updatedAt: now,
patch: {
completion_reason: 'stopped',
},
}); });
} }
} }
@@ -1223,30 +1220,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
chatJid, chatJid,
messagesToSend, messagesToSend,
); );
const formatted = loopPendingTask const {
? buildPairedTurnPrompt({ formatted,
taskId: loopPendingTask.id, botOnlyFollowUpAction,
chatJid,
timezone: deps.timezone,
missedMessages: messagesToSend,
labeledFallbackMessages: labeledMessagesToSend,
turnOutputs: getPairedTurnOutputs(loopPendingTask.id),
})
: formatMessages(labeledMessagesToSend, deps.timezone);
const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn(
chatJid,
messagesToSend,
);
const pendingCursorSource =
rawPendingMessages.length > 0
? rawPendingMessages[rawPendingMessages.length - 1]
: messagesToSend[messagesToSend.length - 1];
const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({
chatJid,
task: loopPendingTask,
isBotOnlyPairedFollowUp, isBotOnlyPairedFollowUp,
lastDeliveredMessages: labeledMessagesToSend, loopCursorKey: dispatchCursorKey,
pendingCursorSource, endSeq,
} = buildQueuedTurnDispatch({
chatJid,
timezone: deps.timezone,
loopPendingTask,
rawPendingMessages,
messagesToSend,
labeledMessagesToSend,
formatMessages,
}); });
if ( if (
await executeBotOnlyPairedFollowUpAction({ await executeBotOnlyPairedFollowUpAction({
@@ -1271,14 +1258,13 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
} }
if (deps.queue.sendMessage(chatJid, formatted)) { if (deps.queue.sendMessage(chatJid, formatted)) {
const endSeq = messagesToSend[messagesToSend.length - 1]?.seq;
if (endSeq != null) { if (endSeq != null) {
advanceLastAgentCursor( advanceLastAgentCursor(
deps.getLastAgentTimestamps(), deps.getLastAgentTimestamps(),
deps.saveState, deps.saveState,
chatJid, chatJid,
endSeq, endSeq,
loopCursorKey, dispatchCursorKey,
); );
} }
logger.debug( logger.debug(

View File

@@ -1,7 +1,9 @@
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js'; import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
import { updatePairedTask } from './db.js';
import { logger } from './logger.js'; import { logger } from './logger.js';
import { classifyArbiterVerdict } from './paired-execution-context-shared.js'; import {
classifyArbiterVerdict,
transitionPairedTaskStatus,
} from './paired-execution-context-shared.js';
import type { PairedTask } from './types.js'; import type { PairedTask } from './types.js';
export function handleFailedArbiterExecution(args: { export function handleFailedArbiterExecution(args: {
@@ -15,7 +17,12 @@ export function handleFailedArbiterExecution(args: {
? 'arbiter_requested' ? 'arbiter_requested'
: task.status; : task.status;
if (fallbackStatus !== task.status) { if (fallbackStatus !== task.status) {
updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: fallbackStatus,
updatedAt: now,
});
logger.warn( logger.warn(
{ {
taskId, taskId,
@@ -45,11 +52,15 @@ export function handleArbiterCompletion(args: {
case 'proceed': case 'proceed':
case 'revise': case 'revise':
case 'reset': case 'reset':
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'active', taskId,
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), currentStatus: 'in_arbitration',
arbiter_verdict: arbiterVerdict, nextStatus: 'active',
updated_at: now, updatedAt: now,
patch: {
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1),
arbiter_verdict: arbiterVerdict,
},
}); });
logger.info( logger.info(
{ taskId, arbiterVerdict }, { taskId, arbiterVerdict },
@@ -57,20 +68,28 @@ export function handleArbiterCompletion(args: {
); );
return; return;
case 'escalate': case 'escalate':
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'completed', taskId,
arbiter_verdict: 'escalate', currentStatus: 'in_arbitration',
completion_reason: 'arbiter_escalated', nextStatus: 'completed',
updated_at: now, updatedAt: now,
patch: {
arbiter_verdict: 'escalate',
completion_reason: 'arbiter_escalated',
},
}); });
logger.info({ taskId }, 'Arbiter escalated to user — task completed'); logger.info({ taskId }, 'Arbiter escalated to user — task completed');
return; return;
default: default:
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'active', taskId,
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1), currentStatus: 'in_arbitration',
arbiter_verdict: 'unknown', nextStatus: 'active',
updated_at: now, updatedAt: now,
patch: {
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1),
arbiter_verdict: 'unknown',
},
}); });
logger.warn( logger.warn(
{ taskId, summary: summary?.slice(0, 200) }, { taskId, summary: summary?.slice(0, 200) },

View File

@@ -14,6 +14,7 @@ import {
hasCodeChangesSinceRef, hasCodeChangesSinceRef,
requestArbiterOrEscalate, requestArbiterOrEscalate,
resolveCanonicalSourceRef, resolveCanonicalSourceRef,
transitionPairedTaskStatus,
} from './paired-execution-context-shared.js'; } from './paired-execution-context-shared.js';
import type { PairedTask } from './types.js'; import type { PairedTask } from './types.js';
@@ -24,7 +25,12 @@ export function handleFailedOwnerExecution(args: {
const { task, taskId } = args; const { task, taskId } = args;
if (task.status !== 'active') { if (task.status !== 'active') {
const now = new Date().toISOString(); const now = new Date().toISOString();
updatePairedTask(taskId, { status: 'active', updated_at: now }); transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: 'active',
updatedAt: now,
});
logger.info( logger.info(
{ taskId, role: 'owner', previousStatus: task.status }, { taskId, role: 'owner', previousStatus: task.status },
'Reset task to active after failed execution', 'Reset task to active after failed execution',
@@ -44,6 +50,7 @@ function handleOwnerFinalizeCompletion(args: {
if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') {
requestArbiterOrEscalate({ requestArbiterOrEscalate({
taskId, taskId,
currentStatus: task.status,
now, now,
arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter', arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter',
escalateLogMessage: 'Owner blocked during finalize — escalating to user', escalateLogMessage: 'Owner blocked during finalize — escalating to user',
@@ -60,6 +67,7 @@ function handleOwnerFinalizeCompletion(args: {
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
requestArbiterOrEscalate({ requestArbiterOrEscalate({
taskId, taskId,
currentStatus: task.status,
now, now,
arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter', arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter',
escalateLogMessage: 'Owner finalize loop detected — escalating to user', escalateLogMessage: 'Owner finalize loop detected — escalating to user',
@@ -71,7 +79,12 @@ function handleOwnerFinalizeCompletion(args: {
}); });
return true; return true;
} }
updatePairedTask(taskId, { status: 'active', updated_at: now }); transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: 'active',
updatedAt: now,
});
logger.info( logger.info(
{ {
taskId, taskId,
@@ -92,6 +105,7 @@ function handleOwnerFinalizeCompletion(args: {
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
requestArbiterOrEscalate({ requestArbiterOrEscalate({
taskId, taskId,
currentStatus: task.status,
now, now,
arbiterLogMessage: arbiterLogMessage:
'Owner finalize DONE loop detected — requesting arbiter', 'Owner finalize DONE loop detected — requesting arbiter',
@@ -116,10 +130,14 @@ function handleOwnerFinalizeCompletion(args: {
return false; return false;
} }
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'completed', taskId,
completion_reason: 'done', currentStatus: task.status,
updated_at: now, nextStatus: 'completed',
updatedAt: now,
patch: {
completion_reason: 'done',
},
}); });
logger.info( logger.info(
{ taskId, hasNewChanges, summary: summary?.slice(0, 100) }, { taskId, hasNewChanges, summary: summary?.slice(0, 100) },
@@ -161,6 +179,7 @@ export function handleOwnerCompletion(args: {
if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') { if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') {
requestArbiterOrEscalate({ requestArbiterOrEscalate({
taskId, taskId,
currentStatus: task.status,
now, now,
arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter', arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter',
escalateLogMessage: 'Owner blocked/needs_context — escalating to user', escalateLogMessage: 'Owner blocked/needs_context — escalating to user',

View File

@@ -5,6 +5,7 @@ import {
classifyVerdict, classifyVerdict,
requestArbiterOrEscalate, requestArbiterOrEscalate,
resolveCanonicalSourceRef, resolveCanonicalSourceRef,
transitionPairedTaskStatus,
} from './paired-execution-context-shared.js'; } from './paired-execution-context-shared.js';
import type { PairedTask } from './types.js'; import type { PairedTask } from './types.js';
@@ -29,11 +30,15 @@ export function handleFailedReviewerExecution(args: {
verdict === 'done' && ownerWs?.workspace_dir verdict === 'done' && ownerWs?.workspace_dir
? resolveCanonicalSourceRef(ownerWs.workspace_dir) ? resolveCanonicalSourceRef(ownerWs.workspace_dir)
: task.source_ref; : task.source_ref;
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: verdict === 'done' ? 'merge_ready' : 'completed', taskId,
...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}), currentStatus: task.status,
...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}), nextStatus: verdict === 'done' ? 'merge_ready' : 'completed',
updated_at: now, updatedAt: now,
patch: {
...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}),
...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}),
},
}); });
logger.info( logger.info(
{ {
@@ -53,7 +58,12 @@ export function handleFailedReviewerExecution(args: {
? 'review_ready' ? 'review_ready'
: task.status; : task.status;
if (fallbackStatus !== task.status) { if (fallbackStatus !== task.status) {
updatePairedTask(taskId, { status: fallbackStatus, updated_at: now }); transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: fallbackStatus,
updatedAt: now,
});
logger.warn( logger.warn(
{ {
taskId, taskId,
@@ -81,10 +91,14 @@ export function handleReviewerCompletion(args: {
const approvedSourceRef = ownerWs?.workspace_dir const approvedSourceRef = ownerWs?.workspace_dir
? resolveCanonicalSourceRef(ownerWs.workspace_dir) ? resolveCanonicalSourceRef(ownerWs.workspace_dir)
: task.source_ref; : task.source_ref;
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'merge_ready', taskId,
source_ref: approvedSourceRef, currentStatus: task.status,
updated_at: now, nextStatus: 'merge_ready',
updatedAt: now,
patch: {
source_ref: approvedSourceRef,
},
}); });
logger.info( logger.info(
{ {
@@ -102,6 +116,7 @@ export function handleReviewerCompletion(args: {
case 'needs_context': case 'needs_context':
requestArbiterOrEscalate({ requestArbiterOrEscalate({
taskId, taskId,
currentStatus: task.status,
now, now,
arbiterLogMessage: arbiterLogMessage:
'Reviewer blocked/needs_context — requesting arbiter before escalating', 'Reviewer blocked/needs_context — requesting arbiter before escalating',
@@ -116,6 +131,7 @@ export function handleReviewerCompletion(args: {
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) { if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
requestArbiterOrEscalate({ requestArbiterOrEscalate({
taskId, taskId,
currentStatus: task.status,
now, now,
arbiterLogMessage: arbiterLogMessage:
'Deadlock detected — requesting arbiter intervention', 'Deadlock detected — requesting arbiter intervention',
@@ -129,7 +145,12 @@ export function handleReviewerCompletion(args: {
}); });
return; return;
} }
updatePairedTask(taskId, { status: 'active', updated_at: now }); transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: 'active',
updatedAt: now,
});
logger.info( logger.info(
{ taskId, verdict }, { taskId, verdict },
'Reviewer has feedback, task set back to active for owner', 'Reviewer has feedback, task set back to active for owner',

View File

@@ -3,6 +3,7 @@ import { execFileSync } from 'child_process';
import { isArbiterEnabled } from './config.js'; import { isArbiterEnabled } from './config.js';
import { updatePairedTask } from './db.js'; import { updatePairedTask } from './db.js';
import { logger } from './logger.js'; import { logger } from './logger.js';
import type { PairedTaskStatus } from './types.js';
export type Verdict = export type Verdict =
| 'done' | 'done'
@@ -93,29 +94,108 @@ export function hasCodeChangesSinceRef(
} }
} }
const ALLOWED_PAIRED_STATUS_TRANSITIONS: Record<
PairedTaskStatus,
ReadonlySet<PairedTaskStatus>
> = {
active: new Set(['review_ready', 'arbiter_requested', 'completed']),
review_ready: new Set([
'active',
'in_review',
'arbiter_requested',
'completed',
]),
in_review: new Set([
'active',
'review_ready',
'merge_ready',
'arbiter_requested',
'completed',
]),
merge_ready: new Set(['active', 'arbiter_requested', 'completed']),
completed: new Set(),
arbiter_requested: new Set(['in_arbitration', 'completed']),
in_arbitration: new Set(['active', 'arbiter_requested', 'completed']),
};
export function assertPairedTaskStatusTransition(args: {
currentStatus: PairedTaskStatus;
nextStatus: PairedTaskStatus;
}): void {
const { currentStatus, nextStatus } = args;
if (currentStatus === nextStatus) {
return;
}
if (ALLOWED_PAIRED_STATUS_TRANSITIONS[currentStatus].has(nextStatus)) {
return;
}
throw new Error(
`Invalid paired task status transition: ${currentStatus} -> ${nextStatus}`,
);
}
export function transitionPairedTaskStatus(args: {
taskId: string;
currentStatus: PairedTaskStatus;
nextStatus: PairedTaskStatus;
updatedAt: string;
patch?: Omit<
Parameters<typeof updatePairedTask>[1],
'status' | 'updated_at'
>;
}): void {
assertPairedTaskStatusTransition({
currentStatus: args.currentStatus,
nextStatus: args.nextStatus,
});
updatePairedTask(args.taskId, {
...args.patch,
status: args.nextStatus,
updated_at: args.updatedAt,
});
}
export function requestArbiterOrEscalate(args: { export function requestArbiterOrEscalate(args: {
taskId: string; taskId: string;
currentStatus: PairedTaskStatus;
now: string; now: string;
arbiterLogMessage: string; arbiterLogMessage: string;
escalateLogMessage: string; escalateLogMessage: string;
logContext?: Record<string, unknown>; logContext?: Record<string, unknown>;
}): void { }): void {
const { taskId, now, arbiterLogMessage, escalateLogMessage, logContext } = const {
args; taskId,
currentStatus,
now,
arbiterLogMessage,
escalateLogMessage,
logContext,
} = args;
if (isArbiterEnabled()) { if (isArbiterEnabled()) {
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'arbiter_requested', taskId,
arbiter_requested_at: now, currentStatus,
updated_at: now, nextStatus: 'arbiter_requested',
updatedAt: now,
patch: {
arbiter_requested_at: now,
},
}); });
logger.info(logContext ?? { taskId }, arbiterLogMessage); logger.info(logContext ?? { taskId }, arbiterLogMessage);
return; return;
} }
updatePairedTask(taskId, { transitionPairedTaskStatus({
status: 'completed', taskId,
completion_reason: 'escalated', currentStatus,
updated_at: now, nextStatus: 'completed',
updatedAt: now,
patch: {
completion_reason: 'escalated',
},
}); });
logger.info(logContext ?? { taskId }, escalateLogMessage); logger.info(logContext ?? { taskId }, escalateLogMessage);
} }

View File

@@ -38,6 +38,7 @@ import {
import { import {
resolveCanonicalSourceRef, resolveCanonicalSourceRef,
requestArbiterOrEscalate, requestArbiterOrEscalate,
transitionPairedTaskStatus,
} from './paired-execution-context-shared.js'; } from './paired-execution-context-shared.js';
import { import {
markPairedTaskReviewReady, markPairedTaskReviewReady,
@@ -226,11 +227,22 @@ export function preparePairedExecutionContext(args: {
latestTask.status === 'review_ready' || latestTask.status === 'review_ready' ||
latestTask.status === 'in_review'; latestTask.status === 'in_review';
if (hasHuman || needsStatusReset) { if (hasHuman || needsStatusReset) {
updatePairedTask(latestTask.id, { if (needsStatusReset) {
...(hasHuman ? { round_trip_count: 0 } : {}), transitionPairedTaskStatus({
...(needsStatusReset ? { status: 'active' as const } : {}), taskId: latestTask.id,
updated_at: now, currentStatus: latestTask.status,
}); nextStatus: 'active',
updatedAt: now,
patch: {
...(hasHuman ? { round_trip_count: 0 } : {}),
},
});
} else {
updatePairedTask(latestTask.id, {
...(hasHuman ? { round_trip_count: 0 } : {}),
updated_at: now,
});
}
} }
// Use a stable per-channel worktree (not per-task) so the Claude SDK // Use a stable per-channel worktree (not per-task) so the Claude SDK
// session persists across tasks. Different channels still get isolation. // session persists across tasks. Different channels still get isolation.
@@ -253,9 +265,11 @@ export function preparePairedExecutionContext(args: {
blockMessage = reviewerWorkspace.blockMessage; blockMessage = reviewerWorkspace.blockMessage;
const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask; const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask;
if (workspace && refreshedTask.status === 'review_ready') { if (workspace && refreshedTask.status === 'review_ready') {
updatePairedTask(latestTask.id, { transitionPairedTaskStatus({
status: 'in_review', taskId: latestTask.id,
updated_at: now, currentStatus: refreshedTask.status,
nextStatus: 'in_review',
updatedAt: now,
}); });
} }
} else if (roomRoleContext.role === 'arbiter') { } else if (roomRoleContext.role === 'arbiter') {
@@ -265,9 +279,11 @@ export function preparePairedExecutionContext(args: {
blockMessage = reviewerWorkspace.blockMessage; blockMessage = reviewerWorkspace.blockMessage;
const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask; const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask;
if (workspace && refreshedTask.status === 'arbiter_requested') { if (workspace && refreshedTask.status === 'arbiter_requested') {
updatePairedTask(latestTask.id, { transitionPairedTaskStatus({
status: 'in_arbitration', taskId: latestTask.id,
updated_at: now, currentStatus: refreshedTask.status,
nextStatus: 'in_arbitration',
updatedAt: now,
}); });
} }
} }

View File

@@ -13,6 +13,7 @@ import {
} from './db.js'; } from './db.js';
import { resolvePairedTaskWorkspacePath } from './group-folder.js'; import { resolvePairedTaskWorkspacePath } from './group-folder.js';
import { logger } from './logger.js'; import { logger } from './logger.js';
import { transitionPairedTaskStatus } from './paired-execution-context-shared.js';
import type { PairedTask, PairedWorkspace } from './types.js'; import type { PairedTask, PairedWorkspace } from './types.js';
import { ensureWorkspaceDependenciesInstalled } from './workspace-package-manager.js'; import { ensureWorkspaceDependenciesInstalled } from './workspace-package-manager.js';
@@ -798,9 +799,15 @@ export function markPairedTaskReviewReady(taskId: string): {
); );
const now = new Date().toISOString(); const now = new Date().toISOString();
updatePairedTask(taskId, { const task = getPairedTaskById(taskId);
status: 'review_ready', if (!task) {
updated_at: now, return null;
}
transitionPairedTaskStatus({
taskId,
currentStatus: task.status,
nextStatus: 'review_ready',
updatedAt: now,
}); });
return { ownerWorkspace, reviewerWorkspace }; return { ownerWorkspace, reviewerWorkspace };

View File

@@ -3,6 +3,7 @@ import { CronExpressionParser } from 'cron-parser';
import fs from 'fs'; import fs from 'fs';
import { getAgentOutputText } from './agent-output.js'; import { getAgentOutputText } from './agent-output.js';
import { createEvaluatedOutputHandler } from './agent-attempt.js'; import { createEvaluatedOutputHandler } from './agent-attempt.js';
import { resolveAttemptRetryAction } from './agent-attempt-retry.js';
import { getErrorMessage } from './utils.js'; import { getErrorMessage } from './utils.js';
import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js'; import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js';
@@ -42,7 +43,6 @@ import {
classifyRotationTrigger, classifyRotationTrigger,
type AgentTriggerReason, type AgentTriggerReason,
type CodexRotationReason, type CodexRotationReason,
isCodexRotationReason,
} from './agent-error-detection.js'; } from './agent-error-detection.js';
import { import {
getTokenCount, getTokenCount,
@@ -512,65 +512,34 @@ async function runTask(
} }
}; };
const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude'; const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude';
{ {
const attempt = await runTaskAttempt(provider); const attempt = await runTaskAttempt(provider);
result = attempt.attemptResult; result = attempt.attemptResult;
error = attempt.attemptError; error = attempt.attemptError;
if ( const retryAction = resolveAttemptRetryAction({
provider === 'claude' && provider,
attempt.streamedTriggerReason && canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0,
!attempt.sawOutput canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1,
) { attempt,
await retryClaudeTaskWithRotation(attempt.streamedTriggerReason); rotationMessage: error,
} else if ( });
provider === 'codex' &&
attempt.streamedTriggerReason && if (retryAction.kind === 'claude') {
!attempt.sawOutput && await retryClaudeTaskWithRotation(
isCodexRotationReason(attempt.streamedTriggerReason.reason) retryAction.trigger,
) { retryAction.rotationMessage,
await retryCodexTaskWithRotation(
{
reason: attempt.streamedTriggerReason.reason,
},
typeof attempt.output.error === 'string'
? attempt.output.error
: undefined,
);
} else if (attempt.output.status === 'error' && provider === 'claude') {
const trigger = attempt.streamedTriggerReason
? {
shouldRetry: true,
reason: attempt.streamedTriggerReason.reason,
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
}
: classifyRotationTrigger(error);
if (trigger.shouldRetry) {
await retryClaudeTaskWithRotation({
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,
});
}
} else if (attempt.output.status === 'error' && provider === 'codex') {
const trigger =
attempt.streamedTriggerReason &&
isCodexRotationReason(attempt.streamedTriggerReason.reason)
? {
shouldRotate: true as const,
reason: attempt.streamedTriggerReason.reason,
}
: detectCodexRotationTrigger(error);
if (trigger.shouldRotate) {
await retryCodexTaskWithRotation(
{ reason: trigger.reason },
error || undefined,
); );
} else if (retryAction.kind === 'codex') {
await retryCodexTaskWithRotation(
retryAction.trigger,
retryAction.rotationMessage,
);
} else if (attempt.output.status === 'error') {
error = attempt.attemptError || 'Unknown error';
} }
} else if (attempt.output.status === 'error') {
error = attempt.attemptError || 'Unknown error';
}
} // end else (non-exhausted path) } // end else (non-exhausted path)
log.info( log.info(