Refactor agent executor retry and follow-up flow
This commit is contained in:
@@ -1485,6 +1485,83 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
expect(outputs).toEqual(['force refresh 뒤 Claude 응답입니다.']);
|
expect(outputs).toEqual(['force refresh 뒤 Claude 응답입니다.']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('marks paired execution as succeeded when Claude rotation recovers after a streamed auth-expired trigger', async () => {
|
||||||
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
|
vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true);
|
||||||
|
vi.mocked(
|
||||||
|
pairedExecutionContext.preparePairedExecutionContext,
|
||||||
|
).mockReturnValue({
|
||||||
|
task: {
|
||||||
|
id: 'paired-task-claude-rotation-success',
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
group_folder: 'test-claude',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-review',
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
round_trip_count: 0,
|
||||||
|
review_requested_at: null,
|
||||||
|
status: 'active',
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
created_at: '2026-04-06T00:00:00.000Z',
|
||||||
|
updated_at: '2026-04-06T00:00:00.000Z',
|
||||||
|
},
|
||||||
|
workspace: null,
|
||||||
|
envOverrides: {},
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'intermediate',
|
||||||
|
result:
|
||||||
|
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
|
||||||
|
});
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result:
|
||||||
|
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: '회전 복구 후 paired success',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-paired-auth-expired-success',
|
||||||
|
onOutput: async () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('success');
|
||||||
|
expect(pairedExecutionContext.completePairedExecutionContext).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
taskId: 'paired-task-claude-rotation-success',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'succeeded',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => {
|
it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => {
|
||||||
const outputs: string[] = [];
|
const outputs: string[] = [];
|
||||||
|
|
||||||
|
|||||||
@@ -774,6 +774,8 @@ export async function runAgentForGroup(
|
|||||||
return 'error';
|
return 'error';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type AgentAttempt = Awaited<ReturnType<typeof runAttempt>>;
|
||||||
|
|
||||||
const retryClaudeWithRotation = async (
|
const retryClaudeWithRotation = async (
|
||||||
initialTrigger: {
|
initialTrigger: {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
@@ -813,9 +815,90 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const resolveClaudeRetryTrigger = (
|
||||||
|
attempt: AgentAttempt,
|
||||||
|
fallbackMessage?: string | null,
|
||||||
|
): { reason: AgentTriggerReason; retryAfterMs?: number } | null => {
|
||||||
|
if (
|
||||||
|
!(
|
||||||
|
canRetryClaudeCredentials &&
|
||||||
|
provider === 'claude' &&
|
||||||
|
!attempt.sawOutput
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt.streamedTriggerReason) {
|
||||||
|
return {
|
||||||
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const trigger = classifyRotationTrigger(fallbackMessage);
|
||||||
|
if (!trigger.shouldRetry) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
reason: trigger.reason,
|
||||||
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const retryClaudeAttemptIfNeeded = async (
|
||||||
|
attempt: AgentAttempt,
|
||||||
|
rotationMessage?: string | null,
|
||||||
|
): Promise<'success' | 'error' | null> => {
|
||||||
|
const trigger = resolveClaudeRetryTrigger(attempt, rotationMessage);
|
||||||
|
if (!trigger) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await retryClaudeWithRotation(
|
||||||
|
trigger,
|
||||||
|
rotationMessage ?? undefined,
|
||||||
|
);
|
||||||
|
if (result === 'error') {
|
||||||
|
return maybeHandoffAfterError(trigger.reason, attempt);
|
||||||
|
}
|
||||||
|
|
||||||
|
pairedExecutionStatus = 'succeeded';
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const retryCodexAttemptIfNeeded = async (
|
||||||
|
attempt: AgentAttempt,
|
||||||
|
rotationMessage?: string | null,
|
||||||
|
): Promise<'success' | 'error' | null> => {
|
||||||
|
if (isClaudeCodeAgent || getCodexAccountCount() <= 1) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const trigger = attempt.streamedTriggerReason
|
||||||
|
? {
|
||||||
|
shouldRotate: true,
|
||||||
|
reason: attempt.streamedTriggerReason.reason as CodexRotationReason,
|
||||||
|
}
|
||||||
|
: detectCodexRotationTrigger(rotationMessage);
|
||||||
|
if (!trigger.shouldRotate) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await retryCodexWithRotation(
|
||||||
|
{ reason: trigger.reason },
|
||||||
|
rotationMessage ?? undefined,
|
||||||
|
);
|
||||||
|
if (result === 'success') {
|
||||||
|
pairedExecutionStatus = 'succeeded';
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
const maybeHandoffAfterError = (
|
const maybeHandoffAfterError = (
|
||||||
reason: AgentTriggerReason,
|
reason: AgentTriggerReason,
|
||||||
attempt: Awaited<ReturnType<typeof runAttempt>>,
|
attempt: AgentAttempt,
|
||||||
): 'success' | 'error' => {
|
): 'success' | 'error' => {
|
||||||
if (maybeHandoffToCodex(reason, attempt.sawVisibleOutput)) {
|
if (maybeHandoffToCodex(reason, attempt.sawVisibleOutput)) {
|
||||||
return 'success';
|
return 'success';
|
||||||
@@ -823,6 +906,38 @@ export async function runAgentForGroup(
|
|||||||
return 'error';
|
return 'error';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type PairedFollowUpQueueAction =
|
||||||
|
| 'generic'
|
||||||
|
| 'pending'
|
||||||
|
| 'skip-inline-finalize'
|
||||||
|
| 'none';
|
||||||
|
type PairedTaskStatus =
|
||||||
|
NonNullable<ReturnType<typeof getPairedTaskById>>['status'];
|
||||||
|
|
||||||
|
const resolvePairedFollowUpQueueAction = (
|
||||||
|
completedRole: PairedRoomRole,
|
||||||
|
executionStatus: 'succeeded' | 'failed',
|
||||||
|
sawOutput: boolean,
|
||||||
|
taskStatus: PairedTaskStatus | null,
|
||||||
|
): PairedFollowUpQueueAction => {
|
||||||
|
if (executionStatus === 'succeeded' && sawOutput) {
|
||||||
|
return completedRole === 'reviewer' && taskStatus === 'merge_ready'
|
||||||
|
? 'skip-inline-finalize'
|
||||||
|
: taskStatus !== 'completed'
|
||||||
|
? 'generic'
|
||||||
|
: 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
const shouldRequeuePendingPairedTurn =
|
||||||
|
(completedRole === 'reviewer' || completedRole === 'arbiter') &&
|
||||||
|
(taskStatus === 'review_ready' ||
|
||||||
|
taskStatus === 'in_review' ||
|
||||||
|
taskStatus === 'arbiter_requested' ||
|
||||||
|
taskStatus === 'in_arbitration' ||
|
||||||
|
taskStatus === 'merge_ready');
|
||||||
|
return shouldRequeuePendingPairedTurn ? 'pending' : 'none';
|
||||||
|
};
|
||||||
|
|
||||||
const provider = 'claude';
|
const provider = 'claude';
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -864,50 +979,21 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (primaryAttempt.error) {
|
if (primaryAttempt.error) {
|
||||||
if (
|
const errMsg = getErrorMessage(primaryAttempt.error);
|
||||||
canRetryClaudeCredentials &&
|
const claudeRetryResult = await retryClaudeAttemptIfNeeded(
|
||||||
provider === 'claude' &&
|
primaryAttempt,
|
||||||
!primaryAttempt.sawOutput
|
errMsg,
|
||||||
) {
|
);
|
||||||
const errMsg = getErrorMessage(primaryAttempt.error);
|
if (claudeRetryResult) {
|
||||||
const trigger = primaryAttempt.streamedTriggerReason
|
return claudeRetryResult;
|
||||||
? {
|
|
||||||
shouldRetry: true,
|
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
}
|
|
||||||
: classifyRotationTrigger(errMsg);
|
|
||||||
if (trigger.shouldRetry) {
|
|
||||||
const result = await retryClaudeWithRotation(
|
|
||||||
{
|
|
||||||
reason: trigger.reason,
|
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
|
||||||
},
|
|
||||||
errMsg,
|
|
||||||
);
|
|
||||||
if (result === 'error') {
|
|
||||||
return maybeHandoffAfterError(trigger.reason, primaryAttempt);
|
|
||||||
}
|
|
||||||
if (result === 'success') {
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isClaudeCodeAgent) {
|
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
||||||
const errMsg = getErrorMessage(primaryAttempt.error);
|
primaryAttempt,
|
||||||
const trigger = detectCodexRotationTrigger(errMsg);
|
errMsg,
|
||||||
if (trigger.shouldRotate && getCodexAccountCount() > 1) {
|
);
|
||||||
const result = await retryCodexWithRotation(
|
if (codexRetryResult) {
|
||||||
{ reason: trigger.reason },
|
return codexRetryResult;
|
||||||
errMsg,
|
|
||||||
);
|
|
||||||
if (result === 'success') {
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.error(
|
log.error(
|
||||||
@@ -937,24 +1023,11 @@ export async function runAgentForGroup(
|
|||||||
: null);
|
: null);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (!primaryAttempt.sawOutput && output.status !== 'error') {
|
||||||
canRetryClaudeCredentials &&
|
const claudeRetryResult = await retryClaudeAttemptIfNeeded(primaryAttempt);
|
||||||
provider === 'claude' &&
|
if (claudeRetryResult) {
|
||||||
!primaryAttempt.sawOutput &&
|
return claudeRetryResult;
|
||||||
primaryAttempt.streamedTriggerReason &&
|
|
||||||
output.status !== 'error'
|
|
||||||
) {
|
|
||||||
const result = await retryClaudeWithRotation({
|
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
});
|
|
||||||
if (result === 'error') {
|
|
||||||
return maybeHandoffAfterError(
|
|
||||||
primaryAttempt.streamedTriggerReason.reason,
|
|
||||||
primaryAttempt,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -969,48 +1042,20 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (output.status === 'error') {
|
if (output.status === 'error') {
|
||||||
if (
|
const claudeRetryResult = await retryClaudeAttemptIfNeeded(
|
||||||
canRetryClaudeCredentials &&
|
primaryAttempt,
|
||||||
provider === 'claude' &&
|
output.error,
|
||||||
!primaryAttempt.sawOutput
|
);
|
||||||
) {
|
if (claudeRetryResult) {
|
||||||
const trigger = primaryAttempt.streamedTriggerReason
|
return claudeRetryResult;
|
||||||
? {
|
|
||||||
shouldRetry: true,
|
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
}
|
|
||||||
: classifyRotationTrigger(output.error);
|
|
||||||
if (trigger.shouldRetry) {
|
|
||||||
const result = await retryClaudeWithRotation(
|
|
||||||
{
|
|
||||||
reason: trigger.reason,
|
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
|
||||||
},
|
|
||||||
output.error ?? undefined,
|
|
||||||
);
|
|
||||||
if (result === 'error') {
|
|
||||||
return maybeHandoffAfterError(trigger.reason, primaryAttempt);
|
|
||||||
}
|
|
||||||
if (result === 'success') {
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isClaudeCodeAgent && getCodexAccountCount() > 1) {
|
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
||||||
const trigger = detectCodexRotationTrigger(output.error);
|
primaryAttempt,
|
||||||
if (trigger.shouldRotate) {
|
output.error,
|
||||||
const result = await retryCodexWithRotation(
|
);
|
||||||
{ reason: trigger.reason },
|
if (codexRetryResult) {
|
||||||
output.error ?? undefined,
|
return codexRetryResult;
|
||||||
);
|
|
||||||
if (result === 'success') {
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.error(
|
log.error(
|
||||||
@@ -1023,22 +1068,12 @@ export async function runAgentForGroup(
|
|||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
const codexRetryResult = await retryCodexAttemptIfNeeded(
|
||||||
!isClaudeCodeAgent &&
|
primaryAttempt,
|
||||||
primaryAttempt.streamedTriggerReason &&
|
output.error ?? output.result,
|
||||||
getCodexAccountCount() > 1
|
);
|
||||||
) {
|
if (codexRetryResult) {
|
||||||
const result = await retryCodexWithRotation(
|
return codexRetryResult;
|
||||||
{
|
|
||||||
reason: primaryAttempt.streamedTriggerReason
|
|
||||||
.reason as CodexRotationReason,
|
|
||||||
},
|
|
||||||
output.error ?? output.result ?? undefined,
|
|
||||||
);
|
|
||||||
if (result === 'success') {
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unresolved streamed trigger — rotation was unavailable or output was
|
// Unresolved streamed trigger — rotation was unavailable or output was
|
||||||
@@ -1144,16 +1179,18 @@ export async function runAgentForGroup(
|
|||||||
// After owner/reviewer completes, enqueue the next turn so
|
// After owner/reviewer completes, enqueue the next turn so
|
||||||
// the message loop picks it up without waiting for a new message.
|
// the message loop picks it up without waiting for a new message.
|
||||||
// Skip if: no output (interrupted), or task already completed (ESCALATE, done, etc.)
|
// Skip if: no output (interrupted), or task already completed (ESCALATE, done, etc.)
|
||||||
if (
|
if (pairedExecutionContext) {
|
||||||
pairedExecutionContext &&
|
|
||||||
pairedExecutionStatus === 'succeeded' &&
|
|
||||||
pairedSawOutput
|
|
||||||
) {
|
|
||||||
const completedRole = roomRoleContext?.role ?? 'owner';
|
const completedRole = roomRoleContext?.role ?? 'owner';
|
||||||
const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id);
|
const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id);
|
||||||
const skipGenericFollowUpForInlineFinalize =
|
const queueAction = resolvePairedFollowUpQueueAction(
|
||||||
completedRole === 'reviewer' && finishedCheck?.status === 'merge_ready';
|
completedRole,
|
||||||
if (skipGenericFollowUpForInlineFinalize) {
|
pairedExecutionStatus,
|
||||||
|
pairedSawOutput,
|
||||||
|
finishedCheck?.status ?? null,
|
||||||
|
);
|
||||||
|
if (queueAction === 'generic') {
|
||||||
|
deps.queue.enqueueMessageCheck(chatJid);
|
||||||
|
} else if (queueAction === 'skip-inline-finalize') {
|
||||||
log.info(
|
log.info(
|
||||||
{
|
{
|
||||||
taskId: pairedExecutionContext.task.id,
|
taskId: pairedExecutionContext.task.id,
|
||||||
@@ -1162,20 +1199,7 @@ export async function runAgentForGroup(
|
|||||||
},
|
},
|
||||||
'Skipping generic follow-up after reviewer approval because inline finalize will handle the owner handoff',
|
'Skipping generic follow-up after reviewer approval because inline finalize will handle the owner handoff',
|
||||||
);
|
);
|
||||||
} else if (finishedCheck?.status !== 'completed') {
|
} else if (queueAction === 'pending') {
|
||||||
deps.queue.enqueueMessageCheck(chatJid);
|
|
||||||
}
|
|
||||||
} else if (pairedExecutionContext) {
|
|
||||||
const completedRole = roomRoleContext?.role ?? 'owner';
|
|
||||||
const finishedCheck = getPairedTaskById(pairedExecutionContext.task.id);
|
|
||||||
const shouldRequeuePendingPairedTurn =
|
|
||||||
(completedRole === 'reviewer' || completedRole === 'arbiter') &&
|
|
||||||
(finishedCheck?.status === 'review_ready' ||
|
|
||||||
finishedCheck?.status === 'in_review' ||
|
|
||||||
finishedCheck?.status === 'arbiter_requested' ||
|
|
||||||
finishedCheck?.status === 'in_arbitration' ||
|
|
||||||
finishedCheck?.status === 'merge_ready');
|
|
||||||
if (shouldRequeuePendingPairedTurn) {
|
|
||||||
deps.queue.enqueueMessageCheck(chatJid);
|
deps.queue.enqueueMessageCheck(chatJid);
|
||||||
log.info(
|
log.info(
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user