Files
EJClaw/src/message-agent-executor.ts
Codex 1bb23a60db fix: park rooms during a transient Claude auth outage instead of failing
ff76d04 told the room about a transient refresh failure but still failed
the execution (completeImmediately 'failed'), which bumps
owner_failure_count and can hand the task to the arbiter after two blips.

- reportClaudeAuthRecovering now parks the turn like the usage-limit
  pause: releaseForClaudeUsagePause (lease released, no failure count, no
  follow-up) plus markRunPausedForClaudeUsage (messages stay unconsumed).
  The room is told once per incident; later blips park silently.
- token-refresh fires setOnClaudeAuthRecovered once a rejected token is
  replaced (deferred so the new token is in memory first); index.ts
  re-queues every parked room so the same turn runs without user input.
- Real-DB test: two parked owner runs leave owner_failure_count/status/
  arbiter_requested_at unchanged (control: a failed run does count).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 11:02:48 +09:00

716 lines
23 KiB
TypeScript

import { AgentOutput } from './agent-runner.js';
import { logger } from './logger.js';
import { GroupQueue } from './group-queue.js';
import type { AgentTriggerReason } from './agent-error-detection.js';
import {
createPairedExecutionLifecycle,
type PairedExecutionLifecycle,
} from './message-agent-executor-paired.js';
import type { PairedTurnIdentity } from './paired-turn-identity.js';
import { executeMessageAgentAttemptLifecycle } from './message-agent-executor-lifecycle.js';
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
import {
formatClaudeUsagePauseMessage,
formatResetTime,
isClaudeUsageLimitReason,
resolveClaudeUsageResume,
type ClaudeUsageResume,
} from './claude-usage-limit.js';
import {
getActiveClaudeUsagePause,
markRunPausedForClaudeUsage,
pauseRoomForClaudeUsage,
} from './claude-usage-pause.js';
import {
failPairedTurn,
getPairedTaskById,
releasePairedTaskExecutionLease,
} from './db.js';
import {
formatUsageAlignmentHoldMessage,
formatUsageLimitAlignedPauseMessage,
getUsageAlignmentHold,
holdUntilSlotAfterLimitReset,
isUsageAlignmentBypassed,
} from './usage-alignment-hold.js';
import { completePairedTaskWithReason } from './paired-task-status.js';
import { parkRoomForClaudeAuthRecovery } from './claude-auth-recovery.js';
import {
forceRefreshToken,
getClaudeAuthRecoveryIncident,
isReloginRequired,
RELOGIN_REQUIRED_MESSAGE,
} from './token-refresh.js';
import { prepareMessageAgentExecutionTarget } from './message-agent-executor-target.js';
import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js';
import { bindPairedSessionToTask } from './paired-session-binding.js';
import {
completeTaskWithoutFinalizeRun,
resolveFastFinalizeText,
} from './paired-finalize.js';
import type { AgentType, PairedRoomRole, RegisteredGroup } from './types.js';
import {
clearCompactRefreshIfUnchanged,
maybeApplyCompactRefresh,
type AppliedCompactRefresh,
} from './compact-refresh.js';
// ── Main executor ─────────────────────────────────────────────────
/** completion_reason of a paired task closed because Claude's safeguards refused it. */
export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal';
/** completion_reason of a paired task closed because the Claude login is gone. */
export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON =
'claude_relogin_required';
export const CLAUDE_AUTH_RECOVERING_NOTICE =
'⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이 방의 작업을 일시정지했습니다. ' +
'봇이 5분마다 자동으로 다시 시도하고, 복구되면 멈춘 작업을 자동으로 이어서 진행합니다. ' +
'30분 넘게 복구되지 않으면 재로그인 안내를 드리겠습니다.';
// chatJid -> auth recovery incident already announced in that room.
const announcedAuthRecovery = new Map<string, number>();
export const CLAUDE_SAFETY_REFUSAL_NOTICE =
'Claude 안전 필터가 이 요청을 차단해서 처리하지 못했습니다. 이 방의 Claude 세션은 초기화했습니다. 요청을 조금 바꿔서 다시 보내 주세요. 에러 원문은 붙이지 말아 주세요.';
function buildPromptWithCompactRefresh(args: {
prompt: string;
sessionFolder: string;
sessionId?: string;
role: PairedRoomRole;
}): { effectivePrompt: string; compactRefresh: AppliedCompactRefresh | null } {
const compactRefresh = maybeApplyCompactRefresh({
sessionFolder: args.sessionFolder,
sessionId: args.sessionId,
role: args.role,
prompt: args.prompt,
});
return {
effectivePrompt: compactRefresh?.prompt ?? args.prompt,
compactRefresh,
};
}
function clearAppliedCompactRefreshAfterSuccess(args: {
result: 'success' | 'error';
sessionFolder: string;
compactRefresh: AppliedCompactRefresh | null;
}): void {
if (args.result !== 'success' || !args.compactRefresh) return;
clearCompactRefreshIfUnchanged({
sessionFolder: args.sessionFolder,
flag: args.compactRefresh.flag,
});
}
interface ClaudeUsageLimitFailure {
reason: AgentTriggerReason;
errorText: string | null;
}
type PauseLifecycle = Pick<
PairedExecutionLifecycle,
'updateSummary' | 'releaseForClaudeUsagePause'
>;
/**
* Claude is out of usage: pause the room instead of failing the turn. The
* paired task is left untouched and the run is marked paused so the queue
* keeps its messages; the scheduler re-queues the room after the reset and the
* same turn runs again.
*/
async function pauseRunForClaudeUsage(args: {
chatJid: string;
runId: string;
resume: ClaudeUsageResume;
/** Paused by the usage-alignment hold instead of a usage limit. */
alignment?: boolean;
summary: string;
pairedExecutionLifecycle: PauseLifecycle;
onOutput?: (output: AgentOutput) => Promise<void>;
}): Promise<void> {
const pause = pauseRoomForClaudeUsage({
chatJid: args.chatJid,
resumeAt: args.resume.resumeAt,
resetKnown: args.resume.resetKnown,
alignment: args.alignment,
});
markRunPausedForClaudeUsage(args.runId);
args.pairedExecutionLifecycle.updateSummary({ outputText: args.summary });
args.pairedExecutionLifecycle.releaseForClaudeUsagePause();
const nowMs = Date.now();
let notice: string;
if (args.alignment) {
notice = formatUsageAlignmentHoldMessage(
formatResetTime(pause.until, nowMs),
);
} else {
// A limit that resets off the slot grid would open an off-slot window
// on the first request after the reset; hold until the next slot.
if (pause.resetKnown) holdUntilSlotAfterLimitReset(pause.until);
const holdAtReset = pause.resetKnown
? getUsageAlignmentHold(pause.until.getTime())
: null;
notice = holdAtReset
? formatUsageLimitAlignedPauseMessage(
formatResetTime(pause.until, nowMs),
formatResetTime(holdAtReset.until, nowMs),
)
: formatClaudeUsagePauseMessage({
resumeAt: pause.until,
resetKnown: pause.resetKnown,
});
}
await args.onOutput?.({
status: 'success',
result: null,
output: { visibility: 'public', text: notice },
phase: 'final',
});
}
/**
* The Claude limit is account-wide: while it lasts, a Claude turn in a room
* that is not paused yet pauses it instead of starting a run that would fail.
* Returns false when no limit is in effect.
*/
async function pauseWhileClaudeUsageLimited(args: {
chatJid: string;
runId: string;
activeRole: PairedRoomRole;
pairedExecutionLifecycle: PauseLifecycle &
Pick<PairedExecutionLifecycle, 'asyncFinalize'>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
const pause = getActiveClaudeUsagePause();
const hold = isUsageAlignmentBypassed(args.chatJid)
? null
: getUsageAlignmentHold();
if (!pause && !hold) return false;
try {
// A real limit wins; its notice names any hold that follows the reset.
if (pause) {
args.log.warn(
{ until: pause.until.toISOString(), role: args.activeRole },
'Claude usage limit still in effect; pausing the room without starting a run',
);
await pauseRunForClaudeUsage({
chatJid: args.chatJid,
runId: args.runId,
resume: { resumeAt: pause.until, resetKnown: pause.resetKnown },
summary: 'Claude usage limit still in effect',
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
});
} else if (hold) {
args.log.warn(
{ until: hold.until.toISOString(), role: args.activeRole },
'Usage alignment hold in effect; pausing the room without starting a run',
);
await pauseRunForClaudeUsage({
chatJid: args.chatJid,
runId: args.runId,
resume: { resumeAt: hold.until, resetKnown: true },
alignment: true,
summary: 'Usage alignment hold in effect',
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
});
}
} finally {
await args.pairedExecutionLifecycle.asyncFinalize();
}
return true;
}
/**
* When a Claude run failed because the account is out of usage, pause the
* room until the limit resets. When the model's safeguards refused the
* request, close the paired task (retrying cannot help; the next user message
* starts a new task) and tell the room; the refusal is never handed to the
* reviewer/arbiter as if it were an answer. Returns false for any other
* failure.
*/
async function reportClaudeUsageLimitToRoom(
args: ClaudeUsageLimitFailure & {
isClaudeCodeAgent: boolean;
activeRole: PairedRoomRole;
chatJid: string;
runId: string;
pairedTaskId?: string;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'completeImmediately' | 'releaseForClaudeUsagePause'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
},
): Promise<boolean> {
if (!args.isClaudeCodeAgent) return false;
if (isClaudeUsageLimitReason(args.reason)) {
args.log.warn(
{ reason: args.reason, role: args.activeRole, errorText: args.errorText },
'Claude usage limit reached; pausing the room until it resets',
);
await pauseRunForClaudeUsage({
chatJid: args.chatJid,
runId: args.runId,
resume: resolveClaudeUsageResume(args.errorText),
summary: args.errorText ?? 'Claude usage limit reached',
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
});
return true;
}
if (args.reason === 'auth-expired') {
return reportClaudeReloginRequired(args);
}
if (args.reason !== 'safety-refusal') return false;
const task = args.pairedTaskId
? getPairedTaskById(args.pairedTaskId)
: undefined;
if (task) {
completePairedTaskWithReason(task, CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON);
}
args.log.warn(
{ reason: args.reason, role: args.activeRole, errorText: args.errorText },
'Claude safeguards refused the request; closed the task and told the room',
);
await deliverNoticeAndFailExecution({
notice: CLAUDE_SAFETY_REFUSAL_NOTICE,
summary: CLAUDE_SAFETY_REFUSAL_NOTICE,
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
});
return true;
}
/**
* A Claude run failed with a 401. The retry path has usually force-checked the
* token already; a failure after visible output has not, so check here. When
* the token is valid again, return false and let the normal retry continue.
* When the login is gone, close the task and tell the room, instead of letting
* owner/arbiter retries fail silently until someone asks why the bot stopped.
*/
async function reportClaudeReloginRequired(args: {
chatJid: string;
runId: string;
activeRole: PairedRoomRole;
pairedTaskId?: string;
errorText?: string | null;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'completeImmediately' | 'releaseForClaudeUsagePause'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
if (!isReloginRequired()) await forceRefreshToken();
if (!isReloginRequired()) {
return reportClaudeAuthRecovering(args);
}
const task = args.pairedTaskId
? getPairedTaskById(args.pairedTaskId)
: undefined;
if (task) {
completePairedTaskWithReason(
task,
CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON,
);
}
args.log.warn(
{ role: args.activeRole, errorText: args.errorText },
'Claude login is gone; closed the task and asked the room to re-login',
);
await deliverNoticeAndFailExecution({
notice: RELOGIN_REQUIRED_MESSAGE,
summary: RELOGIN_REQUIRED_MESSAGE,
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
});
return true;
}
/**
* The token was rejected but refreshing failed only transiently (429/5xx/
* timeout): the login may still be fine. Park the room like the usage-limit
* pause: release the turn without counting a failure (no owner_failure_count,
* no arbiter hand-off), keep the messages unconsumed, and resume the room
* once the token recovers (claude-auth-recovery.ts). The room is told once
* per incident.
*/
async function reportClaudeAuthRecovering(args: {
chatJid: string;
runId: string;
activeRole: PairedRoomRole;
errorText?: string | null;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'releaseForClaudeUsagePause'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
const incident = getClaudeAuthRecoveryIncident();
if (incident === null) return false;
parkRoomForClaudeAuthRecovery(args.chatJid);
markRunPausedForClaudeUsage(args.runId);
args.pairedExecutionLifecycle.updateSummary({
outputText: CLAUDE_AUTH_RECOVERING_NOTICE,
});
args.pairedExecutionLifecycle.releaseForClaudeUsagePause(
'Claude token refresh is failing transiently; room parked until auth recovers',
);
const firstNotice = announcedAuthRecovery.get(args.chatJid) !== incident;
args.log.warn(
{ role: args.activeRole, incident, firstNotice, errorText: args.errorText },
'Claude token refresh failing transiently; parked the room without counting a failure',
);
if (firstNotice) {
announcedAuthRecovery.set(args.chatJid, incident);
await args.onOutput?.({
status: 'success',
result: null,
output: { visibility: 'public', text: CLAUDE_AUTH_RECOVERING_NOTICE },
phase: 'final',
});
}
return true;
}
/**
* Owner finalize without an agent run (see paired-finalize.ts): close the task
* and deliver the already-reviewed answer as the final message. Returns false
* when the finalize turn must run normally.
*/
async function deliverFinalizeWithoutRun(args: {
taskId: string;
pairedExecutionLifecycle: PairedExecutionLifecycle;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { info: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
const task = getPairedTaskById(args.taskId);
const text = task ? resolveFastFinalizeText(task) : null;
if (!task || !text || !completeTaskWithoutFinalizeRun(task)) return false;
args.log.info(
{ pairedTaskId: task.id, textLength: text.length },
'Reviewer approved the first answer unchanged; finalized without an owner run',
);
const { pairedExecutionLifecycle } = args;
pairedExecutionLifecycle.updateSummary({ outputText: text });
if (pairedExecutionLifecycle.recordFinalOutputBeforeDelivery(text)) {
await args.onOutput?.({
status: 'success',
result: text,
output: { visibility: 'public', text },
phase: 'final',
});
}
pairedExecutionLifecycle.markSawOutput(true);
pairedExecutionLifecycle.markStatus('succeeded');
return true;
}
/** Send a final notice to the room and record the execution as failed. */
async function deliverNoticeAndFailExecution(args: {
notice: string;
summary: string;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'completeImmediately'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
}): Promise<void> {
args.pairedExecutionLifecycle.updateSummary({ outputText: args.summary });
await args.onOutput?.({
status: 'success',
result: null,
output: { visibility: 'public', text: args.notice },
phase: 'final',
});
args.pairedExecutionLifecycle.completeImmediately({ status: 'failed' });
}
/**
* The queue claims the paired turn (execution lease + running attempt) before
* this executor runs. If preparing the run throws, nothing else releases that
* claim, so the backoff retry is skipped as "already claimed elsewhere" and the
* room stalls until a human writes again. Release it so the retry can run.
*/
function releaseClaimedTurnAfterPrepareFailure(
turnIdentity: PairedTurnIdentity | undefined,
runId: string,
error: unknown,
): void {
if (!turnIdentity) return;
try {
failPairedTurn({
turnIdentity,
error: error instanceof Error ? error.message : String(error),
});
releasePairedTaskExecutionLease({ taskId: turnIdentity.taskId, runId });
} catch (releaseError) {
logger.error(
{ taskId: turnIdentity.taskId, runId, releaseError },
'Failed to release paired turn claim after execution prepare failure',
);
}
}
export interface MessageAgentExecutorDeps {
assistantName: string;
queue: Pick<GroupQueue, 'registerProcess' | 'enqueueMessageCheck'> &
Partial<
Pick<
GroupQueue,
'getDirectTerminalDeliveryForRun' | 'getCloseReasonForRun'
>
>;
getRoomBindings: () => Record<string, RegisteredGroup>;
getSessions: () => Record<string, string>;
persistSession: (groupFolder: string, sessionId: string) => void;
clearSession: (groupFolder: string) => void;
}
export async function runAgentForGroup(
deps: MessageAgentExecutorDeps,
args: {
group: RegisteredGroup;
prompt: string;
chatJid: string;
runId: string;
startSeq?: number | null;
endSeq?: number | null;
hasHumanMessage?: boolean;
forcedRole?: PairedRoomRole;
forcedAgentType?: AgentType;
pairedTurnIdentity?: PairedTurnIdentity;
onOutput?: (output: AgentOutput) => Promise<void>;
},
): Promise<'success' | 'error'> {
const { group, prompt, chatJid, runId, startSeq, endSeq, onOutput } = args;
let preparedExecution: Awaited<
ReturnType<typeof prepareMessageAgentExecutionTarget>
>;
try {
preparedExecution = await prepareMessageAgentExecutionTarget(deps, {
group,
prompt,
chatJid,
runId,
startSeq,
endSeq,
hasHumanMessage: args.hasHumanMessage,
forcedRole: args.forcedRole,
forcedAgentType: args.forcedAgentType,
pairedTurnIdentity: args.pairedTurnIdentity,
});
} catch (error) {
releaseClaimedTurnAfterPrepareFailure(
args.pairedTurnIdentity,
runId,
error,
);
throw error;
}
const {
activeRole,
effectiveServiceId,
effectiveAgentType,
sessionFolder,
effectiveGroup,
isClaudeCodeAgent,
forceFreshClaudeReviewerSession,
shouldPersistSession,
provider,
memoryBriefing,
canRetryClaudeCredentials,
roomRoleContext,
pairedExecutionContext,
runtimePairedTurnIdentity,
log,
clearRoleSdkSessions,
sessionTaskId,
} = preparedExecution;
let currentSessionId = preparedExecution.currentSessionId;
if (forceFreshClaudeReviewerSession) {
clearRoleSdkSessions();
}
const { effectivePrompt, compactRefresh } = buildPromptWithCompactRefresh({
prompt,
sessionFolder,
sessionId: currentSessionId,
role: activeRole,
});
const pairedExecutionLifecycle = createPairedExecutionLifecycle({
pairedExecutionContext,
pairedTurnIdentity: runtimePairedTurnIdentity,
completedRole: runtimePairedTurnIdentity?.role ?? activeRole,
chatJid,
runId,
enqueueMessageCheck: () => deps.queue.enqueueMessageCheck(chatJid),
getDirectTerminalDeliveryText: () =>
deps.queue.getDirectTerminalDeliveryForRun?.(
chatJid,
runId,
runtimePairedTurnIdentity?.role ?? activeRole,
) ?? null,
getCloseReason: () =>
deps.queue.getCloseReasonForRun?.(chatJid, runId) ?? null,
onOutput,
log,
});
const hasDirectTerminalDelivery = (): boolean =>
!!deps.queue.getDirectTerminalDeliveryForRun?.(
chatJid,
runId,
runtimePairedTurnIdentity?.role ?? activeRole,
);
const reportClaudeUsageLimit = (limit: ClaudeUsageLimitFailure) =>
reportClaudeUsageLimitToRoom({
...limit,
isClaudeCodeAgent,
activeRole,
chatJid,
runId,
pairedTaskId: pairedExecutionContext?.task.id,
pairedExecutionLifecycle,
onOutput,
log,
});
const agentInput = {
prompt: effectivePrompt,
sessionId: currentSessionId,
memoryBriefing,
groupFolder: group.folder,
chatJid,
runId,
assistantName: deps.assistantName,
roomRoleContext,
};
if (pairedExecutionContext?.blockMessage) {
log.warn(
{
roomRoleServiceId: roomRoleContext?.serviceId,
roomRole: roomRoleContext?.role,
},
'Blocked paired execution before runner start',
);
await deliverNoticeAndFailExecution({
notice: pairedExecutionContext.blockMessage,
summary: pairedExecutionContext.blockMessage,
pairedExecutionLifecycle,
onOutput,
});
return 'success';
}
if (
pairedExecutionContext &&
runtimePairedTurnIdentity?.intentKind === 'finalize-owner-turn'
) {
let finalizedWithoutRun = false;
try {
finalizedWithoutRun = await deliverFinalizeWithoutRun({
taskId: pairedExecutionContext.task.id,
pairedExecutionLifecycle,
onOutput,
log,
});
} catch (err) {
await pairedExecutionLifecycle.asyncFinalize();
throw err;
}
if (finalizedWithoutRun) {
await pairedExecutionLifecycle.asyncFinalize();
return 'success';
}
}
if (
isClaudeCodeAgent &&
(await pauseWhileClaudeUsageLimited({
chatJid,
runId,
activeRole,
pairedExecutionLifecycle,
onOutput,
log,
}))
) {
return 'success';
}
const runAttempt = (currentProvider: 'claude' | 'codex') =>
runMessageAgentAttempt({
provider: currentProvider,
currentSessionId,
isClaudeCodeAgent,
canRetryClaudeCredentials,
shouldPersistSession,
effectiveGroup,
agentInput,
activeRole,
effectiveServiceId,
effectiveAgentType,
sessionFolder,
roomRoleContext,
pairedExecutionContext,
fallbackWorkspaceDir: group.workDir ?? null,
onPersistSession: (sessionId) => {
deps.persistSession(sessionFolder, sessionId);
if (sessionTaskId) {
bindPairedSessionToTask(sessionFolder, sessionTaskId);
}
currentSessionId = sessionId;
},
registerProcess: (proc, processName, ipcDir) =>
deps.queue.registerProcess(chatJid, proc, processName, ipcDir),
onOutput,
pairedExecutionLifecycle,
log,
});
try {
const result = await executeMessageAgentAttemptLifecycle({
provider,
runAttempt,
isClaudeCodeAgent,
canRetryClaudeCredentials,
clearStoredSession: () => {
deps.clearSession(sessionFolder);
currentSessionId = undefined;
},
clearRoleSdkSessions,
sessionFolder,
reportClaudeUsageLimit,
hasDirectTerminalDelivery,
pairedExecutionLifecycle,
shouldRetryFreshSessionOnAgentFailure,
retryLogContext: {
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
},
log,
});
clearAppliedCompactRefreshAfterSuccess({
result,
sessionFolder,
compactRefresh,
});
return result;
} finally {
await pairedExecutionLifecycle.asyncFinalize();
}
}