fix: park rooms during a transient Claude auth outage instead of failing

ff76d04 told the room about a transient refresh failure but still failed
the execution (completeImmediately 'failed'), which bumps
owner_failure_count and can hand the task to the arbiter after two blips.

- reportClaudeAuthRecovering now parks the turn like the usage-limit
  pause: releaseForClaudeUsagePause (lease released, no failure count, no
  follow-up) plus markRunPausedForClaudeUsage (messages stay unconsumed).
  The room is told once per incident; later blips park silently.
- token-refresh fires setOnClaudeAuthRecovered once a rejected token is
  replaced (deferred so the new token is in memory first); index.ts
  re-queues every parked room so the same turn runs without user input.
- Real-DB test: two parked owner runs leave owner_failure_count/status/
  arbiter_requested_at unchanged (control: a failed run does count).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-11-02 11:02:48 +09:00
parent ff76d041d3
commit 1bb23a60db
8 changed files with 304 additions and 37 deletions

View File

@@ -0,0 +1,117 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
parkRoomForClaudeAuthRecovery,
resumeRoomsAfterClaudeAuthRecovery,
takeRoomsParkedForClaudeAuthRecovery,
} from './claude-auth-recovery.js';
import {
_initTestDatabase,
createPairedTask,
getPairedTaskById,
} from './db.js';
import { createPairedExecutionLifecycle } from './message-agent-executor-paired.js';
import type { PairedTask } from './types.js';
function makeTask(): PairedTask {
return {
id: 'task-auth-recovery',
chat_jid: 'dc:auth-room',
group_folder: 'auth-room',
owner_service_id: 'claude',
reviewer_service_id: 'codex-review',
owner_agent_type: 'claude-code',
reviewer_agent_type: 'codex',
arbiter_agent_type: null,
title: null,
source_ref: 'HEAD',
plan_notes: null,
review_requested_at: null,
round_trip_count: 0,
owner_failure_count: 0,
owner_step_done_streak: 0,
finalize_step_done_count: 0,
task_done_then_user_reopen_count: 0,
empty_step_done_streak: 0,
status: 'active',
arbiter_verdict: null,
arbiter_requested_at: null,
completion_reason: null,
created_at: '2026-11-02T00:00:00.000Z',
updated_at: '2026-11-02T00:00:00.000Z',
};
}
const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as never;
beforeEach(() => {
_initTestDatabase();
takeRoomsParkedForClaudeAuthRecovery();
});
describe('parking a paired owner turn during a transient Claude auth outage', () => {
it('keeps the stored task untouched across two parked owner runs', async () => {
const task = makeTask();
createPairedTask(task);
for (const runId of ['run-auth-1', 'run-auth-2']) {
const enqueueMessageCheck = vi.fn();
const lifecycle = createPairedExecutionLifecycle({
pairedExecutionContext: { task, workspace: null, envOverrides: {} },
completedRole: 'owner',
chatJid: task.chat_jid,
runId,
enqueueMessageCheck,
log,
});
lifecycle.releaseForClaudeUsagePause('auth recovery test');
await lifecycle.asyncFinalize();
parkRoomForClaudeAuthRecovery(task.chat_jid);
// No paired follow-up is queued: the room waits for the auth recovery.
expect(enqueueMessageCheck).not.toHaveBeenCalled();
}
const stored = getPairedTaskById(task.id);
expect(stored).toMatchObject({
status: 'active',
owner_failure_count: 0,
arbiter_requested_at: null,
completion_reason: null,
updated_at: task.updated_at,
});
});
it('(control) a failed owner execution does count a failure in the same setup', async () => {
const task = makeTask();
createPairedTask(task);
const lifecycle = createPairedExecutionLifecycle({
pairedExecutionContext: { task, workspace: null, envOverrides: {} },
completedRole: 'owner',
chatJid: task.chat_jid,
runId: 'run-auth-control',
enqueueMessageCheck: vi.fn(),
log,
});
lifecycle.completeImmediately({ status: 'failed' });
expect(getPairedTaskById(task.id)?.owner_failure_count).toBe(1);
});
it('re-queues every parked room once the token recovers, without user input', () => {
parkRoomForClaudeAuthRecovery('dc:auth-room');
parkRoomForClaudeAuthRecovery('dc:unbound');
const enqueueMessageCheck = vi.fn();
const resumed = resumeRoomsAfterClaudeAuthRecovery({
resolveIpcDir: (jid) =>
jid === 'dc:auth-room' ? '/ipc/auth-room' : null,
enqueueMessageCheck,
});
expect(resumed).toEqual(['dc:auth-room']);
expect(enqueueMessageCheck).toHaveBeenCalledWith(
'dc:auth-room',
'/ipc/auth-room',
);
expect(takeRoomsParkedForClaudeAuthRecovery()).toEqual([]);
});
});

View File

@@ -0,0 +1,49 @@
/**
* Rooms whose Claude work is parked while a rejected OAuth token is being
* refreshed through a transient outage (see token-refresh.ts). Parking works
* like the usage-limit pause: the paired task, its failure count and the
* unconsumed messages stay as they were. Once the token recovers, every
* parked room is re-queued so the same turn runs again without user input.
*/
import { logger } from './logger.js';
const parkedRooms = new Set<string>();
export function parkRoomForClaudeAuthRecovery(chatJid: string): void {
parkedRooms.add(chatJid);
}
/** Remove and return every parked room. */
export function takeRoomsParkedForClaudeAuthRecovery(): string[] {
const rooms = [...parkedRooms];
parkedRooms.clear();
return rooms;
}
/** Re-queue parked rooms; rooms without a binding are skipped. */
export function resumeRoomsAfterClaudeAuthRecovery(deps: {
resolveIpcDir: (chatJid: string) => string | null;
enqueueMessageCheck: (chatJid: string, ipcDir: string) => void;
}): string[] {
const resumed: string[] = [];
for (const chatJid of takeRoomsParkedForClaudeAuthRecovery()) {
const ipcDir = deps.resolveIpcDir(chatJid);
if (!ipcDir) {
logger.warn(
{ chatJid },
'Skipped resuming a room parked for Claude auth recovery without a room binding',
);
continue;
}
deps.enqueueMessageCheck(chatJid, ipcDir);
resumed.push(chatJid);
}
if (resumed.length > 0) {
logger.info(
{ rooms: resumed },
'Claude OAuth recovered; resumed parked rooms',
);
}
return resumed;
}

View File

@@ -26,6 +26,7 @@ import {
} from './db.js';
import { GroupQueue } from './group-queue.js';
import { resolveGroupIpcPath } from './group-folder.js';
import { resumeRoomsAfterClaudeAuthRecovery } from './claude-auth-recovery.js';
import { startIpcWatcher } from './ipc.js';
import {
deliverCanonicalOutboundMessage,
@@ -65,6 +66,7 @@ export function isTerminalStatusMessage(text: string): boolean {
return parseVisibleVerdict(text) !== 'continue';
}
import {
setOnClaudeAuthRecovered,
startTokenRefreshLoop,
stopTokenRefreshLoop,
} from './token-refresh.js';
@@ -250,6 +252,17 @@ async function main(): Promise<void> {
const processStartedAtMs = Date.now();
initDatabase();
logger.info('Database initialized');
setOnClaudeAuthRecovered(() => {
const bindings = runtimeState.getRoomBindings();
resumeRoomsAfterClaudeAuthRecovery({
resolveIpcDir: (chatJid) => {
const binding = bindings[chatJid];
return binding ? resolveGroupIpcPath(binding.folder) : null;
},
enqueueMessageCheck: (chatJid, ipcDir) =>
queue.enqueueMessageCheck(chatJid, ipcDir),
});
});
startTokenRefreshLoop({
hasActiveAgents: () => queue.getActiveRunCount() > 0,
});

View File

@@ -183,7 +183,7 @@ export interface PairedExecutionLifecycle {
* End the turn without touching the paired task (no failure count, no
* status change, no follow-up) so the same turn can run again later.
*/
releaseForClaudeUsagePause(): void;
releaseForClaudeUsagePause(reason?: string): void;
markStatus(status: 'succeeded' | 'failed'): void;
markSawOutput(sawOutput: boolean): void;
getSummary(): string | null;
@@ -337,7 +337,9 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
this.pairedExecutionCompleted = true;
}
releaseForClaudeUsagePause(): void {
releaseForClaudeUsagePause(
reason = 'Claude hit its usage limit and the room is paused',
): void {
const { pairedExecutionContext, runId, log } = this.args;
this.pausedForClaudeUsage = true;
this.pairedExecutionStatus = 'failed';
@@ -349,7 +351,7 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
pairedExecutionContext.task.id,
runId,
log,
'Claude hit its usage limit and the room is paused',
reason,
);
this.pairedExecutionCompleted = true;
}

View File

@@ -228,6 +228,7 @@ import { buildPairedTurnIdentity } from './paired-turn-identity.js';
import * as sessionRecovery from './session-recovery.js';
import * as serviceRouting from './service-routing.js';
import * as tokenRefresh from './token-refresh.js';
import { takeRoomsParkedForClaudeAuthRecovery } from './claude-auth-recovery.js';
import * as claudeOauthToken from './claude-oauth-token.js';
import { consumeRunPausedForClaudeUsage } from './claude-usage-pause.js';
import * as usageAlignmentHold from './usage-alignment-hold.js';
@@ -3545,11 +3546,35 @@ describe('runAgentForGroup Claude failure recovery', () => {
expect(outputs).toEqual(['test: claude re-login required']);
});
it('keeps the task and tells the room once while the token refresh fails transiently', async () => {
it('parks a paired owner turn without counting a failure while the token refresh fails transiently', async () => {
const revoked =
'Failed to authenticate. API Error: 401 OAuth access token has been revoked.';
const pairedTask = {
id: 'paired-task-auth-recovering',
chat_jid: 'group@recovering',
group_folder: 'test-claude',
owner_service_id: 'claude',
reviewer_service_id: 'codex-review',
title: null,
source_ref: 'HEAD',
plan_notes: null,
round_trip_count: 0,
review_requested_at: null,
status: 'active' as const,
arbiter_verdict: null,
arbiter_requested_at: null,
completion_reason: null,
created_at: '2026-04-06T00:00:00.000Z',
updated_at: '2026-04-06T00:00:00.000Z',
};
vi.mocked(
pairedExecutionContext.preparePairedExecutionContext,
).mockReturnValue({ task: pairedTask, workspace: null, envOverrides: {} });
vi.mocked(db.getPairedTaskById).mockReturnValue(pairedTask);
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(7);
const runOnce = async () => {
takeRoomsParkedForClaudeAuthRecovery();
const runOnce = async (runId: string) => {
const outputs: string[] = [];
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
@@ -3561,7 +3586,7 @@ describe('runAgentForGroup Claude failure recovery', () => {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@recovering',
runId: `run-auth-recovering-${Math.random()}`,
runId,
onOutput: async (output) => {
const text =
output.output && 'text' in output.output
@@ -3573,19 +3598,30 @@ describe('runAgentForGroup Claude failure recovery', () => {
return { result, outputs };
};
const first = await runOnce();
expect(first.result).toBe('success');
expect(first.outputs).toEqual([CLAUDE_AUTH_RECOVERING_NOTICE]);
expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
expect.objectContaining({ completion_reason: 'claude_relogin_required' }),
);
const first = await runOnce('run-auth-recovering-1');
const second = await runOnce('run-auth-recovering-2');
// Same incident: no second notice, the normal retry path takes over.
const second = await runOnce();
expect(second.result).toBe('error');
expect(second.outputs).toEqual([]);
expect(first).toEqual({
result: 'success',
outputs: [CLAUDE_AUTH_RECOVERING_NOTICE],
});
// Same incident: parked again, but the room is not told twice.
expect(second).toEqual({ result: 'success', outputs: [] });
// No failure count, no status change, no arbiter hand-off.
expect(
pairedExecutionContext.completePairedExecutionContext,
).not.toHaveBeenCalled();
expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalled();
expect(db.releasePairedTaskExecutionLease).toHaveBeenCalledWith({
taskId: 'paired-task-auth-recovering',
runId: 'run-auth-recovering-2',
});
// Messages stay unconsumed and the room waits for the auth recovery.
expect(consumeRunPausedForClaudeUsage('run-auth-recovering-1')).toBe(true);
expect(consumeRunPausedForClaudeUsage('run-auth-recovering-2')).toBe(true);
expect(takeRoomsParkedForClaudeAuthRecovery()).toEqual([
'group@recovering',
]);
});
it('returns error after repeated retryable Claude session failures', async () => {

View File

@@ -34,6 +34,7 @@ import {
isUsageAlignmentBypassed,
} from './usage-alignment-hold.js';
import { completePairedTaskWithReason } from './paired-task-status.js';
import { parkRoomForClaudeAuthRecovery } from './claude-auth-recovery.js';
import {
forceRefreshToken,
getClaudeAuthRecoveryIncident,
@@ -64,8 +65,8 @@ export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON =
'claude_relogin_required';
export const CLAUDE_AUTH_RECOVERING_NOTICE =
'⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이번 작업을 멈췄습니다. ' +
'봇이 5분마다 자동으로 다시 시도하고 있습니다. 잠시 후 이 방에 "계속"이라고 보내 주시면 이어서 진행합니다. ' +
'⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이 방의 작업을 일시정지했습니다. ' +
'봇이 5분마다 자동으로 다시 시도하고, 복구되면 멈춘 작업을 자동으로 이어서 진행합니다. ' +
'30분 넘게 복구되지 않으면 재로그인 안내를 드리겠습니다.';
// chatJid -> auth recovery incident already announced in that room.
@@ -296,12 +297,13 @@ async function reportClaudeUsageLimitToRoom(
*/
async function reportClaudeReloginRequired(args: {
chatJid: string;
runId: string;
activeRole: PairedRoomRole;
pairedTaskId?: string;
errorText?: string | null;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'completeImmediately'
'updateSummary' | 'completeImmediately' | 'releaseForClaudeUsagePause'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
@@ -334,35 +336,48 @@ async function reportClaudeReloginRequired(args: {
/**
* The token was rejected but refreshing failed only transiently (429/5xx/
* timeout): the login may still be fine, so keep the task and let the
* automatic retries run. Tell the room once per incident why the work stopped;
* later failures in the same incident take the normal retry path.
* timeout): the login may still be fine. Park the room like the usage-limit
* pause: release the turn without counting a failure (no owner_failure_count,
* no arbiter hand-off), keep the messages unconsumed, and resume the room
* once the token recovers (claude-auth-recovery.ts). The room is told once
* per incident.
*/
async function reportClaudeAuthRecovering(args: {
chatJid: string;
runId: string;
activeRole: PairedRoomRole;
errorText?: string | null;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'completeImmediately'
'updateSummary' | 'releaseForClaudeUsagePause'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
const incident = getClaudeAuthRecoveryIncident();
if (incident === null) return false;
if (announcedAuthRecovery.get(args.chatJid) === incident) return false;
announcedAuthRecovery.set(args.chatJid, incident);
args.log.warn(
{ role: args.activeRole, incident, errorText: args.errorText },
'Claude token refresh failing transiently; told the room and kept the task',
);
await deliverNoticeAndFailExecution({
notice: CLAUDE_AUTH_RECOVERING_NOTICE,
summary: CLAUDE_AUTH_RECOVERING_NOTICE,
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
parkRoomForClaudeAuthRecovery(args.chatJid);
markRunPausedForClaudeUsage(args.runId);
args.pairedExecutionLifecycle.updateSummary({
outputText: CLAUDE_AUTH_RECOVERING_NOTICE,
});
args.pairedExecutionLifecycle.releaseForClaudeUsagePause(
'Claude token refresh is failing transiently; room parked until auth recovers',
);
const firstNotice = announcedAuthRecovery.get(args.chatJid) !== incident;
args.log.warn(
{ role: args.activeRole, incident, firstNotice, errorText: args.errorText },
'Claude token refresh failing transiently; parked the room without counting a failure',
);
if (firstNotice) {
announcedAuthRecovery.set(args.chatJid, incident);
await args.onOutput?.({
status: 'success',
result: null,
output: { visibility: 'public', text: CLAUDE_AUTH_RECOVERING_NOTICE },
phase: 'final',
});
}
return true;
}

View File

@@ -30,6 +30,7 @@ import {
isReloginRequired,
probeAccessToken,
resetAuthFailureStateForTests,
setOnClaudeAuthRecovered,
setRefreshRetryDelaysForTests,
} from './token-refresh.js';
@@ -182,10 +183,16 @@ describe('forceRefreshToken: transient vs definitive refresh failures', () => {
expect(isReloginRequired()).toBe(false);
expect(getClaudeAuthRecoveryIncident()).not.toBeNull();
const resumed = vi.fn();
setOnClaudeAuthRecovered(resumed);
down = false;
expect(await forceRefreshToken()).toBe('fresh');
expect(isReloginRequired()).toBe(false);
expect(getClaudeAuthRecoveryIncident()).toBeNull();
// Parked rooms are resumed once the new token is in place.
await new Promise((resolve) => setTimeout(resolve, 1_100));
expect(resumed).toHaveBeenCalledTimes(1);
setOnClaudeAuthRecovered(null);
});
it('demands re-login at once on invalid_grant, without retrying (401 -> invalid_grant)', async () => {

View File

@@ -85,6 +85,14 @@ let authFailure: AuthFailureState | null = null;
let rejectedAccessToken: string | null = null;
// Bumped for each newly rejected token, so rooms are told once per incident.
let authRecoveryIncident = 0;
// Called once a rejected token has been replaced by a working one.
let onAuthRecovered: (() => void) | null = null;
const AUTH_RECOVERED_RESUME_DELAY_MS = 1_000;
/** Register the hook that resumes rooms parked during an auth outage. */
export function setOnClaudeAuthRecovered(fn: (() => void) | null): void {
onAuthRecovered = fn;
}
/**
* The current token was rejected and refreshing failed for a transient reason;
@@ -107,8 +115,28 @@ export function isReloginRequired(): boolean {
/** Clear failure tracking once a valid token is observed again. */
function recordAuthSuccess(accessToken?: string): void {
if (accessToken && accessToken !== rejectedAccessToken) {
if (
accessToken &&
rejectedAccessToken !== null &&
accessToken !== rejectedAccessToken
) {
rejectedAccessToken = null;
logger.info('Claude OAuth token replaced after a rejection — resuming');
// Deferred: the caller still has to write the new token to disk and into
// memory, and a resumed run must not start with the rejected one.
const hook = onAuthRecovered;
if (hook) {
setTimeout(() => {
try {
hook();
} catch (err) {
logger.warn(
{ err: getErrorMessage(err) },
'Resuming rooms after Claude auth recovery failed',
);
}
}, AUTH_RECOVERED_RESUME_DELAY_MS);
}
}
if (
authFailure?.deadAccessToken &&