diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index f2209a0..8c77d09 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -193,6 +193,7 @@ vi.mock('./claude-oauth-token.js', () => ({ vi.mock('./token-refresh.js', () => ({ forceRefreshToken: vi.fn(async () => null), isReloginRequired: vi.fn(() => false), + getClaudeAuthRecoveryIncident: vi.fn(() => null), RELOGIN_REQUIRED_MESSAGE: 'test: claude re-login required', })); @@ -216,6 +217,7 @@ import * as db from './db.js'; import { logger } from './logger.js'; import { buildRoomMemoryBriefing } from './sqlite-memory-store.js'; import { + CLAUDE_AUTH_RECOVERING_NOTICE, CLAUDE_SAFETY_REFUSAL_NOTICE, runAgentForGroup, } from './message-agent-executor.js'; @@ -2819,6 +2821,7 @@ describe('runAgentForGroup Claude failure recovery', () => { ); vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null); vi.mocked(tokenRefresh.isReloginRequired).mockReturnValue(false); + vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(null); }); it('force-refreshes the Claude token and retries once on auth-expired', async () => { @@ -3542,6 +3545,49 @@ describe('runAgentForGroup Claude failure recovery', () => { expect(outputs).toEqual(['test: claude re-login required']); }); + it('keeps the task and tells the room once while the token refresh fails transiently', async () => { + const revoked = + 'Failed to authenticate. API Error: 401 OAuth access token has been revoked.'; + vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(7); + const runOnce = async () => { + const outputs: string[] = []; + vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ status: 'success', result: revoked }); + return { status: 'success', result: revoked }; + }, + ); + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@recovering', + runId: `run-auth-recovering-${Math.random()}`, + onOutput: async (output) => { + const text = + output.output && 'text' in output.output + ? output.output.text + : output.result; + if (typeof text === 'string') outputs.push(text); + }, + }); + return { result, outputs }; + }; + + const first = await runOnce(); + expect(first.result).toBe('success'); + expect(first.outputs).toEqual([CLAUDE_AUTH_RECOVERING_NOTICE]); + expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.objectContaining({ completion_reason: 'claude_relogin_required' }), + ); + + // Same incident: no second notice, the normal retry path takes over. + const second = await runOnce(); + expect(second.result).toBe('error'); + expect(second.outputs).toEqual([]); + }); + it('returns error after repeated retryable Claude session failures', async () => { vi.mocked( sessionRecovery.shouldRetryFreshSessionOnAgentFailure, diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index d4bd93a..6e188d1 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -36,6 +36,7 @@ import { import { completePairedTaskWithReason } from './paired-task-status.js'; import { forceRefreshToken, + getClaudeAuthRecoveryIncident, isReloginRequired, RELOGIN_REQUIRED_MESSAGE, } from './token-refresh.js'; @@ -62,6 +63,14 @@ export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal'; export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON = 'claude_relogin_required'; +export const CLAUDE_AUTH_RECOVERING_NOTICE = + '⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이번 작업을 멈췄습니다. ' + + '봇이 5분마다 자동으로 다시 시도하고 있습니다. 잠시 후 이 방에 "계속"이라고 보내 주시면 이어서 진행합니다. ' + + '30분 넘게 복구되지 않으면 재로그인 안내를 드리겠습니다.'; + +// chatJid -> auth recovery incident already announced in that room. +const announcedAuthRecovery = new Map(); + export const CLAUDE_SAFETY_REFUSAL_NOTICE = 'Claude 안전 필터가 이 요청을 차단해서 처리하지 못했습니다. 이 방의 Claude 세션은 초기화했습니다. 요청을 조금 바꿔서 다시 보내 주세요. 에러 원문은 붙이지 말아 주세요.'; @@ -286,6 +295,7 @@ async function reportClaudeUsageLimitToRoom( * owner/arbiter retries fail silently until someone asks why the bot stopped. */ async function reportClaudeReloginRequired(args: { + chatJid: string; activeRole: PairedRoomRole; pairedTaskId?: string; errorText?: string | null; @@ -297,7 +307,9 @@ async function reportClaudeReloginRequired(args: { log: { warn: (obj: Record, msg: string) => void }; }): Promise { if (!isReloginRequired()) await forceRefreshToken(); - if (!isReloginRequired()) return false; + if (!isReloginRequired()) { + return reportClaudeAuthRecovering(args); + } const task = args.pairedTaskId ? getPairedTaskById(args.pairedTaskId) : undefined; @@ -320,6 +332,40 @@ async function reportClaudeReloginRequired(args: { return true; } +/** + * The token was rejected but refreshing failed only transiently (429/5xx/ + * timeout): the login may still be fine, so keep the task and let the + * automatic retries run. Tell the room once per incident why the work stopped; + * later failures in the same incident take the normal retry path. + */ +async function reportClaudeAuthRecovering(args: { + chatJid: string; + activeRole: PairedRoomRole; + errorText?: string | null; + pairedExecutionLifecycle: Pick< + PairedExecutionLifecycle, + 'updateSummary' | 'completeImmediately' + >; + onOutput?: (output: AgentOutput) => Promise; + log: { warn: (obj: Record, msg: string) => void }; +}): Promise { + const incident = getClaudeAuthRecoveryIncident(); + if (incident === null) return false; + if (announcedAuthRecovery.get(args.chatJid) === incident) return false; + announcedAuthRecovery.set(args.chatJid, incident); + args.log.warn( + { role: args.activeRole, incident, errorText: args.errorText }, + 'Claude token refresh failing transiently; told the room and kept the task', + ); + await deliverNoticeAndFailExecution({ + notice: CLAUDE_AUTH_RECOVERING_NOTICE, + summary: CLAUDE_AUTH_RECOVERING_NOTICE, + pairedExecutionLifecycle: args.pairedExecutionLifecycle, + onOutput: args.onOutput, + }); + return true; +} + /** * Owner finalize without an agent run (see paired-finalize.ts): close the task * and deliver the already-reviewed answer as the final message. Returns false diff --git a/src/token-refresh-force.test.ts b/src/token-refresh-force.test.ts index a15fb16..a42024f 100644 --- a/src/token-refresh-force.test.ts +++ b/src/token-refresh-force.test.ts @@ -26,9 +26,11 @@ vi.mock('./claude-oauth-token.js', () => ({ import { forceRefreshToken, + getClaudeAuthRecoveryIncident, isReloginRequired, probeAccessToken, resetAuthFailureStateForTests, + setRefreshRetryDelaysForTests, } from './token-refresh.js'; const credsPath = path.join(tmp, 'claude', '.credentials.json'); @@ -48,9 +50,20 @@ function writeCreds(accessToken: string, refreshToken = 'rt-old'): void { ); } +type TokenReply = number | { status: number; body: string }; + +const refreshed = (accessToken: string): TokenReply => ({ + status: 200, + body: JSON.stringify({ + access_token: accessToken, + refresh_token: 'rt-rotated', + expires_in: 8 * 60 * 60, + }), +}); + function mockFetch(handlers: { profile: (token: string) => number; - token?: () => number; + token?: () => TokenReply; }) { const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (url.endsWith('/api/oauth/profile')) { @@ -58,8 +71,11 @@ function mockFetch(handlers: { const status = handlers.profile(auth.replace('Bearer ', '')); return new Response('{}', { status }); } - const status = handlers.token?.() ?? 400; - return new Response('{"error":"invalid_grant"}', { status }); + const reply = handlers.token?.() ?? 400; + if (typeof reply === 'number') { + return new Response('{"error":"invalid_grant"}', { status: reply }); + } + return new Response(reply.body, { status: reply.status }); }); vi.stubGlobal('fetch', fetchMock); return fetchMock; @@ -72,6 +88,7 @@ const tokenCalls = (fetchMock: ReturnType) => beforeEach(() => { process.env.CLAUDE_CREDENTIALS_PATH = credsPath; resetAuthFailureStateForTests(); + setRefreshRetryDelaysForTests([0, 0]); }); afterEach(() => { @@ -132,6 +149,74 @@ describe('forceRefreshToken after a 401', () => { }); }); +describe('forceRefreshToken: transient vs definitive refresh failures', () => { + const unavailable: TokenReply = { + status: 503, + body: '{"error":"overloaded"}', + }; + + it('retries a transient failure right away and recovers (401 -> 503 -> 200)', async () => { + writeCreds('revoked'); + const replies = [unavailable, refreshed('fresh')]; + const fetchMock = mockFetch({ + profile: (t) => (t === 'revoked' ? 401 : 200), + token: () => replies.shift() ?? 500, + }); + + expect(await forceRefreshToken()).toBe('fresh'); + expect(tokenCalls(fetchMock)).toBe(2); + expect(isReloginRequired()).toBe(false); + expect(getClaudeAuthRecoveryIncident()).toBeNull(); + }); + + it('keeps retrying instead of demanding re-login while the endpoint is down (401 -> 503 ... -> next refresh succeeds)', async () => { + writeCreds('revoked'); + let down = true; + const fetchMock = mockFetch({ + profile: (t) => (t === 'revoked' ? 401 : 200), + token: () => (down ? unavailable : refreshed('fresh')), + }); + + expect(await forceRefreshToken()).toBeNull(); + expect(tokenCalls(fetchMock)).toBe(3); // first try + 2 retries + expect(isReloginRequired()).toBe(false); + expect(getClaudeAuthRecoveryIncident()).not.toBeNull(); + + down = false; + expect(await forceRefreshToken()).toBe('fresh'); + expect(isReloginRequired()).toBe(false); + expect(getClaudeAuthRecoveryIncident()).toBeNull(); + }); + + it('demands re-login at once on invalid_grant, without retrying (401 -> invalid_grant)', async () => { + writeCreds('revoked'); + const fetchMock = mockFetch({ + profile: () => 401, + token: () => ({ status: 400, body: '{"error":"invalid_grant"}' }), + }); + + expect(await forceRefreshToken()).toBeNull(); + expect(tokenCalls(fetchMock)).toBe(1); + expect(isReloginRequired()).toBe(true); + expect(getClaudeAuthRecoveryIncident()).toBeNull(); + }); + + it('turns a transient failure that lasts 30 minutes into re-login', async () => { + writeCreds('revoked'); + mockFetch({ profile: () => 401, token: () => unavailable }); + const start = Date.now(); + const now = vi.spyOn(Date, 'now').mockReturnValue(start); + + expect(await forceRefreshToken()).toBeNull(); + expect(isReloginRequired()).toBe(false); + + now.mockReturnValue(start + 31 * 60 * 1000); + expect(await forceRefreshToken()).toBeNull(); + expect(isReloginRequired()).toBe(true); + now.mockRestore(); + }); +}); + afterEach(() => { fs.rmSync(path.join(tmp, 'claude'), { recursive: true, force: true }); }); diff --git a/src/token-refresh.ts b/src/token-refresh.ts index 05fca5e..a138ccf 100644 --- a/src/token-refresh.ts +++ b/src/token-refresh.ts @@ -79,6 +79,23 @@ interface AuthFailureState { // appears again (e.g. after a manual re-login rewrites the creds file). let authFailure: AuthFailureState | null = null; +// Access token the API rejected (401) while refreshing it has only failed +// transiently so far (429/5xx/timeout). The periodic loop keeps retrying it +// every CHECK_INTERVAL_MS; after GIVE_UP_AFTER_MS it becomes re-login-required. +let rejectedAccessToken: string | null = null; +// Bumped for each newly rejected token, so rooms are told once per incident. +let authRecoveryIncident = 0; + +/** + * The current token was rejected and refreshing failed for a transient reason; + * automatic retries are still running. Returns the incident number, or null. + */ +export function getClaudeAuthRecoveryIncident(): number | null { + return rejectedAccessToken !== null && !isReloginRequired() + ? authRecoveryIncident + : null; +} + /** * True when automatic refresh has been failing for longer than * GIVE_UP_AFTER_MS and we have stopped retrying. Request handlers use this to @@ -90,6 +107,9 @@ export function isReloginRequired(): boolean { /** Clear failure tracking once a valid token is observed again. */ function recordAuthSuccess(accessToken?: string): void { + if (accessToken && accessToken !== rejectedAccessToken) { + rejectedAccessToken = null; + } if ( authFailure?.deadAccessToken && (!accessToken || accessToken === authFailure.deadAccessToken) @@ -146,6 +166,41 @@ function markReloginRequired(deadAccessToken: string): void { /** Test hook: reset the module-level failure state. */ export function resetAuthFailureStateForTests(): void { authFailure = null; + rejectedAccessToken = null; + refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS; +} + +/** Test hook: retry delays for a transient refresh failure. */ +export function setRefreshRetryDelaysForTests(delaysMs: number[]): void { + refreshRetryDelaysMs = delaysMs; +} + +// A forced refresh of a rejected token retries a transient failure a few +// times right away before falling back to the periodic loop. +const DEFAULT_REFRESH_RETRY_DELAYS_MS = [2_000, 5_000]; +let refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS; + +/** + * Why the token endpoint refused. `definitive` means the refresh token itself + * was refused (400/401, invalid_grant): only a re-login can help. Anything + * else (429, 5xx, network, timeout) is transient and worth retrying. + */ +export class TokenRefreshError extends Error { + constructor( + readonly definitive: boolean, + readonly status?: number, + ) { + super( + `Token refresh failed${status ? ` (${status})` : ''}${definitive ? ': refresh token refused' : ''}`, + ); + } +} + +export function isDefinitiveRefreshFailure( + status: number, + body: string, +): boolean { + return status === 400 || status === 401 || /invalid_grant/.test(body); } export type AccessTokenProbe = 'valid' | 'rejected' | 'unknown'; @@ -571,14 +626,44 @@ async function refreshToken( { url: TOKEN_URL, status: res.status, body: errText.slice(0, 200) }, 'Token refresh failed at endpoint', ); + throw new TokenRefreshError( + isDefinitiveRefreshFailure(res.status, errText), + res.status, + ); } catch (err) { + if (err instanceof TokenRefreshError) throw err; logger.warn( { url: TOKEN_URL, err: getErrorMessage(err) }, 'Token refresh request error', ); + throw new TokenRefreshError(false); } +} - throw new Error('Token refresh failed'); +/** refreshToken, retrying transient failures after each of `delaysMs`. */ +async function refreshTokenWithRetry( + refreshTokenStr: string, + scopes: string[], + delaysMs: number[], +): Promise { + for (let attempt = 0; ; attempt += 1) { + try { + return await refreshToken(refreshTokenStr, scopes); + } catch (err) { + const delay = delaysMs[attempt]; + if ( + delay === undefined || + (err instanceof TokenRefreshError && err.definitive) + ) { + throw err; + } + logger.warn( + { attempt: attempt + 1, retryInMs: delay }, + 'Claude OAuth refresh failed transiently — retrying', + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } } // Serialize all refreshes within this process so the 5-minute loop and an @@ -638,11 +723,16 @@ async function doCheckAndRefreshCredentials(opts?: { // A forced check comes from a 401. Expiry cannot reveal a revoked token, so // ask the API; a rejected token must be refreshed even though it "looks" // valid, otherwise every retry reuses the same dead token. + const knownRejected = + rejectedAccessToken !== null && rejectedAccessToken === oauth.accessToken; const rejected = - opts?.force === true && - !isReloginRequired() && - (await probeAccessToken(oauth.accessToken)) === 'rejected'; - if (rejected) { + knownRejected || + (opts?.force === true && + !isReloginRequired() && + (await probeAccessToken(oauth.accessToken)) === 'rejected'); + if (rejected && !knownRejected) { + rejectedAccessToken = oauth.accessToken; + authRecoveryIncident += 1; logger.warn( { remainingMin: Math.round(remaining / 60000) }, 'Claude OAuth token was rejected by the API before its expiry', @@ -692,7 +782,11 @@ async function doCheckAndRefreshCredentials(opts?: { ); try { - const response = await refreshToken(rt, oauth.scopes || DEFAULT_SCOPES); + const response = await refreshTokenWithRetry( + rt, + oauth.scopes || DEFAULT_SCOPES, + rejected && opts?.force ? refreshRetryDelaysMs : [], + ); recordAuthSuccess(response.access_token); oauth.accessToken = response.access_token; @@ -738,11 +832,25 @@ async function doCheckAndRefreshCredentials(opts?: { ); return recoveredOauth.accessToken; } - if (rejected) { + const definitive = err instanceof TokenRefreshError && err.definitive; + if (rejected && definitive) { markReloginRequired(oauth.accessToken); return null; } const gaveUp = recordAuthFailure(); + if (rejected) { + // Transient (429/5xx/timeout): keep retrying from the periodic loop; + // only a failure that lasts GIVE_UP_AFTER_MS means the login is gone. + if (gaveUp) { + markReloginRequired(oauth.accessToken); + } else { + logger.warn( + { err: getErrorMessage(err) }, + 'Rejected Claude OAuth token could not be refreshed yet (transient) — retrying every check', + ); + } + return null; + } logger.error( { gaveUp,