fix: tell transient Claude refresh failures apart from a lost login

A 401-rejected token whose refresh failed with 429/5xx/timeout was marked
re-login-required at once, closing tasks and asking for a needless re-login.

- refreshToken throws TokenRefreshError; only 400/401/invalid_grant is
  definitive. A forced refresh of a rejected token retries transient
  failures twice right away; the periodic loop then keeps retrying the
  known-rejected token every check, and only a failure lasting 30 min turns
  into re-login-required.
- While recovering, the room is told once per incident and the paired task
  is kept, so the normal retry path can resume it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-11-02 10:55:31 +09:00
parent 3f2e364c73
commit ff76d041d3
4 changed files with 296 additions and 11 deletions

View File

@@ -193,6 +193,7 @@ vi.mock('./claude-oauth-token.js', () => ({
vi.mock('./token-refresh.js', () => ({
forceRefreshToken: vi.fn(async () => null),
isReloginRequired: vi.fn(() => false),
getClaudeAuthRecoveryIncident: vi.fn(() => null),
RELOGIN_REQUIRED_MESSAGE: 'test: claude re-login required',
}));
@@ -216,6 +217,7 @@ import * as db from './db.js';
import { logger } from './logger.js';
import { buildRoomMemoryBriefing } from './sqlite-memory-store.js';
import {
CLAUDE_AUTH_RECOVERING_NOTICE,
CLAUDE_SAFETY_REFUSAL_NOTICE,
runAgentForGroup,
} from './message-agent-executor.js';
@@ -2819,6 +2821,7 @@ describe('runAgentForGroup Claude failure recovery', () => {
);
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null);
vi.mocked(tokenRefresh.isReloginRequired).mockReturnValue(false);
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(null);
});
it('force-refreshes the Claude token and retries once on auth-expired', async () => {
@@ -3542,6 +3545,49 @@ describe('runAgentForGroup Claude failure recovery', () => {
expect(outputs).toEqual(['test: claude re-login required']);
});
it('keeps the task and tells the room once while the token refresh fails transiently', async () => {
const revoked =
'Failed to authenticate. API Error: 401 OAuth access token has been revoked.';
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(7);
const runOnce = async () => {
const outputs: string[] = [];
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({ status: 'success', result: revoked });
return { status: 'success', result: revoked };
},
);
const result = await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@recovering',
runId: `run-auth-recovering-${Math.random()}`,
onOutput: async (output) => {
const text =
output.output && 'text' in output.output
? output.output.text
: output.result;
if (typeof text === 'string') outputs.push(text);
},
});
return { result, outputs };
};
const first = await runOnce();
expect(first.result).toBe('success');
expect(first.outputs).toEqual([CLAUDE_AUTH_RECOVERING_NOTICE]);
expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
expect.objectContaining({ completion_reason: 'claude_relogin_required' }),
);
// Same incident: no second notice, the normal retry path takes over.
const second = await runOnce();
expect(second.result).toBe('error');
expect(second.outputs).toEqual([]);
});
it('returns error after repeated retryable Claude session failures', async () => {
vi.mocked(
sessionRecovery.shouldRetryFreshSessionOnAgentFailure,

View File

@@ -36,6 +36,7 @@ import {
import { completePairedTaskWithReason } from './paired-task-status.js';
import {
forceRefreshToken,
getClaudeAuthRecoveryIncident,
isReloginRequired,
RELOGIN_REQUIRED_MESSAGE,
} from './token-refresh.js';
@@ -62,6 +63,14 @@ export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal';
export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON =
'claude_relogin_required';
export const CLAUDE_AUTH_RECOVERING_NOTICE =
'⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이번 작업을 멈췄습니다. ' +
'봇이 5분마다 자동으로 다시 시도하고 있습니다. 잠시 후 이 방에 "계속"이라고 보내 주시면 이어서 진행합니다. ' +
'30분 넘게 복구되지 않으면 재로그인 안내를 드리겠습니다.';
// chatJid -> auth recovery incident already announced in that room.
const announcedAuthRecovery = new Map<string, number>();
export const CLAUDE_SAFETY_REFUSAL_NOTICE =
'Claude 안전 필터가 이 요청을 차단해서 처리하지 못했습니다. 이 방의 Claude 세션은 초기화했습니다. 요청을 조금 바꿔서 다시 보내 주세요. 에러 원문은 붙이지 말아 주세요.';
@@ -286,6 +295,7 @@ async function reportClaudeUsageLimitToRoom(
* owner/arbiter retries fail silently until someone asks why the bot stopped.
*/
async function reportClaudeReloginRequired(args: {
chatJid: string;
activeRole: PairedRoomRole;
pairedTaskId?: string;
errorText?: string | null;
@@ -297,7 +307,9 @@ async function reportClaudeReloginRequired(args: {
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
if (!isReloginRequired()) await forceRefreshToken();
if (!isReloginRequired()) return false;
if (!isReloginRequired()) {
return reportClaudeAuthRecovering(args);
}
const task = args.pairedTaskId
? getPairedTaskById(args.pairedTaskId)
: undefined;
@@ -320,6 +332,40 @@ async function reportClaudeReloginRequired(args: {
return true;
}
/**
* The token was rejected but refreshing failed only transiently (429/5xx/
* timeout): the login may still be fine, so keep the task and let the
* automatic retries run. Tell the room once per incident why the work stopped;
* later failures in the same incident take the normal retry path.
*/
async function reportClaudeAuthRecovering(args: {
chatJid: string;
activeRole: PairedRoomRole;
errorText?: string | null;
pairedExecutionLifecycle: Pick<
PairedExecutionLifecycle,
'updateSummary' | 'completeImmediately'
>;
onOutput?: (output: AgentOutput) => Promise<void>;
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
}): Promise<boolean> {
const incident = getClaudeAuthRecoveryIncident();
if (incident === null) return false;
if (announcedAuthRecovery.get(args.chatJid) === incident) return false;
announcedAuthRecovery.set(args.chatJid, incident);
args.log.warn(
{ role: args.activeRole, incident, errorText: args.errorText },
'Claude token refresh failing transiently; told the room and kept the task',
);
await deliverNoticeAndFailExecution({
notice: CLAUDE_AUTH_RECOVERING_NOTICE,
summary: CLAUDE_AUTH_RECOVERING_NOTICE,
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
onOutput: args.onOutput,
});
return true;
}
/**
* Owner finalize without an agent run (see paired-finalize.ts): close the task
* and deliver the already-reviewed answer as the final message. Returns false

View File

@@ -26,9 +26,11 @@ vi.mock('./claude-oauth-token.js', () => ({
import {
forceRefreshToken,
getClaudeAuthRecoveryIncident,
isReloginRequired,
probeAccessToken,
resetAuthFailureStateForTests,
setRefreshRetryDelaysForTests,
} from './token-refresh.js';
const credsPath = path.join(tmp, 'claude', '.credentials.json');
@@ -48,9 +50,20 @@ function writeCreds(accessToken: string, refreshToken = 'rt-old'): void {
);
}
type TokenReply = number | { status: number; body: string };
const refreshed = (accessToken: string): TokenReply => ({
status: 200,
body: JSON.stringify({
access_token: accessToken,
refresh_token: 'rt-rotated',
expires_in: 8 * 60 * 60,
}),
});
function mockFetch(handlers: {
profile: (token: string) => number;
token?: () => number;
token?: () => TokenReply;
}) {
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
if (url.endsWith('/api/oauth/profile')) {
@@ -58,8 +71,11 @@ function mockFetch(handlers: {
const status = handlers.profile(auth.replace('Bearer ', ''));
return new Response('{}', { status });
}
const status = handlers.token?.() ?? 400;
return new Response('{"error":"invalid_grant"}', { status });
const reply = handlers.token?.() ?? 400;
if (typeof reply === 'number') {
return new Response('{"error":"invalid_grant"}', { status: reply });
}
return new Response(reply.body, { status: reply.status });
});
vi.stubGlobal('fetch', fetchMock);
return fetchMock;
@@ -72,6 +88,7 @@ const tokenCalls = (fetchMock: ReturnType<typeof mockFetch>) =>
beforeEach(() => {
process.env.CLAUDE_CREDENTIALS_PATH = credsPath;
resetAuthFailureStateForTests();
setRefreshRetryDelaysForTests([0, 0]);
});
afterEach(() => {
@@ -132,6 +149,74 @@ describe('forceRefreshToken after a 401', () => {
});
});
describe('forceRefreshToken: transient vs definitive refresh failures', () => {
const unavailable: TokenReply = {
status: 503,
body: '{"error":"overloaded"}',
};
it('retries a transient failure right away and recovers (401 -> 503 -> 200)', async () => {
writeCreds('revoked');
const replies = [unavailable, refreshed('fresh')];
const fetchMock = mockFetch({
profile: (t) => (t === 'revoked' ? 401 : 200),
token: () => replies.shift() ?? 500,
});
expect(await forceRefreshToken()).toBe('fresh');
expect(tokenCalls(fetchMock)).toBe(2);
expect(isReloginRequired()).toBe(false);
expect(getClaudeAuthRecoveryIncident()).toBeNull();
});
it('keeps retrying instead of demanding re-login while the endpoint is down (401 -> 503 ... -> next refresh succeeds)', async () => {
writeCreds('revoked');
let down = true;
const fetchMock = mockFetch({
profile: (t) => (t === 'revoked' ? 401 : 200),
token: () => (down ? unavailable : refreshed('fresh')),
});
expect(await forceRefreshToken()).toBeNull();
expect(tokenCalls(fetchMock)).toBe(3); // first try + 2 retries
expect(isReloginRequired()).toBe(false);
expect(getClaudeAuthRecoveryIncident()).not.toBeNull();
down = false;
expect(await forceRefreshToken()).toBe('fresh');
expect(isReloginRequired()).toBe(false);
expect(getClaudeAuthRecoveryIncident()).toBeNull();
});
it('demands re-login at once on invalid_grant, without retrying (401 -> invalid_grant)', async () => {
writeCreds('revoked');
const fetchMock = mockFetch({
profile: () => 401,
token: () => ({ status: 400, body: '{"error":"invalid_grant"}' }),
});
expect(await forceRefreshToken()).toBeNull();
expect(tokenCalls(fetchMock)).toBe(1);
expect(isReloginRequired()).toBe(true);
expect(getClaudeAuthRecoveryIncident()).toBeNull();
});
it('turns a transient failure that lasts 30 minutes into re-login', async () => {
writeCreds('revoked');
mockFetch({ profile: () => 401, token: () => unavailable });
const start = Date.now();
const now = vi.spyOn(Date, 'now').mockReturnValue(start);
expect(await forceRefreshToken()).toBeNull();
expect(isReloginRequired()).toBe(false);
now.mockReturnValue(start + 31 * 60 * 1000);
expect(await forceRefreshToken()).toBeNull();
expect(isReloginRequired()).toBe(true);
now.mockRestore();
});
});
afterEach(() => {
fs.rmSync(path.join(tmp, 'claude'), { recursive: true, force: true });
});

View File

@@ -79,6 +79,23 @@ interface AuthFailureState {
// appears again (e.g. after a manual re-login rewrites the creds file).
let authFailure: AuthFailureState | null = null;
// Access token the API rejected (401) while refreshing it has only failed
// transiently so far (429/5xx/timeout). The periodic loop keeps retrying it
// every CHECK_INTERVAL_MS; after GIVE_UP_AFTER_MS it becomes re-login-required.
let rejectedAccessToken: string | null = null;
// Bumped for each newly rejected token, so rooms are told once per incident.
let authRecoveryIncident = 0;
/**
* The current token was rejected and refreshing failed for a transient reason;
* automatic retries are still running. Returns the incident number, or null.
*/
export function getClaudeAuthRecoveryIncident(): number | null {
return rejectedAccessToken !== null && !isReloginRequired()
? authRecoveryIncident
: null;
}
/**
* True when automatic refresh has been failing for longer than
* GIVE_UP_AFTER_MS and we have stopped retrying. Request handlers use this to
@@ -90,6 +107,9 @@ export function isReloginRequired(): boolean {
/** Clear failure tracking once a valid token is observed again. */
function recordAuthSuccess(accessToken?: string): void {
if (accessToken && accessToken !== rejectedAccessToken) {
rejectedAccessToken = null;
}
if (
authFailure?.deadAccessToken &&
(!accessToken || accessToken === authFailure.deadAccessToken)
@@ -146,6 +166,41 @@ function markReloginRequired(deadAccessToken: string): void {
/** Test hook: reset the module-level failure state. */
export function resetAuthFailureStateForTests(): void {
authFailure = null;
rejectedAccessToken = null;
refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS;
}
/** Test hook: retry delays for a transient refresh failure. */
export function setRefreshRetryDelaysForTests(delaysMs: number[]): void {
refreshRetryDelaysMs = delaysMs;
}
// A forced refresh of a rejected token retries a transient failure a few
// times right away before falling back to the periodic loop.
const DEFAULT_REFRESH_RETRY_DELAYS_MS = [2_000, 5_000];
let refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS;
/**
* Why the token endpoint refused. `definitive` means the refresh token itself
* was refused (400/401, invalid_grant): only a re-login can help. Anything
* else (429, 5xx, network, timeout) is transient and worth retrying.
*/
export class TokenRefreshError extends Error {
constructor(
readonly definitive: boolean,
readonly status?: number,
) {
super(
`Token refresh failed${status ? ` (${status})` : ''}${definitive ? ': refresh token refused' : ''}`,
);
}
}
export function isDefinitiveRefreshFailure(
status: number,
body: string,
): boolean {
return status === 400 || status === 401 || /invalid_grant/.test(body);
}
export type AccessTokenProbe = 'valid' | 'rejected' | 'unknown';
@@ -571,14 +626,44 @@ async function refreshToken(
{ url: TOKEN_URL, status: res.status, body: errText.slice(0, 200) },
'Token refresh failed at endpoint',
);
throw new TokenRefreshError(
isDefinitiveRefreshFailure(res.status, errText),
res.status,
);
} catch (err) {
if (err instanceof TokenRefreshError) throw err;
logger.warn(
{ url: TOKEN_URL, err: getErrorMessage(err) },
'Token refresh request error',
);
throw new TokenRefreshError(false);
}
}
throw new Error('Token refresh failed');
/** refreshToken, retrying transient failures after each of `delaysMs`. */
async function refreshTokenWithRetry(
refreshTokenStr: string,
scopes: string[],
delaysMs: number[],
): Promise<TokenResponse> {
for (let attempt = 0; ; attempt += 1) {
try {
return await refreshToken(refreshTokenStr, scopes);
} catch (err) {
const delay = delaysMs[attempt];
if (
delay === undefined ||
(err instanceof TokenRefreshError && err.definitive)
) {
throw err;
}
logger.warn(
{ attempt: attempt + 1, retryInMs: delay },
'Claude OAuth refresh failed transiently — retrying',
);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
// Serialize all refreshes within this process so the 5-minute loop and an
@@ -638,11 +723,16 @@ async function doCheckAndRefreshCredentials(opts?: {
// A forced check comes from a 401. Expiry cannot reveal a revoked token, so
// ask the API; a rejected token must be refreshed even though it "looks"
// valid, otherwise every retry reuses the same dead token.
const knownRejected =
rejectedAccessToken !== null && rejectedAccessToken === oauth.accessToken;
const rejected =
opts?.force === true &&
!isReloginRequired() &&
(await probeAccessToken(oauth.accessToken)) === 'rejected';
if (rejected) {
knownRejected ||
(opts?.force === true &&
!isReloginRequired() &&
(await probeAccessToken(oauth.accessToken)) === 'rejected');
if (rejected && !knownRejected) {
rejectedAccessToken = oauth.accessToken;
authRecoveryIncident += 1;
logger.warn(
{ remainingMin: Math.round(remaining / 60000) },
'Claude OAuth token was rejected by the API before its expiry',
@@ -692,7 +782,11 @@ async function doCheckAndRefreshCredentials(opts?: {
);
try {
const response = await refreshToken(rt, oauth.scopes || DEFAULT_SCOPES);
const response = await refreshTokenWithRetry(
rt,
oauth.scopes || DEFAULT_SCOPES,
rejected && opts?.force ? refreshRetryDelaysMs : [],
);
recordAuthSuccess(response.access_token);
oauth.accessToken = response.access_token;
@@ -738,11 +832,25 @@ async function doCheckAndRefreshCredentials(opts?: {
);
return recoveredOauth.accessToken;
}
if (rejected) {
const definitive = err instanceof TokenRefreshError && err.definitive;
if (rejected && definitive) {
markReloginRequired(oauth.accessToken);
return null;
}
const gaveUp = recordAuthFailure();
if (rejected) {
// Transient (429/5xx/timeout): keep retrying from the periodic loop;
// only a failure that lasts GIVE_UP_AFTER_MS means the login is gone.
if (gaveUp) {
markReloginRequired(oauth.accessToken);
} else {
logger.warn(
{ err: getErrorMessage(err) },
'Rejected Claude OAuth token could not be refreshed yet (transient) — retrying every check',
);
}
return null;
}
logger.error(
{
gaveUp,