fix: tell transient Claude refresh failures apart from a lost login
A 401-rejected token whose refresh failed with 429/5xx/timeout was marked re-login-required at once, closing tasks and asking for a needless re-login. - refreshToken throws TokenRefreshError; only 400/401/invalid_grant is definitive. A forced refresh of a rejected token retries transient failures twice right away; the periodic loop then keeps retrying the known-rejected token every check, and only a failure lasting 30 min turns into re-login-required. - While recovering, the room is told once per incident and the paired task is kept, so the normal retry path can resume it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -193,6 +193,7 @@ vi.mock('./claude-oauth-token.js', () => ({
|
||||
vi.mock('./token-refresh.js', () => ({
|
||||
forceRefreshToken: vi.fn(async () => null),
|
||||
isReloginRequired: vi.fn(() => false),
|
||||
getClaudeAuthRecoveryIncident: vi.fn(() => null),
|
||||
RELOGIN_REQUIRED_MESSAGE: 'test: claude re-login required',
|
||||
}));
|
||||
|
||||
@@ -216,6 +217,7 @@ import * as db from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import { buildRoomMemoryBriefing } from './sqlite-memory-store.js';
|
||||
import {
|
||||
CLAUDE_AUTH_RECOVERING_NOTICE,
|
||||
CLAUDE_SAFETY_REFUSAL_NOTICE,
|
||||
runAgentForGroup,
|
||||
} from './message-agent-executor.js';
|
||||
@@ -2819,6 +2821,7 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
||||
);
|
||||
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null);
|
||||
vi.mocked(tokenRefresh.isReloginRequired).mockReturnValue(false);
|
||||
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(null);
|
||||
});
|
||||
|
||||
it('force-refreshes the Claude token and retries once on auth-expired', async () => {
|
||||
@@ -3542,6 +3545,49 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
||||
expect(outputs).toEqual(['test: claude re-login required']);
|
||||
});
|
||||
|
||||
it('keeps the task and tells the room once while the token refresh fails transiently', async () => {
|
||||
const revoked =
|
||||
'Failed to authenticate. API Error: 401 OAuth access token has been revoked.';
|
||||
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(7);
|
||||
const runOnce = async () => {
|
||||
const outputs: string[] = [];
|
||||
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||
async (_group, _input, _onProcess, onOutput) => {
|
||||
await onOutput?.({ status: 'success', result: revoked });
|
||||
return { status: 'success', result: revoked };
|
||||
},
|
||||
);
|
||||
const result = await runAgentForGroup(makeDeps(), {
|
||||
group: makeGroup(),
|
||||
prompt: 'hello',
|
||||
chatJid: 'group@recovering',
|
||||
runId: `run-auth-recovering-${Math.random()}`,
|
||||
onOutput: async (output) => {
|
||||
const text =
|
||||
output.output && 'text' in output.output
|
||||
? output.output.text
|
||||
: output.result;
|
||||
if (typeof text === 'string') outputs.push(text);
|
||||
},
|
||||
});
|
||||
return { result, outputs };
|
||||
};
|
||||
|
||||
const first = await runOnce();
|
||||
expect(first.result).toBe('success');
|
||||
expect(first.outputs).toEqual([CLAUDE_AUTH_RECOVERING_NOTICE]);
|
||||
expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
expect.objectContaining({ completion_reason: 'claude_relogin_required' }),
|
||||
);
|
||||
|
||||
// Same incident: no second notice, the normal retry path takes over.
|
||||
const second = await runOnce();
|
||||
expect(second.result).toBe('error');
|
||||
expect(second.outputs).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns error after repeated retryable Claude session failures', async () => {
|
||||
vi.mocked(
|
||||
sessionRecovery.shouldRetryFreshSessionOnAgentFailure,
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
import { completePairedTaskWithReason } from './paired-task-status.js';
|
||||
import {
|
||||
forceRefreshToken,
|
||||
getClaudeAuthRecoveryIncident,
|
||||
isReloginRequired,
|
||||
RELOGIN_REQUIRED_MESSAGE,
|
||||
} from './token-refresh.js';
|
||||
@@ -62,6 +63,14 @@ export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal';
|
||||
export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON =
|
||||
'claude_relogin_required';
|
||||
|
||||
export const CLAUDE_AUTH_RECOVERING_NOTICE =
|
||||
'⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이번 작업을 멈췄습니다. ' +
|
||||
'봇이 5분마다 자동으로 다시 시도하고 있습니다. 잠시 후 이 방에 "계속"이라고 보내 주시면 이어서 진행합니다. ' +
|
||||
'30분 넘게 복구되지 않으면 재로그인 안내를 드리겠습니다.';
|
||||
|
||||
// chatJid -> auth recovery incident already announced in that room.
|
||||
const announcedAuthRecovery = new Map<string, number>();
|
||||
|
||||
export const CLAUDE_SAFETY_REFUSAL_NOTICE =
|
||||
'Claude 안전 필터가 이 요청을 차단해서 처리하지 못했습니다. 이 방의 Claude 세션은 초기화했습니다. 요청을 조금 바꿔서 다시 보내 주세요. 에러 원문은 붙이지 말아 주세요.';
|
||||
|
||||
@@ -286,6 +295,7 @@ async function reportClaudeUsageLimitToRoom(
|
||||
* owner/arbiter retries fail silently until someone asks why the bot stopped.
|
||||
*/
|
||||
async function reportClaudeReloginRequired(args: {
|
||||
chatJid: string;
|
||||
activeRole: PairedRoomRole;
|
||||
pairedTaskId?: string;
|
||||
errorText?: string | null;
|
||||
@@ -297,7 +307,9 @@ async function reportClaudeReloginRequired(args: {
|
||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||
}): Promise<boolean> {
|
||||
if (!isReloginRequired()) await forceRefreshToken();
|
||||
if (!isReloginRequired()) return false;
|
||||
if (!isReloginRequired()) {
|
||||
return reportClaudeAuthRecovering(args);
|
||||
}
|
||||
const task = args.pairedTaskId
|
||||
? getPairedTaskById(args.pairedTaskId)
|
||||
: undefined;
|
||||
@@ -320,6 +332,40 @@ async function reportClaudeReloginRequired(args: {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* The token was rejected but refreshing failed only transiently (429/5xx/
|
||||
* timeout): the login may still be fine, so keep the task and let the
|
||||
* automatic retries run. Tell the room once per incident why the work stopped;
|
||||
* later failures in the same incident take the normal retry path.
|
||||
*/
|
||||
async function reportClaudeAuthRecovering(args: {
|
||||
chatJid: string;
|
||||
activeRole: PairedRoomRole;
|
||||
errorText?: string | null;
|
||||
pairedExecutionLifecycle: Pick<
|
||||
PairedExecutionLifecycle,
|
||||
'updateSummary' | 'completeImmediately'
|
||||
>;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||
}): Promise<boolean> {
|
||||
const incident = getClaudeAuthRecoveryIncident();
|
||||
if (incident === null) return false;
|
||||
if (announcedAuthRecovery.get(args.chatJid) === incident) return false;
|
||||
announcedAuthRecovery.set(args.chatJid, incident);
|
||||
args.log.warn(
|
||||
{ role: args.activeRole, incident, errorText: args.errorText },
|
||||
'Claude token refresh failing transiently; told the room and kept the task',
|
||||
);
|
||||
await deliverNoticeAndFailExecution({
|
||||
notice: CLAUDE_AUTH_RECOVERING_NOTICE,
|
||||
summary: CLAUDE_AUTH_RECOVERING_NOTICE,
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owner finalize without an agent run (see paired-finalize.ts): close the task
|
||||
* and deliver the already-reviewed answer as the final message. Returns false
|
||||
|
||||
@@ -26,9 +26,11 @@ vi.mock('./claude-oauth-token.js', () => ({
|
||||
|
||||
import {
|
||||
forceRefreshToken,
|
||||
getClaudeAuthRecoveryIncident,
|
||||
isReloginRequired,
|
||||
probeAccessToken,
|
||||
resetAuthFailureStateForTests,
|
||||
setRefreshRetryDelaysForTests,
|
||||
} from './token-refresh.js';
|
||||
|
||||
const credsPath = path.join(tmp, 'claude', '.credentials.json');
|
||||
@@ -48,9 +50,20 @@ function writeCreds(accessToken: string, refreshToken = 'rt-old'): void {
|
||||
);
|
||||
}
|
||||
|
||||
type TokenReply = number | { status: number; body: string };
|
||||
|
||||
const refreshed = (accessToken: string): TokenReply => ({
|
||||
status: 200,
|
||||
body: JSON.stringify({
|
||||
access_token: accessToken,
|
||||
refresh_token: 'rt-rotated',
|
||||
expires_in: 8 * 60 * 60,
|
||||
}),
|
||||
});
|
||||
|
||||
function mockFetch(handlers: {
|
||||
profile: (token: string) => number;
|
||||
token?: () => number;
|
||||
token?: () => TokenReply;
|
||||
}) {
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
if (url.endsWith('/api/oauth/profile')) {
|
||||
@@ -58,8 +71,11 @@ function mockFetch(handlers: {
|
||||
const status = handlers.profile(auth.replace('Bearer ', ''));
|
||||
return new Response('{}', { status });
|
||||
}
|
||||
const status = handlers.token?.() ?? 400;
|
||||
return new Response('{"error":"invalid_grant"}', { status });
|
||||
const reply = handlers.token?.() ?? 400;
|
||||
if (typeof reply === 'number') {
|
||||
return new Response('{"error":"invalid_grant"}', { status: reply });
|
||||
}
|
||||
return new Response(reply.body, { status: reply.status });
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
return fetchMock;
|
||||
@@ -72,6 +88,7 @@ const tokenCalls = (fetchMock: ReturnType<typeof mockFetch>) =>
|
||||
beforeEach(() => {
|
||||
process.env.CLAUDE_CREDENTIALS_PATH = credsPath;
|
||||
resetAuthFailureStateForTests();
|
||||
setRefreshRetryDelaysForTests([0, 0]);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -132,6 +149,74 @@ describe('forceRefreshToken after a 401', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('forceRefreshToken: transient vs definitive refresh failures', () => {
|
||||
const unavailable: TokenReply = {
|
||||
status: 503,
|
||||
body: '{"error":"overloaded"}',
|
||||
};
|
||||
|
||||
it('retries a transient failure right away and recovers (401 -> 503 -> 200)', async () => {
|
||||
writeCreds('revoked');
|
||||
const replies = [unavailable, refreshed('fresh')];
|
||||
const fetchMock = mockFetch({
|
||||
profile: (t) => (t === 'revoked' ? 401 : 200),
|
||||
token: () => replies.shift() ?? 500,
|
||||
});
|
||||
|
||||
expect(await forceRefreshToken()).toBe('fresh');
|
||||
expect(tokenCalls(fetchMock)).toBe(2);
|
||||
expect(isReloginRequired()).toBe(false);
|
||||
expect(getClaudeAuthRecoveryIncident()).toBeNull();
|
||||
});
|
||||
|
||||
it('keeps retrying instead of demanding re-login while the endpoint is down (401 -> 503 ... -> next refresh succeeds)', async () => {
|
||||
writeCreds('revoked');
|
||||
let down = true;
|
||||
const fetchMock = mockFetch({
|
||||
profile: (t) => (t === 'revoked' ? 401 : 200),
|
||||
token: () => (down ? unavailable : refreshed('fresh')),
|
||||
});
|
||||
|
||||
expect(await forceRefreshToken()).toBeNull();
|
||||
expect(tokenCalls(fetchMock)).toBe(3); // first try + 2 retries
|
||||
expect(isReloginRequired()).toBe(false);
|
||||
expect(getClaudeAuthRecoveryIncident()).not.toBeNull();
|
||||
|
||||
down = false;
|
||||
expect(await forceRefreshToken()).toBe('fresh');
|
||||
expect(isReloginRequired()).toBe(false);
|
||||
expect(getClaudeAuthRecoveryIncident()).toBeNull();
|
||||
});
|
||||
|
||||
it('demands re-login at once on invalid_grant, without retrying (401 -> invalid_grant)', async () => {
|
||||
writeCreds('revoked');
|
||||
const fetchMock = mockFetch({
|
||||
profile: () => 401,
|
||||
token: () => ({ status: 400, body: '{"error":"invalid_grant"}' }),
|
||||
});
|
||||
|
||||
expect(await forceRefreshToken()).toBeNull();
|
||||
expect(tokenCalls(fetchMock)).toBe(1);
|
||||
expect(isReloginRequired()).toBe(true);
|
||||
expect(getClaudeAuthRecoveryIncident()).toBeNull();
|
||||
});
|
||||
|
||||
it('turns a transient failure that lasts 30 minutes into re-login', async () => {
|
||||
writeCreds('revoked');
|
||||
mockFetch({ profile: () => 401, token: () => unavailable });
|
||||
const start = Date.now();
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(start);
|
||||
|
||||
expect(await forceRefreshToken()).toBeNull();
|
||||
expect(isReloginRequired()).toBe(false);
|
||||
|
||||
now.mockReturnValue(start + 31 * 60 * 1000);
|
||||
expect(await forceRefreshToken()).toBeNull();
|
||||
expect(isReloginRequired()).toBe(true);
|
||||
now.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(path.join(tmp, 'claude'), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -79,6 +79,23 @@ interface AuthFailureState {
|
||||
// appears again (e.g. after a manual re-login rewrites the creds file).
|
||||
let authFailure: AuthFailureState | null = null;
|
||||
|
||||
// Access token the API rejected (401) while refreshing it has only failed
|
||||
// transiently so far (429/5xx/timeout). The periodic loop keeps retrying it
|
||||
// every CHECK_INTERVAL_MS; after GIVE_UP_AFTER_MS it becomes re-login-required.
|
||||
let rejectedAccessToken: string | null = null;
|
||||
// Bumped for each newly rejected token, so rooms are told once per incident.
|
||||
let authRecoveryIncident = 0;
|
||||
|
||||
/**
|
||||
* The current token was rejected and refreshing failed for a transient reason;
|
||||
* automatic retries are still running. Returns the incident number, or null.
|
||||
*/
|
||||
export function getClaudeAuthRecoveryIncident(): number | null {
|
||||
return rejectedAccessToken !== null && !isReloginRequired()
|
||||
? authRecoveryIncident
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when automatic refresh has been failing for longer than
|
||||
* GIVE_UP_AFTER_MS and we have stopped retrying. Request handlers use this to
|
||||
@@ -90,6 +107,9 @@ export function isReloginRequired(): boolean {
|
||||
|
||||
/** Clear failure tracking once a valid token is observed again. */
|
||||
function recordAuthSuccess(accessToken?: string): void {
|
||||
if (accessToken && accessToken !== rejectedAccessToken) {
|
||||
rejectedAccessToken = null;
|
||||
}
|
||||
if (
|
||||
authFailure?.deadAccessToken &&
|
||||
(!accessToken || accessToken === authFailure.deadAccessToken)
|
||||
@@ -146,6 +166,41 @@ function markReloginRequired(deadAccessToken: string): void {
|
||||
/** Test hook: reset the module-level failure state. */
|
||||
export function resetAuthFailureStateForTests(): void {
|
||||
authFailure = null;
|
||||
rejectedAccessToken = null;
|
||||
refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS;
|
||||
}
|
||||
|
||||
/** Test hook: retry delays for a transient refresh failure. */
|
||||
export function setRefreshRetryDelaysForTests(delaysMs: number[]): void {
|
||||
refreshRetryDelaysMs = delaysMs;
|
||||
}
|
||||
|
||||
// A forced refresh of a rejected token retries a transient failure a few
|
||||
// times right away before falling back to the periodic loop.
|
||||
const DEFAULT_REFRESH_RETRY_DELAYS_MS = [2_000, 5_000];
|
||||
let refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS;
|
||||
|
||||
/**
|
||||
* Why the token endpoint refused. `definitive` means the refresh token itself
|
||||
* was refused (400/401, invalid_grant): only a re-login can help. Anything
|
||||
* else (429, 5xx, network, timeout) is transient and worth retrying.
|
||||
*/
|
||||
export class TokenRefreshError extends Error {
|
||||
constructor(
|
||||
readonly definitive: boolean,
|
||||
readonly status?: number,
|
||||
) {
|
||||
super(
|
||||
`Token refresh failed${status ? ` (${status})` : ''}${definitive ? ': refresh token refused' : ''}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function isDefinitiveRefreshFailure(
|
||||
status: number,
|
||||
body: string,
|
||||
): boolean {
|
||||
return status === 400 || status === 401 || /invalid_grant/.test(body);
|
||||
}
|
||||
|
||||
export type AccessTokenProbe = 'valid' | 'rejected' | 'unknown';
|
||||
@@ -571,14 +626,44 @@ async function refreshToken(
|
||||
{ url: TOKEN_URL, status: res.status, body: errText.slice(0, 200) },
|
||||
'Token refresh failed at endpoint',
|
||||
);
|
||||
throw new TokenRefreshError(
|
||||
isDefinitiveRefreshFailure(res.status, errText),
|
||||
res.status,
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenRefreshError) throw err;
|
||||
logger.warn(
|
||||
{ url: TOKEN_URL, err: getErrorMessage(err) },
|
||||
'Token refresh request error',
|
||||
);
|
||||
throw new TokenRefreshError(false);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Token refresh failed');
|
||||
/** refreshToken, retrying transient failures after each of `delaysMs`. */
|
||||
async function refreshTokenWithRetry(
|
||||
refreshTokenStr: string,
|
||||
scopes: string[],
|
||||
delaysMs: number[],
|
||||
): Promise<TokenResponse> {
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
return await refreshToken(refreshTokenStr, scopes);
|
||||
} catch (err) {
|
||||
const delay = delaysMs[attempt];
|
||||
if (
|
||||
delay === undefined ||
|
||||
(err instanceof TokenRefreshError && err.definitive)
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
logger.warn(
|
||||
{ attempt: attempt + 1, retryInMs: delay },
|
||||
'Claude OAuth refresh failed transiently — retrying',
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize all refreshes within this process so the 5-minute loop and an
|
||||
@@ -638,11 +723,16 @@ async function doCheckAndRefreshCredentials(opts?: {
|
||||
// A forced check comes from a 401. Expiry cannot reveal a revoked token, so
|
||||
// ask the API; a rejected token must be refreshed even though it "looks"
|
||||
// valid, otherwise every retry reuses the same dead token.
|
||||
const knownRejected =
|
||||
rejectedAccessToken !== null && rejectedAccessToken === oauth.accessToken;
|
||||
const rejected =
|
||||
opts?.force === true &&
|
||||
!isReloginRequired() &&
|
||||
(await probeAccessToken(oauth.accessToken)) === 'rejected';
|
||||
if (rejected) {
|
||||
knownRejected ||
|
||||
(opts?.force === true &&
|
||||
!isReloginRequired() &&
|
||||
(await probeAccessToken(oauth.accessToken)) === 'rejected');
|
||||
if (rejected && !knownRejected) {
|
||||
rejectedAccessToken = oauth.accessToken;
|
||||
authRecoveryIncident += 1;
|
||||
logger.warn(
|
||||
{ remainingMin: Math.round(remaining / 60000) },
|
||||
'Claude OAuth token was rejected by the API before its expiry',
|
||||
@@ -692,7 +782,11 @@ async function doCheckAndRefreshCredentials(opts?: {
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await refreshToken(rt, oauth.scopes || DEFAULT_SCOPES);
|
||||
const response = await refreshTokenWithRetry(
|
||||
rt,
|
||||
oauth.scopes || DEFAULT_SCOPES,
|
||||
rejected && opts?.force ? refreshRetryDelaysMs : [],
|
||||
);
|
||||
|
||||
recordAuthSuccess(response.access_token);
|
||||
oauth.accessToken = response.access_token;
|
||||
@@ -738,11 +832,25 @@ async function doCheckAndRefreshCredentials(opts?: {
|
||||
);
|
||||
return recoveredOauth.accessToken;
|
||||
}
|
||||
if (rejected) {
|
||||
const definitive = err instanceof TokenRefreshError && err.definitive;
|
||||
if (rejected && definitive) {
|
||||
markReloginRequired(oauth.accessToken);
|
||||
return null;
|
||||
}
|
||||
const gaveUp = recordAuthFailure();
|
||||
if (rejected) {
|
||||
// Transient (429/5xx/timeout): keep retrying from the periodic loop;
|
||||
// only a failure that lasts GIVE_UP_AFTER_MS means the login is gone.
|
||||
if (gaveUp) {
|
||||
markReloginRequired(oauth.accessToken);
|
||||
} else {
|
||||
logger.warn(
|
||||
{ err: getErrorMessage(err) },
|
||||
'Rejected Claude OAuth token could not be refreshed yet (transient) — retrying every check',
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
logger.error(
|
||||
{
|
||||
gaveUp,
|
||||
|
||||
Reference in New Issue
Block a user