fix: tell transient Claude refresh failures apart from a lost login
A 401-rejected token whose refresh failed with 429/5xx/timeout was marked re-login-required at once, closing tasks and asking for a needless re-login. - refreshToken throws TokenRefreshError; only 400/401/invalid_grant is definitive. A forced refresh of a rejected token retries transient failures twice right away; the periodic loop then keeps retrying the known-rejected token every check, and only a failure lasting 30 min turns into re-login-required. - While recovering, the room is told once per incident and the paired task is kept, so the normal retry path can resume it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -193,6 +193,7 @@ vi.mock('./claude-oauth-token.js', () => ({
|
|||||||
vi.mock('./token-refresh.js', () => ({
|
vi.mock('./token-refresh.js', () => ({
|
||||||
forceRefreshToken: vi.fn(async () => null),
|
forceRefreshToken: vi.fn(async () => null),
|
||||||
isReloginRequired: vi.fn(() => false),
|
isReloginRequired: vi.fn(() => false),
|
||||||
|
getClaudeAuthRecoveryIncident: vi.fn(() => null),
|
||||||
RELOGIN_REQUIRED_MESSAGE: 'test: claude re-login required',
|
RELOGIN_REQUIRED_MESSAGE: 'test: claude re-login required',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -216,6 +217,7 @@ import * as db from './db.js';
|
|||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { buildRoomMemoryBriefing } from './sqlite-memory-store.js';
|
import { buildRoomMemoryBriefing } from './sqlite-memory-store.js';
|
||||||
import {
|
import {
|
||||||
|
CLAUDE_AUTH_RECOVERING_NOTICE,
|
||||||
CLAUDE_SAFETY_REFUSAL_NOTICE,
|
CLAUDE_SAFETY_REFUSAL_NOTICE,
|
||||||
runAgentForGroup,
|
runAgentForGroup,
|
||||||
} from './message-agent-executor.js';
|
} from './message-agent-executor.js';
|
||||||
@@ -2819,6 +2821,7 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
|||||||
);
|
);
|
||||||
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null);
|
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null);
|
||||||
vi.mocked(tokenRefresh.isReloginRequired).mockReturnValue(false);
|
vi.mocked(tokenRefresh.isReloginRequired).mockReturnValue(false);
|
||||||
|
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('force-refreshes the Claude token and retries once on auth-expired', async () => {
|
it('force-refreshes the Claude token and retries once on auth-expired', async () => {
|
||||||
@@ -3542,6 +3545,49 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
|||||||
expect(outputs).toEqual(['test: claude re-login required']);
|
expect(outputs).toEqual(['test: claude re-login required']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps the task and tells the room once while the token refresh fails transiently', async () => {
|
||||||
|
const revoked =
|
||||||
|
'Failed to authenticate. API Error: 401 OAuth access token has been revoked.';
|
||||||
|
vi.mocked(tokenRefresh.getClaudeAuthRecoveryIncident).mockReturnValue(7);
|
||||||
|
const runOnce = async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||||
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({ status: 'success', result: revoked });
|
||||||
|
return { status: 'success', result: revoked };
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@recovering',
|
||||||
|
runId: `run-auth-recovering-${Math.random()}`,
|
||||||
|
onOutput: async (output) => {
|
||||||
|
const text =
|
||||||
|
output.output && 'text' in output.output
|
||||||
|
? output.output.text
|
||||||
|
: output.result;
|
||||||
|
if (typeof text === 'string') outputs.push(text);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return { result, outputs };
|
||||||
|
};
|
||||||
|
|
||||||
|
const first = await runOnce();
|
||||||
|
expect(first.result).toBe('success');
|
||||||
|
expect(first.outputs).toEqual([CLAUDE_AUTH_RECOVERING_NOTICE]);
|
||||||
|
expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalledWith(
|
||||||
|
expect.anything(),
|
||||||
|
expect.anything(),
|
||||||
|
expect.objectContaining({ completion_reason: 'claude_relogin_required' }),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Same incident: no second notice, the normal retry path takes over.
|
||||||
|
const second = await runOnce();
|
||||||
|
expect(second.result).toBe('error');
|
||||||
|
expect(second.outputs).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
it('returns error after repeated retryable Claude session failures', async () => {
|
it('returns error after repeated retryable Claude session failures', async () => {
|
||||||
vi.mocked(
|
vi.mocked(
|
||||||
sessionRecovery.shouldRetryFreshSessionOnAgentFailure,
|
sessionRecovery.shouldRetryFreshSessionOnAgentFailure,
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ import {
|
|||||||
import { completePairedTaskWithReason } from './paired-task-status.js';
|
import { completePairedTaskWithReason } from './paired-task-status.js';
|
||||||
import {
|
import {
|
||||||
forceRefreshToken,
|
forceRefreshToken,
|
||||||
|
getClaudeAuthRecoveryIncident,
|
||||||
isReloginRequired,
|
isReloginRequired,
|
||||||
RELOGIN_REQUIRED_MESSAGE,
|
RELOGIN_REQUIRED_MESSAGE,
|
||||||
} from './token-refresh.js';
|
} from './token-refresh.js';
|
||||||
@@ -62,6 +63,14 @@ export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal';
|
|||||||
export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON =
|
export const CLAUDE_RELOGIN_REQUIRED_COMPLETION_REASON =
|
||||||
'claude_relogin_required';
|
'claude_relogin_required';
|
||||||
|
|
||||||
|
export const CLAUDE_AUTH_RECOVERING_NOTICE =
|
||||||
|
'⚠️ 클로드 인증 서버 오류로 토큰 갱신이 잠시 실패해서 이번 작업을 멈췄습니다. ' +
|
||||||
|
'봇이 5분마다 자동으로 다시 시도하고 있습니다. 잠시 후 이 방에 "계속"이라고 보내 주시면 이어서 진행합니다. ' +
|
||||||
|
'30분 넘게 복구되지 않으면 재로그인 안내를 드리겠습니다.';
|
||||||
|
|
||||||
|
// chatJid -> auth recovery incident already announced in that room.
|
||||||
|
const announcedAuthRecovery = new Map<string, number>();
|
||||||
|
|
||||||
export const CLAUDE_SAFETY_REFUSAL_NOTICE =
|
export const CLAUDE_SAFETY_REFUSAL_NOTICE =
|
||||||
'Claude 안전 필터가 이 요청을 차단해서 처리하지 못했습니다. 이 방의 Claude 세션은 초기화했습니다. 요청을 조금 바꿔서 다시 보내 주세요. 에러 원문은 붙이지 말아 주세요.';
|
'Claude 안전 필터가 이 요청을 차단해서 처리하지 못했습니다. 이 방의 Claude 세션은 초기화했습니다. 요청을 조금 바꿔서 다시 보내 주세요. 에러 원문은 붙이지 말아 주세요.';
|
||||||
|
|
||||||
@@ -286,6 +295,7 @@ async function reportClaudeUsageLimitToRoom(
|
|||||||
* owner/arbiter retries fail silently until someone asks why the bot stopped.
|
* owner/arbiter retries fail silently until someone asks why the bot stopped.
|
||||||
*/
|
*/
|
||||||
async function reportClaudeReloginRequired(args: {
|
async function reportClaudeReloginRequired(args: {
|
||||||
|
chatJid: string;
|
||||||
activeRole: PairedRoomRole;
|
activeRole: PairedRoomRole;
|
||||||
pairedTaskId?: string;
|
pairedTaskId?: string;
|
||||||
errorText?: string | null;
|
errorText?: string | null;
|
||||||
@@ -297,7 +307,9 @@ async function reportClaudeReloginRequired(args: {
|
|||||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||||
}): Promise<boolean> {
|
}): Promise<boolean> {
|
||||||
if (!isReloginRequired()) await forceRefreshToken();
|
if (!isReloginRequired()) await forceRefreshToken();
|
||||||
if (!isReloginRequired()) return false;
|
if (!isReloginRequired()) {
|
||||||
|
return reportClaudeAuthRecovering(args);
|
||||||
|
}
|
||||||
const task = args.pairedTaskId
|
const task = args.pairedTaskId
|
||||||
? getPairedTaskById(args.pairedTaskId)
|
? getPairedTaskById(args.pairedTaskId)
|
||||||
: undefined;
|
: undefined;
|
||||||
@@ -320,6 +332,40 @@ async function reportClaudeReloginRequired(args: {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The token was rejected but refreshing failed only transiently (429/5xx/
|
||||||
|
* timeout): the login may still be fine, so keep the task and let the
|
||||||
|
* automatic retries run. Tell the room once per incident why the work stopped;
|
||||||
|
* later failures in the same incident take the normal retry path.
|
||||||
|
*/
|
||||||
|
async function reportClaudeAuthRecovering(args: {
|
||||||
|
chatJid: string;
|
||||||
|
activeRole: PairedRoomRole;
|
||||||
|
errorText?: string | null;
|
||||||
|
pairedExecutionLifecycle: Pick<
|
||||||
|
PairedExecutionLifecycle,
|
||||||
|
'updateSummary' | 'completeImmediately'
|
||||||
|
>;
|
||||||
|
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||||
|
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||||
|
}): Promise<boolean> {
|
||||||
|
const incident = getClaudeAuthRecoveryIncident();
|
||||||
|
if (incident === null) return false;
|
||||||
|
if (announcedAuthRecovery.get(args.chatJid) === incident) return false;
|
||||||
|
announcedAuthRecovery.set(args.chatJid, incident);
|
||||||
|
args.log.warn(
|
||||||
|
{ role: args.activeRole, incident, errorText: args.errorText },
|
||||||
|
'Claude token refresh failing transiently; told the room and kept the task',
|
||||||
|
);
|
||||||
|
await deliverNoticeAndFailExecution({
|
||||||
|
notice: CLAUDE_AUTH_RECOVERING_NOTICE,
|
||||||
|
summary: CLAUDE_AUTH_RECOVERING_NOTICE,
|
||||||
|
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||||
|
onOutput: args.onOutput,
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Owner finalize without an agent run (see paired-finalize.ts): close the task
|
* Owner finalize without an agent run (see paired-finalize.ts): close the task
|
||||||
* and deliver the already-reviewed answer as the final message. Returns false
|
* and deliver the already-reviewed answer as the final message. Returns false
|
||||||
|
|||||||
@@ -26,9 +26,11 @@ vi.mock('./claude-oauth-token.js', () => ({
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
forceRefreshToken,
|
forceRefreshToken,
|
||||||
|
getClaudeAuthRecoveryIncident,
|
||||||
isReloginRequired,
|
isReloginRequired,
|
||||||
probeAccessToken,
|
probeAccessToken,
|
||||||
resetAuthFailureStateForTests,
|
resetAuthFailureStateForTests,
|
||||||
|
setRefreshRetryDelaysForTests,
|
||||||
} from './token-refresh.js';
|
} from './token-refresh.js';
|
||||||
|
|
||||||
const credsPath = path.join(tmp, 'claude', '.credentials.json');
|
const credsPath = path.join(tmp, 'claude', '.credentials.json');
|
||||||
@@ -48,9 +50,20 @@ function writeCreds(accessToken: string, refreshToken = 'rt-old'): void {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TokenReply = number | { status: number; body: string };
|
||||||
|
|
||||||
|
const refreshed = (accessToken: string): TokenReply => ({
|
||||||
|
status: 200,
|
||||||
|
body: JSON.stringify({
|
||||||
|
access_token: accessToken,
|
||||||
|
refresh_token: 'rt-rotated',
|
||||||
|
expires_in: 8 * 60 * 60,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
function mockFetch(handlers: {
|
function mockFetch(handlers: {
|
||||||
profile: (token: string) => number;
|
profile: (token: string) => number;
|
||||||
token?: () => number;
|
token?: () => TokenReply;
|
||||||
}) {
|
}) {
|
||||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||||
if (url.endsWith('/api/oauth/profile')) {
|
if (url.endsWith('/api/oauth/profile')) {
|
||||||
@@ -58,8 +71,11 @@ function mockFetch(handlers: {
|
|||||||
const status = handlers.profile(auth.replace('Bearer ', ''));
|
const status = handlers.profile(auth.replace('Bearer ', ''));
|
||||||
return new Response('{}', { status });
|
return new Response('{}', { status });
|
||||||
}
|
}
|
||||||
const status = handlers.token?.() ?? 400;
|
const reply = handlers.token?.() ?? 400;
|
||||||
return new Response('{"error":"invalid_grant"}', { status });
|
if (typeof reply === 'number') {
|
||||||
|
return new Response('{"error":"invalid_grant"}', { status: reply });
|
||||||
|
}
|
||||||
|
return new Response(reply.body, { status: reply.status });
|
||||||
});
|
});
|
||||||
vi.stubGlobal('fetch', fetchMock);
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
return fetchMock;
|
return fetchMock;
|
||||||
@@ -72,6 +88,7 @@ const tokenCalls = (fetchMock: ReturnType<typeof mockFetch>) =>
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
process.env.CLAUDE_CREDENTIALS_PATH = credsPath;
|
process.env.CLAUDE_CREDENTIALS_PATH = credsPath;
|
||||||
resetAuthFailureStateForTests();
|
resetAuthFailureStateForTests();
|
||||||
|
setRefreshRetryDelaysForTests([0, 0]);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -132,6 +149,74 @@ describe('forceRefreshToken after a 401', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('forceRefreshToken: transient vs definitive refresh failures', () => {
|
||||||
|
const unavailable: TokenReply = {
|
||||||
|
status: 503,
|
||||||
|
body: '{"error":"overloaded"}',
|
||||||
|
};
|
||||||
|
|
||||||
|
it('retries a transient failure right away and recovers (401 -> 503 -> 200)', async () => {
|
||||||
|
writeCreds('revoked');
|
||||||
|
const replies = [unavailable, refreshed('fresh')];
|
||||||
|
const fetchMock = mockFetch({
|
||||||
|
profile: (t) => (t === 'revoked' ? 401 : 200),
|
||||||
|
token: () => replies.shift() ?? 500,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await forceRefreshToken()).toBe('fresh');
|
||||||
|
expect(tokenCalls(fetchMock)).toBe(2);
|
||||||
|
expect(isReloginRequired()).toBe(false);
|
||||||
|
expect(getClaudeAuthRecoveryIncident()).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps retrying instead of demanding re-login while the endpoint is down (401 -> 503 ... -> next refresh succeeds)', async () => {
|
||||||
|
writeCreds('revoked');
|
||||||
|
let down = true;
|
||||||
|
const fetchMock = mockFetch({
|
||||||
|
profile: (t) => (t === 'revoked' ? 401 : 200),
|
||||||
|
token: () => (down ? unavailable : refreshed('fresh')),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await forceRefreshToken()).toBeNull();
|
||||||
|
expect(tokenCalls(fetchMock)).toBe(3); // first try + 2 retries
|
||||||
|
expect(isReloginRequired()).toBe(false);
|
||||||
|
expect(getClaudeAuthRecoveryIncident()).not.toBeNull();
|
||||||
|
|
||||||
|
down = false;
|
||||||
|
expect(await forceRefreshToken()).toBe('fresh');
|
||||||
|
expect(isReloginRequired()).toBe(false);
|
||||||
|
expect(getClaudeAuthRecoveryIncident()).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('demands re-login at once on invalid_grant, without retrying (401 -> invalid_grant)', async () => {
|
||||||
|
writeCreds('revoked');
|
||||||
|
const fetchMock = mockFetch({
|
||||||
|
profile: () => 401,
|
||||||
|
token: () => ({ status: 400, body: '{"error":"invalid_grant"}' }),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await forceRefreshToken()).toBeNull();
|
||||||
|
expect(tokenCalls(fetchMock)).toBe(1);
|
||||||
|
expect(isReloginRequired()).toBe(true);
|
||||||
|
expect(getClaudeAuthRecoveryIncident()).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('turns a transient failure that lasts 30 minutes into re-login', async () => {
|
||||||
|
writeCreds('revoked');
|
||||||
|
mockFetch({ profile: () => 401, token: () => unavailable });
|
||||||
|
const start = Date.now();
|
||||||
|
const now = vi.spyOn(Date, 'now').mockReturnValue(start);
|
||||||
|
|
||||||
|
expect(await forceRefreshToken()).toBeNull();
|
||||||
|
expect(isReloginRequired()).toBe(false);
|
||||||
|
|
||||||
|
now.mockReturnValue(start + 31 * 60 * 1000);
|
||||||
|
expect(await forceRefreshToken()).toBeNull();
|
||||||
|
expect(isReloginRequired()).toBe(true);
|
||||||
|
now.mockRestore();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
fs.rmSync(path.join(tmp, 'claude'), { recursive: true, force: true });
|
fs.rmSync(path.join(tmp, 'claude'), { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -79,6 +79,23 @@ interface AuthFailureState {
|
|||||||
// appears again (e.g. after a manual re-login rewrites the creds file).
|
// appears again (e.g. after a manual re-login rewrites the creds file).
|
||||||
let authFailure: AuthFailureState | null = null;
|
let authFailure: AuthFailureState | null = null;
|
||||||
|
|
||||||
|
// Access token the API rejected (401) while refreshing it has only failed
|
||||||
|
// transiently so far (429/5xx/timeout). The periodic loop keeps retrying it
|
||||||
|
// every CHECK_INTERVAL_MS; after GIVE_UP_AFTER_MS it becomes re-login-required.
|
||||||
|
let rejectedAccessToken: string | null = null;
|
||||||
|
// Bumped for each newly rejected token, so rooms are told once per incident.
|
||||||
|
let authRecoveryIncident = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The current token was rejected and refreshing failed for a transient reason;
|
||||||
|
* automatic retries are still running. Returns the incident number, or null.
|
||||||
|
*/
|
||||||
|
export function getClaudeAuthRecoveryIncident(): number | null {
|
||||||
|
return rejectedAccessToken !== null && !isReloginRequired()
|
||||||
|
? authRecoveryIncident
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* True when automatic refresh has been failing for longer than
|
* True when automatic refresh has been failing for longer than
|
||||||
* GIVE_UP_AFTER_MS and we have stopped retrying. Request handlers use this to
|
* GIVE_UP_AFTER_MS and we have stopped retrying. Request handlers use this to
|
||||||
@@ -90,6 +107,9 @@ export function isReloginRequired(): boolean {
|
|||||||
|
|
||||||
/** Clear failure tracking once a valid token is observed again. */
|
/** Clear failure tracking once a valid token is observed again. */
|
||||||
function recordAuthSuccess(accessToken?: string): void {
|
function recordAuthSuccess(accessToken?: string): void {
|
||||||
|
if (accessToken && accessToken !== rejectedAccessToken) {
|
||||||
|
rejectedAccessToken = null;
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
authFailure?.deadAccessToken &&
|
authFailure?.deadAccessToken &&
|
||||||
(!accessToken || accessToken === authFailure.deadAccessToken)
|
(!accessToken || accessToken === authFailure.deadAccessToken)
|
||||||
@@ -146,6 +166,41 @@ function markReloginRequired(deadAccessToken: string): void {
|
|||||||
/** Test hook: reset the module-level failure state. */
|
/** Test hook: reset the module-level failure state. */
|
||||||
export function resetAuthFailureStateForTests(): void {
|
export function resetAuthFailureStateForTests(): void {
|
||||||
authFailure = null;
|
authFailure = null;
|
||||||
|
rejectedAccessToken = null;
|
||||||
|
refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test hook: retry delays for a transient refresh failure. */
|
||||||
|
export function setRefreshRetryDelaysForTests(delaysMs: number[]): void {
|
||||||
|
refreshRetryDelaysMs = delaysMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A forced refresh of a rejected token retries a transient failure a few
|
||||||
|
// times right away before falling back to the periodic loop.
|
||||||
|
const DEFAULT_REFRESH_RETRY_DELAYS_MS = [2_000, 5_000];
|
||||||
|
let refreshRetryDelaysMs = DEFAULT_REFRESH_RETRY_DELAYS_MS;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Why the token endpoint refused. `definitive` means the refresh token itself
|
||||||
|
* was refused (400/401, invalid_grant): only a re-login can help. Anything
|
||||||
|
* else (429, 5xx, network, timeout) is transient and worth retrying.
|
||||||
|
*/
|
||||||
|
export class TokenRefreshError extends Error {
|
||||||
|
constructor(
|
||||||
|
readonly definitive: boolean,
|
||||||
|
readonly status?: number,
|
||||||
|
) {
|
||||||
|
super(
|
||||||
|
`Token refresh failed${status ? ` (${status})` : ''}${definitive ? ': refresh token refused' : ''}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isDefinitiveRefreshFailure(
|
||||||
|
status: number,
|
||||||
|
body: string,
|
||||||
|
): boolean {
|
||||||
|
return status === 400 || status === 401 || /invalid_grant/.test(body);
|
||||||
}
|
}
|
||||||
|
|
||||||
export type AccessTokenProbe = 'valid' | 'rejected' | 'unknown';
|
export type AccessTokenProbe = 'valid' | 'rejected' | 'unknown';
|
||||||
@@ -571,14 +626,44 @@ async function refreshToken(
|
|||||||
{ url: TOKEN_URL, status: res.status, body: errText.slice(0, 200) },
|
{ url: TOKEN_URL, status: res.status, body: errText.slice(0, 200) },
|
||||||
'Token refresh failed at endpoint',
|
'Token refresh failed at endpoint',
|
||||||
);
|
);
|
||||||
|
throw new TokenRefreshError(
|
||||||
|
isDefinitiveRefreshFailure(res.status, errText),
|
||||||
|
res.status,
|
||||||
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
if (err instanceof TokenRefreshError) throw err;
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ url: TOKEN_URL, err: getErrorMessage(err) },
|
{ url: TOKEN_URL, err: getErrorMessage(err) },
|
||||||
'Token refresh request error',
|
'Token refresh request error',
|
||||||
);
|
);
|
||||||
|
throw new TokenRefreshError(false);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
throw new Error('Token refresh failed');
|
/** refreshToken, retrying transient failures after each of `delaysMs`. */
|
||||||
|
async function refreshTokenWithRetry(
|
||||||
|
refreshTokenStr: string,
|
||||||
|
scopes: string[],
|
||||||
|
delaysMs: number[],
|
||||||
|
): Promise<TokenResponse> {
|
||||||
|
for (let attempt = 0; ; attempt += 1) {
|
||||||
|
try {
|
||||||
|
return await refreshToken(refreshTokenStr, scopes);
|
||||||
|
} catch (err) {
|
||||||
|
const delay = delaysMs[attempt];
|
||||||
|
if (
|
||||||
|
delay === undefined ||
|
||||||
|
(err instanceof TokenRefreshError && err.definitive)
|
||||||
|
) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
logger.warn(
|
||||||
|
{ attempt: attempt + 1, retryInMs: delay },
|
||||||
|
'Claude OAuth refresh failed transiently — retrying',
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Serialize all refreshes within this process so the 5-minute loop and an
|
// Serialize all refreshes within this process so the 5-minute loop and an
|
||||||
@@ -638,11 +723,16 @@ async function doCheckAndRefreshCredentials(opts?: {
|
|||||||
// A forced check comes from a 401. Expiry cannot reveal a revoked token, so
|
// A forced check comes from a 401. Expiry cannot reveal a revoked token, so
|
||||||
// ask the API; a rejected token must be refreshed even though it "looks"
|
// ask the API; a rejected token must be refreshed even though it "looks"
|
||||||
// valid, otherwise every retry reuses the same dead token.
|
// valid, otherwise every retry reuses the same dead token.
|
||||||
|
const knownRejected =
|
||||||
|
rejectedAccessToken !== null && rejectedAccessToken === oauth.accessToken;
|
||||||
const rejected =
|
const rejected =
|
||||||
opts?.force === true &&
|
knownRejected ||
|
||||||
!isReloginRequired() &&
|
(opts?.force === true &&
|
||||||
(await probeAccessToken(oauth.accessToken)) === 'rejected';
|
!isReloginRequired() &&
|
||||||
if (rejected) {
|
(await probeAccessToken(oauth.accessToken)) === 'rejected');
|
||||||
|
if (rejected && !knownRejected) {
|
||||||
|
rejectedAccessToken = oauth.accessToken;
|
||||||
|
authRecoveryIncident += 1;
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ remainingMin: Math.round(remaining / 60000) },
|
{ remainingMin: Math.round(remaining / 60000) },
|
||||||
'Claude OAuth token was rejected by the API before its expiry',
|
'Claude OAuth token was rejected by the API before its expiry',
|
||||||
@@ -692,7 +782,11 @@ async function doCheckAndRefreshCredentials(opts?: {
|
|||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await refreshToken(rt, oauth.scopes || DEFAULT_SCOPES);
|
const response = await refreshTokenWithRetry(
|
||||||
|
rt,
|
||||||
|
oauth.scopes || DEFAULT_SCOPES,
|
||||||
|
rejected && opts?.force ? refreshRetryDelaysMs : [],
|
||||||
|
);
|
||||||
|
|
||||||
recordAuthSuccess(response.access_token);
|
recordAuthSuccess(response.access_token);
|
||||||
oauth.accessToken = response.access_token;
|
oauth.accessToken = response.access_token;
|
||||||
@@ -738,11 +832,25 @@ async function doCheckAndRefreshCredentials(opts?: {
|
|||||||
);
|
);
|
||||||
return recoveredOauth.accessToken;
|
return recoveredOauth.accessToken;
|
||||||
}
|
}
|
||||||
if (rejected) {
|
const definitive = err instanceof TokenRefreshError && err.definitive;
|
||||||
|
if (rejected && definitive) {
|
||||||
markReloginRequired(oauth.accessToken);
|
markReloginRequired(oauth.accessToken);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
const gaveUp = recordAuthFailure();
|
const gaveUp = recordAuthFailure();
|
||||||
|
if (rejected) {
|
||||||
|
// Transient (429/5xx/timeout): keep retrying from the periodic loop;
|
||||||
|
// only a failure that lasts GIVE_UP_AFTER_MS means the login is gone.
|
||||||
|
if (gaveUp) {
|
||||||
|
markReloginRequired(oauth.accessToken);
|
||||||
|
} else {
|
||||||
|
logger.warn(
|
||||||
|
{ err: getErrorMessage(err) },
|
||||||
|
'Rejected Claude OAuth token could not be refreshed yet (transient) — retrying every check',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
logger.error(
|
logger.error(
|
||||||
{
|
{
|
||||||
gaveUp,
|
gaveUp,
|
||||||
|
|||||||
Reference in New Issue
Block a user