diff --git a/src/agent-runner.ts b/src/agent-runner.ts index 0bf1cb0..e3fe5bf 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -17,6 +17,7 @@ import { type PreparedCodexSessionAuth, } from './agent-runner-environment.js'; import { syncCodexSessionAuthBack } from './codex-token-rotation.js'; +import { syncClaudeSessionAuthBack } from './token-refresh.js'; import { runSpawnedAgentProcess } from './agent-runner-process.js'; import { getStoredRoomSkillOverrides } from './db.js'; export { @@ -119,6 +120,31 @@ function createCodexAuthSessionFinalizer( }; } +/** + * After a Claude agent turn, adopt any token the session's Claude CLI refreshed + * (rotated) back into the canonical credentials, so the main refresh loop never + * ends up holding an already-rotated token (which Anthropic revokes as a family). + * No-op for Codex sessions (sessionClaudeDir undefined). + */ +function createClaudeAuthSessionFinalizer( + sessionClaudeDir: string | undefined, +): () => void { + let finalized = false; + return () => { + if (finalized) return; + finalized = true; + if (!sessionClaudeDir) return; + try { + syncClaudeSessionAuthBack(sessionClaudeDir); + } catch (err) { + logger.warn( + { err, sessionClaudeDir }, + 'Failed to sync Claude session auth back to canonical credentials', + ); + } + }; +} + export async function runAgentProcess( group: RegisteredGroup, input: AgentInput, @@ -199,6 +225,15 @@ export async function runAgentProcess( const finalizeCodexAuthSessionOnce = createCodexAuthSessionFinalizer( () => codexSessionAuth, ); + const finalizeClaudeAuthSessionOnce = createClaudeAuthSessionFinalizer( + (group.agentType || 'claude-code') === 'codex' + ? undefined + : env.CLAUDE_CONFIG_DIR, + ); + const finalizeAuthSessionOnce = () => { + finalizeCodexAuthSessionOnce(); + finalizeClaudeAuthSessionOnce(); + }; // Check if runner is built const distEntry = path.join(runnerDir, 'dist', 'index.js'); @@ -255,14 +290,14 @@ export async function runAgentProcess( logsDir, startTime, onOutput, - onTerminalStreamedOutputFlushed: finalizeCodexAuthSessionOnce, + onTerminalStreamedOutputFlushed: finalizeAuthSessionOnce, }) .then((output) => { - finalizeCodexAuthSessionOnce(); + finalizeAuthSessionOnce(); resolve(output); }) .catch((err: unknown) => { - finalizeCodexAuthSessionOnce(); + finalizeAuthSessionOnce(); logger.error( { err, processName, chatJid: input.chatJid, runId: input.runId }, 'Spawned agent process runner failed', diff --git a/src/token-refresh.test.ts b/src/token-refresh.test.ts index 46699bc..66d0384 100644 --- a/src/token-refresh.test.ts +++ b/src/token-refresh.test.ts @@ -2,9 +2,22 @@ import { describe, expect, it } from 'vitest'; import { applyUpdatedTokensToEnvContent, + pickFreshestOAuth, + shouldAdoptSessionOAuth, shouldStartTokenRefreshLoop, } from './token-refresh.js'; +function creds(expiresAt: number, refreshToken = 'rt') { + return { + claudeAiOauth: { + accessToken: `at-${expiresAt}`, + refreshToken, + expiresAt, + scopes: [] as string[], + }, + }; +} + describe('shouldStartTokenRefreshLoop', () => { it('starts refresh for the Claude service', () => { expect(shouldStartTokenRefreshLoop('claude-code')).toBe(true); @@ -42,3 +55,82 @@ describe('shouldStartTokenRefreshLoop', () => { ); }); }); + +describe('pickFreshestOAuth', () => { + it('adopts a session copy that expires later than the source', () => { + const best = pickFreshestOAuth([ + { label: 'source', creds: creds(1000) }, + { label: 'sessionA', creds: creds(5000) }, + { label: 'sessionB', creds: creds(3000) }, + ]); + expect(best?.label).toBe('sessionA'); + }); + + it('keeps the source on ties so we do not churn copies needlessly', () => { + const best = pickFreshestOAuth([ + { label: 'source', creds: creds(5000) }, + { label: 'sessionA', creds: creds(5000) }, + ]); + expect(best?.label).toBe('source'); + }); + + it('ignores candidates without a refresh token or numeric expiry', () => { + const best = pickFreshestOAuth([ + { label: 'source', creds: creds(1000) }, + { label: 'noRefresh', creds: creds(9000, '') }, + { label: 'missing', creds: null }, + ]); + expect(best?.label).toBe('source'); + }); + + it('returns null when no candidate is usable', () => { + expect( + pickFreshestOAuth([ + { label: 'source', creds: null }, + { label: 'noRefresh', creds: creds(9000, '') }, + ]), + ).toBeNull(); + }); +}); + +describe('shouldAdoptSessionOAuth (session→canonical write-back)', () => { + const oauth = (expiresAt: number, extra: Record = {}) => ({ + accessToken: `at-${expiresAt}`, + refreshToken: `rt-${expiresAt}`, + expiresAt, + scopes: [] as string[], + ...extra, + }); + + it('adopts a session token strictly newer than canonical', () => { + expect(shouldAdoptSessionOAuth(oauth(2000), oauth(1000))).toBe(true); + }); + + it('does not regress to an older or equal session token', () => { + expect(shouldAdoptSessionOAuth(oauth(1000), oauth(2000))).toBe(false); + expect(shouldAdoptSessionOAuth(oauth(2000), oauth(2000))).toBe(false); + }); + + it('adopts when canonical is missing/invalid', () => { + expect(shouldAdoptSessionOAuth(oauth(1000), null)).toBe(true); + }); + + it('rejects an invalid session token', () => { + expect(shouldAdoptSessionOAuth(null, oauth(1000))).toBe(false); + expect( + shouldAdoptSessionOAuth( + { accessToken: '', refreshToken: '', expiresAt: 5000, scopes: [] }, + oauth(1000), + ), + ).toBe(false); + }); + + it('refuses to cross subscription types', () => { + expect( + shouldAdoptSessionOAuth( + oauth(2000, { subscriptionType: 'pro' }), + oauth(1000, { subscriptionType: 'max' }), + ), + ).toBe(false); + }); +}); diff --git a/src/token-refresh.ts b/src/token-refresh.ts index a913785..7ed8ec0 100644 --- a/src/token-refresh.ts +++ b/src/token-refresh.ts @@ -12,7 +12,6 @@ * the .env file so new tokens survive restarts. */ import fs from 'fs'; -import os from 'os'; import { getErrorMessage, readJsonFile } from './utils.js'; import path from 'path'; @@ -20,6 +19,10 @@ import { logger } from './logger.js'; import { DATA_DIR } from './config.js'; import { getAllTokens, updateTokenValue } from './token-rotation.js'; import type { AgentType } from './types.js'; +import { + getClaudeCredentialsPath, + hasExplicitClaudeCredentialsPath, +} from './claude-credentials-path.js'; const TOKEN_URL = 'https://api.anthropic.com/v1/oauth/token'; const CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e'; @@ -30,11 +33,85 @@ const DEFAULT_SCOPES = [ 'user:mcp_servers', ]; -// Check every 5 minutes, refresh if within 30 minutes of expiry +// Check every 5 minutes. const CHECK_INTERVAL_MS = 5 * 60 * 1000; -const REFRESH_BEFORE_EXPIRY_MS = 30 * 60 * 1000; +// Refresh when within 60 minutes of expiry. This is intentionally EARLIER than +// a spawned Claude Code agent's own internal refresh threshold: by refreshing +// first and syncing the fresh token into every session dir, the app becomes the +// consistent "winner" of the token-family rotation and agents rarely need to +// rotate on their own. That shrinks the cross-process refresh race that was +// revoking the whole token family (see loadFreshestCredentials below). +const REFRESH_BEFORE_EXPIRY_MS = 60 * 60 * 1000; const REQUEST_TIMEOUT_MS = 15_000; +// After the refresh token itself has expired (or the endpoint keeps rejecting +// us), retrying every CHECK_INTERVAL_MS forever is pointless — only a manual +// browser re-login can recover. Keep trying for ~30 minutes, then give up and +// flag the account as needing manual re-login. Incoming requests can then be +// answered with a "please re-login" message instead of failing opaquely. +const GIVE_UP_AFTER_MS = 30 * 60 * 1000; + +/** User-facing message shown when an account needs a manual re-login. */ +export const RELOGIN_REQUIRED_MESSAGE = + '로그인 정보를 찾을 수 없습니다. 재로그인 해주세요.'; + +interface AuthFailureState { + firstFailureAt: number; + gaveUp: boolean; +} +// Per-account continuous-refresh-failure tracking. Cleared as soon as a valid +// token appears again (e.g. after a manual re-login rewrites the creds file). +const authFailureByAccount = new Map(); + +/** + * True when automatic refresh for this account has been failing for longer than + * GIVE_UP_AFTER_MS and we have stopped retrying. Request handlers use this to + * reply with RELOGIN_REQUIRED_MESSAGE instead of running a doomed agent. + */ +export function isReloginRequired(accountIndex = 0): boolean { + return authFailureByAccount.get(accountIndex)?.gaveUp === true; +} + +/** Clear failure tracking once a valid token is observed again. */ +function recordAuthSuccess(accountIndex: number): void { + if (authFailureByAccount.has(accountIndex)) { + authFailureByAccount.delete(accountIndex); + logger.info( + { accountIndex }, + 'Claude OAuth recovered — cleared re-login-required state', + ); + } +} + +/** + * Record a hard refresh failure. Returns true once we have given up (been + * failing for >= GIVE_UP_AFTER_MS), at which point the caller should stop + * hitting the network. + */ +function recordAuthFailure(accountIndex: number): boolean { + const now = Date.now(); + const state = authFailureByAccount.get(accountIndex); + if (!state) { + authFailureByAccount.set(accountIndex, { + firstFailureAt: now, + gaveUp: false, + }); + return false; + } + if (!state.gaveUp && now - state.firstFailureAt >= GIVE_UP_AFTER_MS) { + state.gaveUp = true; + logger.error( + { + accountIndex, + failingForMin: Math.round((now - state.firstFailureAt) / 60000), + }, + 'Claude OAuth refresh has failed for over 30 min — giving up automatic ' + + 'retries. Manual re-login required (run claude_relogin.sh).', + ); + } + return state.gaveUp; +} + export function shouldStartTokenRefreshLoop( serviceAgentType: AgentType, ): boolean { @@ -67,19 +144,25 @@ interface TokenResponse { * - Index 1+: ~/.claude-accounts/{index}/.credentials.json */ function getCredentialsPath(accountIndex: number): string { - if (accountIndex === 0) { - return path.join(os.homedir(), '.claude', '.credentials.json'); + const credsPath = getClaudeCredentialsPath(accountIndex, { + allowHomeFallback: + process.env.CLAUDE_TOKEN_REFRESH_USE_HOME_CREDENTIALS === 'true', + }); + if (!credsPath) { + throw new Error( + `No Claude credentials path configured for account ${accountIndex}`, + ); } - return path.join( - os.homedir(), - '.claude-accounts', - String(accountIndex), - '.credentials.json', - ); + return credsPath; } function readCredentials(accountIndex: number): CredentialsFile | null { - const credsPath = getCredentialsPath(accountIndex); + let credsPath: string; + try { + credsPath = getCredentialsPath(accountIndex); + } catch { + return null; + } if (!fs.existsSync(credsPath)) return null; const data = readJsonFile(credsPath); if (!data) { @@ -102,19 +185,34 @@ function writeCredentials(accountIndex: number, creds: CredentialsFile): void { syncToSessionDirs(credsPath); } -function syncToSessionDirs(credsPath: string): void { +/** + * Per-group session credential file paths under /sessions. Spawned + * agents run with CLAUDE_CONFIG_DIR pointing at one of these dirs and hold their + * own copy of the credentials, which Claude Code may refresh (rotate) on its own. + */ +function listSessionCredentialPaths(): string[] { const sessionsDir = path.join(DATA_DIR, 'sessions'); try { - if (!fs.existsSync(sessionsDir)) return; - const groups = fs.readdirSync(sessionsDir); - let synced = 0; - for (const group of groups) { - const dest = path.join( - sessionsDir, - group, - '.claude', - '.credentials.json', + if (!fs.existsSync(sessionsDir)) return []; + return fs + .readdirSync(sessionsDir) + .map((group) => + path.join(sessionsDir, group, '.claude', '.credentials.json'), ); + } catch (err) { + logger.warn( + { err: getErrorMessage(err) }, + 'Failed to enumerate session credential dirs', + ); + return []; + } +} + +function syncToSessionDirs(credsPath: string): void { + try { + let synced = 0; + for (const dest of listSessionCredentialPaths()) { + if (dest === credsPath) continue; if (fs.existsSync(path.dirname(dest))) { fs.copyFileSync(credsPath, dest); synced++; @@ -134,6 +232,157 @@ function syncToSessionDirs(credsPath: string): void { } } +/** + * Pure decision for the session→canonical write-back: adopt the session token + * only when it is a valid credential that is strictly newer than canonical (the + * session refreshed mid-turn) and does not cross subscription types. Never + * regress canonical to an older/equal token. + */ +export function shouldAdoptSessionOAuth( + session: OAuthCredentials | undefined | null, + canonical: OAuthCredentials | undefined | null, +): boolean { + if ( + !session?.accessToken || + !session?.refreshToken || + typeof session.expiresAt !== 'number' + ) { + return false; + } + if ( + canonical?.subscriptionType && + session.subscriptionType && + canonical.subscriptionType !== session.subscriptionType + ) { + return false; + } + if ( + canonical && + typeof canonical.expiresAt === 'number' && + session.expiresAt <= canonical.expiresAt + ) { + return false; + } + return true; +} + +/** + * Write-back (session → canonical). After a Claude agent turn, the child Claude + * CLI running with CLAUDE_CONFIG_DIR= may have refreshed + * (rotated) the OAuth token in its own copy. If that copy is strictly newer than + * the canonical credentials, adopt it so the canonical file always holds the + * latest rotated token. Without this the main refresh loop can later refresh an + * already-rotated token, which Anthropic rejects with invalid_grant and revokes + * the whole token family (forcing a manual re-login). Mirrors the Codex + * syncCodexSessionAuthBack write-back. Returns true if canonical was updated. + */ +export function syncClaudeSessionAuthBack( + sessionClaudeDir: string, + accountIndex = 0, +): boolean { + if (!sessionClaudeDir) return false; + const sessionPath = path.join(sessionClaudeDir, '.credentials.json'); + let sessionCreds: CredentialsFile | null = null; + try { + if (!fs.existsSync(sessionPath)) return false; + sessionCreds = JSON.parse( + fs.readFileSync(sessionPath, 'utf-8'), + ) as CredentialsFile; + } catch { + return false; + } + const sOauth = sessionCreds?.claudeAiOauth; + if (!sOauth) return false; + const canonical = readCredentials(accountIndex); + if (!shouldAdoptSessionOAuth(sOauth, canonical?.claudeAiOauth)) { + return false; + } + + // writeCredentials also fans the adopted token out to every session dir, so + // stale copies (old refresh tokens that could trigger family revocation) are + // overwritten with the current one. + writeCredentials(accountIndex, sessionCreds); + recordAuthSuccess(accountIndex); + logger.info( + { + accountIndex, + newExpiryMin: Math.round((sOauth.expiresAt - Date.now()) / 60000), + }, + 'Adopted refreshed Claude session token back into canonical credentials', + ); + return true; +} + +/** + * Pure selector: from a list of candidate credential files, pick the one whose + * access token expires latest (i.e. the most recently rotated, still-live + * member of the OAuth token family). Candidates without a refresh token or a + * numeric expiresAt are ignored. The first candidate should be the source file + * so that ties resolve in its favour (only a strictly-newer copy wins). + */ +export function pickFreshestOAuth( + candidates: Array<{ label: string; creds: CredentialsFile | null }>, +): { label: string; creds: CredentialsFile } | null { + let best: { label: string; creds: CredentialsFile } | null = null; + for (const candidate of candidates) { + const oauth = candidate.creds?.claudeAiOauth; + if (!oauth?.refreshToken || typeof oauth.expiresAt !== 'number') continue; + if (!best || oauth.expiresAt > best.creds.claudeAiOauth.expiresAt) { + best = { + label: candidate.label, + creds: candidate.creds as CredentialsFile, + }; + } + } + return best; +} + +/** + * Only the primary account (index 0 / the service's CLAUDE_CREDENTIALS_PATH) is + * mirrored into session dirs. Scanning them for other accounts would adopt the + * wrong token family. + */ +function isPrimarySessionAccount(accountIndex: number): boolean { + return accountIndex === 0; +} + +interface FreshestCredentials { + creds: CredentialsFile; + /** True when the freshest copy came from a session dir, not the source file. */ + adoptedFromSession: boolean; + label: string; +} + +/** + * Read the source credentials plus every session-dir copy and return the + * freshest (latest-expiring) one. A spawned agent may have already rotated the + * token family in its own session copy; adopting that live token instead of + * re-POSTing our now-stale refresh token is what prevents reuse-detection from + * revoking the entire family. + */ +function loadFreshestCredentials( + accountIndex: number, +): FreshestCredentials | null { + const candidates: Array<{ label: string; creds: CredentialsFile | null }> = [ + { label: 'source', creds: readCredentials(accountIndex) }, + ]; + if (isPrimarySessionAccount(accountIndex)) { + for (const sessionPath of listSessionCredentialPaths()) { + candidates.push({ + label: sessionPath, + creds: readJsonFile(sessionPath), + }); + } + } + const best = pickFreshestOAuth(candidates); + if (!best) return null; + return { + creds: best.creds, + adoptedFromSession: best.label !== 'source', + label: best.label, + }; +} + /** * Update CLAUDE_CODE_OAUTH_TOKENS in .env so refreshed tokens survive restarts. */ @@ -231,30 +480,87 @@ async function refreshToken( throw new Error('Token refresh failed'); } +// Serialize all refreshes within this process so the 5-minute loop and an +// on-demand forceRefreshToken() (triggered by a 401) can never rotate the same +// token family concurrently — a self-inflicted reuse that revokes the family. +let refreshMutex: Promise = Promise.resolve(); +function serializeRefresh(fn: () => Promise): Promise { + const next = refreshMutex.then(fn, fn); + // Keep the chain alive regardless of individual outcomes. + refreshMutex = next.then( + () => undefined, + () => undefined, + ); + return next; +} + /** * Check and refresh a single account's credentials. - * Returns the new access token if refreshed, null otherwise. + * Returns the (new or adopted) access token if it changed, null otherwise. */ async function checkAndRefreshAccount( accountIndex: number, opts?: { force?: boolean }, ): Promise { - const creds = readCredentials(accountIndex); - if (!creds?.claudeAiOauth) return null; + return serializeRefresh(() => doCheckAndRefreshAccount(accountIndex, opts)); +} - const { expiresAt, refreshToken: rt } = creds.claudeAiOauth; +async function doCheckAndRefreshAccount( + accountIndex: number, + opts?: { force?: boolean }, +): Promise { + const freshest = loadFreshestCredentials(accountIndex); + if (!freshest?.creds.claudeAiOauth) return null; + + const creds = freshest.creds; + const oauth = creds.claudeAiOauth; + const rt = oauth.refreshToken; if (!rt) { logger.debug({ accountIndex }, 'No refresh token in credentials, skipping'); return null; } - const now = Date.now(); - const remaining = expiresAt - now; + // A concurrent refresher (typically a spawned agent) may have already rotated + // the family in its session copy. Converge the source file onto that newer + // token so we never re-POST a superseded refresh token. + if (freshest.adoptedFromSession) { + writeCredentials(accountIndex, creds); + logger.info( + { + accountIndex, + remainingMin: Math.round((oauth.expiresAt - Date.now()) / 60000), + }, + 'Adopted newer Claude token from a concurrent refresher (skipped duplicate refresh)', + ); + } - if (!opts?.force && remaining > REFRESH_BEFORE_EXPIRY_MS) { + const now = Date.now(); + const remaining = oauth.expiresAt - now; + + if (remaining > REFRESH_BEFORE_EXPIRY_MS) { + // A valid token is present again (typically after a manual re-login rewrote + // the credentials file) — drop any prior "re-login required" state. + recordAuthSuccess(accountIndex); + if (!opts?.force && !freshest.adoptedFromSession) { + logger.debug( + { accountIndex, remainingMin: Math.round(remaining / 60000) }, + 'Token still valid, no refresh needed', + ); + } + // Token is valid; only report a token when it actually changed (adopted) or + // a caller forced the check and wants a usable token back to retry with. + return freshest.adoptedFromSession || opts?.force + ? oauth.accessToken + : null; + } + + // Already gave up on this account (refresh token dead). Stop hammering the + // endpoint every 5 min — a manual re-login is required, which the + // healthy-token branch above will detect and clear automatically. + if (isReloginRequired(accountIndex)) { logger.debug( - { accountIndex, remainingMin: Math.round(remaining / 60000) }, - 'Token still valid, no refresh needed', + { accountIndex }, + 'Skipping refresh — re-login required until a manual re-login is performed', ); return null; } @@ -271,17 +577,15 @@ async function checkAndRefreshAccount( ); try { - const response = await refreshToken( - rt, - creds.claudeAiOauth.scopes || DEFAULT_SCOPES, - ); + const response = await refreshToken(rt, oauth.scopes || DEFAULT_SCOPES); - creds.claudeAiOauth.accessToken = response.access_token; - creds.claudeAiOauth.refreshToken = response.refresh_token || rt; - creds.claudeAiOauth.expiresAt = now + response.expires_in * 1000; + recordAuthSuccess(accountIndex); + oauth.accessToken = response.access_token; + oauth.refreshToken = response.refresh_token || rt; + oauth.expiresAt = Date.now() + response.expires_in * 1000; if (response.scope) { - creds.claudeAiOauth.scopes = response.scope.split(' '); + oauth.scopes = response.scope.split(' '); } writeCredentials(accountIndex, creds); @@ -294,12 +598,40 @@ async function checkAndRefreshAccount( return response.access_token; } catch (err) { + // We may simply have lost the rotation race: another process refreshed the + // family a moment ago, so our token is now "expired". Re-scan every copy — + // if a valid token exists, adopt it instead of reporting a hard failure. + const recovered = loadFreshestCredentials(accountIndex); + const recoveredOauth = recovered?.creds.claudeAiOauth; + if ( + recoveredOauth && + recoveredOauth.expiresAt - Date.now() > REFRESH_BEFORE_EXPIRY_MS + ) { + if (recovered.adoptedFromSession) { + writeCredentials(accountIndex, recovered.creds); + } + recordAuthSuccess(accountIndex); + logger.warn( + { + accountIndex, + remainingMin: Math.round( + (recoveredOauth.expiresAt - Date.now()) / 60000, + ), + }, + 'Refresh failed but a concurrent refresher produced a valid token — adopted it', + ); + return recoveredOauth.accessToken; + } + const gaveUp = recordAuthFailure(accountIndex); logger.error( { accountIndex, + gaveUp, err: getErrorMessage(err), }, - 'Failed to refresh Claude OAuth token — manual re-login may be required', + gaveUp + ? 'Failed to refresh Claude OAuth token — giving up until manual re-login' + : 'Failed to refresh Claude OAuth token — manual re-login may be required', ); return null; } @@ -339,6 +671,23 @@ async function checkAndRefreshAll(): Promise { let refreshInterval: ReturnType | null = null; export function startTokenRefreshLoop(): void { + const refreshEnabled = + process.env.CLAUDE_TOKEN_REFRESH_USE_CREDENTIALS === 'true' || + process.env.CLAUDE_TOKEN_REFRESH_USE_HOME_CREDENTIALS === 'true'; + if (!refreshEnabled) { + logger.info( + 'Claude token auto-refresh disabled; EJClaw is using its dedicated OAuth token', + ); + return; + } + if ( + process.env.CLAUDE_TOKEN_REFRESH_USE_HOME_CREDENTIALS !== 'true' && + !hasExplicitClaudeCredentialsPath() + ) { + logger.info('Claude token auto-refresh disabled; no credentials path set'); + return; + } + const allTokens = getAllTokens(); // Check if any credentials files exist (for any configured account)