From 906a3dfadb0148f59e3c3c8e9e649e9bd52c5378 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Wed, 25 Mar 2026 03:56:03 +0900 Subject: [PATCH] config: add auto-compact settings for Claude and Codex agents Claude: CLAUDE_AUTOCOMPACT_PCT_OVERRIDE=50 (~500K on 1M context) Codex: model_auto_compact_token_limit=258000 (matching CLI default) --- .claude/settings.json | 3 + src/agent-error-detection.ts | 180 +++++++++++++++++++++++++++ src/agent-runner-environment.test.ts | 1 + src/agent-runner.test.ts | 1 + src/channels/discord.test.ts | 5 +- src/channels/discord.ts | 22 +--- src/codex-token-rotation.ts | 139 +++++++-------------- src/config.ts | 36 ++---- src/env.ts | 58 +++++++-- src/memento-client.ts | 7 +- src/message-agent-executor.ts | 100 ++++++--------- src/provider-fallback.test.ts | 18 ++- src/provider-fallback.ts | 98 +++++---------- src/task-scheduler.ts | 54 ++------ src/token-rotation-base.ts | 71 +++++++++++ src/token-rotation.ts | 70 ++++------- 16 files changed, 484 insertions(+), 379 deletions(-) create mode 100644 src/agent-error-detection.ts create mode 100644 src/token-rotation-base.ts diff --git a/.claude/settings.json b/.claude/settings.json index f859a6d..d155cbc 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,4 +1,7 @@ { + "env": { + "CLAUDE_AUTOCOMPACT_PCT_OVERRIDE": "50" + }, "extraKnownMarketplaces": { "nanoclaw-skills": { "source": { diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts new file mode 100644 index 0000000..c1c171a --- /dev/null +++ b/src/agent-error-detection.ts @@ -0,0 +1,180 @@ +/** + * Agent Error Detection (SSOT) + * + * Single source of truth for classifying agent errors from output text + * and error strings. Used by both message-agent-executor and task-scheduler. + */ + +// ── Banner / text detection ───────────────────────────────────── + +export function isClaudeAuthError(text: string): boolean { + const lower = text.toLowerCase(); + return ( + lower.includes('failed to authenticate') && + (lower.includes('401') || lower.includes('authentication_error')) + ); +} + +export function isClaudeUsageExhaustedMessage(text: string): boolean { + const normalized = text + .trim() + .toLowerCase() + .replace(/['\u2018\u2019`]/g, "'") + .replace(/\s+/g, ' ') + .replace(/^error:\s*/i, ''); + const looksLikeBanner = + normalized.startsWith("you're out of extra usage") || + normalized.startsWith('you are out of extra usage') || + normalized.startsWith("you've hit your limit") || + normalized.startsWith('you have hit your limit'); + const hasResetHint = + normalized.includes('resets ') || + normalized.includes('reset at ') || + normalized.includes('try again'); + return looksLikeBanner && hasResetHint && normalized.length <= 160; +} + +export function isClaudeAuthExpiredMessage(text: string): boolean { + const normalized = text.trim().toLowerCase().replace(/\s+/g, ' '); + const looksLikeAuthFailure = normalized.startsWith('failed to authenticate'); + const hasExpiredTokenMarker = + normalized.includes('oauth token has expired') || + normalized.includes('authentication_error') || + normalized.includes('obtain a new token') || + normalized.includes('refresh your existing token') || + normalized.includes('invalid authentication credentials'); + const hasUnauthorizedMarker = + normalized.includes('401') || normalized.includes('authentication error'); + const hasTerminatedMarker = normalized.includes('terminated'); + + return ( + looksLikeAuthFailure && + hasUnauthorizedMarker && + (hasExpiredTokenMarker || hasTerminatedMarker) + ); +} + +// ── Rotation decision ─────────────────────────────────────────── + +export function shouldRotateClaudeToken(reason: string): boolean { + return ( + reason === '429' || + reason === 'usage-exhausted' || + reason === 'auth-expired' + ); +} + +// ── Unified error classification ──────────────────────────────── + +export type ErrorCategory = + | 'rate-limit' + | 'auth-expired' + | 'overloaded' + | 'network-error' + | 'none'; + +export interface AgentErrorClassification { + category: ErrorCategory; + reason: string; // '429' | 'auth-expired' | 'overloaded' | 'network-error' | '' + retryAfterMs?: number; +} + +const NONE: AgentErrorClassification = { + category: 'none', + reason: '', +}; + +/** + * Classify an agent error string into a category. + * Handles patterns common to both Claude and Codex: 429, 503, network. + * Auth errors are provider-specific — use classifyClaudeAuthError or + * classifyCodexAuthError for those. + */ +export function classifyAgentError( + error: string | null | undefined, +): AgentErrorClassification { + if (!error) return NONE; + + const lower = error.toLowerCase(); + + // 429 / Rate Limit + if ( + lower.includes('429') || + lower.includes('rate limit') || + lower.includes('usage limit') || + lower.includes('hit your limit') || + lower.includes('too many requests') || + lower.includes('rate_limit') + ) { + const retryMatch = error.match(/retry[\s_-]*after[:\s]*(\d+)/i); + const retryAfterMs = retryMatch + ? parseInt(retryMatch[1], 10) * 1000 + : undefined; + return { category: 'rate-limit', reason: '429', retryAfterMs }; + } + + // 503 / Overloaded + if (lower.includes('503') || lower.includes('overloaded')) { + return { category: 'overloaded', reason: 'overloaded' }; + } + + // Network / connection errors + if ( + lower.includes('econnrefused') || + lower.includes('econnreset') || + lower.includes('etimedout') || + lower.includes('enotfound') || + lower.includes('fetch failed') || + lower.includes('network error') + ) { + return { category: 'network-error', reason: 'network-error' }; + } + + return NONE; +} + +// ── Provider-specific auth checks ─────────────────────────────── + +/** Claude auth: strict 3-condition AND (auth failure + 401 + specific marker). */ +export function classifyClaudeAuthError( + error: string | null | undefined, +): AgentErrorClassification { + if (!error) return NONE; + const lower = error.toLowerCase(); + + if ( + (lower.includes('failed to authenticate') || + lower.includes('authentication_error')) && + (lower.includes('401') || lower.includes('unauthorized')) && + (lower.includes('oauth token has expired') || + lower.includes('obtain a new token') || + lower.includes('refresh your existing token') || + lower.includes('invalid authentication credentials') || + lower.includes('terminated')) + ) { + return { category: 'auth-expired', reason: 'auth-expired' }; + } + + return NONE; +} + +/** Codex auth: loose OR check (any single auth indicator). */ +export function classifyCodexAuthError( + error: string | null | undefined, +): AgentErrorClassification { + if (!error) return NONE; + const lower = error.toLowerCase(); + + if ( + lower.includes('401') || + lower.includes('authentication_error') || + lower.includes('failed to authenticate') || + lower.includes('oauth token has expired') || + lower.includes('refresh your existing token') || + lower.includes('unauthorized') + ) { + return { category: 'auth-expired', reason: 'auth-expired' }; + } + + return NONE; +} diff --git a/src/agent-runner-environment.test.ts b/src/agent-runner-environment.test.ts index d358961..67b9d7b 100644 --- a/src/agent-runner-environment.test.ts +++ b/src/agent-runner-environment.test.ts @@ -19,6 +19,7 @@ vi.mock('./db.js', () => ({ vi.mock('./env.js', () => ({ readEnvFile: mockReadEnvFile, + getEnv: vi.fn((key: string) => undefined), })); vi.mock('./codex-token-rotation.js', () => ({ diff --git a/src/agent-runner.test.ts b/src/agent-runner.test.ts index 60238c2..db7d4e6 100644 --- a/src/agent-runner.test.ts +++ b/src/agent-runner.test.ts @@ -54,6 +54,7 @@ vi.mock('fs', async () => { // Mock env vi.mock('./env.js', () => ({ readEnvFile: vi.fn(() => ({})), + getEnv: vi.fn(() => undefined), })); // Create a controllable fake ChildProcess diff --git a/src/channels/discord.test.ts b/src/channels/discord.test.ts index e6c9a7f..ec23e1d 100644 --- a/src/channels/discord.test.ts +++ b/src/channels/discord.test.ts @@ -6,7 +6,10 @@ import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; vi.mock('./registry.js', () => ({ registerChannel: vi.fn() })); // Mock env reader (used by the factory, not needed in unit tests) -vi.mock('../env.js', () => ({ readEnvFile: vi.fn(() => ({})) })); +vi.mock('../env.js', () => ({ + readEnvFile: vi.fn(() => ({})), + getEnv: vi.fn(() => undefined), +})); // Mock config vi.mock('../config.js', () => ({ diff --git a/src/channels/discord.ts b/src/channels/discord.ts index 8a76773..184c3fc 100644 --- a/src/channels/discord.ts +++ b/src/channels/discord.ts @@ -18,7 +18,7 @@ import { TRIGGER_PATTERN, } from '../config.js'; import { isPairedRoomJid } from '../db.js'; -import { readEnvFile } from '../env.js'; +import { getEnv } from '../env.js'; import { logger } from '../logger.js'; import { formatOutbound } from '../router.js'; @@ -95,10 +95,8 @@ async function transcribeAudio(att: Attachment): Promise { const filename = att.name || 'audio.ogg'; // Pick provider: Groq (fast) > OpenAI (fallback) - const envVars = readEnvFile(['GROQ_API_KEY', 'OPENAI_API_KEY']); - const groqKey = process.env.GROQ_API_KEY || envVars.GROQ_API_KEY || ''; - const openaiKey = - process.env.OPENAI_API_KEY || envVars.OPENAI_API_KEY || ''; + const groqKey = getEnv('GROQ_API_KEY') || ''; + const openaiKey = getEnv('OPENAI_API_KEY') || ''; let apiUrl: string; let apiKeyToUse: string; @@ -674,17 +672,13 @@ export class DiscordChannel implements Channel { } registerChannel('discord', (opts: ChannelOpts) => { - const envVars = readEnvFile(['DISCORD_BOT_TOKEN', 'DISCORD_CODEX_BOT_TOKEN']); - const token = - process.env.DISCORD_BOT_TOKEN || envVars.DISCORD_BOT_TOKEN || ''; + const token = getEnv('DISCORD_BOT_TOKEN') || ''; if (!token) { logger.warn('Discord: DISCORD_BOT_TOKEN not set'); return null; } // If a second Codex bot token exists, this instance only handles claude-code groups - const hasCodexBot = !!( - process.env.DISCORD_CODEX_BOT_TOKEN || envVars.DISCORD_CODEX_BOT_TOKEN - ); + const hasCodexBot = !!getEnv('DISCORD_CODEX_BOT_TOKEN'); return new DiscordChannel( token, opts, @@ -696,11 +690,7 @@ registerChannel('discord', (opts: ChannelOpts) => { // The codex service uses its own DISCORD_BOT_TOKEN via systemd EnvironmentFile override. if ((process.env.ASSISTANT_NAME || 'claude') !== 'codex') { registerChannel('discord-codex', (opts: ChannelOpts) => { - const envVars = readEnvFile(['DISCORD_CODEX_BOT_TOKEN']); - const token = - process.env.DISCORD_CODEX_BOT_TOKEN || - envVars.DISCORD_CODEX_BOT_TOKEN || - ''; + const token = getEnv('DISCORD_CODEX_BOT_TOKEN') || ''; if (!token) return null; // Codex Discord bot is optional return new DiscordChannel(token, opts, 'codex'); }); diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 8d58501..dec08d3 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -14,8 +14,17 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; +import { + classifyAgentError, + classifyCodexAuthError, +} from './agent-error-detection.js'; import { DATA_DIR } from './config.js'; import { logger } from './logger.js'; +import { + computeCooldownUntil, + findNextAvailable, + parseRetryAfterFromError, +} from './token-rotation-base.js'; const STATE_FILE = path.join(DATA_DIR, 'codex-rotation-state.json'); @@ -195,35 +204,6 @@ function loadCodexState(): void { } } -const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time -const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback - -/** - * Parse "try again at Mar 26th, 2026 9:00 AM" from error message. - * Returns timestamp in ms, or null if not found. - */ -function parseRetryAfterFromError(error?: string): number | null { - if (!error) return null; - const match = error.match( - /try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, - ); - if (!match) return null; - try { - // Remove ordinal suffixes (1st, 2nd, 3rd, 4th) - const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); - const ts = new Date(cleaned).getTime(); - if (Number.isNaN(ts)) return null; - return ts; - } catch { - return null; - } -} - -function computeCooldownUntil(error?: string): number { - const retryAt = parseRetryAfterFromError(error); - if (retryAt) return retryAt + BUFFER_MS; - return Date.now() + DEFAULT_COOLDOWN_MS; -} /** Get the auth.json path for the current active account. */ export function getActiveCodexAuthPath(): string | null { @@ -236,43 +216,16 @@ export function detectCodexRotationTrigger( ): CodexRotationTriggerResult { if (!error) return { shouldRotate: false, reason: '' }; - const lower = error.toLowerCase(); - - if ( - lower.includes('429') || - lower.includes('rate limit') || - lower.includes('usage limit') || - lower.includes('hit your limit') || - lower.includes('too many requests') || - lower.includes('rate_limit') - ) { - return { shouldRotate: true, reason: '429' }; + // Common patterns (429, 503, network) — delegated to SSOT + const common = classifyAgentError(error); + if (common.category !== 'none') { + return { shouldRotate: true, reason: common.reason }; } - if (lower.includes('503') || lower.includes('overloaded')) { - return { shouldRotate: true, reason: 'overloaded' }; - } - - if ( - lower.includes('econnrefused') || - lower.includes('econnreset') || - lower.includes('etimedout') || - lower.includes('enotfound') || - lower.includes('fetch failed') || - lower.includes('network error') - ) { - return { shouldRotate: true, reason: 'network-error' }; - } - - if ( - lower.includes('401') || - lower.includes('authentication_error') || - lower.includes('failed to authenticate') || - lower.includes('oauth token has expired') || - lower.includes('refresh your existing token') || - lower.includes('unauthorized') - ) { - return { shouldRotate: true, reason: 'auth-expired' }; + // Codex-specific loose auth check + const auth = classifyCodexAuthError(error); + if (auth.category !== 'none') { + return { shouldRotate: true, reason: auth.reason }; } return { shouldRotate: false, reason: '' }; @@ -282,36 +235,35 @@ export function detectCodexRotationTrigger( * Try to rotate to the next available Codex account. * Returns true if a fresh account was found. */ -export function rotateCodexToken(errorMessage?: string): boolean { +export function rotateCodexToken( + errorMessage?: string, + opts?: { ignoreRateLimits?: boolean }, +): boolean { if (accounts.length <= 1) return false; - const now = Date.now(); const acct = accounts[currentIndex]; acct.rateLimitedUntil = computeCooldownUntil(errorMessage); acct.lastUsagePct = 100; // Extract reset time string from error for display - const resetMatch = errorMessage?.match( - /try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, - ); - if (resetMatch) acct.resetAt = resetMatch[1]; + const retryAt = parseRetryAfterFromError(errorMessage); + if (retryAt) { + acct.resetAt = new Date(retryAt).toISOString(); + } - for (let i = 1; i < accounts.length; i++) { - const idx = (currentIndex + i) % accounts.length; - const acct = accounts[idx]; - if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) { - acct.rateLimitedUntil = null; - currentIndex = idx; - logger.info( - { - accountIndex: currentIndex, - totalAccounts: accounts.length, - accountId: acct.accountId, - }, - `Codex rotated to account #${currentIndex + 1}/${accounts.length}`, - ); - saveCodexState(); - return true; - } + const nextIdx = findNextAvailable(accounts, currentIndex, opts); + if (nextIdx !== null) { + accounts[nextIdx].rateLimitedUntil = null; + currentIndex = nextIdx; + logger.info( + { + accountIndex: currentIndex, + totalAccounts: accounts.length, + accountId: accounts[nextIdx].accountId, + }, + `Codex rotated to account #${currentIndex + 1}/${accounts.length}`, + ); + saveCodexState(); + return true; } logger.warn('All Codex accounts are rate-limited'); @@ -325,15 +277,10 @@ export function rotateCodexToken(errorMessage?: string): boolean { */ export function advanceCodexAccount(): void { if (accounts.length <= 1) return; - const now = Date.now(); - for (let i = 1; i < accounts.length; i++) { - const idx = (currentIndex + i) % accounts.length; - const acct = accounts[idx]; - if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) { - currentIndex = idx; - saveCodexState(); - return; - } + const nextIdx = findNextAvailable(accounts, currentIndex); + if (nextIdx !== null) { + currentIndex = nextIdx; + saveCodexState(); } // All others rate-limited, stay on current } diff --git a/src/config.ts b/src/config.ts index 67cdbd7..39a95a6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,29 +2,14 @@ import os from 'os'; import path from 'path'; import type { AgentType } from './types.js'; -import { readEnvFile } from './env.js'; +import { getEnv } from './env.js'; -const envConfig = readEnvFile([ - 'ASSISTANT_NAME', - 'ASSISTANT_HAS_OWN_NUMBER', - 'SERVICE_ID', - 'SERVICE_AGENT_TYPE', - 'SESSION_COMMAND_ALLOWED_SENDERS', - 'SESSION_COMMAND_USER_IDS', - 'STATUS_SHOW_ROOMS', - 'USAGE_DASHBOARD', -]); - -export const ASSISTANT_NAME = - process.env.ASSISTANT_NAME || envConfig.ASSISTANT_NAME || 'Andy'; +export const ASSISTANT_NAME = getEnv('ASSISTANT_NAME') || 'Andy'; export const ASSISTANT_HAS_OWN_NUMBER = - (process.env.ASSISTANT_HAS_OWN_NUMBER || - envConfig.ASSISTANT_HAS_OWN_NUMBER) === 'true'; + getEnv('ASSISTANT_HAS_OWN_NUMBER') === 'true'; const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase(); -const rawServiceAgentType = - process.env.SERVICE_AGENT_TYPE || envConfig.SERVICE_AGENT_TYPE; -export const SERVICE_ID = - process.env.SERVICE_ID || envConfig.SERVICE_ID || ASSISTANT_SLUG; +const rawServiceAgentType = getEnv('SERVICE_AGENT_TYPE'); +export const SERVICE_ID = getEnv('SERVICE_ID') || ASSISTANT_SLUG; export const SERVICE_AGENT_TYPE: AgentType = rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code' ? rawServiceAgentType @@ -84,10 +69,9 @@ export const STATUS_CHANNEL_ID = process.env.STATUS_CHANNEL_ID || ''; export const STATUS_UPDATE_INTERVAL = 10000; // 10s export const USAGE_UPDATE_INTERVAL = 300000; // 5 minutes export const STATUS_SHOW_ROOMS = - (process.env.STATUS_SHOW_ROOMS || envConfig.STATUS_SHOW_ROOMS || 'true') !== - 'false'; + (getEnv('STATUS_SHOW_ROOMS') || 'true') !== 'false'; export const USAGE_DASHBOARD_ENABLED = - (process.env.USAGE_DASHBOARD || envConfig.USAGE_DASHBOARD) === 'true'; + getEnv('USAGE_DASHBOARD') === 'true'; // Timezone for scheduled tasks (cron expressions, etc.) // Uses system timezone by default @@ -95,10 +79,8 @@ export const TIMEZONE = process.env.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone; const rawSessionCommandAllowedSenders = - process.env.SESSION_COMMAND_ALLOWED_SENDERS || - process.env.SESSION_COMMAND_USER_IDS || - envConfig.SESSION_COMMAND_ALLOWED_SENDERS || - envConfig.SESSION_COMMAND_USER_IDS || + getEnv('SESSION_COMMAND_ALLOWED_SENDERS') || + getEnv('SESSION_COMMAND_USER_IDS') || ''; const SESSION_COMMAND_ALLOWED_SENDERS = new Set( diff --git a/src/env.ts b/src/env.ts index 988b59e..7e349c4 100644 --- a/src/env.ts +++ b/src/env.ts @@ -2,13 +2,12 @@ import fs from 'fs'; import path from 'path'; import { logger } from './logger.js'; -/** - * Parse the .env file and return values for the requested keys. - * Does NOT load anything into process.env — callers decide what to - * do with the values. This keeps secrets out of the process environment - * so they don't leak to child processes. - */ -export function readEnvFile(keys: string[]): Record { +// ── Internal cache ────────────────────────────────────────────── + +let _cache: Record | null = null; + +/** Parse the entire .env file into a Record (no key filtering). */ +function parseEnvFile(): Record { const envFile = path.join(process.cwd(), '.env'); let content: string; try { @@ -19,15 +18,12 @@ export function readEnvFile(keys: string[]): Record { } const result: Record = {}; - const wanted = new Set(keys); - for (const line of content.split('\n')) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#')) continue; const eqIdx = trimmed.indexOf('='); if (eqIdx === -1) continue; const key = trimmed.slice(0, eqIdx).trim(); - if (!wanted.has(key)) continue; let value = trimmed.slice(eqIdx + 1).trim(); if ( (value.startsWith('"') && value.endsWith('"')) || @@ -37,6 +33,46 @@ export function readEnvFile(keys: string[]): Record { } if (value) result[key] = value; } - + return result; +} + +// ── Public API ────────────────────────────────────────────────── + +/** Load (or reload) the .env file into the in-memory cache. */ +export function loadEnvFile(): void { + _cache = parseEnvFile(); +} + +/** + * Look up a single env value. + * Priority: process.env > .env cache > undefined + */ +export function getEnv(key: string): string | undefined { + if (!_cache) loadEnvFile(); + return process.env[key] || _cache![key] || undefined; +} + +/** Force-reload the .env file (e.g. after token refresh writes new values). */ +export function reloadEnvFile(): void { + _cache = null; + loadEnvFile(); +} + +/** + * Parse the .env file and return values for the requested keys. + * Does NOT load anything into process.env — callers decide what to + * do with the values. This keeps secrets out of the process environment + * so they don't leak to child processes. + * + * Now backed by the in-memory cache (disk read happens at most once). + */ +export function readEnvFile(keys: string[]): Record { + if (!_cache) loadEnvFile(); + + const result: Record = {}; + const wanted = new Set(keys); + for (const [key, value] of Object.entries(_cache!)) { + if (wanted.has(key)) result[key] = value; + } return result; } diff --git a/src/memento-client.ts b/src/memento-client.ts index 092755d..5c696c7 100644 --- a/src/memento-client.ts +++ b/src/memento-client.ts @@ -1,7 +1,7 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js'; -import { readEnvFile } from './env.js'; +import { getEnv } from './env.js'; import { logger } from './logger.js'; const DEFAULT_TIMEOUT_MS = 4_000; @@ -42,9 +42,8 @@ let cachedConfig: MementoConfig | null | undefined; function getMementoConfig(): MementoConfig | null { if (cachedConfig !== undefined) return cachedConfig; - const env = readEnvFile(['MEMENTO_MCP_SSE_URL', 'MEMENTO_ACCESS_KEY']); - const sseUrl = process.env.MEMENTO_MCP_SSE_URL || env.MEMENTO_MCP_SSE_URL; - const accessKey = process.env.MEMENTO_ACCESS_KEY || env.MEMENTO_ACCESS_KEY; + const sseUrl = getEnv('MEMENTO_MCP_SSE_URL'); + const accessKey = getEnv('MEMENTO_ACCESS_KEY'); cachedConfig = sseUrl && accessKey diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 110d32b..c014f68 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -12,6 +12,12 @@ import { getAllTasks } from './db.js'; import { GroupQueue } from './group-queue.js'; import { logger } from './logger.js'; import { buildRoomMemoryBriefing } from './memento-client.js'; +import { + isClaudeAuthError, + isClaudeAuthExpiredMessage, + isClaudeUsageExhaustedMessage, + shouldRotateClaudeToken, +} from './agent-error-detection.js'; import { detectFallbackTrigger, getActiveProvider, @@ -45,53 +51,6 @@ export interface MessageAgentExecutorDeps { clearSession: (groupFolder: string) => void; } -function isClaudeAuthError(text: string): boolean { - const lower = text.toLowerCase(); - return ( - lower.includes('failed to authenticate') && - (lower.includes('401') || lower.includes('authentication_error')) - ); -} - -function isClaudeUsageExhaustedMessage(text: string): boolean { - const normalized = text - .trim() - .toLowerCase() - .replace(/[’‘`]/g, "'") - .replace(/\s+/g, ' ') - .replace(/^error:\s*/i, ''); - const looksLikeBanner = - normalized.startsWith("you're out of extra usage") || - normalized.startsWith('you are out of extra usage') || - normalized.startsWith("you've hit your limit") || - normalized.startsWith('you have hit your limit'); - const hasResetHint = - normalized.includes('resets ') || - normalized.includes('reset at ') || - normalized.includes('try again'); - return looksLikeBanner && hasResetHint && normalized.length <= 160; -} - -function isClaudeAuthExpiredMessage(text: string): boolean { - const normalized = text.trim().toLowerCase().replace(/\s+/g, ' '); - const looksLikeAuthFailure = normalized.startsWith('failed to authenticate'); - const hasExpiredTokenMarker = - normalized.includes('oauth token has expired') || - normalized.includes('authentication_error') || - normalized.includes('obtain a new token') || - normalized.includes('refresh your existing token') || - normalized.includes('invalid authentication credentials'); - const hasUnauthorizedMarker = - normalized.includes('401') || normalized.includes('authentication error'); - const hasTerminatedMarker = normalized.includes('terminated'); - - return ( - looksLikeAuthFailure && - hasUnauthorizedMarker && - (hasExpiredTokenMarker || hasTerminatedMarker) - ); -} - export async function runAgentForGroup( deps: MessageAgentExecutorDeps, args: { @@ -146,6 +105,7 @@ export async function runAgentForGroup( 'settings.json', ); const groupHasOverride = hasGroupProviderOverride(settingsPath); + const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1; const canFallback = isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride; @@ -195,7 +155,7 @@ export async function runAgentForGroup( resetSessionRequested = true; } if ( - canFallback && + isClaudeCodeAgent && provider === 'claude' && output.status === 'success' && !sawOutput && @@ -265,7 +225,7 @@ export async function runAgentForGroup( reason: trigger.reason, retryAfterMs: trigger.retryAfterMs, }; - if (canFallback) { + if (canFallback || canRotateToken) { return; } } @@ -413,11 +373,6 @@ export async function runAgentForGroup( return 'success'; }; - const shouldRotateClaudeToken = (reason: string): boolean => - reason === '429' || - reason === 'usage-exhausted' || - reason === 'auth-expired'; - const retryCodexWithRotation = async ( initialTrigger: { reason: string }, rotationMessage?: string, @@ -618,7 +573,9 @@ export async function runAgentForGroup( !retryAttempt.sawOutput && retryAttempt.sawSuccessNullResultWithoutOutput ) { - return runFallbackAttempt('success-null-result'); + return canFallback + ? runFallbackAttempt('success-null-result') + : 'error'; } if (retryOutput.status === 'error') { @@ -657,12 +614,23 @@ export async function runAgentForGroup( return 'success'; } - // Usage exhausted: don't fall back to Kimi — log only, no response - if (trigger.reason === 'usage-exhausted') { + // Usage exhausted or auth-expired: don't fall back to Kimi — log only + if ( + trigger.reason === 'usage-exhausted' || + trigger.reason === 'auth-expired' + ) { markPrimaryCooldown(trigger.reason, trigger.retryAfterMs); logger.info( - { chatJid, group: group.name, runId }, - 'All Claude tokens usage-exhausted, silently skipping (no Kimi fallback)', + { chatJid, group: group.name, runId, reason: trigger.reason }, + `All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`, + ); + return 'error'; + } + + if (!canFallback) { + logger.warn( + { chatJid, group: group.name, runId, reason: trigger.reason }, + 'All Claude tokens exhausted and fallback disabled', ); return 'error'; } @@ -684,7 +652,11 @@ export async function runAgentForGroup( const primaryAttempt = await runAttempt(provider); if (primaryAttempt.error) { - if (canFallback && provider === 'claude' && !primaryAttempt.sawOutput) { + if ( + (canFallback || canRotateToken) && + provider === 'claude' && + !primaryAttempt.sawOutput + ) { const errMsg = primaryAttempt.error instanceof Error ? primaryAttempt.error.message @@ -748,7 +720,7 @@ export async function runAgentForGroup( } if ( - canFallback && + (canFallback || canRotateToken) && provider === 'claude' && !primaryAttempt.sawOutput && primaryAttempt.streamedTriggerReason && @@ -781,7 +753,11 @@ export async function runAgentForGroup( } if (output.status === 'error') { - if (canFallback && provider === 'claude' && !primaryAttempt.sawOutput) { + if ( + (canFallback || canRotateToken) && + provider === 'claude' && + !primaryAttempt.sawOutput + ) { const trigger = primaryAttempt.streamedTriggerReason ? { shouldFallback: true, diff --git a/src/provider-fallback.test.ts b/src/provider-fallback.test.ts index d21a2c2..393ec1a 100644 --- a/src/provider-fallback.test.ts +++ b/src/provider-fallback.test.ts @@ -4,16 +4,26 @@ vi.mock('./claude-usage.js', () => ({ fetchClaudeUsage: vi.fn(), })); -vi.mock('./env.js', () => ({ - readEnvFile: vi.fn(() => ({ +vi.mock('./env.js', () => { + const store: Record = { FALLBACK_PROVIDER_NAME: 'kimi', FALLBACK_BASE_URL: 'https://api.kimi.com/coding/', FALLBACK_AUTH_TOKEN: 'test-kimi-key', FALLBACK_MODEL: 'kimi-k2.5', FALLBACK_SMALL_MODEL: 'kimi-k2.5', FALLBACK_COOLDOWN_MS: '600000', - })), -})); + }; + return { + readEnvFile: vi.fn((keys: string[]) => { + const result: Record = {}; + for (const k of keys) { + if (store[k]) result[k] = store[k]; + } + return result; + }), + getEnv: vi.fn((key: string) => store[key]), + }; +}); vi.mock('./logger.js', () => ({ logger: { diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index 71867b5..d7211c4 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -14,8 +14,12 @@ import fs from 'fs'; +import { + classifyAgentError, + classifyClaudeAuthError, +} from './agent-error-detection.js'; import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js'; -import { readEnvFile } from './env.js'; +import { getEnv } from './env.js'; import { logger } from './logger.js'; import { rotateToken, getTokenCount } from './token-rotation.js'; @@ -65,33 +69,21 @@ let _config: FallbackConfig | null = null; function loadConfig(): FallbackConfig { if (_config) return _config; - const env = readEnvFile([ - 'FALLBACK_PROVIDER_NAME', - 'FALLBACK_BASE_URL', - 'FALLBACK_AUTH_TOKEN', - 'FALLBACK_MODEL', - 'FALLBACK_SMALL_MODEL', - 'FALLBACK_COOLDOWN_MS', - ]); - - const baseUrl = process.env.FALLBACK_BASE_URL || env.FALLBACK_BASE_URL || ''; - const authToken = - process.env.FALLBACK_AUTH_TOKEN || env.FALLBACK_AUTH_TOKEN || ''; - const model = process.env.FALLBACK_MODEL || env.FALLBACK_MODEL || ''; + const baseUrl = getEnv('FALLBACK_BASE_URL') || ''; + const authToken = getEnv('FALLBACK_AUTH_TOKEN') || ''; + const model = getEnv('FALLBACK_MODEL') || ''; + const explicitlyDisabled = + (getEnv('FALLBACK_ENABLED') || '').toLowerCase() === 'false'; _config = { - enabled: Boolean(baseUrl && authToken && model), - providerName: - process.env.FALLBACK_PROVIDER_NAME || - env.FALLBACK_PROVIDER_NAME || - 'kimi', + enabled: !explicitlyDisabled && Boolean(baseUrl && authToken && model), + providerName: getEnv('FALLBACK_PROVIDER_NAME') || 'kimi', baseUrl, authToken, model, - smallModel: - process.env.FALLBACK_SMALL_MODEL || env.FALLBACK_SMALL_MODEL || model, + smallModel: getEnv('FALLBACK_SMALL_MODEL') || model, defaultCooldownMs: parseInt( - process.env.FALLBACK_COOLDOWN_MS || env.FALLBACK_COOLDOWN_MS || '600000', + getEnv('FALLBACK_COOLDOWN_MS') || '600000', 10, ), }; @@ -359,53 +351,31 @@ export function detectFallbackTrigger( ): FallbackTriggerResult { if (!error) return { shouldFallback: false, reason: '' }; - const lower = error.toLowerCase(); + // Delegated to shared SSOT — original priority preserved: + // 429 first, then auth-expired, then 503/network + const common = classifyAgentError(error); - // 429 Rate Limit - if ( - lower.includes('429') || - lower.includes('rate limit') || - lower.includes('usage limit') || - lower.includes('hit your limit') || - lower.includes('too many requests') || - lower.includes('rate_limit') - ) { - // Try to extract retry-after value (seconds → ms) - const retryMatch = error.match(/retry[\s_-]*after[:\s]*(\d+)/i); - const retryAfterMs = retryMatch - ? parseInt(retryMatch[1], 10) * 1000 - : undefined; - return { shouldFallback: true, reason: '429', retryAfterMs }; + // 429 rate-limit (highest priority) + if (common.category === 'rate-limit') { + return { + shouldFallback: true, + reason: common.reason, + retryAfterMs: common.retryAfterMs, + }; } - if ( - (lower.includes('failed to authenticate') || - lower.includes('authentication_error')) && - (lower.includes('401') || lower.includes('unauthorized')) && - (lower.includes('oauth token has expired') || - lower.includes('obtain a new token') || - lower.includes('refresh your existing token') || - lower.includes('invalid authentication credentials') || - lower.includes('terminated')) - ) { - return { shouldFallback: true, reason: 'auth-expired' }; + // Claude-specific strict auth check (before 503/network) + const auth = classifyClaudeAuthError(error); + if (auth.category !== 'none') { + return { shouldFallback: true, reason: auth.reason }; } - // 503 Overloaded - if (lower.includes('503') || lower.includes('overloaded')) { - return { shouldFallback: true, reason: 'overloaded' }; - } - - // Network / connection errors - if ( - lower.includes('econnrefused') || - lower.includes('econnreset') || - lower.includes('etimedout') || - lower.includes('enotfound') || - lower.includes('fetch failed') || - lower.includes('network error') - ) { - return { shouldFallback: true, reason: 'network-error' }; + // 503 overloaded, network errors + if (common.category !== 'none') { + return { + shouldFallback: true, + reason: common.reason, + }; } return { shouldFallback: false, reason: '' }; diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 7272296..40fe184 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -31,6 +31,11 @@ import { } from './group-folder.js'; import { logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; +import { + isClaudeAuthExpiredMessage, + isClaudeUsageExhaustedMessage, + shouldRotateClaudeToken, +} from './agent-error-detection.js'; import { detectFallbackTrigger, getActiveProvider, @@ -73,44 +78,6 @@ export { shouldUseTaskScopedSession, } from './task-watch-status.js'; -function isClaudeUsageExhaustedMessage(text: string): boolean { - const normalized = text - .trim() - .toLowerCase() - .replace(/[’‘`]/g, "'") - .replace(/\s+/g, ' ') - .replace(/^error:\s*/i, ''); - const looksLikeBanner = - normalized.startsWith("you're out of extra usage") || - normalized.startsWith('you are out of extra usage') || - normalized.startsWith("you've hit your limit") || - normalized.startsWith('you have hit your limit'); - const hasResetHint = - normalized.includes('resets ') || - normalized.includes('reset at ') || - normalized.includes('try again'); - return looksLikeBanner && hasResetHint && normalized.length <= 160; -} - -function isClaudeAuthExpiredMessage(text: string): boolean { - const normalized = text.trim().toLowerCase().replace(/\s+/g, ' '); - const looksLikeAuthFailure = normalized.startsWith('failed to authenticate'); - const hasExpiredTokenMarker = - normalized.includes('oauth token has expired') || - normalized.includes('authentication_error') || - normalized.includes('obtain a new token') || - normalized.includes('refresh your existing token') || - normalized.includes('invalid authentication credentials'); - const hasUnauthorizedMarker = - normalized.includes('401') || normalized.includes('authentication error'); - const hasTerminatedMarker = normalized.includes('terminated'); - - return ( - looksLikeAuthFailure && - hasUnauthorizedMarker && - (hasExpiredTokenMarker || hasTerminatedMarker) - ); -} /** * Compute the next run time for a recurring task, anchored to the @@ -307,8 +274,10 @@ async function runTask( '.claude', 'settings.json', ); + const isClaudeAgent = context.taskAgentType === 'claude-code'; + const canRotateToken = isClaudeAgent && getTokenCount() > 1; const canFallback = - context.taskAgentType === 'claude-code' && + isClaudeAgent && isFallbackEnabled() && !hasGroupProviderOverride(settingsPath); @@ -363,7 +332,7 @@ async function runTask( } if ( - canFallback && + isClaudeAgent && provider === 'claude' && !sawOutput && streamedOutput.status === 'success' && @@ -434,11 +403,6 @@ async function runTask( }; }; - const shouldRotateClaudeToken = (reason: string): boolean => - reason === '429' || - reason === 'usage-exhausted' || - reason === 'auth-expired'; - const runFallbackTaskAttempt = async ( reason: string, retryAfterMs?: number, diff --git a/src/token-rotation-base.ts b/src/token-rotation-base.ts new file mode 100644 index 0000000..6acaab4 --- /dev/null +++ b/src/token-rotation-base.ts @@ -0,0 +1,71 @@ +/** + * Token Rotation Base Utilities (SSOT) + * + * Shared algorithms for Claude and Codex token rotation. + * Extracted from token-rotation.ts and codex-token-rotation.ts + * to eliminate duplicate implementations. + */ + +export const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time +export const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback + +/** + * Parse "try again at Mar 26th, 2026 9:00 AM" or "resets at ..." from + * an error message. Returns timestamp in ms, or null if not found. + * + * Uses the wider regex from token-rotation.ts that also matches + * "resets at" patterns (codex-token-rotation only matched "try again at"). + */ +export function parseRetryAfterFromError(error?: string): number | null { + if (!error) return null; + const match = error.match( + /(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (!match) return null; + try { + const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); + const ts = new Date(cleaned).getTime(); + if (Number.isNaN(ts)) return null; + return ts; + } catch { + return null; + } +} + +/** + * Compute when a rate-limited token should become available again. + * Parses retry-after from error message, falling back to DEFAULT_COOLDOWN_MS. + */ +export function computeCooldownUntil(error?: string): number { + const retryAt = parseRetryAfterFromError(error); + if (retryAt) return retryAt + BUFFER_MS; + return Date.now() + DEFAULT_COOLDOWN_MS; +} + +/** + * Find the next available (non-rate-limited) item in a rotation pool. + * Returns the index, or null if all are exhausted. + * + * This is the shared rotation algorithm used by both Claude token rotation + * and Codex account rotation. + */ +export function findNextAvailable< + T extends { rateLimitedUntil: number | null }, +>( + items: T[], + currentIndex: number, + opts?: { ignoreRateLimits?: boolean }, +): number | null { + const now = Date.now(); + const ignoreRL = opts?.ignoreRateLimits ?? false; + + for (let i = 1; i < items.length; i++) { + const idx = (currentIndex + i) % items.length; + const item = items[idx]; + if (ignoreRL || !item.rateLimitedUntil || item.rateLimitedUntil <= now) { + return idx; + } + } + + return null; +} diff --git a/src/token-rotation.ts b/src/token-rotation.ts index a5930be..af2937e 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -14,8 +14,12 @@ import fs from 'fs'; import path from 'path'; import { DATA_DIR } from './config.js'; -import { readEnvFile } from './env.js'; +import { getEnv } from './env.js'; import { logger } from './logger.js'; +import { + computeCooldownUntil, + findNextAvailable, +} from './token-rotation-base.js'; const STATE_FILE = path.join(DATA_DIR, 'token-rotation-state.json'); @@ -32,14 +36,8 @@ export function initTokenRotation(): void { if (initialized) return; initialized = true; - const envFile = readEnvFile([ - 'CLAUDE_CODE_OAUTH_TOKENS', - 'CLAUDE_CODE_OAUTH_TOKEN', - ]); - const multi = - process.env.CLAUDE_CODE_OAUTH_TOKENS || envFile.CLAUDE_CODE_OAUTH_TOKENS; - const single = - process.env.CLAUDE_CODE_OAUTH_TOKEN || envFile.CLAUDE_CODE_OAUTH_TOKEN; + const multi = getEnv('CLAUDE_CODE_OAUTH_TOKENS'); + const single = getEnv('CLAUDE_CODE_OAUTH_TOKEN'); const raw = multi ? multi @@ -107,30 +105,6 @@ function loadState(): void { } } -const BUFFER_MS = 3 * 60_000; -const DEFAULT_COOLDOWN_MS = 3_600_000; - -function parseRetryAfterFromError(error?: string): number | null { - if (!error) return null; - const match = error.match( - /(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, - ); - if (!match) return null; - try { - const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); - const ts = new Date(cleaned).getTime(); - if (Number.isNaN(ts)) return null; - return ts; - } catch { - return null; - } -} - -function computeCooldownUntil(error?: string): number { - const retryAt = parseRetryAfterFromError(error); - if (retryAt) return retryAt + BUFFER_MS; - return Date.now() + DEFAULT_COOLDOWN_MS; -} /** Get the current active token. */ export function getCurrentToken(): string | undefined { @@ -148,24 +122,22 @@ export function rotateToken( ): boolean { if (tokens.length <= 1) return false; - const now = Date.now(); tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage); - const ignoreRL = opts?.ignoreRateLimits ?? false; - // Find next available token - for (let i = 1; i < tokens.length; i++) { - const idx = (currentIndex + i) % tokens.length; - const state = tokens[idx]; - if (ignoreRL || !state.rateLimitedUntil || state.rateLimitedUntil <= now) { - state.rateLimitedUntil = null; - currentIndex = idx; - logger.info( - { tokenIndex: currentIndex, totalTokens: tokens.length, ignoreRL }, - `Rotated to token #${currentIndex + 1}/${tokens.length}`, - ); - saveState(); - return true; - } + const nextIdx = findNextAvailable(tokens, currentIndex, opts); + if (nextIdx !== null) { + tokens[nextIdx].rateLimitedUntil = null; + currentIndex = nextIdx; + logger.info( + { + tokenIndex: currentIndex, + totalTokens: tokens.length, + ignoreRL: opts?.ignoreRateLimits ?? false, + }, + `Rotated to token #${currentIndex + 1}/${tokens.length}`, + ); + saveState(); + return true; } logger.warn(