config: add auto-compact settings for Claude and Codex agents
Claude: CLAUDE_AUTOCOMPACT_PCT_OVERRIDE=50 (~500K on 1M context) Codex: model_auto_compact_token_limit=258000 (matching CLI default)
This commit is contained in:
@@ -1,4 +1,7 @@
|
|||||||
{
|
{
|
||||||
|
"env": {
|
||||||
|
"CLAUDE_AUTOCOMPACT_PCT_OVERRIDE": "50"
|
||||||
|
},
|
||||||
"extraKnownMarketplaces": {
|
"extraKnownMarketplaces": {
|
||||||
"nanoclaw-skills": {
|
"nanoclaw-skills": {
|
||||||
"source": {
|
"source": {
|
||||||
|
|||||||
180
src/agent-error-detection.ts
Normal file
180
src/agent-error-detection.ts
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
/**
|
||||||
|
* Agent Error Detection (SSOT)
|
||||||
|
*
|
||||||
|
* Single source of truth for classifying agent errors from output text
|
||||||
|
* and error strings. Used by both message-agent-executor and task-scheduler.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// ── Banner / text detection ─────────────────────────────────────
|
||||||
|
|
||||||
|
export function isClaudeAuthError(text: string): boolean {
|
||||||
|
const lower = text.toLowerCase();
|
||||||
|
return (
|
||||||
|
lower.includes('failed to authenticate') &&
|
||||||
|
(lower.includes('401') || lower.includes('authentication_error'))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isClaudeUsageExhaustedMessage(text: string): boolean {
|
||||||
|
const normalized = text
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/['\u2018\u2019`]/g, "'")
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.replace(/^error:\s*/i, '');
|
||||||
|
const looksLikeBanner =
|
||||||
|
normalized.startsWith("you're out of extra usage") ||
|
||||||
|
normalized.startsWith('you are out of extra usage') ||
|
||||||
|
normalized.startsWith("you've hit your limit") ||
|
||||||
|
normalized.startsWith('you have hit your limit');
|
||||||
|
const hasResetHint =
|
||||||
|
normalized.includes('resets ') ||
|
||||||
|
normalized.includes('reset at ') ||
|
||||||
|
normalized.includes('try again');
|
||||||
|
return looksLikeBanner && hasResetHint && normalized.length <= 160;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isClaudeAuthExpiredMessage(text: string): boolean {
|
||||||
|
const normalized = text.trim().toLowerCase().replace(/\s+/g, ' ');
|
||||||
|
const looksLikeAuthFailure = normalized.startsWith('failed to authenticate');
|
||||||
|
const hasExpiredTokenMarker =
|
||||||
|
normalized.includes('oauth token has expired') ||
|
||||||
|
normalized.includes('authentication_error') ||
|
||||||
|
normalized.includes('obtain a new token') ||
|
||||||
|
normalized.includes('refresh your existing token') ||
|
||||||
|
normalized.includes('invalid authentication credentials');
|
||||||
|
const hasUnauthorizedMarker =
|
||||||
|
normalized.includes('401') || normalized.includes('authentication error');
|
||||||
|
const hasTerminatedMarker = normalized.includes('terminated');
|
||||||
|
|
||||||
|
return (
|
||||||
|
looksLikeAuthFailure &&
|
||||||
|
hasUnauthorizedMarker &&
|
||||||
|
(hasExpiredTokenMarker || hasTerminatedMarker)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Rotation decision ───────────────────────────────────────────
|
||||||
|
|
||||||
|
export function shouldRotateClaudeToken(reason: string): boolean {
|
||||||
|
return (
|
||||||
|
reason === '429' ||
|
||||||
|
reason === 'usage-exhausted' ||
|
||||||
|
reason === 'auth-expired'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Unified error classification ────────────────────────────────
|
||||||
|
|
||||||
|
export type ErrorCategory =
|
||||||
|
| 'rate-limit'
|
||||||
|
| 'auth-expired'
|
||||||
|
| 'overloaded'
|
||||||
|
| 'network-error'
|
||||||
|
| 'none';
|
||||||
|
|
||||||
|
export interface AgentErrorClassification {
|
||||||
|
category: ErrorCategory;
|
||||||
|
reason: string; // '429' | 'auth-expired' | 'overloaded' | 'network-error' | ''
|
||||||
|
retryAfterMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const NONE: AgentErrorClassification = {
|
||||||
|
category: 'none',
|
||||||
|
reason: '',
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify an agent error string into a category.
|
||||||
|
* Handles patterns common to both Claude and Codex: 429, 503, network.
|
||||||
|
* Auth errors are provider-specific — use classifyClaudeAuthError or
|
||||||
|
* classifyCodexAuthError for those.
|
||||||
|
*/
|
||||||
|
export function classifyAgentError(
|
||||||
|
error: string | null | undefined,
|
||||||
|
): AgentErrorClassification {
|
||||||
|
if (!error) return NONE;
|
||||||
|
|
||||||
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
|
// 429 / Rate Limit
|
||||||
|
if (
|
||||||
|
lower.includes('429') ||
|
||||||
|
lower.includes('rate limit') ||
|
||||||
|
lower.includes('usage limit') ||
|
||||||
|
lower.includes('hit your limit') ||
|
||||||
|
lower.includes('too many requests') ||
|
||||||
|
lower.includes('rate_limit')
|
||||||
|
) {
|
||||||
|
const retryMatch = error.match(/retry[\s_-]*after[:\s]*(\d+)/i);
|
||||||
|
const retryAfterMs = retryMatch
|
||||||
|
? parseInt(retryMatch[1], 10) * 1000
|
||||||
|
: undefined;
|
||||||
|
return { category: 'rate-limit', reason: '429', retryAfterMs };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 503 / Overloaded
|
||||||
|
if (lower.includes('503') || lower.includes('overloaded')) {
|
||||||
|
return { category: 'overloaded', reason: 'overloaded' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Network / connection errors
|
||||||
|
if (
|
||||||
|
lower.includes('econnrefused') ||
|
||||||
|
lower.includes('econnreset') ||
|
||||||
|
lower.includes('etimedout') ||
|
||||||
|
lower.includes('enotfound') ||
|
||||||
|
lower.includes('fetch failed') ||
|
||||||
|
lower.includes('network error')
|
||||||
|
) {
|
||||||
|
return { category: 'network-error', reason: 'network-error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Provider-specific auth checks ───────────────────────────────
|
||||||
|
|
||||||
|
/** Claude auth: strict 3-condition AND (auth failure + 401 + specific marker). */
|
||||||
|
export function classifyClaudeAuthError(
|
||||||
|
error: string | null | undefined,
|
||||||
|
): AgentErrorClassification {
|
||||||
|
if (!error) return NONE;
|
||||||
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
|
if (
|
||||||
|
(lower.includes('failed to authenticate') ||
|
||||||
|
lower.includes('authentication_error')) &&
|
||||||
|
(lower.includes('401') || lower.includes('unauthorized')) &&
|
||||||
|
(lower.includes('oauth token has expired') ||
|
||||||
|
lower.includes('obtain a new token') ||
|
||||||
|
lower.includes('refresh your existing token') ||
|
||||||
|
lower.includes('invalid authentication credentials') ||
|
||||||
|
lower.includes('terminated'))
|
||||||
|
) {
|
||||||
|
return { category: 'auth-expired', reason: 'auth-expired' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Codex auth: loose OR check (any single auth indicator). */
|
||||||
|
export function classifyCodexAuthError(
|
||||||
|
error: string | null | undefined,
|
||||||
|
): AgentErrorClassification {
|
||||||
|
if (!error) return NONE;
|
||||||
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
|
if (
|
||||||
|
lower.includes('401') ||
|
||||||
|
lower.includes('authentication_error') ||
|
||||||
|
lower.includes('failed to authenticate') ||
|
||||||
|
lower.includes('oauth token has expired') ||
|
||||||
|
lower.includes('refresh your existing token') ||
|
||||||
|
lower.includes('unauthorized')
|
||||||
|
) {
|
||||||
|
return { category: 'auth-expired', reason: 'auth-expired' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return NONE;
|
||||||
|
}
|
||||||
@@ -19,6 +19,7 @@ vi.mock('./db.js', () => ({
|
|||||||
|
|
||||||
vi.mock('./env.js', () => ({
|
vi.mock('./env.js', () => ({
|
||||||
readEnvFile: mockReadEnvFile,
|
readEnvFile: mockReadEnvFile,
|
||||||
|
getEnv: vi.fn((key: string) => undefined),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./codex-token-rotation.js', () => ({
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ vi.mock('fs', async () => {
|
|||||||
// Mock env
|
// Mock env
|
||||||
vi.mock('./env.js', () => ({
|
vi.mock('./env.js', () => ({
|
||||||
readEnvFile: vi.fn(() => ({})),
|
readEnvFile: vi.fn(() => ({})),
|
||||||
|
getEnv: vi.fn(() => undefined),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Create a controllable fake ChildProcess
|
// Create a controllable fake ChildProcess
|
||||||
|
|||||||
@@ -6,7 +6,10 @@ import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
|
|||||||
vi.mock('./registry.js', () => ({ registerChannel: vi.fn() }));
|
vi.mock('./registry.js', () => ({ registerChannel: vi.fn() }));
|
||||||
|
|
||||||
// Mock env reader (used by the factory, not needed in unit tests)
|
// Mock env reader (used by the factory, not needed in unit tests)
|
||||||
vi.mock('../env.js', () => ({ readEnvFile: vi.fn(() => ({})) }));
|
vi.mock('../env.js', () => ({
|
||||||
|
readEnvFile: vi.fn(() => ({})),
|
||||||
|
getEnv: vi.fn(() => undefined),
|
||||||
|
}));
|
||||||
|
|
||||||
// Mock config
|
// Mock config
|
||||||
vi.mock('../config.js', () => ({
|
vi.mock('../config.js', () => ({
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import {
|
|||||||
TRIGGER_PATTERN,
|
TRIGGER_PATTERN,
|
||||||
} from '../config.js';
|
} from '../config.js';
|
||||||
import { isPairedRoomJid } from '../db.js';
|
import { isPairedRoomJid } from '../db.js';
|
||||||
import { readEnvFile } from '../env.js';
|
import { getEnv } from '../env.js';
|
||||||
import { logger } from '../logger.js';
|
import { logger } from '../logger.js';
|
||||||
import { formatOutbound } from '../router.js';
|
import { formatOutbound } from '../router.js';
|
||||||
|
|
||||||
@@ -95,10 +95,8 @@ async function transcribeAudio(att: Attachment): Promise<string> {
|
|||||||
const filename = att.name || 'audio.ogg';
|
const filename = att.name || 'audio.ogg';
|
||||||
|
|
||||||
// Pick provider: Groq (fast) > OpenAI (fallback)
|
// Pick provider: Groq (fast) > OpenAI (fallback)
|
||||||
const envVars = readEnvFile(['GROQ_API_KEY', 'OPENAI_API_KEY']);
|
const groqKey = getEnv('GROQ_API_KEY') || '';
|
||||||
const groqKey = process.env.GROQ_API_KEY || envVars.GROQ_API_KEY || '';
|
const openaiKey = getEnv('OPENAI_API_KEY') || '';
|
||||||
const openaiKey =
|
|
||||||
process.env.OPENAI_API_KEY || envVars.OPENAI_API_KEY || '';
|
|
||||||
|
|
||||||
let apiUrl: string;
|
let apiUrl: string;
|
||||||
let apiKeyToUse: string;
|
let apiKeyToUse: string;
|
||||||
@@ -674,17 +672,13 @@ export class DiscordChannel implements Channel {
|
|||||||
}
|
}
|
||||||
|
|
||||||
registerChannel('discord', (opts: ChannelOpts) => {
|
registerChannel('discord', (opts: ChannelOpts) => {
|
||||||
const envVars = readEnvFile(['DISCORD_BOT_TOKEN', 'DISCORD_CODEX_BOT_TOKEN']);
|
const token = getEnv('DISCORD_BOT_TOKEN') || '';
|
||||||
const token =
|
|
||||||
process.env.DISCORD_BOT_TOKEN || envVars.DISCORD_BOT_TOKEN || '';
|
|
||||||
if (!token) {
|
if (!token) {
|
||||||
logger.warn('Discord: DISCORD_BOT_TOKEN not set');
|
logger.warn('Discord: DISCORD_BOT_TOKEN not set');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
// If a second Codex bot token exists, this instance only handles claude-code groups
|
// If a second Codex bot token exists, this instance only handles claude-code groups
|
||||||
const hasCodexBot = !!(
|
const hasCodexBot = !!getEnv('DISCORD_CODEX_BOT_TOKEN');
|
||||||
process.env.DISCORD_CODEX_BOT_TOKEN || envVars.DISCORD_CODEX_BOT_TOKEN
|
|
||||||
);
|
|
||||||
return new DiscordChannel(
|
return new DiscordChannel(
|
||||||
token,
|
token,
|
||||||
opts,
|
opts,
|
||||||
@@ -696,11 +690,7 @@ registerChannel('discord', (opts: ChannelOpts) => {
|
|||||||
// The codex service uses its own DISCORD_BOT_TOKEN via systemd EnvironmentFile override.
|
// The codex service uses its own DISCORD_BOT_TOKEN via systemd EnvironmentFile override.
|
||||||
if ((process.env.ASSISTANT_NAME || 'claude') !== 'codex') {
|
if ((process.env.ASSISTANT_NAME || 'claude') !== 'codex') {
|
||||||
registerChannel('discord-codex', (opts: ChannelOpts) => {
|
registerChannel('discord-codex', (opts: ChannelOpts) => {
|
||||||
const envVars = readEnvFile(['DISCORD_CODEX_BOT_TOKEN']);
|
const token = getEnv('DISCORD_CODEX_BOT_TOKEN') || '';
|
||||||
const token =
|
|
||||||
process.env.DISCORD_CODEX_BOT_TOKEN ||
|
|
||||||
envVars.DISCORD_CODEX_BOT_TOKEN ||
|
|
||||||
'';
|
|
||||||
if (!token) return null; // Codex Discord bot is optional
|
if (!token) return null; // Codex Discord bot is optional
|
||||||
return new DiscordChannel(token, opts, 'codex');
|
return new DiscordChannel(token, opts, 'codex');
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -14,8 +14,17 @@ import fs from 'fs';
|
|||||||
import os from 'os';
|
import os from 'os';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
|
import {
|
||||||
|
classifyAgentError,
|
||||||
|
classifyCodexAuthError,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
import { DATA_DIR } from './config.js';
|
import { DATA_DIR } from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import {
|
||||||
|
computeCooldownUntil,
|
||||||
|
findNextAvailable,
|
||||||
|
parseRetryAfterFromError,
|
||||||
|
} from './token-rotation-base.js';
|
||||||
|
|
||||||
const STATE_FILE = path.join(DATA_DIR, 'codex-rotation-state.json');
|
const STATE_FILE = path.join(DATA_DIR, 'codex-rotation-state.json');
|
||||||
|
|
||||||
@@ -195,35 +204,6 @@ function loadCodexState(): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time
|
|
||||||
const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse "try again at Mar 26th, 2026 9:00 AM" from error message.
|
|
||||||
* Returns timestamp in ms, or null if not found.
|
|
||||||
*/
|
|
||||||
function parseRetryAfterFromError(error?: string): number | null {
|
|
||||||
if (!error) return null;
|
|
||||||
const match = error.match(
|
|
||||||
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
|
||||||
);
|
|
||||||
if (!match) return null;
|
|
||||||
try {
|
|
||||||
// Remove ordinal suffixes (1st, 2nd, 3rd, 4th)
|
|
||||||
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
|
|
||||||
const ts = new Date(cleaned).getTime();
|
|
||||||
if (Number.isNaN(ts)) return null;
|
|
||||||
return ts;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function computeCooldownUntil(error?: string): number {
|
|
||||||
const retryAt = parseRetryAfterFromError(error);
|
|
||||||
if (retryAt) return retryAt + BUFFER_MS;
|
|
||||||
return Date.now() + DEFAULT_COOLDOWN_MS;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Get the auth.json path for the current active account. */
|
/** Get the auth.json path for the current active account. */
|
||||||
export function getActiveCodexAuthPath(): string | null {
|
export function getActiveCodexAuthPath(): string | null {
|
||||||
@@ -236,43 +216,16 @@ export function detectCodexRotationTrigger(
|
|||||||
): CodexRotationTriggerResult {
|
): CodexRotationTriggerResult {
|
||||||
if (!error) return { shouldRotate: false, reason: '' };
|
if (!error) return { shouldRotate: false, reason: '' };
|
||||||
|
|
||||||
const lower = error.toLowerCase();
|
// Common patterns (429, 503, network) — delegated to SSOT
|
||||||
|
const common = classifyAgentError(error);
|
||||||
if (
|
if (common.category !== 'none') {
|
||||||
lower.includes('429') ||
|
return { shouldRotate: true, reason: common.reason };
|
||||||
lower.includes('rate limit') ||
|
|
||||||
lower.includes('usage limit') ||
|
|
||||||
lower.includes('hit your limit') ||
|
|
||||||
lower.includes('too many requests') ||
|
|
||||||
lower.includes('rate_limit')
|
|
||||||
) {
|
|
||||||
return { shouldRotate: true, reason: '429' };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (lower.includes('503') || lower.includes('overloaded')) {
|
// Codex-specific loose auth check
|
||||||
return { shouldRotate: true, reason: 'overloaded' };
|
const auth = classifyCodexAuthError(error);
|
||||||
}
|
if (auth.category !== 'none') {
|
||||||
|
return { shouldRotate: true, reason: auth.reason };
|
||||||
if (
|
|
||||||
lower.includes('econnrefused') ||
|
|
||||||
lower.includes('econnreset') ||
|
|
||||||
lower.includes('etimedout') ||
|
|
||||||
lower.includes('enotfound') ||
|
|
||||||
lower.includes('fetch failed') ||
|
|
||||||
lower.includes('network error')
|
|
||||||
) {
|
|
||||||
return { shouldRotate: true, reason: 'network-error' };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
lower.includes('401') ||
|
|
||||||
lower.includes('authentication_error') ||
|
|
||||||
lower.includes('failed to authenticate') ||
|
|
||||||
lower.includes('oauth token has expired') ||
|
|
||||||
lower.includes('refresh your existing token') ||
|
|
||||||
lower.includes('unauthorized')
|
|
||||||
) {
|
|
||||||
return { shouldRotate: true, reason: 'auth-expired' };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return { shouldRotate: false, reason: '' };
|
return { shouldRotate: false, reason: '' };
|
||||||
@@ -282,36 +235,35 @@ export function detectCodexRotationTrigger(
|
|||||||
* Try to rotate to the next available Codex account.
|
* Try to rotate to the next available Codex account.
|
||||||
* Returns true if a fresh account was found.
|
* Returns true if a fresh account was found.
|
||||||
*/
|
*/
|
||||||
export function rotateCodexToken(errorMessage?: string): boolean {
|
export function rotateCodexToken(
|
||||||
|
errorMessage?: string,
|
||||||
|
opts?: { ignoreRateLimits?: boolean },
|
||||||
|
): boolean {
|
||||||
if (accounts.length <= 1) return false;
|
if (accounts.length <= 1) return false;
|
||||||
|
|
||||||
const now = Date.now();
|
|
||||||
const acct = accounts[currentIndex];
|
const acct = accounts[currentIndex];
|
||||||
acct.rateLimitedUntil = computeCooldownUntil(errorMessage);
|
acct.rateLimitedUntil = computeCooldownUntil(errorMessage);
|
||||||
acct.lastUsagePct = 100;
|
acct.lastUsagePct = 100;
|
||||||
// Extract reset time string from error for display
|
// Extract reset time string from error for display
|
||||||
const resetMatch = errorMessage?.match(
|
const retryAt = parseRetryAfterFromError(errorMessage);
|
||||||
/try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
if (retryAt) {
|
||||||
);
|
acct.resetAt = new Date(retryAt).toISOString();
|
||||||
if (resetMatch) acct.resetAt = resetMatch[1];
|
}
|
||||||
|
|
||||||
for (let i = 1; i < accounts.length; i++) {
|
const nextIdx = findNextAvailable(accounts, currentIndex, opts);
|
||||||
const idx = (currentIndex + i) % accounts.length;
|
if (nextIdx !== null) {
|
||||||
const acct = accounts[idx];
|
accounts[nextIdx].rateLimitedUntil = null;
|
||||||
if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) {
|
currentIndex = nextIdx;
|
||||||
acct.rateLimitedUntil = null;
|
logger.info(
|
||||||
currentIndex = idx;
|
{
|
||||||
logger.info(
|
accountIndex: currentIndex,
|
||||||
{
|
totalAccounts: accounts.length,
|
||||||
accountIndex: currentIndex,
|
accountId: accounts[nextIdx].accountId,
|
||||||
totalAccounts: accounts.length,
|
},
|
||||||
accountId: acct.accountId,
|
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
||||||
},
|
);
|
||||||
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
saveCodexState();
|
||||||
);
|
return true;
|
||||||
saveCodexState();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.warn('All Codex accounts are rate-limited');
|
logger.warn('All Codex accounts are rate-limited');
|
||||||
@@ -325,15 +277,10 @@ export function rotateCodexToken(errorMessage?: string): boolean {
|
|||||||
*/
|
*/
|
||||||
export function advanceCodexAccount(): void {
|
export function advanceCodexAccount(): void {
|
||||||
if (accounts.length <= 1) return;
|
if (accounts.length <= 1) return;
|
||||||
const now = Date.now();
|
const nextIdx = findNextAvailable(accounts, currentIndex);
|
||||||
for (let i = 1; i < accounts.length; i++) {
|
if (nextIdx !== null) {
|
||||||
const idx = (currentIndex + i) % accounts.length;
|
currentIndex = nextIdx;
|
||||||
const acct = accounts[idx];
|
saveCodexState();
|
||||||
if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) {
|
|
||||||
currentIndex = idx;
|
|
||||||
saveCodexState();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// All others rate-limited, stay on current
|
// All others rate-limited, stay on current
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,29 +2,14 @@ import os from 'os';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import type { AgentType } from './types.js';
|
import type { AgentType } from './types.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { getEnv } from './env.js';
|
||||||
|
|
||||||
const envConfig = readEnvFile([
|
export const ASSISTANT_NAME = getEnv('ASSISTANT_NAME') || 'Andy';
|
||||||
'ASSISTANT_NAME',
|
|
||||||
'ASSISTANT_HAS_OWN_NUMBER',
|
|
||||||
'SERVICE_ID',
|
|
||||||
'SERVICE_AGENT_TYPE',
|
|
||||||
'SESSION_COMMAND_ALLOWED_SENDERS',
|
|
||||||
'SESSION_COMMAND_USER_IDS',
|
|
||||||
'STATUS_SHOW_ROOMS',
|
|
||||||
'USAGE_DASHBOARD',
|
|
||||||
]);
|
|
||||||
|
|
||||||
export const ASSISTANT_NAME =
|
|
||||||
process.env.ASSISTANT_NAME || envConfig.ASSISTANT_NAME || 'Andy';
|
|
||||||
export const ASSISTANT_HAS_OWN_NUMBER =
|
export const ASSISTANT_HAS_OWN_NUMBER =
|
||||||
(process.env.ASSISTANT_HAS_OWN_NUMBER ||
|
getEnv('ASSISTANT_HAS_OWN_NUMBER') === 'true';
|
||||||
envConfig.ASSISTANT_HAS_OWN_NUMBER) === 'true';
|
|
||||||
const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase();
|
const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase();
|
||||||
const rawServiceAgentType =
|
const rawServiceAgentType = getEnv('SERVICE_AGENT_TYPE');
|
||||||
process.env.SERVICE_AGENT_TYPE || envConfig.SERVICE_AGENT_TYPE;
|
export const SERVICE_ID = getEnv('SERVICE_ID') || ASSISTANT_SLUG;
|
||||||
export const SERVICE_ID =
|
|
||||||
process.env.SERVICE_ID || envConfig.SERVICE_ID || ASSISTANT_SLUG;
|
|
||||||
export const SERVICE_AGENT_TYPE: AgentType =
|
export const SERVICE_AGENT_TYPE: AgentType =
|
||||||
rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code'
|
rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code'
|
||||||
? rawServiceAgentType
|
? rawServiceAgentType
|
||||||
@@ -84,10 +69,9 @@ export const STATUS_CHANNEL_ID = process.env.STATUS_CHANNEL_ID || '';
|
|||||||
export const STATUS_UPDATE_INTERVAL = 10000; // 10s
|
export const STATUS_UPDATE_INTERVAL = 10000; // 10s
|
||||||
export const USAGE_UPDATE_INTERVAL = 300000; // 5 minutes
|
export const USAGE_UPDATE_INTERVAL = 300000; // 5 minutes
|
||||||
export const STATUS_SHOW_ROOMS =
|
export const STATUS_SHOW_ROOMS =
|
||||||
(process.env.STATUS_SHOW_ROOMS || envConfig.STATUS_SHOW_ROOMS || 'true') !==
|
(getEnv('STATUS_SHOW_ROOMS') || 'true') !== 'false';
|
||||||
'false';
|
|
||||||
export const USAGE_DASHBOARD_ENABLED =
|
export const USAGE_DASHBOARD_ENABLED =
|
||||||
(process.env.USAGE_DASHBOARD || envConfig.USAGE_DASHBOARD) === 'true';
|
getEnv('USAGE_DASHBOARD') === 'true';
|
||||||
|
|
||||||
// Timezone for scheduled tasks (cron expressions, etc.)
|
// Timezone for scheduled tasks (cron expressions, etc.)
|
||||||
// Uses system timezone by default
|
// Uses system timezone by default
|
||||||
@@ -95,10 +79,8 @@ export const TIMEZONE =
|
|||||||
process.env.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone;
|
process.env.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone;
|
||||||
|
|
||||||
const rawSessionCommandAllowedSenders =
|
const rawSessionCommandAllowedSenders =
|
||||||
process.env.SESSION_COMMAND_ALLOWED_SENDERS ||
|
getEnv('SESSION_COMMAND_ALLOWED_SENDERS') ||
|
||||||
process.env.SESSION_COMMAND_USER_IDS ||
|
getEnv('SESSION_COMMAND_USER_IDS') ||
|
||||||
envConfig.SESSION_COMMAND_ALLOWED_SENDERS ||
|
|
||||||
envConfig.SESSION_COMMAND_USER_IDS ||
|
|
||||||
'';
|
'';
|
||||||
|
|
||||||
const SESSION_COMMAND_ALLOWED_SENDERS = new Set(
|
const SESSION_COMMAND_ALLOWED_SENDERS = new Set(
|
||||||
|
|||||||
58
src/env.ts
58
src/env.ts
@@ -2,13 +2,12 @@ import fs from 'fs';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
|
||||||
/**
|
// ── Internal cache ──────────────────────────────────────────────
|
||||||
* Parse the .env file and return values for the requested keys.
|
|
||||||
* Does NOT load anything into process.env — callers decide what to
|
let _cache: Record<string, string> | null = null;
|
||||||
* do with the values. This keeps secrets out of the process environment
|
|
||||||
* so they don't leak to child processes.
|
/** Parse the entire .env file into a Record (no key filtering). */
|
||||||
*/
|
function parseEnvFile(): Record<string, string> {
|
||||||
export function readEnvFile(keys: string[]): Record<string, string> {
|
|
||||||
const envFile = path.join(process.cwd(), '.env');
|
const envFile = path.join(process.cwd(), '.env');
|
||||||
let content: string;
|
let content: string;
|
||||||
try {
|
try {
|
||||||
@@ -19,15 +18,12 @@ export function readEnvFile(keys: string[]): Record<string, string> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const result: Record<string, string> = {};
|
const result: Record<string, string> = {};
|
||||||
const wanted = new Set(keys);
|
|
||||||
|
|
||||||
for (const line of content.split('\n')) {
|
for (const line of content.split('\n')) {
|
||||||
const trimmed = line.trim();
|
const trimmed = line.trim();
|
||||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||||
const eqIdx = trimmed.indexOf('=');
|
const eqIdx = trimmed.indexOf('=');
|
||||||
if (eqIdx === -1) continue;
|
if (eqIdx === -1) continue;
|
||||||
const key = trimmed.slice(0, eqIdx).trim();
|
const key = trimmed.slice(0, eqIdx).trim();
|
||||||
if (!wanted.has(key)) continue;
|
|
||||||
let value = trimmed.slice(eqIdx + 1).trim();
|
let value = trimmed.slice(eqIdx + 1).trim();
|
||||||
if (
|
if (
|
||||||
(value.startsWith('"') && value.endsWith('"')) ||
|
(value.startsWith('"') && value.endsWith('"')) ||
|
||||||
@@ -37,6 +33,46 @@ export function readEnvFile(keys: string[]): Record<string, string> {
|
|||||||
}
|
}
|
||||||
if (value) result[key] = value;
|
if (value) result[key] = value;
|
||||||
}
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Public API ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Load (or reload) the .env file into the in-memory cache. */
|
||||||
|
export function loadEnvFile(): void {
|
||||||
|
_cache = parseEnvFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Look up a single env value.
|
||||||
|
* Priority: process.env > .env cache > undefined
|
||||||
|
*/
|
||||||
|
export function getEnv(key: string): string | undefined {
|
||||||
|
if (!_cache) loadEnvFile();
|
||||||
|
return process.env[key] || _cache![key] || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Force-reload the .env file (e.g. after token refresh writes new values). */
|
||||||
|
export function reloadEnvFile(): void {
|
||||||
|
_cache = null;
|
||||||
|
loadEnvFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the .env file and return values for the requested keys.
|
||||||
|
* Does NOT load anything into process.env — callers decide what to
|
||||||
|
* do with the values. This keeps secrets out of the process environment
|
||||||
|
* so they don't leak to child processes.
|
||||||
|
*
|
||||||
|
* Now backed by the in-memory cache (disk read happens at most once).
|
||||||
|
*/
|
||||||
|
export function readEnvFile(keys: string[]): Record<string, string> {
|
||||||
|
if (!_cache) loadEnvFile();
|
||||||
|
|
||||||
|
const result: Record<string, string> = {};
|
||||||
|
const wanted = new Set(keys);
|
||||||
|
for (const [key, value] of Object.entries(_cache!)) {
|
||||||
|
if (wanted.has(key)) result[key] = value;
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
||||||
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
|
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
|
||||||
|
|
||||||
import { readEnvFile } from './env.js';
|
import { getEnv } from './env.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT_MS = 4_000;
|
const DEFAULT_TIMEOUT_MS = 4_000;
|
||||||
@@ -42,9 +42,8 @@ let cachedConfig: MementoConfig | null | undefined;
|
|||||||
function getMementoConfig(): MementoConfig | null {
|
function getMementoConfig(): MementoConfig | null {
|
||||||
if (cachedConfig !== undefined) return cachedConfig;
|
if (cachedConfig !== undefined) return cachedConfig;
|
||||||
|
|
||||||
const env = readEnvFile(['MEMENTO_MCP_SSE_URL', 'MEMENTO_ACCESS_KEY']);
|
const sseUrl = getEnv('MEMENTO_MCP_SSE_URL');
|
||||||
const sseUrl = process.env.MEMENTO_MCP_SSE_URL || env.MEMENTO_MCP_SSE_URL;
|
const accessKey = getEnv('MEMENTO_ACCESS_KEY');
|
||||||
const accessKey = process.env.MEMENTO_ACCESS_KEY || env.MEMENTO_ACCESS_KEY;
|
|
||||||
|
|
||||||
cachedConfig =
|
cachedConfig =
|
||||||
sseUrl && accessKey
|
sseUrl && accessKey
|
||||||
|
|||||||
@@ -12,6 +12,12 @@ import { getAllTasks } from './db.js';
|
|||||||
import { GroupQueue } from './group-queue.js';
|
import { GroupQueue } from './group-queue.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { buildRoomMemoryBriefing } from './memento-client.js';
|
import { buildRoomMemoryBriefing } from './memento-client.js';
|
||||||
|
import {
|
||||||
|
isClaudeAuthError,
|
||||||
|
isClaudeAuthExpiredMessage,
|
||||||
|
isClaudeUsageExhaustedMessage,
|
||||||
|
shouldRotateClaudeToken,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
detectFallbackTrigger,
|
detectFallbackTrigger,
|
||||||
getActiveProvider,
|
getActiveProvider,
|
||||||
@@ -45,53 +51,6 @@ export interface MessageAgentExecutorDeps {
|
|||||||
clearSession: (groupFolder: string) => void;
|
clearSession: (groupFolder: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isClaudeAuthError(text: string): boolean {
|
|
||||||
const lower = text.toLowerCase();
|
|
||||||
return (
|
|
||||||
lower.includes('failed to authenticate') &&
|
|
||||||
(lower.includes('401') || lower.includes('authentication_error'))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isClaudeUsageExhaustedMessage(text: string): boolean {
|
|
||||||
const normalized = text
|
|
||||||
.trim()
|
|
||||||
.toLowerCase()
|
|
||||||
.replace(/[’‘`]/g, "'")
|
|
||||||
.replace(/\s+/g, ' ')
|
|
||||||
.replace(/^error:\s*/i, '');
|
|
||||||
const looksLikeBanner =
|
|
||||||
normalized.startsWith("you're out of extra usage") ||
|
|
||||||
normalized.startsWith('you are out of extra usage') ||
|
|
||||||
normalized.startsWith("you've hit your limit") ||
|
|
||||||
normalized.startsWith('you have hit your limit');
|
|
||||||
const hasResetHint =
|
|
||||||
normalized.includes('resets ') ||
|
|
||||||
normalized.includes('reset at ') ||
|
|
||||||
normalized.includes('try again');
|
|
||||||
return looksLikeBanner && hasResetHint && normalized.length <= 160;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isClaudeAuthExpiredMessage(text: string): boolean {
|
|
||||||
const normalized = text.trim().toLowerCase().replace(/\s+/g, ' ');
|
|
||||||
const looksLikeAuthFailure = normalized.startsWith('failed to authenticate');
|
|
||||||
const hasExpiredTokenMarker =
|
|
||||||
normalized.includes('oauth token has expired') ||
|
|
||||||
normalized.includes('authentication_error') ||
|
|
||||||
normalized.includes('obtain a new token') ||
|
|
||||||
normalized.includes('refresh your existing token') ||
|
|
||||||
normalized.includes('invalid authentication credentials');
|
|
||||||
const hasUnauthorizedMarker =
|
|
||||||
normalized.includes('401') || normalized.includes('authentication error');
|
|
||||||
const hasTerminatedMarker = normalized.includes('terminated');
|
|
||||||
|
|
||||||
return (
|
|
||||||
looksLikeAuthFailure &&
|
|
||||||
hasUnauthorizedMarker &&
|
|
||||||
(hasExpiredTokenMarker || hasTerminatedMarker)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function runAgentForGroup(
|
export async function runAgentForGroup(
|
||||||
deps: MessageAgentExecutorDeps,
|
deps: MessageAgentExecutorDeps,
|
||||||
args: {
|
args: {
|
||||||
@@ -146,6 +105,7 @@ export async function runAgentForGroup(
|
|||||||
'settings.json',
|
'settings.json',
|
||||||
);
|
);
|
||||||
const groupHasOverride = hasGroupProviderOverride(settingsPath);
|
const groupHasOverride = hasGroupProviderOverride(settingsPath);
|
||||||
|
const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1;
|
||||||
const canFallback =
|
const canFallback =
|
||||||
isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride;
|
isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride;
|
||||||
|
|
||||||
@@ -195,7 +155,7 @@ export async function runAgentForGroup(
|
|||||||
resetSessionRequested = true;
|
resetSessionRequested = true;
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
canFallback &&
|
isClaudeCodeAgent &&
|
||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
output.status === 'success' &&
|
output.status === 'success' &&
|
||||||
!sawOutput &&
|
!sawOutput &&
|
||||||
@@ -265,7 +225,7 @@ export async function runAgentForGroup(
|
|||||||
reason: trigger.reason,
|
reason: trigger.reason,
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
};
|
};
|
||||||
if (canFallback) {
|
if (canFallback || canRotateToken) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -413,11 +373,6 @@ export async function runAgentForGroup(
|
|||||||
return 'success';
|
return 'success';
|
||||||
};
|
};
|
||||||
|
|
||||||
const shouldRotateClaudeToken = (reason: string): boolean =>
|
|
||||||
reason === '429' ||
|
|
||||||
reason === 'usage-exhausted' ||
|
|
||||||
reason === 'auth-expired';
|
|
||||||
|
|
||||||
const retryCodexWithRotation = async (
|
const retryCodexWithRotation = async (
|
||||||
initialTrigger: { reason: string },
|
initialTrigger: { reason: string },
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
@@ -618,7 +573,9 @@ export async function runAgentForGroup(
|
|||||||
!retryAttempt.sawOutput &&
|
!retryAttempt.sawOutput &&
|
||||||
retryAttempt.sawSuccessNullResultWithoutOutput
|
retryAttempt.sawSuccessNullResultWithoutOutput
|
||||||
) {
|
) {
|
||||||
return runFallbackAttempt('success-null-result');
|
return canFallback
|
||||||
|
? runFallbackAttempt('success-null-result')
|
||||||
|
: 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
if (retryOutput.status === 'error') {
|
if (retryOutput.status === 'error') {
|
||||||
@@ -657,12 +614,23 @@ export async function runAgentForGroup(
|
|||||||
return 'success';
|
return 'success';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Usage exhausted: don't fall back to Kimi — log only, no response
|
// Usage exhausted or auth-expired: don't fall back to Kimi — log only
|
||||||
if (trigger.reason === 'usage-exhausted') {
|
if (
|
||||||
|
trigger.reason === 'usage-exhausted' ||
|
||||||
|
trigger.reason === 'auth-expired'
|
||||||
|
) {
|
||||||
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid, group: group.name, runId },
|
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||||
'All Claude tokens usage-exhausted, silently skipping (no Kimi fallback)',
|
`All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`,
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!canFallback) {
|
||||||
|
logger.warn(
|
||||||
|
{ chatJid, group: group.name, runId, reason: trigger.reason },
|
||||||
|
'All Claude tokens exhausted and fallback disabled',
|
||||||
);
|
);
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
@@ -684,7 +652,11 @@ export async function runAgentForGroup(
|
|||||||
const primaryAttempt = await runAttempt(provider);
|
const primaryAttempt = await runAttempt(provider);
|
||||||
|
|
||||||
if (primaryAttempt.error) {
|
if (primaryAttempt.error) {
|
||||||
if (canFallback && provider === 'claude' && !primaryAttempt.sawOutput) {
|
if (
|
||||||
|
(canFallback || canRotateToken) &&
|
||||||
|
provider === 'claude' &&
|
||||||
|
!primaryAttempt.sawOutput
|
||||||
|
) {
|
||||||
const errMsg =
|
const errMsg =
|
||||||
primaryAttempt.error instanceof Error
|
primaryAttempt.error instanceof Error
|
||||||
? primaryAttempt.error.message
|
? primaryAttempt.error.message
|
||||||
@@ -748,7 +720,7 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
canFallback &&
|
(canFallback || canRotateToken) &&
|
||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
!primaryAttempt.sawOutput &&
|
!primaryAttempt.sawOutput &&
|
||||||
primaryAttempt.streamedTriggerReason &&
|
primaryAttempt.streamedTriggerReason &&
|
||||||
@@ -781,7 +753,11 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (output.status === 'error') {
|
if (output.status === 'error') {
|
||||||
if (canFallback && provider === 'claude' && !primaryAttempt.sawOutput) {
|
if (
|
||||||
|
(canFallback || canRotateToken) &&
|
||||||
|
provider === 'claude' &&
|
||||||
|
!primaryAttempt.sawOutput
|
||||||
|
) {
|
||||||
const trigger = primaryAttempt.streamedTriggerReason
|
const trigger = primaryAttempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldFallback: true,
|
||||||
|
|||||||
@@ -4,16 +4,26 @@ vi.mock('./claude-usage.js', () => ({
|
|||||||
fetchClaudeUsage: vi.fn(),
|
fetchClaudeUsage: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./env.js', () => ({
|
vi.mock('./env.js', () => {
|
||||||
readEnvFile: vi.fn(() => ({
|
const store: Record<string, string> = {
|
||||||
FALLBACK_PROVIDER_NAME: 'kimi',
|
FALLBACK_PROVIDER_NAME: 'kimi',
|
||||||
FALLBACK_BASE_URL: 'https://api.kimi.com/coding/',
|
FALLBACK_BASE_URL: 'https://api.kimi.com/coding/',
|
||||||
FALLBACK_AUTH_TOKEN: 'test-kimi-key',
|
FALLBACK_AUTH_TOKEN: 'test-kimi-key',
|
||||||
FALLBACK_MODEL: 'kimi-k2.5',
|
FALLBACK_MODEL: 'kimi-k2.5',
|
||||||
FALLBACK_SMALL_MODEL: 'kimi-k2.5',
|
FALLBACK_SMALL_MODEL: 'kimi-k2.5',
|
||||||
FALLBACK_COOLDOWN_MS: '600000',
|
FALLBACK_COOLDOWN_MS: '600000',
|
||||||
})),
|
};
|
||||||
}));
|
return {
|
||||||
|
readEnvFile: vi.fn((keys: string[]) => {
|
||||||
|
const result: Record<string, string> = {};
|
||||||
|
for (const k of keys) {
|
||||||
|
if (store[k]) result[k] = store[k];
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}),
|
||||||
|
getEnv: vi.fn((key: string) => store[key]),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock('./logger.js', () => ({
|
vi.mock('./logger.js', () => ({
|
||||||
logger: {
|
logger: {
|
||||||
|
|||||||
@@ -14,8 +14,12 @@
|
|||||||
|
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
|
||||||
|
import {
|
||||||
|
classifyAgentError,
|
||||||
|
classifyClaudeAuthError,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js';
|
import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { getEnv } from './env.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { rotateToken, getTokenCount } from './token-rotation.js';
|
import { rotateToken, getTokenCount } from './token-rotation.js';
|
||||||
|
|
||||||
@@ -65,33 +69,21 @@ let _config: FallbackConfig | null = null;
|
|||||||
function loadConfig(): FallbackConfig {
|
function loadConfig(): FallbackConfig {
|
||||||
if (_config) return _config;
|
if (_config) return _config;
|
||||||
|
|
||||||
const env = readEnvFile([
|
const baseUrl = getEnv('FALLBACK_BASE_URL') || '';
|
||||||
'FALLBACK_PROVIDER_NAME',
|
const authToken = getEnv('FALLBACK_AUTH_TOKEN') || '';
|
||||||
'FALLBACK_BASE_URL',
|
const model = getEnv('FALLBACK_MODEL') || '';
|
||||||
'FALLBACK_AUTH_TOKEN',
|
const explicitlyDisabled =
|
||||||
'FALLBACK_MODEL',
|
(getEnv('FALLBACK_ENABLED') || '').toLowerCase() === 'false';
|
||||||
'FALLBACK_SMALL_MODEL',
|
|
||||||
'FALLBACK_COOLDOWN_MS',
|
|
||||||
]);
|
|
||||||
|
|
||||||
const baseUrl = process.env.FALLBACK_BASE_URL || env.FALLBACK_BASE_URL || '';
|
|
||||||
const authToken =
|
|
||||||
process.env.FALLBACK_AUTH_TOKEN || env.FALLBACK_AUTH_TOKEN || '';
|
|
||||||
const model = process.env.FALLBACK_MODEL || env.FALLBACK_MODEL || '';
|
|
||||||
|
|
||||||
_config = {
|
_config = {
|
||||||
enabled: Boolean(baseUrl && authToken && model),
|
enabled: !explicitlyDisabled && Boolean(baseUrl && authToken && model),
|
||||||
providerName:
|
providerName: getEnv('FALLBACK_PROVIDER_NAME') || 'kimi',
|
||||||
process.env.FALLBACK_PROVIDER_NAME ||
|
|
||||||
env.FALLBACK_PROVIDER_NAME ||
|
|
||||||
'kimi',
|
|
||||||
baseUrl,
|
baseUrl,
|
||||||
authToken,
|
authToken,
|
||||||
model,
|
model,
|
||||||
smallModel:
|
smallModel: getEnv('FALLBACK_SMALL_MODEL') || model,
|
||||||
process.env.FALLBACK_SMALL_MODEL || env.FALLBACK_SMALL_MODEL || model,
|
|
||||||
defaultCooldownMs: parseInt(
|
defaultCooldownMs: parseInt(
|
||||||
process.env.FALLBACK_COOLDOWN_MS || env.FALLBACK_COOLDOWN_MS || '600000',
|
getEnv('FALLBACK_COOLDOWN_MS') || '600000',
|
||||||
10,
|
10,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
@@ -359,53 +351,31 @@ export function detectFallbackTrigger(
|
|||||||
): FallbackTriggerResult {
|
): FallbackTriggerResult {
|
||||||
if (!error) return { shouldFallback: false, reason: '' };
|
if (!error) return { shouldFallback: false, reason: '' };
|
||||||
|
|
||||||
const lower = error.toLowerCase();
|
// Delegated to shared SSOT — original priority preserved:
|
||||||
|
// 429 first, then auth-expired, then 503/network
|
||||||
|
const common = classifyAgentError(error);
|
||||||
|
|
||||||
// 429 Rate Limit
|
// 429 rate-limit (highest priority)
|
||||||
if (
|
if (common.category === 'rate-limit') {
|
||||||
lower.includes('429') ||
|
return {
|
||||||
lower.includes('rate limit') ||
|
shouldFallback: true,
|
||||||
lower.includes('usage limit') ||
|
reason: common.reason,
|
||||||
lower.includes('hit your limit') ||
|
retryAfterMs: common.retryAfterMs,
|
||||||
lower.includes('too many requests') ||
|
};
|
||||||
lower.includes('rate_limit')
|
|
||||||
) {
|
|
||||||
// Try to extract retry-after value (seconds → ms)
|
|
||||||
const retryMatch = error.match(/retry[\s_-]*after[:\s]*(\d+)/i);
|
|
||||||
const retryAfterMs = retryMatch
|
|
||||||
? parseInt(retryMatch[1], 10) * 1000
|
|
||||||
: undefined;
|
|
||||||
return { shouldFallback: true, reason: '429', retryAfterMs };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
// Claude-specific strict auth check (before 503/network)
|
||||||
(lower.includes('failed to authenticate') ||
|
const auth = classifyClaudeAuthError(error);
|
||||||
lower.includes('authentication_error')) &&
|
if (auth.category !== 'none') {
|
||||||
(lower.includes('401') || lower.includes('unauthorized')) &&
|
return { shouldFallback: true, reason: auth.reason };
|
||||||
(lower.includes('oauth token has expired') ||
|
|
||||||
lower.includes('obtain a new token') ||
|
|
||||||
lower.includes('refresh your existing token') ||
|
|
||||||
lower.includes('invalid authentication credentials') ||
|
|
||||||
lower.includes('terminated'))
|
|
||||||
) {
|
|
||||||
return { shouldFallback: true, reason: 'auth-expired' };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 503 Overloaded
|
// 503 overloaded, network errors
|
||||||
if (lower.includes('503') || lower.includes('overloaded')) {
|
if (common.category !== 'none') {
|
||||||
return { shouldFallback: true, reason: 'overloaded' };
|
return {
|
||||||
}
|
shouldFallback: true,
|
||||||
|
reason: common.reason,
|
||||||
// Network / connection errors
|
};
|
||||||
if (
|
|
||||||
lower.includes('econnrefused') ||
|
|
||||||
lower.includes('econnreset') ||
|
|
||||||
lower.includes('etimedout') ||
|
|
||||||
lower.includes('enotfound') ||
|
|
||||||
lower.includes('fetch failed') ||
|
|
||||||
lower.includes('network error')
|
|
||||||
) {
|
|
||||||
return { shouldFallback: true, reason: 'network-error' };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return { shouldFallback: false, reason: '' };
|
return { shouldFallback: false, reason: '' };
|
||||||
|
|||||||
@@ -31,6 +31,11 @@ import {
|
|||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||||
|
import {
|
||||||
|
isClaudeAuthExpiredMessage,
|
||||||
|
isClaudeUsageExhaustedMessage,
|
||||||
|
shouldRotateClaudeToken,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
detectFallbackTrigger,
|
detectFallbackTrigger,
|
||||||
getActiveProvider,
|
getActiveProvider,
|
||||||
@@ -73,44 +78,6 @@ export {
|
|||||||
shouldUseTaskScopedSession,
|
shouldUseTaskScopedSession,
|
||||||
} from './task-watch-status.js';
|
} from './task-watch-status.js';
|
||||||
|
|
||||||
function isClaudeUsageExhaustedMessage(text: string): boolean {
|
|
||||||
const normalized = text
|
|
||||||
.trim()
|
|
||||||
.toLowerCase()
|
|
||||||
.replace(/[’‘`]/g, "'")
|
|
||||||
.replace(/\s+/g, ' ')
|
|
||||||
.replace(/^error:\s*/i, '');
|
|
||||||
const looksLikeBanner =
|
|
||||||
normalized.startsWith("you're out of extra usage") ||
|
|
||||||
normalized.startsWith('you are out of extra usage') ||
|
|
||||||
normalized.startsWith("you've hit your limit") ||
|
|
||||||
normalized.startsWith('you have hit your limit');
|
|
||||||
const hasResetHint =
|
|
||||||
normalized.includes('resets ') ||
|
|
||||||
normalized.includes('reset at ') ||
|
|
||||||
normalized.includes('try again');
|
|
||||||
return looksLikeBanner && hasResetHint && normalized.length <= 160;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isClaudeAuthExpiredMessage(text: string): boolean {
|
|
||||||
const normalized = text.trim().toLowerCase().replace(/\s+/g, ' ');
|
|
||||||
const looksLikeAuthFailure = normalized.startsWith('failed to authenticate');
|
|
||||||
const hasExpiredTokenMarker =
|
|
||||||
normalized.includes('oauth token has expired') ||
|
|
||||||
normalized.includes('authentication_error') ||
|
|
||||||
normalized.includes('obtain a new token') ||
|
|
||||||
normalized.includes('refresh your existing token') ||
|
|
||||||
normalized.includes('invalid authentication credentials');
|
|
||||||
const hasUnauthorizedMarker =
|
|
||||||
normalized.includes('401') || normalized.includes('authentication error');
|
|
||||||
const hasTerminatedMarker = normalized.includes('terminated');
|
|
||||||
|
|
||||||
return (
|
|
||||||
looksLikeAuthFailure &&
|
|
||||||
hasUnauthorizedMarker &&
|
|
||||||
(hasExpiredTokenMarker || hasTerminatedMarker)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute the next run time for a recurring task, anchored to the
|
* Compute the next run time for a recurring task, anchored to the
|
||||||
@@ -307,8 +274,10 @@ async function runTask(
|
|||||||
'.claude',
|
'.claude',
|
||||||
'settings.json',
|
'settings.json',
|
||||||
);
|
);
|
||||||
|
const isClaudeAgent = context.taskAgentType === 'claude-code';
|
||||||
|
const canRotateToken = isClaudeAgent && getTokenCount() > 1;
|
||||||
const canFallback =
|
const canFallback =
|
||||||
context.taskAgentType === 'claude-code' &&
|
isClaudeAgent &&
|
||||||
isFallbackEnabled() &&
|
isFallbackEnabled() &&
|
||||||
!hasGroupProviderOverride(settingsPath);
|
!hasGroupProviderOverride(settingsPath);
|
||||||
|
|
||||||
@@ -363,7 +332,7 @@ async function runTask(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
canFallback &&
|
isClaudeAgent &&
|
||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
!sawOutput &&
|
!sawOutput &&
|
||||||
streamedOutput.status === 'success' &&
|
streamedOutput.status === 'success' &&
|
||||||
@@ -434,11 +403,6 @@ async function runTask(
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const shouldRotateClaudeToken = (reason: string): boolean =>
|
|
||||||
reason === '429' ||
|
|
||||||
reason === 'usage-exhausted' ||
|
|
||||||
reason === 'auth-expired';
|
|
||||||
|
|
||||||
const runFallbackTaskAttempt = async (
|
const runFallbackTaskAttempt = async (
|
||||||
reason: string,
|
reason: string,
|
||||||
retryAfterMs?: number,
|
retryAfterMs?: number,
|
||||||
|
|||||||
71
src/token-rotation-base.ts
Normal file
71
src/token-rotation-base.ts
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
/**
|
||||||
|
* Token Rotation Base Utilities (SSOT)
|
||||||
|
*
|
||||||
|
* Shared algorithms for Claude and Codex token rotation.
|
||||||
|
* Extracted from token-rotation.ts and codex-token-rotation.ts
|
||||||
|
* to eliminate duplicate implementations.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time
|
||||||
|
export const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse "try again at Mar 26th, 2026 9:00 AM" or "resets at ..." from
|
||||||
|
* an error message. Returns timestamp in ms, or null if not found.
|
||||||
|
*
|
||||||
|
* Uses the wider regex from token-rotation.ts that also matches
|
||||||
|
* "resets at" patterns (codex-token-rotation only matched "try again at").
|
||||||
|
*/
|
||||||
|
export function parseRetryAfterFromError(error?: string): number | null {
|
||||||
|
if (!error) return null;
|
||||||
|
const match = error.match(
|
||||||
|
/(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
||||||
|
);
|
||||||
|
if (!match) return null;
|
||||||
|
try {
|
||||||
|
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
|
||||||
|
const ts = new Date(cleaned).getTime();
|
||||||
|
if (Number.isNaN(ts)) return null;
|
||||||
|
return ts;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute when a rate-limited token should become available again.
|
||||||
|
* Parses retry-after from error message, falling back to DEFAULT_COOLDOWN_MS.
|
||||||
|
*/
|
||||||
|
export function computeCooldownUntil(error?: string): number {
|
||||||
|
const retryAt = parseRetryAfterFromError(error);
|
||||||
|
if (retryAt) return retryAt + BUFFER_MS;
|
||||||
|
return Date.now() + DEFAULT_COOLDOWN_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the next available (non-rate-limited) item in a rotation pool.
|
||||||
|
* Returns the index, or null if all are exhausted.
|
||||||
|
*
|
||||||
|
* This is the shared rotation algorithm used by both Claude token rotation
|
||||||
|
* and Codex account rotation.
|
||||||
|
*/
|
||||||
|
export function findNextAvailable<
|
||||||
|
T extends { rateLimitedUntil: number | null },
|
||||||
|
>(
|
||||||
|
items: T[],
|
||||||
|
currentIndex: number,
|
||||||
|
opts?: { ignoreRateLimits?: boolean },
|
||||||
|
): number | null {
|
||||||
|
const now = Date.now();
|
||||||
|
const ignoreRL = opts?.ignoreRateLimits ?? false;
|
||||||
|
|
||||||
|
for (let i = 1; i < items.length; i++) {
|
||||||
|
const idx = (currentIndex + i) % items.length;
|
||||||
|
const item = items[idx];
|
||||||
|
if (ignoreRL || !item.rateLimitedUntil || item.rateLimitedUntil <= now) {
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
@@ -14,8 +14,12 @@ import fs from 'fs';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { DATA_DIR } from './config.js';
|
import { DATA_DIR } from './config.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { getEnv } from './env.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import {
|
||||||
|
computeCooldownUntil,
|
||||||
|
findNextAvailable,
|
||||||
|
} from './token-rotation-base.js';
|
||||||
|
|
||||||
const STATE_FILE = path.join(DATA_DIR, 'token-rotation-state.json');
|
const STATE_FILE = path.join(DATA_DIR, 'token-rotation-state.json');
|
||||||
|
|
||||||
@@ -32,14 +36,8 @@ export function initTokenRotation(): void {
|
|||||||
if (initialized) return;
|
if (initialized) return;
|
||||||
initialized = true;
|
initialized = true;
|
||||||
|
|
||||||
const envFile = readEnvFile([
|
const multi = getEnv('CLAUDE_CODE_OAUTH_TOKENS');
|
||||||
'CLAUDE_CODE_OAUTH_TOKENS',
|
const single = getEnv('CLAUDE_CODE_OAUTH_TOKEN');
|
||||||
'CLAUDE_CODE_OAUTH_TOKEN',
|
|
||||||
]);
|
|
||||||
const multi =
|
|
||||||
process.env.CLAUDE_CODE_OAUTH_TOKENS || envFile.CLAUDE_CODE_OAUTH_TOKENS;
|
|
||||||
const single =
|
|
||||||
process.env.CLAUDE_CODE_OAUTH_TOKEN || envFile.CLAUDE_CODE_OAUTH_TOKEN;
|
|
||||||
|
|
||||||
const raw = multi
|
const raw = multi
|
||||||
? multi
|
? multi
|
||||||
@@ -107,30 +105,6 @@ function loadState(): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const BUFFER_MS = 3 * 60_000;
|
|
||||||
const DEFAULT_COOLDOWN_MS = 3_600_000;
|
|
||||||
|
|
||||||
function parseRetryAfterFromError(error?: string): number | null {
|
|
||||||
if (!error) return null;
|
|
||||||
const match = error.match(
|
|
||||||
/(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i,
|
|
||||||
);
|
|
||||||
if (!match) return null;
|
|
||||||
try {
|
|
||||||
const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1');
|
|
||||||
const ts = new Date(cleaned).getTime();
|
|
||||||
if (Number.isNaN(ts)) return null;
|
|
||||||
return ts;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function computeCooldownUntil(error?: string): number {
|
|
||||||
const retryAt = parseRetryAfterFromError(error);
|
|
||||||
if (retryAt) return retryAt + BUFFER_MS;
|
|
||||||
return Date.now() + DEFAULT_COOLDOWN_MS;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Get the current active token. */
|
/** Get the current active token. */
|
||||||
export function getCurrentToken(): string | undefined {
|
export function getCurrentToken(): string | undefined {
|
||||||
@@ -148,24 +122,22 @@ export function rotateToken(
|
|||||||
): boolean {
|
): boolean {
|
||||||
if (tokens.length <= 1) return false;
|
if (tokens.length <= 1) return false;
|
||||||
|
|
||||||
const now = Date.now();
|
|
||||||
tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage);
|
tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage);
|
||||||
const ignoreRL = opts?.ignoreRateLimits ?? false;
|
|
||||||
|
|
||||||
// Find next available token
|
const nextIdx = findNextAvailable(tokens, currentIndex, opts);
|
||||||
for (let i = 1; i < tokens.length; i++) {
|
if (nextIdx !== null) {
|
||||||
const idx = (currentIndex + i) % tokens.length;
|
tokens[nextIdx].rateLimitedUntil = null;
|
||||||
const state = tokens[idx];
|
currentIndex = nextIdx;
|
||||||
if (ignoreRL || !state.rateLimitedUntil || state.rateLimitedUntil <= now) {
|
logger.info(
|
||||||
state.rateLimitedUntil = null;
|
{
|
||||||
currentIndex = idx;
|
tokenIndex: currentIndex,
|
||||||
logger.info(
|
totalTokens: tokens.length,
|
||||||
{ tokenIndex: currentIndex, totalTokens: tokens.length, ignoreRL },
|
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||||
`Rotated to token #${currentIndex + 1}/${tokens.length}`,
|
},
|
||||||
);
|
`Rotated to token #${currentIndex + 1}/${tokens.length}`,
|
||||||
saveState();
|
);
|
||||||
return true;
|
saveState();
|
||||||
}
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.warn(
|
logger.warn(
|
||||||
|
|||||||
Reference in New Issue
Block a user