From 21a6be84b4a31a6656f120a36ccd21d0c0e60d48 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 19:46:08 +0900 Subject: [PATCH 01/27] feat: multi-agent progress tracking with per-subagent display Track each subagent separately via agentId from SDK task events. Single subagent shows detailed view with activity sub-lines, multiple subagents show compact one-line-each format. --- runners/agent-runner/src/index.ts | 17 ++++ src/agent-runner.ts | 3 + src/message-turn-controller.ts | 158 ++++++++++++++++++++++++------ 3 files changed, 147 insertions(+), 31 deletions(-) diff --git a/runners/agent-runner/src/index.ts b/runners/agent-runner/src/index.ts index a951ac2..d10a8b9 100644 --- a/runners/agent-runner/src/index.ts +++ b/runners/agent-runner/src/index.ts @@ -33,6 +33,9 @@ interface ContainerInput { interface ContainerOutput { status: 'success' | 'error'; phase?: 'progress' | 'final' | 'tool-activity' | 'intermediate'; + agentId?: string; + agentLabel?: string; + agentDone?: boolean; result: string | null; newSessionId?: string; error?: string; @@ -561,10 +564,21 @@ async function runQuery( if (message.type === 'system' && (message as { subtype?: string }).subtype === 'task_notification') { const tn = message as { task_id: string; status: string; summary: string }; log(`Task notification: task=${tn.task_id} status=${tn.status} summary=${tn.summary}`); + if (tn.status === 'completed' || tn.status === 'error' || tn.status === 'cancelled') { + writeOutput({ + status: 'success', + phase: 'progress', + agentId: tn.task_id, + agentDone: true, + result: tn.summary || null, + newSessionId, + }); + } } if (message.type === 'system' && (message as { subtype?: string }).subtype === 'task_progress') { const tp = message as Record; + const taskId = typeof tp.task_id === 'string' ? tp.task_id : undefined; const summary = typeof tp.summary === 'string' ? tp.summary : ''; const description = typeof tp.description === 'string' ? tp.description : ''; if (description) { @@ -572,6 +586,7 @@ async function runQuery( status: 'success', phase: 'tool-activity', result: description, + agentId: taskId, newSessionId, }); } @@ -586,6 +601,8 @@ async function runQuery( status: 'success', phase: 'progress', result: `πŸ”„ ${desc}`, + agentId: ts.task_id, + agentLabel: desc, newSessionId, }); } diff --git a/src/agent-runner.ts b/src/agent-runner.ts index b3a7b55..74842cf 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -42,6 +42,9 @@ export interface AgentOutput { status: 'success' | 'error'; result: string | null; phase?: 'progress' | 'final' | 'tool-activity' | 'intermediate'; + agentId?: string; + agentLabel?: string; + agentDone?: boolean; newSessionId?: string; error?: string; } diff --git a/src/message-turn-controller.ts b/src/message-turn-controller.ts index 348f21a..32c0b4d 100644 --- a/src/message-turn-controller.ts +++ b/src/message-turn-controller.ts @@ -7,6 +7,11 @@ import { type Channel, type RegisteredGroup } from './types.js'; export type VisiblePhase = 'silent' | 'progress' | 'final'; +interface SubagentTrack { + label: string; + activities: string[]; +} + interface MessageTurnControllerOptions { chatJid: string; group: RegisteredGroup; @@ -36,6 +41,7 @@ export class MessageTurnController { private progressTicker: ReturnType | null = null; private progressEditFailCount = 0; private latestProgressTextForFinal: string | null = null; + private subagents = new Map(); private poisonedSessionDetected = false; private closeRequested = false; @@ -117,20 +123,29 @@ export class MessageTurnController { } if (result.phase === 'tool-activity') { - // Ensure a progress message exists for tool activity sub-lines - if (!this.progressMessageId && !this.progressCreating) { - this.progressCreating = true; - const heading = this.pendingProgressText || 'μž‘μ—… 쀑...'; - this.sendProgressMessage(heading).then(() => { - this.progressCreating = false; - this.ensureProgressTicker(); - // Replay any queued tool activities now that the message exists - if (this.toolActivities.length > 0 && this.progressMessageId) { - void this.syncTrackedProgressMessage(); + if (result.agentId) { + // Subagent tool activity + let track = this.subagents.get(result.agentId); + if (!track) { + track = { label: 'μž‘μ—… 쀑...', activities: [] }; + this.subagents.set(result.agentId, track); + } + if (text) { + const MAX = 2; + track.activities.push(text); + if (track.activities.length > MAX) { + track.activities = track.activities.slice(-MAX); } - }); - this.pendingProgressText = null; + } + this.ensureProgressMessageExists(); + this.ensureProgressTicker(); + if (!this.poisonedSessionDetected) { + this.resetIdleTimer(); + } + return; } + // Main agent tool activity + this.ensureProgressMessageExists(); if (text) { this.addToolActivity(text); } @@ -141,6 +156,39 @@ export class MessageTurnController { } if (result.phase === 'progress') { + if (result.agentId) { + if (result.agentDone) { + this.subagents.delete(result.agentId); + } else { + const label = + text || + (result.agentLabel ? `πŸ”„ ${result.agentLabel}` : 'μž‘μ—… 쀑...'); + const existing = this.subagents.get(result.agentId); + if (existing) { + existing.label = label; + existing.activities = []; + } else { + this.subagents.set(result.agentId, { label, activities: [] }); + } + } + if (!this.latestProgressText) { + this.latestProgressText = 'μž‘μ—… 쀑...'; + this.latestProgressTextForFinal = 'μž‘μ—… 쀑...'; + } + this.ensureProgressMessageExists(); + this.ensureProgressTicker(); + if (this.progressMessageId) { + void this.syncTrackedProgressMessage(); + } + if (!this.poisonedSessionDetected) { + this.resetIdleTimer(); + } + if (result.status === 'error') { + this.hadError = true; + } + return; + } + // Main agent progress if (text) { if (this.progressMessageId) { // Progress message already visible β€” update heading directly @@ -262,27 +310,48 @@ export class MessageTurnController { if (minutes > 0) elapsedParts.push(`${minutes}λΆ„`); elapsedParts.push(`${seconds}초`); - const activityLines = - this.toolActivities.length > 0 - ? '\n' + - this.toolActivities - .map((a, i) => { - const isLast = i === this.toolActivities.length - 1; - const connector = isLast ? 'β””' : 'β”œ'; - const isSummary = a.startsWith('πŸ“‹'); - return isSummary ? `${connector} ${a}` : `${connector} ${a}`; - }) - .join('\n') - : ''; const suffix = `\n\n${elapsedParts.join(' ')}`; - const maxText = - 2000 - - TASK_STATUS_MESSAGE_PREFIX.length - - activityLines.length - - suffix.length; + let body: string; + + if (this.subagents.size > 1) { + // Compact: one line per subagent with latest activity + const lines: string[] = []; + for (const [, track] of this.subagents) { + const latest = track.activities[track.activities.length - 1]; + lines.push(latest ? `${track.label} Β· ${latest}` : track.label); + } + body = lines.join('\n'); + } else if (this.subagents.size === 1) { + // Single subagent: detailed view with activity sub-lines + const [, track] = this.subagents.entries().next().value!; + const lines: string[] = [track.label]; + for (let i = 0; i < track.activities.length; i++) { + const isLast = i === track.activities.length - 1; + lines.push(`${isLast ? 'β””' : 'β”œ'} ${track.activities[i]}`); + } + body = lines.join('\n'); + } else { + // Single agent rendering + const activityLines = + this.toolActivities.length > 0 + ? '\n' + + this.toolActivities + .map((a, i) => { + const isLast = i === this.toolActivities.length - 1; + const connector = isLast ? 'β””' : 'β”œ'; + const isSummary = a.startsWith('πŸ“‹'); + return isSummary ? `${connector} ${a}` : `${connector} ${a}`; + }) + .join('\n') + : ''; + body = text + activityLines; + } + + const maxBody = + 2000 - TASK_STATUS_MESSAGE_PREFIX.length - suffix.length; const truncated = - text.length > maxText ? text.slice(0, maxText - 1) + '…' : text; - return `${TASK_STATUS_MESSAGE_PREFIX}${truncated}${activityLines}${suffix}`; + body.length > maxBody ? body.slice(0, maxBody - 1) + '…' : body; + return `${TASK_STATUS_MESSAGE_PREFIX}${truncated}${suffix}`; } private clearProgressTicker(): void { @@ -296,6 +365,7 @@ export class MessageTurnController { this.pendingProgressText = null; this.progressCreating = false; this.toolActivities = []; + this.subagents.clear(); this.latestProgressText = null; this.previousProgressText = null; this.latestProgressRendered = null; @@ -304,6 +374,32 @@ export class MessageTurnController { this.progressEditFailCount = 0; } + /** + * Ensure a progress message exists in Discord. + * Creates one if needed, using pending or default text. + */ + private ensureProgressMessageExists(): void { + if (this.progressMessageId || this.progressCreating) return; + this.progressCreating = true; + const heading = + this.pendingProgressText || this.latestProgressText || 'μž‘μ—… 쀑...'; + if (!this.latestProgressText) { + this.latestProgressText = heading; + this.latestProgressTextForFinal = heading; + } + void this.sendProgressMessage(heading).then(() => { + this.progressCreating = false; + this.ensureProgressTicker(); + if ( + (this.toolActivities.length > 0 || this.subagents.size > 0) && + this.progressMessageId + ) { + void this.syncTrackedProgressMessage(); + } + }); + this.pendingProgressText = null; + } + /** * Buffer a progress update. The previous pending text gets flushed * immediately, and the new text waits until the next event arrives. From 2161b555ecb1cefbcd229157c19e5579d2211de9 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 20:16:54 +0900 Subject: [PATCH 02/27] fix: dedup intermediate/final, filter long descriptions, update SDKs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Track lastIntermediateText to prevent duplicate Discord delivery when same text arrives as both intermediate and final - Filter task_progress descriptions >80 chars (AI summaries) - Mark completed subagents with βœ… instead of removing - Update claude-agent-sdk 0.2.76β†’0.2.81, codex 0.115β†’0.116 --- runners/agent-runner/package.json | 2 +- runners/agent-runner/pnpm-lock.yaml | 10 ++--- runners/agent-runner/src/index.ts | 6 ++- runners/codex-runner/package.json | 2 +- runners/codex-runner/pnpm-lock.yaml | 68 +++++++++++++---------------- src/message-turn-controller.ts | 26 ++++++++--- 6 files changed, 62 insertions(+), 52 deletions(-) diff --git a/runners/agent-runner/package.json b/runners/agent-runner/package.json index 1154b6e..14b3084 100644 --- a/runners/agent-runner/package.json +++ b/runners/agent-runner/package.json @@ -9,7 +9,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.76", + "@anthropic-ai/claude-agent-sdk": "^0.2.81", "@modelcontextprotocol/sdk": "^1.12.1", "cron-parser": "^5.0.0", "zod": "^4.0.0" diff --git a/runners/agent-runner/pnpm-lock.yaml b/runners/agent-runner/pnpm-lock.yaml index a5dcc26..a0ac373 100644 --- a/runners/agent-runner/pnpm-lock.yaml +++ b/runners/agent-runner/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@anthropic-ai/claude-agent-sdk': - specifier: ^0.2.76 - version: 0.2.76(zod@4.3.6) + specifier: ^0.2.81 + version: 0.2.81(zod@4.3.6) '@modelcontextprotocol/sdk': specifier: ^1.12.1 version: 1.27.1(zod@4.3.6) @@ -30,8 +30,8 @@ importers: packages: - '@anthropic-ai/claude-agent-sdk@0.2.76': - resolution: {integrity: sha512-HZxvnT8ZWkzCnQygaYCA0dl8RSUzuVbxE1YG4ecy6vh4nQbTT36CxUxBy+QVdR12pPQluncC0mCOLhI2918Eaw==} + '@anthropic-ai/claude-agent-sdk@0.2.81': + resolution: {integrity: sha512-CBeebgibBEN/DWOQGZN67vhuTG55RbI1hlsFSSoZ4uA/Io3lw04eHTE2ISCmdbqyJaefYTt6GKZei1nP0TQMNw==} engines: {node: '>=18.0.0'} peerDependencies: zod: ^4.0.0 @@ -512,7 +512,7 @@ packages: snapshots: - '@anthropic-ai/claude-agent-sdk@0.2.76(zod@4.3.6)': + '@anthropic-ai/claude-agent-sdk@0.2.81(zod@4.3.6)': dependencies: zod: 4.3.6 optionalDependencies: diff --git a/runners/agent-runner/src/index.ts b/runners/agent-runner/src/index.ts index d10a8b9..6b810f6 100644 --- a/runners/agent-runner/src/index.ts +++ b/runners/agent-runner/src/index.ts @@ -581,7 +581,8 @@ async function runQuery( const taskId = typeof tp.task_id === 'string' ? tp.task_id : undefined; const summary = typeof tp.summary === 'string' ? tp.summary : ''; const description = typeof tp.description === 'string' ? tp.description : ''; - if (description) { + if (description && description.length <= 80) { + // Short tool description β†’ show as sub-line in progress writeOutput({ status: 'success', phase: 'tool-activity', @@ -589,6 +590,9 @@ async function runQuery( agentId: taskId, newSessionId, }); + } else if (description) { + // Long AI summary β†’ skip (too long for progress sub-line) + log(`Skipping long task_progress description (${description.length} chars)`); } } diff --git a/runners/codex-runner/package.json b/runners/codex-runner/package.json index a5bd352..01aaca7 100644 --- a/runners/codex-runner/package.json +++ b/runners/codex-runner/package.json @@ -9,7 +9,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@openai/codex": "^0.115.0" + "@openai/codex": "^0.116.0" }, "devDependencies": { "@types/node": "^22.10.7", diff --git a/runners/codex-runner/pnpm-lock.yaml b/runners/codex-runner/pnpm-lock.yaml index e397a43..a59b118 100644 --- a/runners/codex-runner/pnpm-lock.yaml +++ b/runners/codex-runner/pnpm-lock.yaml @@ -8,9 +8,9 @@ importers: .: dependencies: - '@openai/codex-sdk': - specifier: ^0.115.0 - version: 0.115.0 + '@openai/codex': + specifier: ^0.116.0 + version: 0.116.0 devDependencies: '@types/node': specifier: ^22.10.7 @@ -21,47 +21,43 @@ importers: packages: - '@openai/codex-sdk@0.115.0': - resolution: {integrity: sha512-BPoPhim0uUm3rzugY7JFaFJ+rPG/wMRhvKNFii//Dp3kTb+gFy3jrip7ijXqhswsnqXM3nwTiv7kYHt1+TMUPg==} - engines: {node: '>=18'} - - '@openai/codex@0.115.0': - resolution: {integrity: sha512-uu689DHUzvuPcb39hJ+7fqy++TAvY32w9VggDpcz3HS0Sx0WadWoAPPcMK547P2T6AqfMsAtA8kspkR3tqErOg==} + '@openai/codex@0.116.0': + resolution: {integrity: sha512-K6q9P2ZmpnzGmpS6Ybjvsdtvu8AbJx3f/Z4KmjH1u85StSS9TWMSQB8z0PPObKMejbtiIkHwhGyEIHi4iBYjig==} engines: {node: '>=16'} hasBin: true - '@openai/codex@0.115.0-darwin-arm64': - resolution: {integrity: sha512-wTWV+YlDTL0y0mL+FMmbzhilm+dPkbIZTe/lH3LwBzcEMhgSGLsPdZLfAiMND4wFE21HS7H0pjMogNQEMLQmqg==} + '@openai/codex@0.116.0-darwin-arm64': + resolution: {integrity: sha512-WkdL083p8uMeASpg8bwV0DPGgzkm48LjN3MyU2m/YukujbiLnknAmG29O2q2rFCLm0oLSDIGUK8EnXA4ZcAF9Q==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] - '@openai/codex@0.115.0-darwin-x64': - resolution: {integrity: sha512-EPzgymU4CFp83qjv29wXFwhWib3zC8g6SLTJGh2OpcJiOYyLY0bO53FB+QchL1jC9gm1uLyD5j5F3ddI0AbjIg==} + '@openai/codex@0.116.0-darwin-x64': + resolution: {integrity: sha512-Ax8uTwYSNIwGrzcNRcn0jJQhZzNcKGDbbn00Emde7gGOemjSLhRALjUaKjckAaW5xWnNqHTGdtzzPB4phNlDYg==} engines: {node: '>=16'} cpu: [x64] os: [darwin] - '@openai/codex@0.115.0-linux-arm64': - resolution: {integrity: sha512-40+SCyI+LvVx/iX30qH7dTQzWt9MZxDaK2E6YRB4hoYej5UALrhkFNzweHa5uvqvhmqGZCuagZOYB8285eGCgw==} + '@openai/codex@0.116.0-linux-arm64': + resolution: {integrity: sha512-X7cL8rBSGDB+RSZc2FoKiqcMVeLPMmo06bkss/en4lLQsV1XG2DZI56WuXg92IOX3SjYl6Av/eOWgsb1t3UeLQ==} engines: {node: '>=16'} cpu: [arm64] os: [linux] - '@openai/codex@0.115.0-linux-x64': - resolution: {integrity: sha512-+6eRd2p4KMrhQvuF7XpjYIdCA2Ok2LbhOq+ywZdLpHbmwQ/Yynd0gJ/Q90xCeo2vloCwl9WsbsiVVSahG5FyWg==} + '@openai/codex@0.116.0-linux-x64': + resolution: {integrity: sha512-S9InOgJT3tj6uQp55NqrCA1k5tklwFaH00JdC2ElbRmxchm7ard4WxHSJZX9TiY8enj4cQoLIC04NFTUCO+/PQ==} engines: {node: '>=16'} cpu: [x64] os: [linux] - '@openai/codex@0.115.0-win32-arm64': - resolution: {integrity: sha512-yaYhQ0kPL9Kithmrr1vh5A4c+gqAMhMZeA/htTtkpWW8RQkmwgcYYpX/Ky2cEzu0w51pvxQQy63LgHftc+pg1Q==} + '@openai/codex@0.116.0-win32-arm64': + resolution: {integrity: sha512-kX2oAUzkgZX9OsYpd4omv9IGf+9VWj4Vy3UtIAnQKBu1DTSzmTJmXDuDn87mkyUciSZadm2QbeqQQzm2NC0NYw==} engines: {node: '>=16'} cpu: [arm64] os: [win32] - '@openai/codex@0.115.0-win32-x64': - resolution: {integrity: sha512-wtYf2HJCB+p+Uj7o1W08fRgZMzKCVGRQ4YdSfLRNfF4wwPqX5XsK9blsv7brukn5J9lclYxagiR6qGvbfHbD7w==} + '@openai/codex@0.116.0-win32-x64': + resolution: {integrity: sha512-6sBIMOoA9FNuxQvCCnK0P548Wqrlk3I9SMdtOCUg2zYzYU7jOF2mWS1VpRQ6R+Jvo2x50dxeJZ+W37dBmXfprw==} engines: {node: '>=16'} cpu: [x64] os: [win32] @@ -79,35 +75,31 @@ packages: snapshots: - '@openai/codex-sdk@0.115.0': - dependencies: - '@openai/codex': 0.115.0 - - '@openai/codex@0.115.0': + '@openai/codex@0.116.0': optionalDependencies: - '@openai/codex-darwin-arm64': '@openai/codex@0.115.0-darwin-arm64' - '@openai/codex-darwin-x64': '@openai/codex@0.115.0-darwin-x64' - '@openai/codex-linux-arm64': '@openai/codex@0.115.0-linux-arm64' - '@openai/codex-linux-x64': '@openai/codex@0.115.0-linux-x64' - '@openai/codex-win32-arm64': '@openai/codex@0.115.0-win32-arm64' - '@openai/codex-win32-x64': '@openai/codex@0.115.0-win32-x64' + '@openai/codex-darwin-arm64': '@openai/codex@0.116.0-darwin-arm64' + '@openai/codex-darwin-x64': '@openai/codex@0.116.0-darwin-x64' + '@openai/codex-linux-arm64': '@openai/codex@0.116.0-linux-arm64' + '@openai/codex-linux-x64': '@openai/codex@0.116.0-linux-x64' + '@openai/codex-win32-arm64': '@openai/codex@0.116.0-win32-arm64' + '@openai/codex-win32-x64': '@openai/codex@0.116.0-win32-x64' - '@openai/codex@0.115.0-darwin-arm64': + '@openai/codex@0.116.0-darwin-arm64': optional: true - '@openai/codex@0.115.0-darwin-x64': + '@openai/codex@0.116.0-darwin-x64': optional: true - '@openai/codex@0.115.0-linux-arm64': + '@openai/codex@0.116.0-linux-arm64': optional: true - '@openai/codex@0.115.0-linux-x64': + '@openai/codex@0.116.0-linux-x64': optional: true - '@openai/codex@0.115.0-win32-arm64': + '@openai/codex@0.116.0-win32-arm64': optional: true - '@openai/codex@0.115.0-win32-x64': + '@openai/codex@0.116.0-win32-x64': optional: true '@types/node@22.19.15': diff --git a/src/message-turn-controller.ts b/src/message-turn-controller.ts index 32c0b4d..72623af 100644 --- a/src/message-turn-controller.ts +++ b/src/message-turn-controller.ts @@ -42,6 +42,7 @@ export class MessageTurnController { private progressEditFailCount = 0; private latestProgressTextForFinal: string | null = null; private subagents = new Map(); + private lastIntermediateText: string | null = null; private poisonedSessionDetected = false; private closeRequested = false; @@ -114,6 +115,7 @@ export class MessageTurnController { if (result.phase === 'intermediate') { // Send as standalone message without touching progress state if (text) { + this.lastIntermediateText = text; await this.options.channel.sendMessage(this.options.chatJid, text); } if (!this.poisonedSessionDetected) { @@ -158,7 +160,11 @@ export class MessageTurnController { if (result.phase === 'progress') { if (result.agentId) { if (result.agentDone) { - this.subagents.delete(result.agentId); + const done = this.subagents.get(result.agentId); + if (done) { + done.label = done.label.replace('πŸ”„', 'βœ…'); + done.activities = []; + } } else { const label = text || @@ -212,9 +218,18 @@ export class MessageTurnController { // Final arrived β€” flush any buffered progress that isn't the same text, // then discard the pending buffer so it never shows up. if (text) { - await this.flushPendingProgress(text); - await this.finalizeProgressMessage(); - await this.deliverFinalText(text); + if (this.lastIntermediateText && text === this.lastIntermediateText) { + // Already sent as intermediate β€” skip duplicate, just finalize + this.lastIntermediateText = null; + await this.finalizeProgressMessage(); + this.visiblePhase = 'final'; + this.latestProgressTextForFinal = null; + } else { + this.lastIntermediateText = null; + await this.flushPendingProgress(text); + await this.finalizeProgressMessage(); + await this.deliverFinalText(text); + } } else if (raw) { logger.info( { @@ -347,8 +362,7 @@ export class MessageTurnController { body = text + activityLines; } - const maxBody = - 2000 - TASK_STATUS_MESSAGE_PREFIX.length - suffix.length; + const maxBody = 2000 - TASK_STATUS_MESSAGE_PREFIX.length - suffix.length; const truncated = body.length > maxBody ? body.slice(0, maxBody - 1) + '…' : body; return `${TASK_STATUS_MESSAGE_PREFIX}${truncated}${suffix}`; From 961b8ccf65199813316447bb71ad3a1f48c47b24 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 20:17:52 +0900 Subject: [PATCH 03/27] docs: add SDK version badges to README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 61a6aa8..c97064e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # EJClaw +![Claude Agent SDK](https://img.shields.io/badge/Claude_Agent_SDK-0.2.81-blueviolet) +![Codex SDK](https://img.shields.io/badge/Codex_SDK-0.116.0-green) +![Node](https://img.shields.io/badge/Node-20+-339933?logo=nodedotjs&logoColor=white) +![Discord](https://img.shields.io/badge/Discord-Bot-5865F2?logo=discord&logoColor=white) + Dual-agent AI assistant (Claude Code + Codex) over Discord. Originally derived from [qwibitai/nanoclaw](https://github.com/qwibitai/nanoclaw), now maintained as EJClaw for personal production use. From 45a332f6bb5206799129a496eee8ff7452ab4415 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 20:30:29 +0900 Subject: [PATCH 04/27] docs: update Node requirement to 24+ --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c97064e..77b71dd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Claude Agent SDK](https://img.shields.io/badge/Claude_Agent_SDK-0.2.81-blueviolet) ![Codex SDK](https://img.shields.io/badge/Codex_SDK-0.116.0-green) -![Node](https://img.shields.io/badge/Node-20+-339933?logo=nodedotjs&logoColor=white) +![Node](https://img.shields.io/badge/Node-24+-339933?logo=nodedotjs&logoColor=white) ![Discord](https://img.shields.io/badge/Discord-Bot-5865F2?logo=discord&logoColor=white) Dual-agent AI assistant (Claude Code + Codex) over Discord. @@ -52,7 +52,7 @@ Each agent has access to: ### Prerequisites - Linux (Ubuntu 22.04+) or macOS -- Node.js 20+ (fnm recommended) +- Node.js 24+ (fnm recommended) - [Claude Code CLI](https://docs.anthropic.com/en/docs/claude-code) - [Codex CLI](https://github.com/openai/codex) (`npm install -g @openai/codex`) - Bun 1.0+ (for browser automation) From ee65330e6665121413c723061c2b30c4b7030faf Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 20:38:30 +0900 Subject: [PATCH 05/27] fix: rollback codex SDK to 0.115.0 (0.116.0 causes stuck turns) --- README.md | 2 +- runners/codex-runner/package.json | 2 +- runners/codex-runner/pnpm-lock.yaml | 58 ++++++++++++++--------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 77b71dd..e4a4e52 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # EJClaw ![Claude Agent SDK](https://img.shields.io/badge/Claude_Agent_SDK-0.2.81-blueviolet) -![Codex SDK](https://img.shields.io/badge/Codex_SDK-0.116.0-green) +![Codex SDK](https://img.shields.io/badge/Codex_SDK-0.115.0-green) ![Node](https://img.shields.io/badge/Node-24+-339933?logo=nodedotjs&logoColor=white) ![Discord](https://img.shields.io/badge/Discord-Bot-5865F2?logo=discord&logoColor=white) diff --git a/runners/codex-runner/package.json b/runners/codex-runner/package.json index 01aaca7..a5bd352 100644 --- a/runners/codex-runner/package.json +++ b/runners/codex-runner/package.json @@ -9,7 +9,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@openai/codex": "^0.116.0" + "@openai/codex": "^0.115.0" }, "devDependencies": { "@types/node": "^22.10.7", diff --git a/runners/codex-runner/pnpm-lock.yaml b/runners/codex-runner/pnpm-lock.yaml index a59b118..572f06f 100644 --- a/runners/codex-runner/pnpm-lock.yaml +++ b/runners/codex-runner/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@openai/codex': - specifier: ^0.116.0 - version: 0.116.0 + specifier: ^0.115.0 + version: 0.115.0 devDependencies: '@types/node': specifier: ^22.10.7 @@ -21,43 +21,43 @@ importers: packages: - '@openai/codex@0.116.0': - resolution: {integrity: sha512-K6q9P2ZmpnzGmpS6Ybjvsdtvu8AbJx3f/Z4KmjH1u85StSS9TWMSQB8z0PPObKMejbtiIkHwhGyEIHi4iBYjig==} + '@openai/codex@0.115.0': + resolution: {integrity: sha512-uu689DHUzvuPcb39hJ+7fqy++TAvY32w9VggDpcz3HS0Sx0WadWoAPPcMK547P2T6AqfMsAtA8kspkR3tqErOg==} engines: {node: '>=16'} hasBin: true - '@openai/codex@0.116.0-darwin-arm64': - resolution: {integrity: sha512-WkdL083p8uMeASpg8bwV0DPGgzkm48LjN3MyU2m/YukujbiLnknAmG29O2q2rFCLm0oLSDIGUK8EnXA4ZcAF9Q==} + '@openai/codex@0.115.0-darwin-arm64': + resolution: {integrity: sha512-wTWV+YlDTL0y0mL+FMmbzhilm+dPkbIZTe/lH3LwBzcEMhgSGLsPdZLfAiMND4wFE21HS7H0pjMogNQEMLQmqg==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] - '@openai/codex@0.116.0-darwin-x64': - resolution: {integrity: sha512-Ax8uTwYSNIwGrzcNRcn0jJQhZzNcKGDbbn00Emde7gGOemjSLhRALjUaKjckAaW5xWnNqHTGdtzzPB4phNlDYg==} + '@openai/codex@0.115.0-darwin-x64': + resolution: {integrity: sha512-EPzgymU4CFp83qjv29wXFwhWib3zC8g6SLTJGh2OpcJiOYyLY0bO53FB+QchL1jC9gm1uLyD5j5F3ddI0AbjIg==} engines: {node: '>=16'} cpu: [x64] os: [darwin] - '@openai/codex@0.116.0-linux-arm64': - resolution: {integrity: sha512-X7cL8rBSGDB+RSZc2FoKiqcMVeLPMmo06bkss/en4lLQsV1XG2DZI56WuXg92IOX3SjYl6Av/eOWgsb1t3UeLQ==} + '@openai/codex@0.115.0-linux-arm64': + resolution: {integrity: sha512-40+SCyI+LvVx/iX30qH7dTQzWt9MZxDaK2E6YRB4hoYej5UALrhkFNzweHa5uvqvhmqGZCuagZOYB8285eGCgw==} engines: {node: '>=16'} cpu: [arm64] os: [linux] - '@openai/codex@0.116.0-linux-x64': - resolution: {integrity: sha512-S9InOgJT3tj6uQp55NqrCA1k5tklwFaH00JdC2ElbRmxchm7ard4WxHSJZX9TiY8enj4cQoLIC04NFTUCO+/PQ==} + '@openai/codex@0.115.0-linux-x64': + resolution: {integrity: sha512-+6eRd2p4KMrhQvuF7XpjYIdCA2Ok2LbhOq+ywZdLpHbmwQ/Yynd0gJ/Q90xCeo2vloCwl9WsbsiVVSahG5FyWg==} engines: {node: '>=16'} cpu: [x64] os: [linux] - '@openai/codex@0.116.0-win32-arm64': - resolution: {integrity: sha512-kX2oAUzkgZX9OsYpd4omv9IGf+9VWj4Vy3UtIAnQKBu1DTSzmTJmXDuDn87mkyUciSZadm2QbeqQQzm2NC0NYw==} + '@openai/codex@0.115.0-win32-arm64': + resolution: {integrity: sha512-yaYhQ0kPL9Kithmrr1vh5A4c+gqAMhMZeA/htTtkpWW8RQkmwgcYYpX/Ky2cEzu0w51pvxQQy63LgHftc+pg1Q==} engines: {node: '>=16'} cpu: [arm64] os: [win32] - '@openai/codex@0.116.0-win32-x64': - resolution: {integrity: sha512-6sBIMOoA9FNuxQvCCnK0P548Wqrlk3I9SMdtOCUg2zYzYU7jOF2mWS1VpRQ6R+Jvo2x50dxeJZ+W37dBmXfprw==} + '@openai/codex@0.115.0-win32-x64': + resolution: {integrity: sha512-wtYf2HJCB+p+Uj7o1W08fRgZMzKCVGRQ4YdSfLRNfF4wwPqX5XsK9blsv7brukn5J9lclYxagiR6qGvbfHbD7w==} engines: {node: '>=16'} cpu: [x64] os: [win32] @@ -75,31 +75,31 @@ packages: snapshots: - '@openai/codex@0.116.0': + '@openai/codex@0.115.0': optionalDependencies: - '@openai/codex-darwin-arm64': '@openai/codex@0.116.0-darwin-arm64' - '@openai/codex-darwin-x64': '@openai/codex@0.116.0-darwin-x64' - '@openai/codex-linux-arm64': '@openai/codex@0.116.0-linux-arm64' - '@openai/codex-linux-x64': '@openai/codex@0.116.0-linux-x64' - '@openai/codex-win32-arm64': '@openai/codex@0.116.0-win32-arm64' - '@openai/codex-win32-x64': '@openai/codex@0.116.0-win32-x64' + '@openai/codex-darwin-arm64': '@openai/codex@0.115.0-darwin-arm64' + '@openai/codex-darwin-x64': '@openai/codex@0.115.0-darwin-x64' + '@openai/codex-linux-arm64': '@openai/codex@0.115.0-linux-arm64' + '@openai/codex-linux-x64': '@openai/codex@0.115.0-linux-x64' + '@openai/codex-win32-arm64': '@openai/codex@0.115.0-win32-arm64' + '@openai/codex-win32-x64': '@openai/codex@0.115.0-win32-x64' - '@openai/codex@0.116.0-darwin-arm64': + '@openai/codex@0.115.0-darwin-arm64': optional: true - '@openai/codex@0.116.0-darwin-x64': + '@openai/codex@0.115.0-darwin-x64': optional: true - '@openai/codex@0.116.0-linux-arm64': + '@openai/codex@0.115.0-linux-arm64': optional: true - '@openai/codex@0.116.0-linux-x64': + '@openai/codex@0.115.0-linux-x64': optional: true - '@openai/codex@0.116.0-win32-arm64': + '@openai/codex@0.115.0-win32-arm64': optional: true - '@openai/codex@0.116.0-win32-x64': + '@openai/codex@0.115.0-win32-x64': optional: true '@types/node@22.19.15': From d378598f2e6042be0b1629f87089b212bc16dd0b Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 22:01:06 +0900 Subject: [PATCH 06/27] feat: Claude OAuth token rotation on rate-limit - New token-rotation module: stores multiple tokens from CLAUDE_CODE_OAUTH_TOKENS env var (comma-separated) - On rate-limit, rotates to next healthy token before falling back to Kimi provider - Also fixes: intermediate text routes to progress message when no subagents active (prevents message flooding) --- src/agent-runner-environment.ts | 15 +++-- src/index.ts | 4 ++ src/message-agent-executor.ts | 24 +++++++ src/message-turn-controller.ts | 18 ++++- src/task-suspension.ts | 5 +- src/token-rotation.ts | 116 ++++++++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 src/token-rotation.ts diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index 1f3f204..c32587e 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -5,6 +5,7 @@ import path from 'path'; import { GROUPS_DIR, TIMEZONE } from './config.js'; import { isPairedRoomJid } from './db.js'; import { readEnvFile } from './env.js'; +import { getCurrentToken } from './token-rotation.js'; import { resolveGroupFolderPath, resolveGroupIpcPath, @@ -120,14 +121,14 @@ function prepareClaudeEnvironment(args: { args.env.ANTHROPIC_BASE_URL = args.envVars.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL || ''; } - if ( - args.envVars.CLAUDE_CODE_OAUTH_TOKEN || - process.env.CLAUDE_CODE_OAUTH_TOKEN - ) { - args.env.CLAUDE_CODE_OAUTH_TOKEN = + { + const oauthToken = args.envVars.CLAUDE_CODE_OAUTH_TOKEN || - process.env.CLAUDE_CODE_OAUTH_TOKEN || - ''; + getCurrentToken() || + process.env.CLAUDE_CODE_OAUTH_TOKEN; + if (oauthToken) { + args.env.CLAUDE_CODE_OAUTH_TOKEN = oauthToken; + } } for (const key of [ 'CLAUDE_MODEL', diff --git a/src/index.ts b/src/index.ts index 8a33e6e..8e5c216 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,11 +63,15 @@ import { startUnifiedDashboard } from './unified-dashboard.js'; import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { normalizeStoredSeqCursor } from './message-cursor.js'; +import { initTokenRotation } from './token-rotation.js'; // Re-export for backwards compatibility during refactor export { escapeXml, formatMessages } from './router.js'; export { composeDashboardContent } from './dashboard-render.js'; +// Initialize token rotation early (reads CLAUDE_CODE_OAUTH_TOKENS from env) +initTokenRotation(); + export async function sendFormattedChannelMessage( channels: Channel[], jid: string, diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index c1b4d8f..968bf1c 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -21,6 +21,7 @@ import { markPrimaryCooldown, } from './provider-fallback.js'; import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; +import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; export interface MessageAgentExecutorDeps { @@ -299,6 +300,18 @@ export async function runAgentForGroup( } : detectFallbackTrigger(errMsg); if (trigger.shouldFallback) { + // Try rotating token before falling back to another provider + if (getTokenCount() > 1 && rotateToken()) { + logger.info( + { chatJid, group: group.name, runId, reason: trigger.reason }, + 'Rate-limited, retrying with rotated token', + ); + const retryAttempt = await runAttempt('claude'); + if (!retryAttempt.error) { + markTokenHealthy(); + return 'success'; + } + } return runFallbackAttempt(trigger.reason, trigger.retryAfterMs); } } @@ -362,6 +375,17 @@ export async function runAgentForGroup( } : detectFallbackTrigger(output.error); if (trigger.shouldFallback) { + if (getTokenCount() > 1 && rotateToken()) { + logger.info( + { chatJid, group: group.name, runId, reason: trigger.reason }, + 'Rate-limited (output error), retrying with rotated token', + ); + const retryAttempt = await runAttempt('claude'); + if (!retryAttempt.error) { + markTokenHealthy(); + return 'success'; + } + } return runFallbackAttempt(trigger.reason, trigger.retryAfterMs); } } diff --git a/src/message-turn-controller.ts b/src/message-turn-controller.ts index 72623af..7f72ae0 100644 --- a/src/message-turn-controller.ts +++ b/src/message-turn-controller.ts @@ -113,10 +113,22 @@ export class MessageTurnController { } if (result.phase === 'intermediate') { - // Send as standalone message without touching progress state if (text) { - this.lastIntermediateText = text; - await this.options.channel.sendMessage(this.options.chatJid, text); + if (this.subagents.size > 0) { + // Subagents active β€” standalone to not disrupt progress + this.lastIntermediateText = text; + await this.options.channel.sendMessage(this.options.chatJid, text); + } else if (this.progressMessageId) { + // Progress exists β€” update heading + this.previousProgressText = this.latestProgressText; + this.latestProgressText = text; + this.latestProgressTextForFinal = text; + this.toolActivities = []; + void this.syncTrackedProgressMessage(); + } else { + // No progress yet β€” buffer (creates on next event) + this.bufferProgress(text); + } } if (!this.poisonedSessionDetected) { this.resetIdleTimer(); diff --git a/src/task-suspension.ts b/src/task-suspension.ts index 514f808..b3f4555 100644 --- a/src/task-suspension.ts +++ b/src/task-suspension.ts @@ -105,10 +105,7 @@ export function evaluateTaskSuspension( /** * Apply suspension to a task in the DB. */ -export function suspendTask( - taskId: string, - suspendedUntil: string, -): void { +export function suspendTask(taskId: string, suspendedUntil: string): void { updateTask(taskId, { suspended_until: suspendedUntil }); logger.info( { taskId, suspendedUntil }, diff --git a/src/token-rotation.ts b/src/token-rotation.ts new file mode 100644 index 0000000..944c7a3 --- /dev/null +++ b/src/token-rotation.ts @@ -0,0 +1,116 @@ +/** + * OAuth Token Rotation + * + * Rotates between multiple CLAUDE_CODE_OAUTH_TOKEN values when + * rate-limited. Tokens are stored as comma-separated values in + * CLAUDE_CODE_OAUTH_TOKENS env var. Falls through to the single + * CLAUDE_CODE_OAUTH_TOKEN if multi-token is not configured. + * + * On rate-limit: rotate to next token + * All exhausted: fall through to provider fallback (Kimi etc.) + */ + +import { logger } from './logger.js'; + +interface TokenState { + token: string; + rateLimitedUntil: number | null; +} + +const tokens: TokenState[] = []; +let currentIndex = 0; +let initialized = false; + +export function initTokenRotation(): void { + if (initialized) return; + initialized = true; + + const multi = process.env.CLAUDE_CODE_OAUTH_TOKENS; + const single = process.env.CLAUDE_CODE_OAUTH_TOKEN; + + const raw = multi + ? multi.split(',').map((t) => t.trim()).filter(Boolean) + : single + ? [single] + : []; + + for (const token of raw) { + tokens.push({ token, rateLimitedUntil: null }); + } + + if (tokens.length > 1) { + logger.info( + { count: tokens.length }, + `Token rotation initialized with ${tokens.length} tokens`, + ); + } +} + +/** Get the current active token. */ +export function getCurrentToken(): string | undefined { + if (tokens.length === 0) return process.env.CLAUDE_CODE_OAUTH_TOKEN; + return tokens[currentIndex % tokens.length]?.token; +} + +/** + * Try to rotate to the next available (non-rate-limited) token. + * Returns true if a fresh token was found, false if all are exhausted. + */ +export function rotateToken(): boolean { + if (tokens.length <= 1) return false; + + const now = Date.now(); + // Mark current as rate-limited (default 1 hour) + tokens[currentIndex].rateLimitedUntil = now + 3_600_000; + + // Find next available token + for (let i = 1; i < tokens.length; i++) { + const idx = (currentIndex + i) % tokens.length; + const state = tokens[idx]; + if (!state.rateLimitedUntil || state.rateLimitedUntil <= now) { + state.rateLimitedUntil = null; + currentIndex = idx; + logger.info( + { tokenIndex: currentIndex, totalTokens: tokens.length }, + `Rotated to token #${currentIndex + 1}/${tokens.length}`, + ); + return true; + } + } + + logger.warn( + { totalTokens: tokens.length }, + 'All tokens are rate-limited, falling through to provider fallback', + ); + return false; +} + +/** Clear rate-limit flag for the current token (on successful response). */ +export function markTokenHealthy(): void { + if (tokens.length === 0) return; + const state = tokens[currentIndex]; + if (state?.rateLimitedUntil) { + state.rateLimitedUntil = null; + } +} + +/** Number of configured tokens. */ +export function getTokenCount(): number { + return tokens.length; +} + +/** Diagnostic info. */ +export function getTokenRotationInfo(): { + total: number; + currentIndex: number; + rateLimited: number; +} { + const now = Date.now(); + return { + total: tokens.length, + currentIndex, + rateLimited: tokens.filter( + (t) => t.rateLimitedUntil && t.rateLimitedUntil > now, + ).length, + }; +} From 83b25fb7bdcec6434f8274d40c361fa17eac4cbc Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 22:15:06 +0900 Subject: [PATCH 07/27] feat: Claude usage API + multi-account dashboard display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace expect/CLI hack with direct HTTP API call (GET api.anthropic.com/api/oauth/usage with OAuth token) - Add fetchAllClaudeUsage() for per-token usage fetching - Export getAllTokens() from token-rotation for usage queries - Dashboard shows each account separately (Claude1⚑, Claude2) with ⚑ for active token, 🚫 for rate-limited --- src/claude-usage.test.ts | 63 +++------ src/claude-usage.ts | 279 +++++++++++++++++---------------------- src/token-rotation.ts | 17 ++- src/unified-dashboard.ts | 40 +++--- 4 files changed, 181 insertions(+), 218 deletions(-) diff --git a/src/claude-usage.test.ts b/src/claude-usage.test.ts index e868e2b..ce661c7 100644 --- a/src/claude-usage.test.ts +++ b/src/claude-usage.test.ts @@ -1,53 +1,28 @@ import { describe, expect, it } from 'vitest'; -import { parseClaudeUsagePanel } from './claude-usage.js'; +import type { ClaudeUsageData } from './claude-usage.js'; -describe('parseClaudeUsagePanel', () => { - it('parses session and weekly usage from Claude CLI panel output', () => { - const sample = ` -Settings: Status Config Usage -Loading usage data... +describe('ClaudeUsageData', () => { + it('represents the API response structure correctly', () => { + const data: ClaudeUsageData = { + five_hour: { utilization: 45.2, resets_at: '2026-03-23T17:00:00Z' }, + seven_day: { utilization: 72.1, resets_at: '2026-03-29T00:00:00Z' }, + seven_day_sonnet: { utilization: 60.0, resets_at: '2026-03-29T00:00:00Z' }, + seven_day_opus: { utilization: 80.0, resets_at: '2026-03-29T00:00:00Z' }, + }; -Current session -β–ˆβ–ˆβ–ˆβ–ˆ 4% used -Resets in 8m (Asia/Seoul) - -Current week (all models) -β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ 78% used -Resets Mar 17 at 10pm (Asia/Seoul) - -Current week (Sonnet only) -β–ˆβ–ˆβ–ˆ 6% used -Resets Mar 17 at 11pm (Asia/Seoul) - -Extra usage -Extra usage not enabled β€’ /extra-usage to enable -`; - - expect(parseClaudeUsagePanel(sample)).toEqual({ - five_hour: { - utilization: 4, - resets_at: 'Resets in 8m (Asia/Seoul)', - }, - seven_day: { - utilization: 78, - resets_at: 'Resets Mar 17 at 10pm (Asia/Seoul)', - }, - }); + expect(data.five_hour?.utilization).toBe(45.2); + expect(data.seven_day?.utilization).toBe(72.1); + expect(data.seven_day_sonnet?.utilization).toBe(60.0); + expect(data.seven_day_opus?.utilization).toBe(80.0); }); - it('converts percent left into used percent', () => { - const sample = ` -Current session -60% left -Resets in 1h -`; + it('allows partial data (only five_hour)', () => { + const data: ClaudeUsageData = { + five_hour: { utilization: 10, resets_at: '2026-03-23T17:00:00Z' }, + }; - expect(parseClaudeUsagePanel(sample)).toEqual({ - five_hour: { - utilization: 40, - resets_at: 'Resets in 1h', - }, - }); + expect(data.five_hour?.utilization).toBe(10); + expect(data.seven_day).toBeUndefined(); }); }); diff --git a/src/claude-usage.ts b/src/claude-usage.ts index 3ed83c1..b286eae 100644 --- a/src/claude-usage.ts +++ b/src/claude-usage.ts @@ -1,183 +1,148 @@ -import { spawn } from 'child_process'; +/** + * Claude Usage API + * + * Fetches usage data directly from the Anthropic OAuth API. + * Supports multiple tokens for rotation-aware usage checking. + */ import { logger } from './logger.js'; +import { getCurrentToken, getAllTokens } from './token-rotation.js'; export interface ClaudeUsageData { five_hour?: { utilization: number; resets_at: string }; seven_day?: { utilization: number; resets_at: string }; + seven_day_sonnet?: { utilization: number; resets_at: string }; + seven_day_opus?: { utilization: number; resets_at: string }; } -const CLAUDE_EXPECT_TIMEOUT_MS = 25000; -const ANSI_RE = /\u001b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g; +const USAGE_ENDPOINT = 'https://api.anthropic.com/api/oauth/usage'; +const FETCH_TIMEOUT_MS = 10_000; -const EXPECT_PROGRAM = ` -set timeout 20 -log_user 1 -match_max 200000 -set binary $env(CLAUDE_BINARY) -spawn -noecho -- $binary --setting-sources user --allowed-tools "" -expect { - -re "Do you trust the files in this folder\\\\?" { send "y\\r"; exp_continue } - -re "Quick safety check:" { send "\\r"; exp_continue } - -re "Yes, I trust this folder" { send "\\r"; exp_continue } - -re "Ready to code here\\\\?" { send "\\r"; exp_continue } - -re "Press Enter to continue" { send "\\r"; exp_continue } - timeout {} -} -send "/usage\\r" -set deadline [expr {[clock seconds] + 20}] -while {[clock seconds] < $deadline} { - expect { - -re "Do you trust the files in this folder\\\\?" { send "y\\r"; exp_continue } - -re "Quick safety check:" { send "\\r"; exp_continue } - -re "Yes, I trust this folder" { send "\\r"; exp_continue } - -re "Ready to code here\\\\?" { send "\\r"; exp_continue } - -re "Press Enter to continue" { send "\\r"; exp_continue } - -re "Current session" { after 2000; exit 0 } - -re "Failed to load usage data" { after 500; exit 2 } - eof { exit 3 } - timeout { send "\\r" } - } -} -exit 4 -`; - -function normalizeLines(rawText: string): string[] { - return rawText - .replace(ANSI_RE, '') - .replace(/\r/g, '\n') - .split('\n') - .map((line) => line.replace(/\s+/g, ' ').trim()) - .filter(Boolean); +interface UsageApiResponse { + five_hour?: { utilization: number; resets_at?: string }; + seven_day?: { utilization: number; resets_at?: string }; + seven_day_sonnet?: { utilization: number; resets_at?: string }; + seven_day_opus?: { utilization: number; resets_at?: string }; } -function parsePercent(windowText: string): number | null { - const match = windowText.match(/(\d{1,3})%\s*(used|left)\b/i); - if (!match) return null; - const value = parseInt(match[1], 10); - if (Number.isNaN(value)) return null; - return match[2].toLowerCase() === 'left' ? 100 - value : value; +function mapWindow( + w?: { utilization: number; resets_at?: string }, +): { utilization: number; resets_at: string } | undefined { + if (!w) return undefined; + return { utilization: w.utilization, resets_at: w.resets_at || '' }; } -function parseWindow( - lines: string[], - labels: string[], -): { utilization: number; resets_at: string } | null { - const normalizedLabels = labels.map((label) => label.toLowerCase()); - for (let i = 0; i < lines.length; i++) { - const line = lines[i].toLowerCase(); - if (!normalizedLabels.some((label) => line.includes(label))) continue; +async function fetchUsageForToken( + token: string, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); - const windowLines = lines.slice(i, i + 6); - const windowText = windowLines.join('\n'); - const utilization = parsePercent(windowText); - if (utilization === null) continue; + try { + const res = await fetch(USAGE_ENDPOINT, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'anthropic-beta': 'oauth-2025-04-20', + 'User-Agent': 'ejclaw/1.0', + }, + signal: controller.signal, + }); - const resetLine = windowLines.find((candidate) => - candidate.toLowerCase().startsWith('resets'), - ); + if (res.status === 401) { + logger.warn('Claude usage API: token expired or invalid (401)'); + return null; + } + if (res.status === 429) { + logger.warn('Claude usage API: rate limited (429)'); + return null; + } + if (!res.ok) { + logger.warn( + { status: res.status }, + `Claude usage API: unexpected status ${res.status}`, + ); + return null; + } + + const data = (await res.json()) as UsageApiResponse; return { - utilization, - resets_at: resetLine || '', + five_hour: mapWindow(data.five_hour), + seven_day: mapWindow(data.seven_day), + seven_day_sonnet: mapWindow(data.seven_day_sonnet), + seven_day_opus: mapWindow(data.seven_day_opus), }; + } catch (err) { + if ((err as Error).name === 'AbortError') { + logger.warn('Claude usage API: request timed out'); + } else { + logger.warn( + { err }, + 'Claude usage API: fetch failed', + ); + } + return null; + } finally { + clearTimeout(timer); } - - return null; } -export function parseClaudeUsagePanel(rawText: string): ClaudeUsageData | null { - const lines = normalizeLines(rawText); - if (lines.length === 0) return null; - if ( - lines.some((line) => - line.toLowerCase().includes('failed to load usage data'), - ) - ) { +/** + * Fetch Claude usage via the OAuth API. + * Uses the current active token from rotation. + */ +export async function fetchClaudeUsage(): Promise { + const token = + getCurrentToken() || process.env.CLAUDE_CODE_OAUTH_TOKEN; + if (!token) { + logger.debug('No Claude OAuth token available for usage check'); return null; } - - const fiveHour = parseWindow(lines, ['Current session']); - if (!fiveHour) return null; - - const sevenDay = - parseWindow(lines, ['Current week (all models)']) || - parseWindow(lines, [ - 'Current week (Sonnet only)', - 'Current week (Sonnet)', - ]) || - parseWindow(lines, ['Current week (Opus)']); - - return { - five_hour: fiveHour, - ...(sevenDay ? { seven_day: sevenDay } : {}), - }; + return fetchUsageForToken(token); } -export async function fetchClaudeUsageViaCli( - binary = 'claude', -): Promise { - return new Promise((resolve) => { - let output = ''; - let finished = false; - - const finish = (value: ClaudeUsageData | null) => { - if (finished) return; - finished = true; - clearTimeout(timer); - resolve(value); - }; - - let proc: ReturnType | null = null; - try { - proc = spawn('expect', ['-c', EXPECT_PROGRAM], { - stdio: ['ignore', 'pipe', 'pipe'], - env: { - ...(process.env as Record), - CLAUDE_BINARY: binary, - }, - }); - } catch (err) { - logger.debug({ err }, 'Claude CLI PTY probe unavailable'); - finish(null); - return; - } - - const timer = setTimeout(() => { - try { - proc?.kill('SIGTERM'); - } catch { - /* ignore */ - } - finish(null); - }, CLAUDE_EXPECT_TIMEOUT_MS); - - if (!proc.stdout || !proc.stderr) { - finish(null); - return; - } - - proc.stdout.setEncoding('utf8'); - proc.stderr.setEncoding('utf8'); - proc.stdout.on('data', (chunk: string) => { - output += chunk; - }); - proc.stderr.on('data', (chunk: string) => { - output += chunk; - }); - proc.on('error', (err) => { - logger.debug({ err }, 'Claude CLI PTY probe failed to start'); - finish(null); - }); - proc.on('close', () => { - const parsed = parseClaudeUsagePanel(output); - if (!parsed && output.trim()) { - logger.debug( - { tail: output.slice(-400) }, - 'Claude CLI PTY probe produced unparsable output', - ); - } - finish(parsed); - }); - }); +export interface ClaudeAccountUsage { + index: number; + masked: string; + isActive: boolean; + isRateLimited: boolean; + usage: ClaudeUsageData | null; } + +/** + * Fetch usage for ALL configured tokens. + * Returns per-account usage for dashboard display. + */ +export async function fetchAllClaudeUsage(): Promise { + const allTokens = getAllTokens(); + if (allTokens.length === 0) { + // Single token fallback + const token = process.env.CLAUDE_CODE_OAUTH_TOKEN; + if (!token) return []; + const usage = await fetchUsageForToken(token); + return [{ + index: 0, + masked: `${token.slice(0, 20)}...${token.slice(-4)}`, + isActive: true, + isRateLimited: false, + usage, + }]; + } + + const results: ClaudeAccountUsage[] = []; + for (const t of allTokens) { + const usage = await fetchUsageForToken(t.token); + results.push({ + index: t.index, + masked: t.masked, + isActive: t.isActive, + isRateLimited: t.isRateLimited, + usage, + }); + } + return results; +} + +// Legacy alias +export const fetchClaudeUsageViaCli = fetchClaudeUsage; diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 944c7a3..997a970 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -29,7 +29,10 @@ export function initTokenRotation(): void { const single = process.env.CLAUDE_CODE_OAUTH_TOKEN; const raw = multi - ? multi.split(',').map((t) => t.trim()).filter(Boolean) + ? multi + .split(',') + .map((t) => t.trim()) + .filter(Boolean) : single ? [single] : []; @@ -99,6 +102,18 @@ export function getTokenCount(): number { return tokens.length; } +/** Get all configured tokens (masked for display, raw for API calls). */ +export function getAllTokens(): { index: number; token: string; masked: string; isActive: boolean; isRateLimited: boolean }[] { + const now = Date.now(); + return tokens.map((t, i) => ({ + index: i, + token: t.token, + masked: `${t.token.slice(0, 20)}...${t.token.slice(-4)}`, + isActive: i === currentIndex, + isRateLimited: Boolean(t.rateLimitedUntil && t.rateLimitedUntil > now), + })); +} + /** Diagnostic info. */ export function getTokenRotationInfo(): { total: number; diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 2e42cf7..0fbf0bd 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -5,8 +5,10 @@ import path from 'path'; import { STATUS_SHOW_ROOMS, USAGE_DASHBOARD_ENABLED } from './config.js'; import { - fetchClaudeUsageViaCli, + fetchClaudeUsage as fetchClaudeUsageApi, + fetchAllClaudeUsage, type ClaudeUsageData, + type ClaudeAccountUsage, } from './claude-usage.js'; import { composeDashboardContent, @@ -349,10 +351,10 @@ async function fetchClaudeUsage(): Promise { return null; } - const cliUsage = await fetchClaudeUsageViaCli(); - if (cliUsage) { + const apiUsage = await fetchClaudeUsageApi(); + if (apiUsage) { usageApi429Streak = 0; - return cliUsage; + return apiUsage; } try { @@ -520,20 +522,18 @@ async function buildUsageContent(): Promise { ); const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED; - const [liveClaudeUsage, codexUsage] = await Promise.all([ + const [liveClaudeAccounts, codexUsage] = await Promise.all([ shouldFetchClaudeUsage && !hasActiveClaudeWork - ? fetchClaudeUsage() + ? fetchAllClaudeUsage() : Promise.resolve(null), fetchCodexUsage(), ]); - const claudeUsage = shouldFetchClaudeUsage - ? liveClaudeUsage || cachedClaudeUsageData - : null; + // Update cache with first account's data for backward compat const claudeUsageIsCached = - shouldFetchClaudeUsage && !liveClaudeUsage && !!cachedClaudeUsageData; - if (shouldFetchClaudeUsage && liveClaudeUsage) { - cachedClaudeUsageData = liveClaudeUsage; + shouldFetchClaudeUsage && !liveClaudeAccounts && !!cachedClaudeUsageData; + if (shouldFetchClaudeUsage && liveClaudeAccounts?.length) { + cachedClaudeUsageData = liveClaudeAccounts[0].usage; } const lines: string[] = ['πŸ“Š *μ‚¬μš©λŸ‰*']; @@ -551,11 +551,19 @@ async function buildUsageContent(): Promise { }; const rows: UsageRow[] = []; - if (claudeUsage) { - const h5 = claudeUsage.five_hour; - const d7 = claudeUsage.seven_day; + const claudeAccounts = liveClaudeAccounts + || (cachedClaudeUsageData ? [{ index: 0, masked: '', isActive: true, isRateLimited: false, usage: cachedClaudeUsageData }] : []); + + for (const account of claudeAccounts) { + const u = account.usage; + if (!u) continue; + const h5 = u.five_hour; + const d7 = u.seven_day; + const label = claudeAccounts.length > 1 + ? `Claude${account.index + 1}${account.isActive ? '⚑' : ''}${account.isRateLimited ? '🚫' : ''}` + : claudeUsageIsCached ? 'Claude*' : 'Claude'; rows.push({ - name: claudeUsageIsCached ? 'Claude*' : 'Claude', + name: label, h5pct: h5 ? h5.utilization > 1 ? Math.round(h5.utilization) From 0b86f937f4265c090abe0e7ef2e3f68a46e487c3 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 22:41:07 +0900 Subject: [PATCH 08/27] fix: read tokens from .env file, clean up debug logs - token-rotation reads from readEnvFile() (systemd doesn't load .env) - fetchAllClaudeUsage polling log demoted to debug level - Remove verbose dashboard send/create logs --- src/claude-usage.test.ts | 5 ++++- src/claude-usage.ts | 32 ++++++++++++++++---------------- src/token-rotation.ts | 21 ++++++++++++++++++--- src/unified-dashboard.ts | 30 ++++++++++++++++++++++++------ 4 files changed, 62 insertions(+), 26 deletions(-) diff --git a/src/claude-usage.test.ts b/src/claude-usage.test.ts index ce661c7..884a7bb 100644 --- a/src/claude-usage.test.ts +++ b/src/claude-usage.test.ts @@ -7,7 +7,10 @@ describe('ClaudeUsageData', () => { const data: ClaudeUsageData = { five_hour: { utilization: 45.2, resets_at: '2026-03-23T17:00:00Z' }, seven_day: { utilization: 72.1, resets_at: '2026-03-29T00:00:00Z' }, - seven_day_sonnet: { utilization: 60.0, resets_at: '2026-03-29T00:00:00Z' }, + seven_day_sonnet: { + utilization: 60.0, + resets_at: '2026-03-29T00:00:00Z', + }, seven_day_opus: { utilization: 80.0, resets_at: '2026-03-29T00:00:00Z' }, }; diff --git a/src/claude-usage.ts b/src/claude-usage.ts index b286eae..4c863e1 100644 --- a/src/claude-usage.ts +++ b/src/claude-usage.ts @@ -25,9 +25,10 @@ interface UsageApiResponse { seven_day_opus?: { utilization: number; resets_at?: string }; } -function mapWindow( - w?: { utilization: number; resets_at?: string }, -): { utilization: number; resets_at: string } | undefined { +function mapWindow(w?: { + utilization: number; + resets_at?: string; +}): { utilization: number; resets_at: string } | undefined { if (!w) return undefined; return { utilization: w.utilization, resets_at: w.resets_at || '' }; } @@ -77,10 +78,7 @@ async function fetchUsageForToken( if ((err as Error).name === 'AbortError') { logger.warn('Claude usage API: request timed out'); } else { - logger.warn( - { err }, - 'Claude usage API: fetch failed', - ); + logger.warn({ err }, 'Claude usage API: fetch failed'); } return null; } finally { @@ -93,8 +91,7 @@ async function fetchUsageForToken( * Uses the current active token from rotation. */ export async function fetchClaudeUsage(): Promise { - const token = - getCurrentToken() || process.env.CLAUDE_CODE_OAUTH_TOKEN; + const token = getCurrentToken() || process.env.CLAUDE_CODE_OAUTH_TOKEN; if (!token) { logger.debug('No Claude OAuth token available for usage check'); return null; @@ -116,18 +113,21 @@ export interface ClaudeAccountUsage { */ export async function fetchAllClaudeUsage(): Promise { const allTokens = getAllTokens(); + logger.debug({ tokenCount: allTokens.length }, 'fetchAllClaudeUsage called'); if (allTokens.length === 0) { // Single token fallback const token = process.env.CLAUDE_CODE_OAUTH_TOKEN; if (!token) return []; const usage = await fetchUsageForToken(token); - return [{ - index: 0, - masked: `${token.slice(0, 20)}...${token.slice(-4)}`, - isActive: true, - isRateLimited: false, - usage, - }]; + return [ + { + index: 0, + masked: `${token.slice(0, 20)}...${token.slice(-4)}`, + isActive: true, + isRateLimited: false, + usage, + }, + ]; } const results: ClaudeAccountUsage[] = []; diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 997a970..9874732 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -10,6 +10,7 @@ * All exhausted: fall through to provider fallback (Kimi etc.) */ +import { readEnvFile } from './env.js'; import { logger } from './logger.js'; interface TokenState { @@ -25,8 +26,16 @@ export function initTokenRotation(): void { if (initialized) return; initialized = true; - const multi = process.env.CLAUDE_CODE_OAUTH_TOKENS; - const single = process.env.CLAUDE_CODE_OAUTH_TOKEN; + const envFile = readEnvFile([ + 'CLAUDE_CODE_OAUTH_TOKENS', + 'CLAUDE_CODE_OAUTH_TOKEN', + ]); + const multi = + process.env.CLAUDE_CODE_OAUTH_TOKENS || + envFile.CLAUDE_CODE_OAUTH_TOKENS; + const single = + process.env.CLAUDE_CODE_OAUTH_TOKEN || + envFile.CLAUDE_CODE_OAUTH_TOKEN; const raw = multi ? multi @@ -103,7 +112,13 @@ export function getTokenCount(): number { } /** Get all configured tokens (masked for display, raw for API calls). */ -export function getAllTokens(): { index: number; token: string; masked: string; isActive: boolean; isRateLimited: boolean }[] { +export function getAllTokens(): { + index: number; + token: string; + masked: string; + isActive: boolean; + isRateLimited: boolean; +}[] { const now = Date.now(); return tokens.map((t, i) => ({ index: i, diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 0fbf0bd..3d83766 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -551,17 +551,31 @@ async function buildUsageContent(): Promise { }; const rows: UsageRow[] = []; - const claudeAccounts = liveClaudeAccounts - || (cachedClaudeUsageData ? [{ index: 0, masked: '', isActive: true, isRateLimited: false, usage: cachedClaudeUsageData }] : []); + const claudeAccounts = + liveClaudeAccounts || + (cachedClaudeUsageData + ? [ + { + index: 0, + masked: '', + isActive: true, + isRateLimited: false, + usage: cachedClaudeUsageData, + }, + ] + : []); for (const account of claudeAccounts) { const u = account.usage; if (!u) continue; const h5 = u.five_hour; const d7 = u.seven_day; - const label = claudeAccounts.length > 1 - ? `Claude${account.index + 1}${account.isActive ? '⚑' : ''}${account.isRateLimited ? '🚫' : ''}` - : claudeUsageIsCached ? 'Claude*' : 'Claude'; + const label = + claudeAccounts.length > 1 + ? `Claude${account.index + 1}${account.isActive ? '⚑' : ''}${account.isRateLimited ? '🚫' : ''}` + : claudeUsageIsCached + ? 'Claude*' + : 'Claude'; rows.push({ name: label, h5pct: h5 @@ -717,6 +731,10 @@ export async function startUnifiedDashboard( await refreshChannelMeta(opts); const content = buildUnifiedDashboardContent(); if (!content) { + logger.warn( + { cachedUsageLength: cachedUsageContent.length, statusShowRooms: STATUS_SHOW_ROOMS }, + 'Dashboard content empty, skipping render', + ); statusMessageId = null; return; } @@ -728,7 +746,7 @@ export async function startUnifiedDashboard( if (id) statusMessageId = id; } } catch (err) { - logger.debug({ err }, 'Dashboard update failed'); + logger.warn({ err }, 'Dashboard update failed'); statusMessageId = null; } }; From 816b0a39ccf5202f8b6a5f8a85eee2ad5b8f8ce5 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 23:06:17 +0900 Subject: [PATCH 09/27] feat: Codex account rotation + fix rate-limit detection - New codex-token-rotation module: rotates between multiple ~/.codex-accounts/{n}/auth.json on rate-limit - Copies active account auth to session dir before each spawn - Fix detectFallbackTrigger to match "usage limit" / "hit your limit" - Fix streamed error detection: remove claude-only guard so codex rate-limit errors are caught even when output.status is "success" - Add rotation in both task-scheduler and message-agent-executor --- src/agent-runner-environment.ts | 6 +- src/codex-token-rotation.ts | 128 ++++++++++++++++++++++++++++++++ src/index.ts | 6 +- src/message-agent-executor.ts | 48 +++++++++++- src/provider-fallback.ts | 2 + src/task-scheduler.ts | 32 ++++++++ 6 files changed, 217 insertions(+), 5 deletions(-) create mode 100644 src/codex-token-rotation.ts diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index c32587e..43b548e 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -5,6 +5,7 @@ import path from 'path'; import { GROUPS_DIR, TIMEZONE } from './config.js'; import { isPairedRoomJid } from './db.js'; import { readEnvFile } from './env.js'; +import { getActiveCodexAuthPath } from './codex-token-rotation.js'; import { getCurrentToken } from './token-rotation.js'; import { resolveGroupFolderPath, @@ -182,7 +183,10 @@ function prepareCodexSessionEnvironment(args: { const sessionCodexDir = path.join(args.sessionRootDir, '.codex'); fs.mkdirSync(sessionCodexDir, { recursive: true }); - const authSrc = path.join(hostCodexDir, 'auth.json'); + const rotatedAuthSrc = getActiveCodexAuthPath(); + const authSrc = rotatedAuthSrc && fs.existsSync(rotatedAuthSrc) + ? rotatedAuthSrc + : path.join(hostCodexDir, 'auth.json'); const authDst = path.join(sessionCodexDir, 'auth.json'); if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst); for (const file of ['config.toml', 'config.json']) { diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts new file mode 100644 index 0000000..092d642 --- /dev/null +++ b/src/codex-token-rotation.ts @@ -0,0 +1,128 @@ +/** + * Codex OAuth Token Rotation + * + * Rotates between multiple Codex (ChatGPT) OAuth accounts when + * rate-limited. Each account is stored as a separate auth.json in + * ~/.codex-accounts/{n}/auth.json. + * + * The active account's auth.json is copied to the session directory + * before each agent spawn (existing behavior in agent-runner-environment). + * On rate-limit, we rotate to the next account. + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { logger } from './logger.js'; + +interface CodexAccount { + index: number; + authPath: string; + accountId: string; + rateLimitedUntil: number | null; +} + +const accounts: CodexAccount[] = []; +let currentIndex = 0; +let initialized = false; + +const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts'); + +export function initCodexTokenRotation(): void { + if (initialized) return; + initialized = true; + + if (!fs.existsSync(ACCOUNTS_DIR)) { + logger.info({ dir: ACCOUNTS_DIR }, 'Codex accounts dir not found, skipping'); + return; + } + + const dirs = fs.readdirSync(ACCOUNTS_DIR) + .filter((d) => /^\d+$/.test(d)) + .sort((a, b) => parseInt(a) - parseInt(b)); + + for (const dir of dirs) { + const authPath = path.join(ACCOUNTS_DIR, dir, 'auth.json'); + if (!fs.existsSync(authPath)) continue; + + try { + const data = JSON.parse(fs.readFileSync(authPath, 'utf-8')); + const accountId = data?.tokens?.account_id || `account-${dir}`; + accounts.push({ + index: accounts.length, + authPath, + accountId, + rateLimitedUntil: null, + }); + } catch { + logger.warn({ authPath }, 'Failed to parse codex account auth.json'); + } + } + + logger.info( + { count: accounts.length, dir: ACCOUNTS_DIR }, + `Codex token rotation: ${accounts.length} account(s) found`, + ); +} + +/** Get the auth.json path for the current active account. */ +export function getActiveCodexAuthPath(): string | null { + if (accounts.length === 0) return null; + return accounts[currentIndex]?.authPath ?? null; +} + +/** + * Try to rotate to the next available Codex account. + * Returns true if a fresh account was found. + */ +export function rotateCodexToken(): boolean { + if (accounts.length <= 1) return false; + + const now = Date.now(); + accounts[currentIndex].rateLimitedUntil = now + 3_600_000; + + for (let i = 1; i < accounts.length; i++) { + const idx = (currentIndex + i) % accounts.length; + const acct = accounts[idx]; + if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) { + acct.rateLimitedUntil = null; + currentIndex = idx; + logger.info( + { accountIndex: currentIndex, totalAccounts: accounts.length, accountId: acct.accountId }, + `Codex rotated to account #${currentIndex + 1}/${accounts.length}`, + ); + return true; + } + } + + logger.warn('All Codex accounts are rate-limited'); + return false; +} + +export function markCodexTokenHealthy(): void { + if (accounts.length === 0) return; + const acct = accounts[currentIndex]; + if (acct?.rateLimitedUntil) { + acct.rateLimitedUntil = null; + } +} + +export function getCodexAccountCount(): number { + return accounts.length; +} + +export function getAllCodexAccounts(): { + index: number; + accountId: string; + isActive: boolean; + isRateLimited: boolean; +}[] { + const now = Date.now(); + return accounts.map((a, i) => ({ + index: i, + accountId: a.accountId, + isActive: i === currentIndex, + isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), + })); +} diff --git a/src/index.ts b/src/index.ts index 8e5c216..45db441 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,14 +63,14 @@ import { startUnifiedDashboard } from './unified-dashboard.js'; import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { normalizeStoredSeqCursor } from './message-cursor.js'; +import { initCodexTokenRotation } from './codex-token-rotation.js'; import { initTokenRotation } from './token-rotation.js'; // Re-export for backwards compatibility during refactor export { escapeXml, formatMessages } from './router.js'; export { composeDashboardContent } from './dashboard-render.js'; -// Initialize token rotation early (reads CLAUDE_CODE_OAUTH_TOKENS from env) -initTokenRotation(); +// Token rotation is initialized lazily on first use or at startup below export async function sendFormattedChannelMessage( channels: Channel[], @@ -297,6 +297,8 @@ async function main(): Promise { const processStartedAtMs = Date.now(); initDatabase(); logger.info('Database initialized'); + initTokenRotation(); + initCodexTokenRotation(); loadState(); // Graceful shutdown handlers diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 968bf1c..5cd7b5e 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -21,7 +21,16 @@ import { markPrimaryCooldown, } from './provider-fallback.js'; import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; -import { rotateToken, getTokenCount, markTokenHealthy } from './token-rotation.js'; +import { + rotateCodexToken, + getCodexAccountCount, + markCodexTokenHealthy, +} from './codex-token-rotation.js'; +import { + rotateToken, + getTokenCount, + markTokenHealthy, +} from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; export interface MessageAgentExecutorDeps { @@ -138,7 +147,6 @@ export async function runAgentForGroup( sawSuccessNullResultWithoutOutput = true; } if ( - provider === 'claude' && output.status === 'error' && !sawOutput && !streamedTriggerReason @@ -390,6 +398,22 @@ export async function runAgentForGroup( } } + // Codex rate-limit rotation (non-Claude agents) + if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { + const trigger = detectFallbackTrigger(output.error); + if (trigger.shouldFallback && rotateCodexToken()) { + logger.info( + { chatJid, group: group.name, runId, reason: trigger.reason }, + 'Codex rate-limited, retrying with rotated account', + ); + const retryAttempt = await runAttempt('codex'); + if (!retryAttempt.error) { + markCodexTokenHealthy(); + return 'success'; + } + } + } + logger.error( { group: group.name, @@ -403,5 +427,25 @@ export async function runAgentForGroup( return 'error'; } + // Codex may report success but have streamed a rate-limit error. + // Rotate token so the NEXT request uses a fresh account. + if ( + !isClaudeCodeAgent && + primaryAttempt.streamedTriggerReason && + getCodexAccountCount() > 1 + ) { + if (rotateCodexToken()) { + logger.info( + { + chatJid, + group: group.name, + runId, + reason: primaryAttempt.streamedTriggerReason.reason, + }, + 'Codex rate-limited (streamed), rotated account for next request', + ); + } + } + return 'success'; } diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index 451f29d..f79d6c0 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -247,6 +247,8 @@ export function detectFallbackTrigger( if ( lower.includes('429') || lower.includes('rate limit') || + lower.includes('usage limit') || + lower.includes('hit your limit') || lower.includes('too many requests') || lower.includes('rate_limit') ) { diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 04895fc..42f0b30 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -29,6 +29,17 @@ import { } from './group-folder.js'; import { logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; +import { detectFallbackTrigger } from './provider-fallback.js'; +import { + rotateCodexToken, + getCodexAccountCount, + markCodexTokenHealthy, +} from './codex-token-rotation.js'; +import { + rotateToken, + getTokenCount, + markTokenHealthy, +} from './token-rotation.js'; import { evaluateTaskSuspension, formatSuspensionNotice, @@ -315,6 +326,27 @@ async function runTask( updateTask(task.id, { suspended_until: null }); } + // Try token rotation before suspending + if (error) { + const trigger = detectFallbackTrigger(error); + if (trigger.shouldFallback) { + const isCodex = SERVICE_AGENT_TYPE === 'codex'; + const rotated = isCodex + ? getCodexAccountCount() > 1 && rotateCodexToken() + : getTokenCount() > 1 && rotateToken(); + if (rotated) { + logger.info( + { taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason }, + 'Task rate-limited, rotated token β€” will retry on next schedule', + ); + if (isCodex) markCodexTokenHealthy(); + else markTokenHealthy(); + // Clear the error so suspension doesn't trigger + error = null; + } + } + } + // Check for repeated quota/auth errors β†’ auto-suspend let suspended = false; if (error) { From 30be180f4b5a079d27676d2bcce26ced03bcf88f Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 23:07:35 +0900 Subject: [PATCH 10/27] fix: codex rate-limit retry immediately instead of next request --- src/message-agent-executor.ts | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 5cd7b5e..07f09a1 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -428,22 +428,26 @@ export async function runAgentForGroup( } // Codex may report success but have streamed a rate-limit error. - // Rotate token so the NEXT request uses a fresh account. + // Rotate token and retry immediately with the new account. if ( !isClaudeCodeAgent && primaryAttempt.streamedTriggerReason && - getCodexAccountCount() > 1 + getCodexAccountCount() > 1 && + rotateCodexToken() ) { - if (rotateCodexToken()) { - logger.info( - { - chatJid, - group: group.name, - runId, - reason: primaryAttempt.streamedTriggerReason.reason, - }, - 'Codex rate-limited (streamed), rotated account for next request', - ); + logger.info( + { + chatJid, + group: group.name, + runId, + reason: primaryAttempt.streamedTriggerReason.reason, + }, + 'Codex rate-limited (streamed), retrying with rotated account', + ); + const retryAttempt = await runAttempt('codex'); + if (!retryAttempt.error) { + markCodexTokenHealthy(); + return 'success'; } } From 74016263568ddf25ffb665929f3234cf7cff5b75 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 23:16:54 +0900 Subject: [PATCH 11/27] feat: persist rotation state across restarts, fix dashboard alignment - Save currentIndex + rateLimits to JSON file in DATA_DIR - Restore on startup so rotation survives service restarts - Replace emoji indicators with text (*/!) for monospace alignment - Dynamic column width based on longest name - Show Codex accounts with plan type on dashboard --- src/codex-token-rotation.ts | 70 ++++++++++++++++++++++++++++++++++--- src/token-rotation.ts | 47 ++++++++++++++++++++++--- src/unified-dashboard.ts | 51 ++++++++++++++++++++++++--- 3 files changed, 154 insertions(+), 14 deletions(-) diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 092d642..12de6e4 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -14,15 +14,32 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; +import { DATA_DIR } from './config.js'; import { logger } from './logger.js'; +const STATE_FILE = path.join(DATA_DIR, 'codex-rotation-state.json'); + interface CodexAccount { index: number; authPath: string; accountId: string; + planType: string; rateLimitedUntil: number | null; } +function parsePlanFromJwt(idToken: string): string { + try { + const parts = idToken.split('.'); + if (parts.length < 2) return '?'; + const payload = JSON.parse( + Buffer.from(parts[1], 'base64url').toString('utf-8'), + ); + return payload?.['https://api.openai.com/auth']?.chatgpt_plan_type || '?'; + } catch { + return '?'; + } +} + const accounts: CodexAccount[] = []; let currentIndex = 0; let initialized = false; @@ -34,11 +51,15 @@ export function initCodexTokenRotation(): void { initialized = true; if (!fs.existsSync(ACCOUNTS_DIR)) { - logger.info({ dir: ACCOUNTS_DIR }, 'Codex accounts dir not found, skipping'); + logger.info( + { dir: ACCOUNTS_DIR }, + 'Codex accounts dir not found, skipping', + ); return; } - const dirs = fs.readdirSync(ACCOUNTS_DIR) + const dirs = fs + .readdirSync(ACCOUNTS_DIR) .filter((d) => /^\d+$/.test(d)) .sort((a, b) => parseInt(a) - parseInt(b)); @@ -49,10 +70,12 @@ export function initCodexTokenRotation(): void { try { const data = JSON.parse(fs.readFileSync(authPath, 'utf-8')); const accountId = data?.tokens?.account_id || `account-${dir}`; + const planType = parsePlanFromJwt(data?.tokens?.id_token || ''); accounts.push({ index: accounts.length, authPath, accountId, + planType, rateLimitedUntil: null, }); } catch { @@ -60,12 +83,43 @@ export function initCodexTokenRotation(): void { } } + if (accounts.length > 1) loadCodexState(); logger.info( - { count: accounts.length, dir: ACCOUNTS_DIR }, + { count: accounts.length, dir: ACCOUNTS_DIR, activeIndex: currentIndex }, `Codex token rotation: ${accounts.length} account(s) found`, ); } +function saveCodexState(): void { + try { + const state = { + currentIndex, + rateLimits: accounts.map((a) => a.rateLimitedUntil), + }; + fs.writeFileSync(STATE_FILE, JSON.stringify(state)); + } catch { /* best effort */ } +} + +function loadCodexState(): void { + try { + if (!fs.existsSync(STATE_FILE)) return; + const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); + const now = Date.now(); + if (typeof state.currentIndex === 'number' && state.currentIndex < accounts.length) { + currentIndex = state.currentIndex; + } + if (Array.isArray(state.rateLimits)) { + for (let i = 0; i < Math.min(state.rateLimits.length, accounts.length); i++) { + const until = state.rateLimits[i]; + if (typeof until === 'number' && until > now) { + accounts[i].rateLimitedUntil = until; + } + } + } + logger.info({ currentIndex, accountCount: accounts.length }, 'Codex rotation state restored'); + } catch { /* start fresh */ } +} + /** Get the auth.json path for the current active account. */ export function getActiveCodexAuthPath(): string | null { if (accounts.length === 0) return null; @@ -89,9 +143,14 @@ export function rotateCodexToken(): boolean { acct.rateLimitedUntil = null; currentIndex = idx; logger.info( - { accountIndex: currentIndex, totalAccounts: accounts.length, accountId: acct.accountId }, + { + accountIndex: currentIndex, + totalAccounts: accounts.length, + accountId: acct.accountId, + }, `Codex rotated to account #${currentIndex + 1}/${accounts.length}`, ); + saveCodexState(); return true; } } @@ -105,6 +164,7 @@ export function markCodexTokenHealthy(): void { const acct = accounts[currentIndex]; if (acct?.rateLimitedUntil) { acct.rateLimitedUntil = null; + saveCodexState(); } } @@ -115,6 +175,7 @@ export function getCodexAccountCount(): number { export function getAllCodexAccounts(): { index: number; accountId: string; + planType: string; isActive: boolean; isRateLimited: boolean; }[] { @@ -122,6 +183,7 @@ export function getAllCodexAccounts(): { return accounts.map((a, i) => ({ index: i, accountId: a.accountId, + planType: a.planType, isActive: i === currentIndex, isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), })); diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 9874732..da9f289 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -10,9 +10,15 @@ * All exhausted: fall through to provider fallback (Kimi etc.) */ +import fs from 'fs'; +import path from 'path'; + +import { DATA_DIR } from './config.js'; import { readEnvFile } from './env.js'; import { logger } from './logger.js'; +const STATE_FILE = path.join(DATA_DIR, 'token-rotation-state.json'); + interface TokenState { token: string; rateLimitedUntil: number | null; @@ -31,11 +37,9 @@ export function initTokenRotation(): void { 'CLAUDE_CODE_OAUTH_TOKEN', ]); const multi = - process.env.CLAUDE_CODE_OAUTH_TOKENS || - envFile.CLAUDE_CODE_OAUTH_TOKENS; + process.env.CLAUDE_CODE_OAUTH_TOKENS || envFile.CLAUDE_CODE_OAUTH_TOKENS; const single = - process.env.CLAUDE_CODE_OAUTH_TOKEN || - envFile.CLAUDE_CODE_OAUTH_TOKEN; + process.env.CLAUDE_CODE_OAUTH_TOKEN || envFile.CLAUDE_CODE_OAUTH_TOKEN; const raw = multi ? multi @@ -51,13 +55,44 @@ export function initTokenRotation(): void { } if (tokens.length > 1) { + loadState(); logger.info( - { count: tokens.length }, + { count: tokens.length, activeIndex: currentIndex }, `Token rotation initialized with ${tokens.length} tokens`, ); } } +function saveState(): void { + try { + const state = { + currentIndex, + rateLimits: tokens.map((t) => t.rateLimitedUntil), + }; + fs.writeFileSync(STATE_FILE, JSON.stringify(state)); + } catch { /* best effort */ } +} + +function loadState(): void { + try { + if (!fs.existsSync(STATE_FILE)) return; + const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); + const now = Date.now(); + if (typeof state.currentIndex === 'number' && state.currentIndex < tokens.length) { + currentIndex = state.currentIndex; + } + if (Array.isArray(state.rateLimits)) { + for (let i = 0; i < Math.min(state.rateLimits.length, tokens.length); i++) { + const until = state.rateLimits[i]; + if (typeof until === 'number' && until > now) { + tokens[i].rateLimitedUntil = until; + } + } + } + logger.info({ currentIndex, tokenCount: tokens.length }, 'Token rotation state restored'); + } catch { /* start fresh */ } +} + /** Get the current active token. */ export function getCurrentToken(): string | undefined { if (tokens.length === 0) return process.env.CLAUDE_CODE_OAUTH_TOKEN; @@ -86,6 +121,7 @@ export function rotateToken(): boolean { { tokenIndex: currentIndex, totalTokens: tokens.length }, `Rotated to token #${currentIndex + 1}/${tokens.length}`, ); + saveState(); return true; } } @@ -103,6 +139,7 @@ export function markTokenHealthy(): void { const state = tokens[currentIndex]; if (state?.rateLimitedUntil) { state.rateLimitedUntil = null; + saveState(); } } diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 3d83766..94f40b5 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -10,6 +10,7 @@ import { type ClaudeUsageData, type ClaudeAccountUsage, } from './claude-usage.js'; +import { getAllCodexAccounts } from './codex-token-rotation.js'; import { composeDashboardContent, formatElapsed, @@ -572,7 +573,7 @@ async function buildUsageContent(): Promise { const d7 = u.seven_day; const label = claudeAccounts.length > 1 - ? `Claude${account.index + 1}${account.isActive ? '⚑' : ''}${account.isRateLimited ? '🚫' : ''}` + ? `Claude${account.index + 1}${account.isActive ? '*' : ''}${account.isRateLimited ? '!' : ''}` : claudeUsageIsCached ? 'Claude*' : 'Claude'; @@ -593,7 +594,39 @@ async function buildUsageContent(): Promise { }); } - if (codexUsage && Array.isArray(codexUsage)) { + const codexAccounts = getAllCodexAccounts(); + if (codexAccounts.length > 1) { + // Multi-account: show each account with plan + status + for (const acct of codexAccounts) { + const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' '; + const label = `Codex${acct.index + 1}${icon}`; + if (acct.isActive && codexUsage && Array.isArray(codexUsage)) { + const relevant = codexUsage.filter( + (limit) => + limit.primary.usedPercent > 0 || limit.secondary.usedPercent > 0, + ); + const display = relevant.length > 0 ? relevant : codexUsage.slice(0, 1); + for (const limit of display) { + rows.push({ + name: `${label} ${acct.planType}`, + h5pct: Math.round(limit.primary.usedPercent), + h5reset: formatResetKST(limit.primary.resetsAt), + d7pct: Math.round(limit.secondary.usedPercent), + d7reset: formatResetKST(limit.secondary.resetsAt), + }); + } + } else { + rows.push({ + name: `${label} ${acct.planType}`, + h5pct: -1, + h5reset: '', + d7pct: -1, + d7reset: '', + }); + } + } + } else if (codexUsage && Array.isArray(codexUsage)) { + // Single account: existing behavior const relevant = codexUsage.filter( (limit) => limit.primary.usedPercent > 0 || limit.secondary.usedPercent > 0, @@ -611,8 +644,13 @@ async function buildUsageContent(): Promise { } if (rows.length > 0) { + // Emoji characters take 2 columns in monospace β€” count visual width + const visualWidth = (s: string) => + [...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0); + const maxNameWidth = Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1; + const padName = (s: string) => s + ' '.repeat(maxNameWidth - visualWidth(s)); lines.push('```'); - lines.push(' 5-Hour 7-Day'); + lines.push(`${' '.repeat(maxNameWidth)}5-Hour 7-Day`); for (const row of rows) { const h5 = row.h5pct >= 0 @@ -622,7 +660,7 @@ async function buildUsageContent(): Promise { row.d7pct >= 0 ? `${bar(row.d7pct)} ${String(row.d7pct).padStart(3)}%` : ' β€” '; - lines.push(`${row.name.padEnd(8)}${h5} ${d7}`); + lines.push(`${padName(row.name)}${h5} ${d7}`); } lines.push('```'); } else { @@ -732,7 +770,10 @@ export async function startUnifiedDashboard( const content = buildUnifiedDashboardContent(); if (!content) { logger.warn( - { cachedUsageLength: cachedUsageContent.length, statusShowRooms: STATUS_SHOW_ROOMS }, + { + cachedUsageLength: cachedUsageContent.length, + statusShowRooms: STATUS_SHOW_ROOMS, + }, 'Dashboard content empty, skipping render', ); statusMessageId = null; From 2e6fc4a08ec9a40038471aab0015dfcd2e82aab3 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Mon, 23 Mar 2026 23:18:47 +0900 Subject: [PATCH 12/27] fix: add warn log for usage content build failures --- src/unified-dashboard.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 94f40b5..fbd5410 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -647,8 +647,10 @@ async function buildUsageContent(): Promise { // Emoji characters take 2 columns in monospace β€” count visual width const visualWidth = (s: string) => [...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0); - const maxNameWidth = Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1; - const padName = (s: string) => s + ' '.repeat(maxNameWidth - visualWidth(s)); + const maxNameWidth = + Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1; + const padName = (s: string) => + s + ' '.repeat(maxNameWidth - visualWidth(s)); lines.push('```'); lines.push(`${' '.repeat(maxNameWidth)}5-Hour 7-Day`); for (const row of rows) { @@ -735,8 +737,8 @@ async function refreshUsageCache(): Promise { usageUpdateInProgress = true; try { cachedUsageContent = await buildUsageContent(); - } catch { - /* keep previous cache */ + } catch (err) { + logger.warn({ err }, 'Failed to build usage content'); } finally { usageUpdateInProgress = false; } From 60dab43a3fa2f9b4608ce21a858c01f8b27dc89b Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:16:54 +0900 Subject: [PATCH 13/27] feat: parse retry-after from errors, cache codex usage per account - Parse "try again at" time from rate-limit errors instead of fixed 1-hour cooldown, with 3-min buffer after reset - Pass error message to rotateToken/rotateCodexToken for parsing - Cache usage % and reset time per Codex account for dashboard - Persist cached usage in rotation state file across restarts - Show rate-limited Codex accounts with cached 100% + reset time - Fix dashboard text indicators (*!/space) for monospace alignment --- src/codex-token-rotation.ts | 83 ++++++++++++++++++++++++++++++++--- src/message-agent-executor.ts | 8 ++-- src/task-scheduler.ts | 10 +++-- src/token-rotation.ts | 54 +++++++++++++++++++---- src/unified-dashboard.ts | 8 +++- 5 files changed, 139 insertions(+), 24 deletions(-) diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 12de6e4..d57b90c 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -25,6 +25,8 @@ interface CodexAccount { accountId: string; planType: string; rateLimitedUntil: number | null; + lastUsagePct?: number; + resetAt?: string; } function parsePlanFromJwt(idToken: string): string { @@ -95,9 +97,13 @@ function saveCodexState(): void { const state = { currentIndex, rateLimits: accounts.map((a) => a.rateLimitedUntil), + usagePcts: accounts.map((a) => a.lastUsagePct ?? null), + resetAts: accounts.map((a) => a.resetAt ?? null), }; fs.writeFileSync(STATE_FILE, JSON.stringify(state)); - } catch { /* best effort */ } + } catch { + /* best effort */ + } } function loadCodexState(): void { @@ -105,19 +111,71 @@ function loadCodexState(): void { if (!fs.existsSync(STATE_FILE)) return; const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); const now = Date.now(); - if (typeof state.currentIndex === 'number' && state.currentIndex < accounts.length) { + if ( + typeof state.currentIndex === 'number' && + state.currentIndex < accounts.length + ) { currentIndex = state.currentIndex; } if (Array.isArray(state.rateLimits)) { - for (let i = 0; i < Math.min(state.rateLimits.length, accounts.length); i++) { + for ( + let i = 0; + i < Math.min(state.rateLimits.length, accounts.length); + i++ + ) { const until = state.rateLimits[i]; if (typeof until === 'number' && until > now) { accounts[i].rateLimitedUntil = until; } } } - logger.info({ currentIndex, accountCount: accounts.length }, 'Codex rotation state restored'); - } catch { /* start fresh */ } + if (Array.isArray(state.usagePcts)) { + for (let i = 0; i < Math.min(state.usagePcts.length, accounts.length); i++) { + if (typeof state.usagePcts[i] === 'number') accounts[i].lastUsagePct = state.usagePcts[i]; + } + } + if (Array.isArray(state.resetAts)) { + for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) { + if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]; + } + } + logger.info( + { currentIndex, accountCount: accounts.length }, + 'Codex rotation state restored', + ); + } catch { + /* start fresh */ + } +} + +const BUFFER_MS = 3 * 60_000; // 3 min buffer after reset time +const DEFAULT_COOLDOWN_MS = 3_600_000; // 1 hour fallback + +/** + * Parse "try again at Mar 26th, 2026 9:00 AM" from error message. + * Returns timestamp in ms, or null if not found. + */ +function parseRetryAfterFromError(error?: string): number | null { + if (!error) return null; + const match = error.match( + /try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (!match) return null; + try { + // Remove ordinal suffixes (1st, 2nd, 3rd, 4th) + const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); + const ts = new Date(cleaned).getTime(); + if (Number.isNaN(ts)) return null; + return ts; + } catch { + return null; + } +} + +function computeCooldownUntil(error?: string): number { + const retryAt = parseRetryAfterFromError(error); + if (retryAt) return retryAt + BUFFER_MS; + return Date.now() + DEFAULT_COOLDOWN_MS; } /** Get the auth.json path for the current active account. */ @@ -130,11 +188,18 @@ export function getActiveCodexAuthPath(): string | null { * Try to rotate to the next available Codex account. * Returns true if a fresh account was found. */ -export function rotateCodexToken(): boolean { +export function rotateCodexToken(errorMessage?: string): boolean { if (accounts.length <= 1) return false; const now = Date.now(); - accounts[currentIndex].rateLimitedUntil = now + 3_600_000; + const acct = accounts[currentIndex]; + acct.rateLimitedUntil = computeCooldownUntil(errorMessage); + acct.lastUsagePct = 100; + // Extract reset time string from error for display + const resetMatch = errorMessage?.match( + /try again at\s+(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (resetMatch) acct.resetAt = resetMatch[1]; for (let i = 1; i < accounts.length; i++) { const idx = (currentIndex + i) % accounts.length; @@ -178,6 +243,8 @@ export function getAllCodexAccounts(): { planType: string; isActive: boolean; isRateLimited: boolean; + cachedUsagePct?: number; + resetAt?: string; }[] { const now = Date.now(); return accounts.map((a, i) => ({ @@ -186,5 +253,7 @@ export function getAllCodexAccounts(): { planType: a.planType, isActive: i === currentIndex, isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), + cachedUsagePct: a.lastUsagePct, + resetAt: a.resetAt, })); } diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 07f09a1..17c1c55 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -309,7 +309,7 @@ export async function runAgentForGroup( : detectFallbackTrigger(errMsg); if (trigger.shouldFallback) { // Try rotating token before falling back to another provider - if (getTokenCount() > 1 && rotateToken()) { + if (getTokenCount() > 1 && rotateToken(errMsg)) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Rate-limited, retrying with rotated token', @@ -383,7 +383,7 @@ export async function runAgentForGroup( } : detectFallbackTrigger(output.error); if (trigger.shouldFallback) { - if (getTokenCount() > 1 && rotateToken()) { + if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Rate-limited (output error), retrying with rotated token', @@ -401,7 +401,7 @@ export async function runAgentForGroup( // Codex rate-limit rotation (non-Claude agents) if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { const trigger = detectFallbackTrigger(output.error); - if (trigger.shouldFallback && rotateCodexToken()) { + if (trigger.shouldFallback && rotateCodexToken(output.error ?? undefined)) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Codex rate-limited, retrying with rotated account', @@ -433,7 +433,7 @@ export async function runAgentForGroup( !isClaudeCodeAgent && primaryAttempt.streamedTriggerReason && getCodexAccountCount() > 1 && - rotateCodexToken() + rotateCodexToken(output.error ?? undefined) ) { logger.info( { diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 42f0b30..3542c0a 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -332,11 +332,15 @@ async function runTask( if (trigger.shouldFallback) { const isCodex = SERVICE_AGENT_TYPE === 'codex'; const rotated = isCodex - ? getCodexAccountCount() > 1 && rotateCodexToken() - : getTokenCount() > 1 && rotateToken(); + ? getCodexAccountCount() > 1 && rotateCodexToken(error) + : getTokenCount() > 1 && rotateToken(error); if (rotated) { logger.info( - { taskId: task.id, agent: SERVICE_AGENT_TYPE, reason: trigger.reason }, + { + taskId: task.id, + agent: SERVICE_AGENT_TYPE, + reason: trigger.reason, + }, 'Task rate-limited, rotated token β€” will retry on next schedule', ); if (isCodex) markCodexTokenHealthy(); diff --git a/src/token-rotation.ts b/src/token-rotation.ts index da9f289..2e73dbc 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -70,7 +70,9 @@ function saveState(): void { rateLimits: tokens.map((t) => t.rateLimitedUntil), }; fs.writeFileSync(STATE_FILE, JSON.stringify(state)); - } catch { /* best effort */ } + } catch { + /* best effort */ + } } function loadState(): void { @@ -78,19 +80,56 @@ function loadState(): void { if (!fs.existsSync(STATE_FILE)) return; const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); const now = Date.now(); - if (typeof state.currentIndex === 'number' && state.currentIndex < tokens.length) { + if ( + typeof state.currentIndex === 'number' && + state.currentIndex < tokens.length + ) { currentIndex = state.currentIndex; } if (Array.isArray(state.rateLimits)) { - for (let i = 0; i < Math.min(state.rateLimits.length, tokens.length); i++) { + for ( + let i = 0; + i < Math.min(state.rateLimits.length, tokens.length); + i++ + ) { const until = state.rateLimits[i]; if (typeof until === 'number' && until > now) { tokens[i].rateLimitedUntil = until; } } } - logger.info({ currentIndex, tokenCount: tokens.length }, 'Token rotation state restored'); - } catch { /* start fresh */ } + logger.info( + { currentIndex, tokenCount: tokens.length }, + 'Token rotation state restored', + ); + } catch { + /* start fresh */ + } +} + +const BUFFER_MS = 3 * 60_000; +const DEFAULT_COOLDOWN_MS = 3_600_000; + +function parseRetryAfterFromError(error?: string): number | null { + if (!error) return null; + const match = error.match( + /(?:try again at|resets?\s+(?:at\s+)?)\s*(\w+ \d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\s+\d{1,2}:\d{2}\s*(?:AM|PM)?)/i, + ); + if (!match) return null; + try { + const cleaned = match[1].replace(/(\d+)(?:st|nd|rd|th)/i, '$1'); + const ts = new Date(cleaned).getTime(); + if (Number.isNaN(ts)) return null; + return ts; + } catch { + return null; + } +} + +function computeCooldownUntil(error?: string): number { + const retryAt = parseRetryAfterFromError(error); + if (retryAt) return retryAt + BUFFER_MS; + return Date.now() + DEFAULT_COOLDOWN_MS; } /** Get the current active token. */ @@ -103,12 +142,11 @@ export function getCurrentToken(): string | undefined { * Try to rotate to the next available (non-rate-limited) token. * Returns true if a fresh token was found, false if all are exhausted. */ -export function rotateToken(): boolean { +export function rotateToken(errorMessage?: string): boolean { if (tokens.length <= 1) return false; const now = Date.now(); - // Mark current as rate-limited (default 1 hour) - tokens[currentIndex].rateLimitedUntil = now + 3_600_000; + tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage); // Find next available token for (let i = 1; i < tokens.length; i++) { diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index fbd5410..b3146da 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -616,10 +616,14 @@ async function buildUsageContent(): Promise { }); } } else { + // Show cached usage for rate-limited accounts + const pct = acct.isRateLimited && acct.cachedUsagePct != null + ? acct.cachedUsagePct : -1; + const reset = acct.resetAt || ''; rows.push({ name: `${label} ${acct.planType}`, - h5pct: -1, - h5reset: '', + h5pct: pct, + h5reset: reset, d7pct: -1, d7reset: '', }); From c7a3a563865a08e5657e8fcf148c9a66051d8b21 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:25:02 +0900 Subject: [PATCH 14/27] feat: scan all Codex accounts on startup + hourly, cache 5h/7d usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Scan all Codex accounts by spawning app-server with each auth dir - Cache primary (5h) and secondary (7d) usage per account - Show cached usage + reset remaining time on dashboard - Persist d7 usage in rotation state file - formatResetRemaining: "Xh Ym ν›„" / "N일 ν›„" format - Remove isRateLimited guard for cached display --- src/codex-token-rotation.ts | 81 ++++++++++++++++++++++++--- src/unified-dashboard.ts | 107 +++++++++++++++++++++++++++++++----- 2 files changed, 166 insertions(+), 22 deletions(-) diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index d57b90c..98c12be 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -24,21 +24,27 @@ interface CodexAccount { authPath: string; accountId: string; planType: string; + subscriptionUntil: string | null; rateLimitedUntil: number | null; lastUsagePct?: number; + lastUsageD7Pct?: number; resetAt?: string; } -function parsePlanFromJwt(idToken: string): string { +function parseJwtAuth(idToken: string): { planType: string; expiresAt: string | null } { try { const parts = idToken.split('.'); - if (parts.length < 2) return '?'; + if (parts.length < 2) return { planType: '?', expiresAt: null }; const payload = JSON.parse( Buffer.from(parts[1], 'base64url').toString('utf-8'), ); - return payload?.['https://api.openai.com/auth']?.chatgpt_plan_type || '?'; + const auth = payload?.['https://api.openai.com/auth'] || {}; + return { + planType: auth.chatgpt_plan_type || '?', + expiresAt: auth.chatgpt_subscription_active_until || null, + }; } catch { - return '?'; + return { planType: '?', expiresAt: null }; } } @@ -72,12 +78,14 @@ export function initCodexTokenRotation(): void { try { const data = JSON.parse(fs.readFileSync(authPath, 'utf-8')); const accountId = data?.tokens?.account_id || `account-${dir}`; - const planType = parsePlanFromJwt(data?.tokens?.id_token || ''); + const jwt = parseJwtAuth(data?.tokens?.id_token || ''); + const planType = jwt.planType; accounts.push({ index: accounts.length, authPath, accountId, planType, + subscriptionUntil: jwt.expiresAt, rateLimitedUntil: null, }); } catch { @@ -98,6 +106,7 @@ function saveCodexState(): void { currentIndex, rateLimits: accounts.map((a) => a.rateLimitedUntil), usagePcts: accounts.map((a) => a.lastUsagePct ?? null), + usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null), resetAts: accounts.map((a) => a.resetAt ?? null), }; fs.writeFileSync(STATE_FILE, JSON.stringify(state)); @@ -130,12 +139,26 @@ function loadCodexState(): void { } } if (Array.isArray(state.usagePcts)) { - for (let i = 0; i < Math.min(state.usagePcts.length, accounts.length); i++) { - if (typeof state.usagePcts[i] === 'number') accounts[i].lastUsagePct = state.usagePcts[i]; + for ( + let i = 0; + i < Math.min(state.usagePcts.length, accounts.length); + i++ + ) { + if (typeof state.usagePcts[i] === 'number') + accounts[i].lastUsagePct = state.usagePcts[i]; + } + } + if (Array.isArray(state.usageD7Pcts)) { + for (let i = 0; i < Math.min(state.usageD7Pcts.length, accounts.length); i++) { + if (typeof state.usageD7Pcts[i] === 'number') accounts[i].lastUsageD7Pct = state.usageD7Pcts[i]; } } if (Array.isArray(state.resetAts)) { - for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) { + for ( + let i = 0; + i < Math.min(state.resetAts.length, accounts.length); + i++ + ) { if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]; } } @@ -224,6 +247,46 @@ export function rotateCodexToken(errorMessage?: string): boolean { return false; } +/** + * Advance to the next healthy account (round-robin). + * Called after each successful request to spread load evenly + * and keep usage data fresh for all accounts. + */ +export function advanceCodexAccount(): void { + if (accounts.length <= 1) return; + const now = Date.now(); + for (let i = 1; i < accounts.length; i++) { + const idx = (currentIndex + i) % accounts.length; + const acct = accounts[idx]; + if (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now) { + currentIndex = idx; + saveCodexState(); + return; + } + } + // All others rate-limited, stay on current +} + +/** + * Update cached usage info for a specific account (or current if index omitted). + */ +export function updateCodexAccountUsage( + usagePct: number, + resetAt?: string, + accountIndex?: number, + d7Pct?: number, +): void { + if (accounts.length === 0) return; + const idx = accountIndex ?? currentIndex; + const acct = accounts[idx]; + if (acct) { + acct.lastUsagePct = usagePct; + if (d7Pct != null) acct.lastUsageD7Pct = d7Pct; + if (resetAt) acct.resetAt = resetAt; + saveCodexState(); + } +} + export function markCodexTokenHealthy(): void { if (accounts.length === 0) return; const acct = accounts[currentIndex]; @@ -244,6 +307,7 @@ export function getAllCodexAccounts(): { isActive: boolean; isRateLimited: boolean; cachedUsagePct?: number; + cachedUsageD7Pct?: number; resetAt?: string; }[] { const now = Date.now(); @@ -254,6 +318,7 @@ export function getAllCodexAccounts(): { isActive: i === currentIndex, isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now), cachedUsagePct: a.lastUsagePct, + cachedUsageD7Pct: a.lastUsageD7Pct, resetAt: a.resetAt, })); } diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index b3146da..7b16933 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -10,7 +10,10 @@ import { type ClaudeUsageData, type ClaudeAccountUsage, } from './claude-usage.js'; -import { getAllCodexAccounts } from './codex-token-rotation.js'; +import { + getAllCodexAccounts, + updateCodexAccountUsage, +} from './codex-token-rotation.js'; import { composeDashboardContent, formatElapsed, @@ -88,6 +91,25 @@ function formatResetKST(value: string | number): string { } } +function formatResetRemaining(value: string | number): string { + try { + const date = + typeof value === 'number' ? new Date(value * 1000) : new Date(value); + const diffMs = date.getTime() - Date.now(); + if (diffMs <= 0) return '리셋됨'; + const hours = Math.floor(diffMs / 3_600_000); + const minutes = Math.floor((diffMs % 3_600_000) / 60_000); + if (hours > 24) { + const days = Math.floor(hours / 24); + return `${days}일 ν›„`; + } + if (hours > 0) return `${hours}h ${minutes}m ν›„`; + return `${minutes}m ν›„`; + } catch { + return String(value); + } +} + function findDiscordChannel(channels: Channel[]): Channel | undefined { return channels.find( (channel) => channel.name.startsWith('discord') && channel.isConnected(), @@ -421,7 +443,9 @@ async function fetchClaudeUsage(): Promise { } } -async function fetchCodexUsage(): Promise { +async function fetchCodexUsage( + codexHomeOverride?: string, +): Promise { const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex'); const codexBin = fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex'; @@ -444,17 +468,22 @@ async function fetchCodexUsage(): Promise { const timer = setTimeout(() => finish(null), 20_000); + const spawnEnv: Record = { + ...(process.env as Record), + PATH: [ + path.dirname(process.execPath), + path.join(os.homedir(), '.npm-global', 'bin'), + process.env.PATH || '', + ].join(':'), + }; + if (codexHomeOverride) { + spawnEnv.CODEX_HOME = codexHomeOverride; + } + try { proc = spawn(codexBin, ['app-server'], { stdio: ['pipe', 'pipe', 'pipe'], - env: { - ...(process.env as Record), - PATH: [ - path.dirname(process.execPath), - path.join(os.homedir(), '.npm-global', 'bin'), - process.env.PATH || '', - ].join(':'), - }, + env: spawnEnv, }); } catch { resolve(null); @@ -616,15 +645,17 @@ async function buildUsageContent(): Promise { }); } } else { - // Show cached usage for rate-limited accounts - const pct = acct.isRateLimited && acct.cachedUsagePct != null - ? acct.cachedUsagePct : -1; + // Show cached usage from last scan + const pct = + acct.cachedUsagePct != null ? acct.cachedUsagePct : -1; + const d7pct = + acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1; const reset = acct.resetAt || ''; rows.push({ name: `${label} ${acct.planType}`, h5pct: pct, h5reset: reset, - d7pct: -1, + d7pct, d7reset: '', }); } @@ -736,6 +767,51 @@ function buildUnifiedDashboardContent(): string { return composeDashboardContent(sections); } +const CODEX_ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts'); +const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour + +/** + * Scan ALL Codex accounts by spawning app-server with each auth. + * Called on startup and every hour to keep cached usage fresh. + */ +async function refreshAllCodexAccountUsage(): Promise { + const codexAccounts = getAllCodexAccounts(); + if (codexAccounts.length <= 1) return; + + logger.info( + { accountCount: codexAccounts.length }, + 'Scanning all Codex accounts for usage data', + ); + + for (const acct of codexAccounts) { + const accountDir = path.join(CODEX_ACCOUNTS_DIR, String(acct.index + 1)); + if (!fs.existsSync(accountDir)) continue; + + try { + const usage = await fetchCodexUsage(accountDir); + if (usage && Array.isArray(usage)) { + const relevant = usage.filter( + (l) => l.primary.usedPercent > 0 || l.secondary.usedPercent > 0, + ); + const display = relevant.length > 0 ? relevant : usage.slice(0, 1); + if (display.length > 0) { + const pct = Math.round(display[0].primary.usedPercent); + const resetVal = display[0].primary.resetsAt; + const resetStr = resetVal ? formatResetRemaining(resetVal) : undefined; + const d7Pct = Math.round(display[0].secondary.usedPercent); + updateCodexAccountUsage(pct, resetStr, acct.index, d7Pct); + logger.info( + { account: acct.index + 1, usagePct: pct, reset: resetStr }, + `Codex account #${acct.index + 1} usage: ${pct}%`, + ); + } + } + } catch (err) { + logger.debug({ err, account: acct.index + 1 }, 'Failed to fetch usage for Codex account'); + } + } +} + async function refreshUsageCache(): Promise { if (usageUpdateInProgress) return; usageUpdateInProgress = true; @@ -803,6 +879,9 @@ export async function startUnifiedDashboard( if (isRenderer) { setInterval(refreshUsageCache, opts.usageUpdateInterval); + // Full scan of all Codex accounts on startup + hourly + void refreshAllCodexAccountUsage(); + setInterval(() => void refreshAllCodexAccountUsage(), CODEX_FULL_SCAN_INTERVAL); } logger.info( From 9e091df6f6437844fff1a1ca4d06bbd4ec6ab9f5 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:28:48 +0900 Subject: [PATCH 15/27] fix: scan all Codex limits for max 5h/7d, persist d7 reset times - Iterate all rate limits per account instead of just display[0] - Track and persist 7-day reset time separately (resetD7At) - Show both 5h and 7d cached usage for inactive Codex accounts --- src/codex-token-rotation.ts | 31 ++++++++++++++++------ src/unified-dashboard.ts | 51 +++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 98c12be..185092d 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -29,9 +29,13 @@ interface CodexAccount { lastUsagePct?: number; lastUsageD7Pct?: number; resetAt?: string; + resetD7At?: string; } -function parseJwtAuth(idToken: string): { planType: string; expiresAt: string | null } { +function parseJwtAuth(idToken: string): { + planType: string; + expiresAt: string | null; +} { try { const parts = idToken.split('.'); if (parts.length < 2) return { planType: '?', expiresAt: null }; @@ -108,6 +112,7 @@ function saveCodexState(): void { usagePcts: accounts.map((a) => a.lastUsagePct ?? null), usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null), resetAts: accounts.map((a) => a.resetAt ?? null), + resetD7Ats: accounts.map((a) => a.resetD7At ?? null), }; fs.writeFileSync(STATE_FILE, JSON.stringify(state)); } catch { @@ -149,19 +154,25 @@ function loadCodexState(): void { } } if (Array.isArray(state.usageD7Pcts)) { - for (let i = 0; i < Math.min(state.usageD7Pcts.length, accounts.length); i++) { - if (typeof state.usageD7Pcts[i] === 'number') accounts[i].lastUsageD7Pct = state.usageD7Pcts[i]; + for ( + let i = 0; + i < Math.min(state.usageD7Pcts.length, accounts.length); + i++ + ) { + if (typeof state.usageD7Pcts[i] === 'number') + accounts[i].lastUsageD7Pct = state.usageD7Pcts[i]; } } if (Array.isArray(state.resetAts)) { - for ( - let i = 0; - i < Math.min(state.resetAts.length, accounts.length); - i++ - ) { + for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) { if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]; } } + if (Array.isArray(state.resetD7Ats)) { + for (let i = 0; i < Math.min(state.resetD7Ats.length, accounts.length); i++) { + if (state.resetD7Ats[i]) accounts[i].resetD7At = state.resetD7Ats[i]; + } + } logger.info( { currentIndex, accountCount: accounts.length }, 'Codex rotation state restored', @@ -275,6 +286,7 @@ export function updateCodexAccountUsage( resetAt?: string, accountIndex?: number, d7Pct?: number, + resetD7At?: string, ): void { if (accounts.length === 0) return; const idx = accountIndex ?? currentIndex; @@ -283,6 +295,7 @@ export function updateCodexAccountUsage( acct.lastUsagePct = usagePct; if (d7Pct != null) acct.lastUsageD7Pct = d7Pct; if (resetAt) acct.resetAt = resetAt; + if (resetD7At) acct.resetD7At = resetD7At; saveCodexState(); } } @@ -309,6 +322,7 @@ export function getAllCodexAccounts(): { cachedUsagePct?: number; cachedUsageD7Pct?: number; resetAt?: string; + resetD7At?: string; }[] { const now = Date.now(); return accounts.map((a, i) => ({ @@ -320,5 +334,6 @@ export function getAllCodexAccounts(): { cachedUsagePct: a.lastUsagePct, cachedUsageD7Pct: a.lastUsageD7Pct, resetAt: a.resetAt, + resetD7At: a.resetD7At, })); } diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 7b16933..6cc35b0 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -646,17 +646,17 @@ async function buildUsageContent(): Promise { } } else { // Show cached usage from last scan - const pct = - acct.cachedUsagePct != null ? acct.cachedUsagePct : -1; + const pct = acct.cachedUsagePct != null ? acct.cachedUsagePct : -1; const d7pct = acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1; const reset = acct.resetAt || ''; + const d7reset = acct.resetD7At || ''; rows.push({ name: `${label} ${acct.planType}`, h5pct: pct, h5reset: reset, d7pct, - d7reset: '', + d7reset, }); } } @@ -794,20 +794,42 @@ async function refreshAllCodexAccountUsage(): Promise { (l) => l.primary.usedPercent > 0 || l.secondary.usedPercent > 0, ); const display = relevant.length > 0 ? relevant : usage.slice(0, 1); - if (display.length > 0) { - const pct = Math.round(display[0].primary.usedPercent); - const resetVal = display[0].primary.resetsAt; - const resetStr = resetVal ? formatResetRemaining(resetVal) : undefined; - const d7Pct = Math.round(display[0].secondary.usedPercent); - updateCodexAccountUsage(pct, resetStr, acct.index, d7Pct); + if (usage.length > 0) { + // Find max primary (5h) and secondary (7d) across all limits + let maxH5 = 0; + let maxD7 = 0; + let h5Reset: string | number | undefined; + let d7Reset: string | number | undefined; + for (const limit of usage) { + if (limit.primary.usedPercent > maxH5) { + maxH5 = limit.primary.usedPercent; + h5Reset = limit.primary.resetsAt; + } + if (limit.secondary.usedPercent > maxD7) { + maxD7 = limit.secondary.usedPercent; + d7Reset = limit.secondary.resetsAt; + } + } + const pct = Math.round(maxH5); + const d7Pct = Math.round(maxD7); + const resetStr = h5Reset + ? formatResetRemaining(h5Reset) + : undefined; + const resetD7Str = d7Reset + ? formatResetRemaining(d7Reset) + : undefined; + updateCodexAccountUsage(pct, resetStr, acct.index, d7Pct, resetD7Str); logger.info( - { account: acct.index + 1, usagePct: pct, reset: resetStr }, - `Codex account #${acct.index + 1} usage: ${pct}%`, + { account: acct.index + 1, h5: pct, d7: d7Pct, reset: resetStr }, + `Codex account #${acct.index + 1} usage: 5h=${pct}% 7d=${d7Pct}%`, ); } } } catch (err) { - logger.debug({ err, account: acct.index + 1 }, 'Failed to fetch usage for Codex account'); + logger.debug( + { err, account: acct.index + 1 }, + 'Failed to fetch usage for Codex account', + ); } } } @@ -881,7 +903,10 @@ export async function startUnifiedDashboard( setInterval(refreshUsageCache, opts.usageUpdateInterval); // Full scan of all Codex accounts on startup + hourly void refreshAllCodexAccountUsage(); - setInterval(() => void refreshAllCodexAccountUsage(), CODEX_FULL_SCAN_INTERVAL); + setInterval( + () => void refreshAllCodexAccountUsage(), + CODEX_FULL_SCAN_INTERVAL, + ); } logger.info( From 0375912c2b022ce75bb3b2d1efee8fe3b1f17f1f Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:29:32 +0900 Subject: [PATCH 16/27] fix: render reset times in dashboard usage rows --- src/unified-dashboard.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 6cc35b0..4e333b8 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -697,7 +697,10 @@ async function buildUsageContent(): Promise { row.d7pct >= 0 ? `${bar(row.d7pct)} ${String(row.d7pct).padStart(3)}%` : ' β€” '; - lines.push(`${padName(row.name)}${h5} ${d7}`); + const reset = row.h5reset || row.d7reset + ? ` ${row.h5reset || ''}${row.d7reset ? ` / ${row.d7reset}` : ''}` + : ''; + lines.push(`${padName(row.name)}${h5} ${d7}${reset}`); } lines.push('```'); } else { @@ -812,9 +815,7 @@ async function refreshAllCodexAccountUsage(): Promise { } const pct = Math.round(maxH5); const d7Pct = Math.round(maxD7); - const resetStr = h5Reset - ? formatResetRemaining(h5Reset) - : undefined; + const resetStr = h5Reset ? formatResetRemaining(h5Reset) : undefined; const resetD7Str = d7Reset ? formatResetRemaining(d7Reset) : undefined; From 6ac706f9c5521bfd91a3f87405c95ac4d1eb64d1 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:31:10 +0900 Subject: [PATCH 17/27] =?UTF-8?q?fix:=20unify=20reset=20time=20format=20to?= =?UTF-8?q?=20relative=20(Xh=20Ym=20=ED=9B=84)=20for=20all=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/unified-dashboard.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 4e333b8..a1368ae 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -613,13 +613,13 @@ async function buildUsageContent(): Promise { ? Math.round(h5.utilization) : Math.round(h5.utilization * 100) : -1, - h5reset: h5 ? formatResetKST(h5.resets_at) : '', + h5reset: h5 ? formatResetRemaining(h5.resets_at) : '', d7pct: d7 ? d7.utilization > 1 ? Math.round(d7.utilization) : Math.round(d7.utilization * 100) : -1, - d7reset: d7 ? formatResetKST(d7.resets_at) : '', + d7reset: d7 ? formatResetRemaining(d7.resets_at) : '', }); } @@ -639,9 +639,9 @@ async function buildUsageContent(): Promise { rows.push({ name: `${label} ${acct.planType}`, h5pct: Math.round(limit.primary.usedPercent), - h5reset: formatResetKST(limit.primary.resetsAt), + h5reset: formatResetRemaining(limit.primary.resetsAt), d7pct: Math.round(limit.secondary.usedPercent), - d7reset: formatResetKST(limit.secondary.resetsAt), + d7reset: formatResetRemaining(limit.secondary.resetsAt), }); } } else { @@ -697,9 +697,10 @@ async function buildUsageContent(): Promise { row.d7pct >= 0 ? `${bar(row.d7pct)} ${String(row.d7pct).padStart(3)}%` : ' β€” '; - const reset = row.h5reset || row.d7reset - ? ` ${row.h5reset || ''}${row.d7reset ? ` / ${row.d7reset}` : ''}` - : ''; + const reset = + row.h5reset || row.d7reset + ? ` ${row.h5reset || ''}${row.d7reset ? ` / ${row.d7reset}` : ''}` + : ''; lines.push(`${padName(row.name)}${h5} ${d7}${reset}`); } lines.push('```'); From 2ffb4d56965899e04df71e4a3fc2fd40318ca5df Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:32:18 +0900 Subject: [PATCH 18/27] fix: fixed-width English reset format for monospace alignment --- src/unified-dashboard.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index a1368ae..e5f29dd 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -96,17 +96,16 @@ function formatResetRemaining(value: string | number): string { const date = typeof value === 'number' ? new Date(value * 1000) : new Date(value); const diffMs = date.getTime() - Date.now(); - if (diffMs <= 0) return '리셋됨'; + if (diffMs <= 0) return ' reset'; const hours = Math.floor(diffMs / 3_600_000); const minutes = Math.floor((diffMs % 3_600_000) / 60_000); - if (hours > 24) { + if (hours >= 24) { const days = Math.floor(hours / 24); - return `${days}일 ν›„`; + return `${String(days).padStart(2)}d`.padStart(6); } - if (hours > 0) return `${hours}h ${minutes}m ν›„`; - return `${minutes}m ν›„`; + return `${hours}h${String(minutes).padStart(2, '0')}m`.padStart(6); } catch { - return String(value); + return String(value).padStart(6); } } From 9f8b4ae0aa35e7583fd486ffa515986e28397531 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:33:48 +0900 Subject: [PATCH 19/27] fix: space between h/m in reset format, padStart(7) --- src/unified-dashboard.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index e5f29dd..d692e5f 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -101,9 +101,9 @@ function formatResetRemaining(value: string | number): string { const minutes = Math.floor((diffMs % 3_600_000) / 60_000); if (hours >= 24) { const days = Math.floor(hours / 24); - return `${String(days).padStart(2)}d`.padStart(6); + return `${String(days).padStart(2)}d`.padStart(7); } - return `${hours}h${String(minutes).padStart(2, '0')}m`.padStart(6); + return `${hours}h ${String(minutes).padStart(2, '0')}m`.padStart(7); } catch { return String(value).padStart(6); } From 92c82c192a4c97ff60856299318b6af9937be66e Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:35:00 +0900 Subject: [PATCH 20/27] fix: fixed-width reset format Xd Yh / Xh Ym for alignment --- src/unified-dashboard.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index d692e5f..6ebd378 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -101,9 +101,10 @@ function formatResetRemaining(value: string | number): string { const minutes = Math.floor((diffMs % 3_600_000) / 60_000); if (hours >= 24) { const days = Math.floor(hours / 24); - return `${String(days).padStart(2)}d`.padStart(7); + const remH = hours % 24; + return `${String(days).padStart(2)}d ${String(remH).padStart(2)}h`; } - return `${hours}h ${String(minutes).padStart(2, '0')}m`.padStart(7); + return `${String(hours).padStart(2)}h ${String(minutes).padStart(2)}m`; } catch { return String(value).padStart(6); } From c51d80bd076a006100ca14509c6c72b15675f626 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:37:14 +0900 Subject: [PATCH 21/27] style: prettier formatting --- src/agent-runner-environment.ts | 7 ++++--- src/codex-token-rotation.ts | 12 ++++++++++-- src/message-agent-executor.ts | 5 ++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index 43b548e..6608b7f 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -184,9 +184,10 @@ function prepareCodexSessionEnvironment(args: { fs.mkdirSync(sessionCodexDir, { recursive: true }); const rotatedAuthSrc = getActiveCodexAuthPath(); - const authSrc = rotatedAuthSrc && fs.existsSync(rotatedAuthSrc) - ? rotatedAuthSrc - : path.join(hostCodexDir, 'auth.json'); + const authSrc = + rotatedAuthSrc && fs.existsSync(rotatedAuthSrc) + ? rotatedAuthSrc + : path.join(hostCodexDir, 'auth.json'); const authDst = path.join(sessionCodexDir, 'auth.json'); if (fs.existsSync(authSrc)) fs.copyFileSync(authSrc, authDst); for (const file of ['config.toml', 'config.json']) { diff --git a/src/codex-token-rotation.ts b/src/codex-token-rotation.ts index 185092d..f02932a 100644 --- a/src/codex-token-rotation.ts +++ b/src/codex-token-rotation.ts @@ -164,12 +164,20 @@ function loadCodexState(): void { } } if (Array.isArray(state.resetAts)) { - for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) { + for ( + let i = 0; + i < Math.min(state.resetAts.length, accounts.length); + i++ + ) { if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]; } } if (Array.isArray(state.resetD7Ats)) { - for (let i = 0; i < Math.min(state.resetD7Ats.length, accounts.length); i++) { + for ( + let i = 0; + i < Math.min(state.resetD7Ats.length, accounts.length); + i++ + ) { if (state.resetD7Ats[i]) accounts[i].resetD7At = state.resetD7Ats[i]; } } diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 17c1c55..9b21d3c 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -401,7 +401,10 @@ export async function runAgentForGroup( // Codex rate-limit rotation (non-Claude agents) if (!isClaudeCodeAgent && getCodexAccountCount() > 1) { const trigger = detectFallbackTrigger(output.error); - if (trigger.shouldFallback && rotateCodexToken(output.error ?? undefined)) { + if ( + trigger.shouldFallback && + rotateCodexToken(output.error ?? undefined) + ) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, 'Codex rate-limited, retrying with rotated account', From ed9123d66c4a139936cb425ae6ba857b5b1898c4 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:38:28 +0900 Subject: [PATCH 22/27] fix: always capture reset times even at 0% usage (>= instead of >) --- src/unified-dashboard.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 6ebd378..dc8f3cc 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -800,16 +800,17 @@ async function refreshAllCodexAccountUsage(): Promise { const display = relevant.length > 0 ? relevant : usage.slice(0, 1); if (usage.length > 0) { // Find max primary (5h) and secondary (7d) across all limits + // Always capture reset times from first limit as baseline let maxH5 = 0; let maxD7 = 0; - let h5Reset: string | number | undefined; - let d7Reset: string | number | undefined; + let h5Reset: string | number | undefined = usage[0].primary.resetsAt; + let d7Reset: string | number | undefined = usage[0].secondary.resetsAt; for (const limit of usage) { - if (limit.primary.usedPercent > maxH5) { + if (limit.primary.usedPercent >= maxH5) { maxH5 = limit.primary.usedPercent; h5Reset = limit.primary.resetsAt; } - if (limit.secondary.usedPercent > maxD7) { + if (limit.secondary.usedPercent >= maxD7) { maxD7 = limit.secondary.usedPercent; d7Reset = limit.secondary.resetsAt; } From 16f4e6ccb91a9c131714c91ea5a7daa227cf444e Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:39:15 +0900 Subject: [PATCH 23/27] fix: force dashboard refresh after Codex account scan completes --- src/unified-dashboard.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index dc8f3cc..4adc783 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -804,7 +804,8 @@ async function refreshAllCodexAccountUsage(): Promise { let maxH5 = 0; let maxD7 = 0; let h5Reset: string | number | undefined = usage[0].primary.resetsAt; - let d7Reset: string | number | undefined = usage[0].secondary.resetsAt; + let d7Reset: string | number | undefined = + usage[0].secondary.resetsAt; for (const limit of usage) { if (limit.primary.usedPercent >= maxH5) { maxH5 = limit.primary.usedPercent; @@ -905,9 +906,13 @@ export async function startUnifiedDashboard( if (isRenderer) { setInterval(refreshUsageCache, opts.usageUpdateInterval); // Full scan of all Codex accounts on startup + hourly - void refreshAllCodexAccountUsage(); + // After scan, refresh dashboard so cached data is visible immediately + void refreshAllCodexAccountUsage().then(() => { + void refreshUsageCache().then(() => void updateStatus()); + }); setInterval( - () => void refreshAllCodexAccountUsage(), + () => + void refreshAllCodexAccountUsage().then(() => void refreshUsageCache()), CODEX_FULL_SCAN_INTERVAL, ); } From 2ab6c25f0e60a7c31426bb10e3e28a4616486696 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 01:41:23 +0900 Subject: [PATCH 24/27] feat: fetch Claude account profiles (plan type) on startup - Call /api/oauth/profile for each token, cache in memory - Display plan type (max/pro/free) next to Claude account names - Profiles fetched once on startup, no periodic refresh needed --- src/claude-usage.ts | 62 ++++++++++++++++++++++++++++++++++++++++ src/unified-dashboard.ts | 11 +++++-- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/claude-usage.ts b/src/claude-usage.ts index 4c863e1..0f265b4 100644 --- a/src/claude-usage.ts +++ b/src/claude-usage.ts @@ -8,6 +8,8 @@ import { logger } from './logger.js'; import { getCurrentToken, getAllTokens } from './token-rotation.js'; +const PROFILE_ENDPOINT = 'https://api.anthropic.com/api/oauth/profile'; + export interface ClaudeUsageData { five_hour?: { utilization: number; resets_at: string }; seven_day?: { utilization: number; resets_at: string }; @@ -99,6 +101,66 @@ export async function fetchClaudeUsage(): Promise { return fetchUsageForToken(token); } +export interface ClaudeAccountProfile { + email: string; + planType: string; // "max", "pro", "free" +} + +const profileCache = new Map(); + +async function fetchProfileForToken(token: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const res = await fetch(PROFILE_ENDPOINT, { + method: 'GET', + headers: { + Authorization: `Bearer ${token}`, + 'anthropic-beta': 'oauth-2025-04-20', + }, + signal: controller.signal, + }); + if (!res.ok) return null; + const data = await res.json() as { + account?: { email?: string; has_claude_max?: boolean; has_claude_pro?: boolean }; + organization?: { organization_type?: string }; + }; + const orgType = data.organization?.organization_type || ''; + const planType = data.account?.has_claude_max ? 'max' + : data.account?.has_claude_pro ? 'pro' + : orgType.replace('claude_', '') || 'free'; + return { + email: data.account?.email || '?', + planType, + }; + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +/** + * Fetch profiles for all Claude tokens (cached, called once on startup). + */ +export async function fetchAllClaudeProfiles(): Promise { + const allTokens = getAllTokens(); + for (const t of allTokens) { + const profile = await fetchProfileForToken(t.token); + if (profile) { + profileCache.set(t.index, profile); + logger.info( + { account: t.index + 1, plan: profile.planType, email: profile.email }, + `Claude account #${t.index + 1}: ${profile.planType}`, + ); + } + } +} + +export function getClaudeProfile(index: number): ClaudeAccountProfile | undefined { + return profileCache.get(index); +} + export interface ClaudeAccountUsage { index: number; masked: string; diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 4adc783..f26fdac 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -7,6 +7,8 @@ import { STATUS_SHOW_ROOMS, USAGE_DASHBOARD_ENABLED } from './config.js'; import { fetchClaudeUsage as fetchClaudeUsageApi, fetchAllClaudeUsage, + fetchAllClaudeProfiles, + getClaudeProfile, type ClaudeUsageData, type ClaudeAccountUsage, } from './claude-usage.js'; @@ -600,12 +602,14 @@ async function buildUsageContent(): Promise { if (!u) continue; const h5 = u.five_hour; const d7 = u.seven_day; + const profile = getClaudeProfile(account.index); + const planSuffix = profile ? ` ${profile.planType}` : ''; const label = claudeAccounts.length > 1 - ? `Claude${account.index + 1}${account.isActive ? '*' : ''}${account.isRateLimited ? '!' : ''}` + ? `Claude${account.index + 1}${account.isActive ? '*' : ''}${account.isRateLimited ? '!' : ''}${planSuffix}` : claudeUsageIsCached - ? 'Claude*' - : 'Claude'; + ? `Claude*${planSuffix}` + : `Claude${planSuffix}`; rows.push({ name: label, h5pct: h5 @@ -863,6 +867,7 @@ export async function startUnifiedDashboard( } if (isRenderer) { + await fetchAllClaudeProfiles(); await refreshUsageCache(); } From 120f16e9e1836c013932428971409adbc33c1e4d Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 03:40:51 +0900 Subject: [PATCH 25/27] fix: Claude token rotation, usage-exhausted fallback, and usage API caching - Fix token rotation not taking effect: getCurrentToken() was shadowed by static .env CLAUDE_CODE_OAUTH_TOKEN value in agent environment - Don't fall back to Kimi on usage-exhausted (only on transient 429/network) - Add disk cache for Claude usage API data (survives restarts, 429s) - Rate-limit usage API calls to 1 per token per 5 minutes - Add ignoreRateLimits option to rotateToken() for exhausted recovery - Rotate to next token in getActiveProvider() when current is exhausted - Add isUsageExhausted() helper to provider-fallback --- src/agent-runner-environment.ts | 3 +- src/claude-usage.ts | 89 ++++++-- src/message-agent-executor.test.ts | 269 ++++++++++++++++++++++ src/message-agent-executor.ts | 255 ++++++++++++++++++--- src/message-runtime.test.ts | 169 +++++++++++++- src/provider-fallback.test.ts | 103 +++++++++ src/provider-fallback.ts | 123 ++++++++++- src/task-scheduler.test.ts | 142 ++++++++++++ src/task-scheduler.ts | 344 +++++++++++++++++++++++++---- src/token-rotation.ts | 14 +- src/unified-dashboard.test.ts | 140 ++++++++++++ src/unified-dashboard.ts | 240 ++++++-------------- 12 files changed, 1633 insertions(+), 258 deletions(-) create mode 100644 src/message-agent-executor.test.ts create mode 100644 src/provider-fallback.test.ts create mode 100644 src/unified-dashboard.test.ts diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index 6608b7f..59687f5 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -123,9 +123,10 @@ function prepareClaudeEnvironment(args: { args.envVars.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL || ''; } { + // Token rotation takes priority over static .env value const oauthToken = - args.envVars.CLAUDE_CODE_OAUTH_TOKEN || getCurrentToken() || + args.envVars.CLAUDE_CODE_OAUTH_TOKEN || process.env.CLAUDE_CODE_OAUTH_TOKEN; if (oauthToken) { args.env.CLAUDE_CODE_OAUTH_TOKEN = oauthToken; diff --git a/src/claude-usage.ts b/src/claude-usage.ts index 0f265b4..0c5def7 100644 --- a/src/claude-usage.ts +++ b/src/claude-usage.ts @@ -5,9 +5,15 @@ * Supports multiple tokens for rotation-aware usage checking. */ +import fs from 'fs'; +import path from 'path'; + +import { DATA_DIR } from './config.js'; import { logger } from './logger.js'; import { getCurrentToken, getAllTokens } from './token-rotation.js'; +const USAGE_CACHE_FILE = path.join(DATA_DIR, 'claude-usage-cache.json'); + const PROFILE_ENDPOINT = 'https://api.anthropic.com/api/oauth/profile'; export interface ClaudeUsageData { @@ -35,9 +41,50 @@ function mapWindow(w?: { return { utilization: w.utilization, resets_at: w.resets_at || '' }; } +// ── Disk cache for usage data (survives restarts, 429s) ── + +interface UsageCacheEntry { + usage: ClaudeUsageData; + fetchedAt: number; +} + +let usageDiskCache: Record = {}; +let diskCacheLoaded = false; + +function loadUsageDiskCache(): void { + if (diskCacheLoaded) return; + diskCacheLoaded = true; + try { + if (fs.existsSync(USAGE_CACHE_FILE)) { + usageDiskCache = JSON.parse(fs.readFileSync(USAGE_CACHE_FILE, 'utf-8')); + } + } catch { /* start fresh */ } +} + +function saveUsageDiskCache(): void { + try { + fs.writeFileSync(USAGE_CACHE_FILE, JSON.stringify(usageDiskCache)); + } catch { /* best effort */ } +} + +function cacheKey(token: string): string { + return token.slice(0, 12); +} + +// Rate limit: at most one API call per token per 5 minutes +const MIN_FETCH_INTERVAL_MS = 300_000; + async function fetchUsageForToken( token: string, ): Promise { + loadUsageDiskCache(); + + // Return cached data if fetched recently (avoid API rate-limit) + const key = cacheKey(token); + const cached = usageDiskCache[key]; + if (cached && Date.now() - cached.fetchedAt < MIN_FETCH_INTERVAL_MS) { + return cached.usage; + } const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); @@ -57,32 +104,38 @@ async function fetchUsageForToken( return null; } if (res.status === 429) { - logger.warn('Claude usage API: rate limited (429)'); - return null; + logger.warn('Claude usage API: rate limited (429), returning cached data'); + return usageDiskCache[cacheKey(token)]?.usage ?? null; } if (!res.ok) { logger.warn( { status: res.status }, `Claude usage API: unexpected status ${res.status}`, ); - return null; + return usageDiskCache[cacheKey(token)]?.usage ?? null; } const data = (await res.json()) as UsageApiResponse; - return { + const result: ClaudeUsageData = { five_hour: mapWindow(data.five_hour), seven_day: mapWindow(data.seven_day), seven_day_sonnet: mapWindow(data.seven_day_sonnet), seven_day_opus: mapWindow(data.seven_day_opus), }; + + // Persist to disk cache + usageDiskCache[cacheKey(token)] = { usage: result, fetchedAt: Date.now() }; + saveUsageDiskCache(); + + return result; } catch (err) { if ((err as Error).name === 'AbortError') { logger.warn('Claude usage API: request timed out'); } else { logger.warn({ err }, 'Claude usage API: fetch failed'); } - return null; + return usageDiskCache[cacheKey(token)]?.usage ?? null; } finally { clearTimeout(timer); } @@ -103,12 +156,14 @@ export async function fetchClaudeUsage(): Promise { export interface ClaudeAccountProfile { email: string; - planType: string; // "max", "pro", "free" + planType: string; // "max", "pro", "free" } const profileCache = new Map(); -async function fetchProfileForToken(token: string): Promise { +async function fetchProfileForToken( + token: string, +): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); try { @@ -121,14 +176,20 @@ async function fetchProfileForToken(token: string): Promise { } } -export function getClaudeProfile(index: number): ClaudeAccountProfile | undefined { +export function getClaudeProfile( + index: number, +): ClaudeAccountProfile | undefined { return profileCache.get(index); } diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts new file mode 100644 index 0000000..52e4bf4 --- /dev/null +++ b/src/message-agent-executor.test.ts @@ -0,0 +1,269 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./agent-runner.js', () => ({ + runAgentProcess: vi.fn(), + writeGroupsSnapshot: vi.fn(), + writeTasksSnapshot: vi.fn(), +})); + +vi.mock('./available-groups.js', () => ({ + listAvailableGroups: vi.fn(() => []), +})); + +vi.mock('./config.js', () => ({ + DATA_DIR: '/tmp/ejclaw-test-data', +})); + +vi.mock('./db.js', () => ({ + getAllTasks: vi.fn(() => []), +})); + +vi.mock('./logger.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +vi.mock('./provider-fallback.js', () => ({ + detectFallbackTrigger: vi.fn((error?: string | null) => { + const lower = (error || '').toLowerCase(); + if ( + lower.includes('429') || + lower.includes('rate limit') || + lower.includes('hit your limit') + ) { + return { shouldFallback: true, reason: '429' }; + } + return { shouldFallback: false, reason: '' }; + }), + getActiveProvider: vi.fn(async () => 'claude'), + getFallbackEnvOverrides: vi.fn(() => ({ + ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', + ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', + ANTHROPIC_MODEL: 'kimi-k2.5', + })), + getFallbackProviderName: vi.fn(() => 'kimi'), + hasGroupProviderOverride: vi.fn(() => false), + isFallbackEnabled: vi.fn(() => true), + markPrimaryCooldown: vi.fn(), +})); + +vi.mock('./session-recovery.js', () => ({ + shouldResetSessionOnAgentFailure: vi.fn(() => false), +})); + +vi.mock('./token-rotation.js', () => ({ + rotateToken: vi.fn(() => false), + getTokenCount: vi.fn(() => 1), + markTokenHealthy: vi.fn(), +})); + +vi.mock('./codex-token-rotation.js', () => ({ + rotateCodexToken: vi.fn(() => false), + getCodexAccountCount: vi.fn(() => 1), + markCodexTokenHealthy: vi.fn(), +})); + +import * as agentRunner from './agent-runner.js'; +import { runAgentForGroup } from './message-agent-executor.js'; +import * as providerFallback from './provider-fallback.js'; +import * as tokenRotation from './token-rotation.js'; +import type { RegisteredGroup } from './types.js'; + +function makeGroup(): RegisteredGroup { + return { + name: 'Test Group', + folder: 'test-claude', + trigger: '@Andy', + added_at: new Date().toISOString(), + requiresTrigger: false, + agentType: 'claude-code', + }; +} + +function makeDeps() { + return { + assistantName: 'Andy', + queue: { + registerProcess: vi.fn(), + }, + getRegisteredGroups: () => ({}), + getSessions: () => ({}), + persistSession: vi.fn(), + clearSession: vi.fn(), + }; +} + +describe('runAgentForGroup Claude rotation', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); + vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true); + vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); + }); + + it('rotates to another Claude account before falling back to Kimi', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'You’re out of extra usage Β· resets 4am (Asia/Seoul)', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'νšŒμ „λœ Claude μ‘λ‹΅μž…λ‹ˆλ‹€.', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-rotate-claude', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); + expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); + expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + expect(outputs).toEqual(['νšŒμ „λœ Claude μ‘λ‹΅μž…λ‹ˆλ‹€.']); + }); + + it('falls back to Kimi only after all Claude accounts are exhausted', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'You’re out of extra usage Β· resets 4am (Asia/Seoul)', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'Kimi 폴백 μ‘λ‹΅μž…λ‹ˆλ‹€.', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-fallback-after-rotation', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(3); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2); + expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( + 'usage-exhausted', + undefined, + ); + expect(agentRunner.runAgentProcess).toHaveBeenNthCalledWith( + 3, + expect.anything(), + expect.anything(), + expect.any(Function), + expect.any(Function), + expect.objectContaining({ + ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', + ANTHROPIC_MODEL: 'kimi-k2.5', + }), + ); + expect(outputs).toEqual(['Kimi 폴백 μ‘λ‹΅μž…λ‹ˆλ‹€.']); + }); + + it('does not mistake a normal response quoting the banner text for a usage error', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + + vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: + "μƒνƒœ 문ꡬ μ˜ˆμ‹œ: You're out of extra usage Β· resets 4am (Asia/Seoul) λΌλŠ” λ°°λ„ˆκ°€ 뜰 수 μžˆμŠ΅λ‹ˆλ‹€.", + }); + return { + status: 'success', + result: null, + }; + }, + ); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-normal-quoted-banner', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(tokenRotation.rotateToken).not.toHaveBeenCalled(); + expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + expect(outputs).toEqual([ + "μƒνƒœ 문ꡬ μ˜ˆμ‹œ: You're out of extra usage Β· resets 4am (Asia/Seoul) λΌλŠ” λ°°λ„ˆκ°€ 뜰 수 μžˆμŠ΅λ‹ˆλ‹€.", + ]); + }); +}); diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 9b21d3c..82b9568 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -18,6 +18,7 @@ import { getFallbackProviderName, hasGroupProviderOverride, isFallbackEnabled, + isUsageExhausted, markPrimaryCooldown, } from './provider-fallback.js'; import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; @@ -42,6 +43,25 @@ export interface MessageAgentExecutorDeps { clearSession: (groupFolder: string) => void; } +function isClaudeUsageExhaustedMessage(text: string): boolean { + const normalized = text + .trim() + .toLowerCase() + .replace(/[β€™β€˜`]/g, "'") + .replace(/\s+/g, ' ') + .replace(/^error:\s*/i, ''); + const looksLikeBanner = + normalized.startsWith("you're out of extra usage") || + normalized.startsWith('you are out of extra usage') || + normalized.startsWith("you've hit your limit") || + normalized.startsWith('you have hit your limit'); + const hasResetHint = + normalized.includes('resets ') || + normalized.includes('reset at ') || + normalized.includes('try again'); + return looksLikeBanner && hasResetHint && normalized.length <= 160; +} + export async function runAgentForGroup( deps: MessageAgentExecutorDeps, args: { @@ -137,6 +157,30 @@ export async function runAgentForGroup( ) { resetSessionRequested = true; } + if ( + canFallback && + provider === 'claude' && + output.status === 'success' && + !sawOutput && + typeof output.result === 'string' && + isClaudeUsageExhaustedMessage(output.result) + ) { + if (!streamedTriggerReason) { + logger.warn( + { + chatJid, + group: group.name, + runId, + resultPreview: output.result.slice(0, 120), + }, + 'Detected Claude usage exhaustion banner in successful output', + ); + } + streamedTriggerReason = { + reason: 'usage-exhausted', + }; + return; + } if (output.result !== null && output.result !== undefined) { sawOutput = true; } else if ( @@ -291,7 +335,164 @@ export async function runAgentForGroup( return 'success'; }; - const provider = canFallback ? getActiveProvider() : 'claude'; + const shouldRotateClaudeToken = (reason: string): boolean => + reason === '429' || reason === 'usage-exhausted'; + + const retryClaudeWithRotation = async ( + initialTrigger: { + reason: string; + retryAfterMs?: number; + }, + rotationMessage?: string, + ): Promise<'success' | 'error'> => { + let trigger = initialTrigger; + let lastRotationMessage = rotationMessage; + + while ( + shouldRotateClaudeToken(trigger.reason) && + getTokenCount() > 1 && + rotateToken(lastRotationMessage) + ) { + logger.info( + { chatJid, group: group.name, runId, reason: trigger.reason }, + 'Claude rate-limited, retrying with rotated account', + ); + + const retryAttempt = await runAttempt('claude'); + + if (retryAttempt.error) { + if (!retryAttempt.sawOutput) { + const errMsg = + retryAttempt.error instanceof Error + ? retryAttempt.error.message + : String(retryAttempt.error); + const retryTrigger = retryAttempt.streamedTriggerReason + ? { + shouldFallback: true, + reason: retryAttempt.streamedTriggerReason.reason, + retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs, + } + : detectFallbackTrigger(errMsg); + if (retryTrigger.shouldFallback) { + trigger = { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }; + lastRotationMessage = errMsg; + continue; + } + } + + logger.error( + { + chatJid, + group: group.name, + groupFolder: group.folder, + runId, + provider: 'claude', + err: retryAttempt.error, + }, + 'Rotated Claude account also threw', + ); + return 'error'; + } + + const retryOutput = retryAttempt.output; + if (!retryOutput) { + logger.error( + { + chatJid, + group: group.name, + groupFolder: group.folder, + runId, + provider: 'claude', + }, + 'Rotated Claude account produced no output object', + ); + return 'error'; + } + + if ( + !retryAttempt.sawOutput && + retryAttempt.streamedTriggerReason && + retryOutput.status !== 'error' + ) { + trigger = { + reason: retryAttempt.streamedTriggerReason.reason, + retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs, + }; + lastRotationMessage = + typeof retryOutput.result === 'string' ? retryOutput.result : undefined; + continue; + } + + if ( + !retryAttempt.sawOutput && + retryAttempt.sawSuccessNullResultWithoutOutput + ) { + return runFallbackAttempt('success-null-result'); + } + + if (retryOutput.status === 'error') { + if (!retryAttempt.sawOutput) { + const retryTrigger = retryAttempt.streamedTriggerReason + ? { + shouldFallback: true, + reason: retryAttempt.streamedTriggerReason.reason, + retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs, + } + : detectFallbackTrigger(retryOutput.error); + if (retryTrigger.shouldFallback) { + trigger = { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }; + lastRotationMessage = retryOutput.error ?? undefined; + continue; + } + } + + logger.error( + { + group: group.name, + chatJid, + runId, + provider: 'claude', + error: retryOutput.error, + }, + 'Rotated Claude account failed', + ); + return 'error'; + } + + markTokenHealthy(); + return 'success'; + } + + // Usage exhausted: don't fall back to Kimi β€” log only, no response + if (trigger.reason === 'usage-exhausted') { + markPrimaryCooldown(trigger.reason, trigger.retryAfterMs); + logger.info( + { chatJid, group: group.name, runId }, + 'All Claude tokens usage-exhausted, silently skipping (no Kimi fallback)', + ); + return 'error'; + } + + return runFallbackAttempt(trigger.reason, trigger.retryAfterMs); + }; + + const provider = canFallback ? await getActiveProvider() : 'claude'; + + // Already in usage-exhausted cooldown β€” log only, no response + if (provider !== 'claude' && isUsageExhausted()) { + logger.info( + { chatJid, group: group.name, runId, provider }, + 'Claude usage exhausted (cooldown active), silently skipping', + ); + return 'error'; + } + const primaryAttempt = await runAttempt(provider); if (primaryAttempt.error) { @@ -308,19 +509,13 @@ export async function runAgentForGroup( } : detectFallbackTrigger(errMsg); if (trigger.shouldFallback) { - // Try rotating token before falling back to another provider - if (getTokenCount() > 1 && rotateToken(errMsg)) { - logger.info( - { chatJid, group: group.name, runId, reason: trigger.reason }, - 'Rate-limited, retrying with rotated token', - ); - const retryAttempt = await runAttempt('claude'); - if (!retryAttempt.error) { - markTokenHealthy(); - return 'success'; - } - } - return runFallbackAttempt(trigger.reason, trigger.retryAfterMs); + return retryClaudeWithRotation( + { + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }, + errMsg, + ); } } @@ -353,6 +548,19 @@ export async function runAgentForGroup( return 'error'; } + if ( + canFallback && + provider === 'claude' && + !primaryAttempt.sawOutput && + primaryAttempt.streamedTriggerReason && + output.status !== 'error' + ) { + return retryClaudeWithRotation({ + reason: primaryAttempt.streamedTriggerReason.reason, + retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, + }); + } + if ( canFallback && provider === 'claude' && @@ -383,18 +591,13 @@ export async function runAgentForGroup( } : detectFallbackTrigger(output.error); if (trigger.shouldFallback) { - if (getTokenCount() > 1 && rotateToken(output.error ?? undefined)) { - logger.info( - { chatJid, group: group.name, runId, reason: trigger.reason }, - 'Rate-limited (output error), retrying with rotated token', - ); - const retryAttempt = await runAttempt('claude'); - if (!retryAttempt.error) { - markTokenHealthy(); - return 'success'; - } - } - return runFallbackAttempt(trigger.reason, trigger.retryAfterMs); + return retryClaudeWithRotation( + { + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }, + output.error ?? undefined, + ); } } diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index 1af4b31..cadd29f 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -122,7 +122,7 @@ vi.mock('./logger.js', () => ({ vi.mock('./provider-fallback.js', () => ({ detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })), - getActiveProvider: vi.fn(() => 'claude'), + getActiveProvider: vi.fn(async () => 'claude'), getFallbackEnvOverrides: vi.fn(() => ({ ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', @@ -181,7 +181,7 @@ function makeChannel(chatJid: string): Channel { describe('createMessageRuntime', () => { beforeEach(() => { vi.clearAllMocks(); - vi.mocked(providerFallback.getActiveProvider).mockReturnValue('claude'); + vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); vi.mocked(providerFallback.getFallbackProviderName).mockReturnValue('kimi'); vi.mocked(providerFallback.getFallbackEnvOverrides).mockReturnValue({ ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', @@ -1957,6 +1957,171 @@ describe('createMessageRuntime', () => { ); }); + it('retries with the fallback provider when Claude returns only a usage exhaustion banner', async () => { + const chatJid = 'group@test'; + const group = makeGroup('claude-code'); + const channel = makeChannel(chatJid); + + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-24T00:00:00.000Z', + }, + ]); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'usage fallback μ‘λ‹΅μž…λ‹ˆλ‹€.', + }); + return { + status: 'success', + result: null, + }; + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => ({}), + saveState: vi.fn(), + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-fallback-usage-exhausted', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( + 'usage-exhausted', + undefined, + ); + expect(channel.sendMessage).toHaveBeenCalledTimes(1); + expect(channel.sendMessage).toHaveBeenCalledWith( + chatJid, + 'usage fallback μ‘λ‹΅μž…λ‹ˆλ‹€.', + ); + }); + + it('suppresses duplicate streamed usage banners before retrying the fallback provider', async () => { + const chatJid = 'group@test'; + const group = makeGroup('claude-code'); + const channel = makeChannel(chatJid); + + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-24T00:00:00.000Z', + }, + ]); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + await onOutput?.({ + status: 'success', + phase: 'final', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'duplicate banner fallback μ‘λ‹΅μž…λ‹ˆλ‹€.', + }); + return { + status: 'success', + result: null, + }; + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => ({}), + saveState: vi.fn(), + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-fallback-usage-exhausted-duplicate', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( + 'usage-exhausted', + undefined, + ); + expect(channel.sendMessage).toHaveBeenCalledTimes(1); + expect(channel.sendMessage).toHaveBeenCalledWith( + chatJid, + 'duplicate banner fallback μ‘λ‹΅μž…λ‹ˆλ‹€.', + ); + }); + it('retries with the fallback provider when Claude ends with success-null-result before any output', async () => { const chatJid = 'group@test'; const group = makeGroup('claude-code'); diff --git a/src/provider-fallback.test.ts b/src/provider-fallback.test.ts new file mode 100644 index 0000000..54a9cc9 --- /dev/null +++ b/src/provider-fallback.test.ts @@ -0,0 +1,103 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./claude-usage.js', () => ({ + fetchClaudeUsage: vi.fn(), +})); + +vi.mock('./env.js', () => ({ + readEnvFile: vi.fn(() => ({ + FALLBACK_PROVIDER_NAME: 'kimi', + FALLBACK_BASE_URL: 'https://api.kimi.com/coding/', + FALLBACK_AUTH_TOKEN: 'test-kimi-key', + FALLBACK_MODEL: 'kimi-k2.5', + FALLBACK_SMALL_MODEL: 'kimi-k2.5', + FALLBACK_COOLDOWN_MS: '600000', + })), +})); + +vi.mock('./logger.js', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); + +import { fetchClaudeUsage } from './claude-usage.js'; +import { + clearPrimaryCooldown, + getActiveProvider, + getCooldownInfo, + markPrimaryCooldown, + resetFallbackConfig, +} from './provider-fallback.js'; + +describe('provider fallback usage recovery', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-03-24T00:00:00.000Z')); + vi.clearAllMocks(); + clearPrimaryCooldown(); + resetFallbackConfig(); + delete process.env.FALLBACK_PROVIDER_NAME; + delete process.env.FALLBACK_BASE_URL; + delete process.env.FALLBACK_AUTH_TOKEN; + delete process.env.FALLBACK_MODEL; + delete process.env.FALLBACK_SMALL_MODEL; + delete process.env.FALLBACK_COOLDOWN_MS; + }); + + afterEach(() => { + clearPrimaryCooldown(); + resetFallbackConfig(); + vi.useRealTimers(); + }); + + it('keeps the fallback provider active while Claude usage is still exhausted', async () => { + vi.mocked(fetchClaudeUsage).mockResolvedValue({ + five_hour: { + utilization: 100, + resets_at: '2026-03-24T04:00:00.000+09:00', + }, + }); + + markPrimaryCooldown('usage-exhausted', 1_000); + vi.advanceTimersByTime(5_000); + + await expect(getActiveProvider()).resolves.toBe('kimi'); + expect(getCooldownInfo()).toMatchObject({ + active: true, + reason: 'usage-exhausted', + remainingMs: 0, + }); + }); + + it('returns to Claude immediately when usage is no longer exhausted', async () => { + vi.mocked(fetchClaudeUsage).mockResolvedValue({ + five_hour: { + utilization: 72, + resets_at: '2026-03-24T04:00:00.000+09:00', + }, + seven_day: { + utilization: 55, + resets_at: '2026-03-31T04:00:00.000+09:00', + }, + }); + + markPrimaryCooldown('usage-exhausted', 600_000); + + await expect(getActiveProvider()).resolves.toBe('claude'); + expect(getCooldownInfo()).toEqual({ active: false }); + }); + + it('falls back to time-based retry when usage status cannot be fetched', async () => { + vi.mocked(fetchClaudeUsage).mockResolvedValue(null); + + markPrimaryCooldown('usage-exhausted', 1_000); + vi.advanceTimersByTime(5_000); + + await expect(getActiveProvider()).resolves.toBe('claude'); + expect(getCooldownInfo()).toEqual({ active: false }); + }); +}); diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index f79d6c0..5140bdb 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -14,8 +14,10 @@ import fs from 'fs'; +import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js'; import { readEnvFile } from './env.js'; import { logger } from './logger.js'; +import { rotateToken, getTokenCount } from './token-rotation.js'; // ── Types ──────────────────────────────────────────────────────── @@ -46,6 +48,17 @@ interface FallbackConfig { // ── State ──────────────────────────────────────────────────────── let cooldown: CooldownState | null = null; +let lastUsageAvailabilityCheck: + | { + checkedAt: number; + result: 'available' | 'exhausted' | 'unknown'; + } + | null = null; +let usageAvailabilityCheckPromise: + | Promise<'available' | 'exhausted' | 'unknown'> + | null = null; + +const USAGE_RECOVERY_RECHECK_MS = 30_000; // ── Config ─────────────────────────────────────────────────────── @@ -116,16 +129,109 @@ export function getFallbackProviderName(): string { return loadConfig().providerName; } +function normalizeUtilization(utilization: number): number { + return utilization > 1 ? utilization : utilization * 100; +} + +type ClaudeUsageWindow = NonNullable; + +function hasExhaustedClaudeUsageWindow( + usage: ClaudeUsageData | null, +): boolean | null { + if (!usage) return null; + const windows: ClaudeUsageWindow[] = []; + if (usage.five_hour) windows.push(usage.five_hour); + if (usage.seven_day) windows.push(usage.seven_day); + if (usage.seven_day_sonnet) windows.push(usage.seven_day_sonnet); + if (usage.seven_day_opus) windows.push(usage.seven_day_opus); + if (windows.length === 0) return null; + return windows.some( + (window) => normalizeUtilization(window.utilization) >= 100, + ); +} + +function clearUsageAvailabilityCache(): void { + lastUsageAvailabilityCheck = null; + usageAvailabilityCheckPromise = null; +} + +async function getClaudeUsageAvailability(): Promise< + 'available' | 'exhausted' | 'unknown' +> { + const now = Date.now(); + if ( + lastUsageAvailabilityCheck && + now - lastUsageAvailabilityCheck.checkedAt < USAGE_RECOVERY_RECHECK_MS + ) { + return lastUsageAvailabilityCheck.result; + } + + if (!usageAvailabilityCheckPromise) { + usageAvailabilityCheckPromise = (async () => { + const usage = await fetchClaudeUsage(); + const exhausted = hasExhaustedClaudeUsageWindow(usage); + const result = + exhausted === null ? 'unknown' : exhausted ? 'exhausted' : 'available'; + lastUsageAvailabilityCheck = { + checkedAt: Date.now(), + result, + }; + return result; + })(); + + void usageAvailabilityCheckPromise.finally(() => { + usageAvailabilityCheckPromise = null; + }); + } + + return usageAvailabilityCheckPromise; +} + /** * Determine which provider should be used for the next request. * Returns 'claude' when Claude is healthy or cooldown has expired, * or the fallback provider name during an active cooldown. */ -export function getActiveProvider(): string { +export async function getActiveProvider(): Promise { const config = loadConfig(); if (!config.enabled) return 'claude'; if (cooldown) { + if (cooldown.reason === 'usage-exhausted') { + const usageAvailability = await getClaudeUsageAvailability(); + if (usageAvailability === 'available') { + logger.info( + { + provider: 'claude', + reason: cooldown.reason, + }, + 'Claude usage recovered, retrying primary provider', + ); + cooldown = null; + clearUsageAvailabilityCache(); + return 'claude'; + } + if (usageAvailability === 'exhausted') { + // Current token exhausted β€” try rotating to another token (ignore cooldowns) + if (getTokenCount() > 1 && rotateToken(undefined, { ignoreRateLimits: true })) { + logger.info( + 'Claude current token exhausted, rotated to next token β€” retrying', + ); + cooldown = null; + clearUsageAvailabilityCache(); + return 'claude'; + } + logger.debug( + { + provider: config.providerName, + reason: cooldown.reason, + }, + 'All Claude tokens exhausted, keeping cooldown active', + ); + return config.providerName; + } + } + if (Date.now() < cooldown.expiresAt) { return config.providerName; } @@ -161,6 +267,7 @@ export function markPrimaryCooldown( expiresAt: now + durationMs, reason, }; + clearUsageAvailabilityCache(); logger.info( { @@ -175,6 +282,7 @@ export function markPrimaryCooldown( /** Manually clear cooldown (e.g. after a successful Claude response). */ export function clearPrimaryCooldown(): void { + clearUsageAvailabilityCache(); if (cooldown) { logger.info( { reason: cooldown.reason }, @@ -184,6 +292,11 @@ export function clearPrimaryCooldown(): void { } } +/** Check if Claude is currently in usage-exhausted cooldown. */ +export function isUsageExhausted(): boolean { + return cooldown?.reason === 'usage-exhausted'; +} + /** Get current cooldown info (for diagnostics / status dashboard). */ export function getCooldownInfo(): { active: boolean; @@ -191,14 +304,18 @@ export function getCooldownInfo(): { expiresAt?: string; remainingMs?: number; } { - if (!cooldown || Date.now() >= cooldown.expiresAt) { + if (!cooldown) { + return { active: false }; + } + const remainingMs = Math.max(cooldown.expiresAt - Date.now(), 0); + if (cooldown.reason !== 'usage-exhausted' && remainingMs === 0) { return { active: false }; } return { active: true, reason: cooldown.reason, expiresAt: new Date(cooldown.expiresAt).toISOString(), - remainingMs: cooldown.expiresAt - Date.now(), + remainingMs, }; } diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index 97fa40d..6bf9a37 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -8,6 +8,42 @@ const { runAgentProcessMock, writeTasksSnapshotMock } = vi.hoisted(() => ({ writeTasksSnapshotMock: vi.fn(), })); +vi.mock('./provider-fallback.js', () => ({ + detectFallbackTrigger: vi.fn((error?: string | null) => { + const lower = (error || '').toLowerCase(); + if ( + lower.includes('429') || + lower.includes('rate limit') || + lower.includes('hit your limit') + ) { + return { shouldFallback: true, reason: '429' }; + } + return { shouldFallback: false, reason: '' }; + }), + getActiveProvider: vi.fn(async () => 'claude'), + getFallbackEnvOverrides: vi.fn(() => ({ + ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', + ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', + ANTHROPIC_MODEL: 'kimi-k2.5', + })), + getFallbackProviderName: vi.fn(() => 'kimi'), + hasGroupProviderOverride: vi.fn(() => false), + isFallbackEnabled: vi.fn(() => true), + markPrimaryCooldown: vi.fn(), +})); + +vi.mock('./token-rotation.js', () => ({ + rotateToken: vi.fn(() => false), + getTokenCount: vi.fn(() => 1), + markTokenHealthy: vi.fn(), +})); + +vi.mock('./codex-token-rotation.js', () => ({ + rotateCodexToken: vi.fn(() => false), + getCodexAccountCount: vi.fn(() => 1), + markCodexTokenHealthy: vi.fn(), +})); + vi.mock('./agent-runner.js', async () => { const actual = await vi.importActual( @@ -21,8 +57,10 @@ vi.mock('./agent-runner.js', async () => { }); import { _initTestDatabase, createTask, getTaskById } from './db.js'; +import * as providerFallback from './provider-fallback.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js'; +import * as tokenRotation from './token-rotation.js'; import { _resetSchedulerLoopForTests, computeNextRun, @@ -39,6 +77,11 @@ describe('task scheduler', () => { _resetSchedulerLoopForTests(); runAgentProcessMock.mockClear(); writeTasksSnapshotMock.mockClear(); + vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); + vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true); + vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); vi.useFakeTimers(); }); @@ -232,6 +275,103 @@ Check the run. expect(enqueueTask.mock.calls[0][1]).toBe('task-watch-group'); }); + it('suppresses Claude usage banners for scheduled tasks and retries with a rotated account', async () => { + const dueAt = new Date(Date.now() - 60_000).toISOString(); + createTask({ + id: 'task-usage-banner', + group_folder: 'shared-group', + chat_jid: 'shared@g.us', + agent_type: 'claude-code', + prompt: 'claude task', + schedule_type: 'once', + schedule_value: dueAt, + context_mode: 'isolated', + next_run: dueAt, + status: 'active', + created_at: '2026-02-22T00:00:00.000Z', + }); + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true); + + (runAgentProcessMock as any) + .mockImplementationOnce( + async ( + _group: unknown, + _input: unknown, + _onProcess: unknown, + onOutput?: (output: Record) => Promise, + ) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + await onOutput?.({ + status: 'success', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + return { + status: 'success', + result: null, + }; + }, + ) + .mockImplementationOnce( + async ( + _group: unknown, + _input: unknown, + _onProcess: unknown, + onOutput?: (output: Record) => Promise, + ) => { + await onOutput?.({ + status: 'success', + result: 'rotated scheduled task response', + }); + return { + status: 'success', + result: null, + }; + }, + ); + + const enqueueTask = vi.fn( + (_groupJid: string, _taskId: string, fn: () => Promise) => { + void fn(); + }, + ); + const sendMessage = vi.fn(async () => {}); + + startSchedulerLoop({ + serviceAgentType: 'claude-code', + registeredGroups: () => ({ + 'shared@g.us': { + name: 'Shared', + folder: 'shared-group', + trigger: '@Claude', + added_at: '2026-02-22T00:00:00.000Z', + agentType: 'claude-code', + }, + }), + getSessions: () => ({}), + queue: { enqueueTask } as any, + onProcess: () => {}, + sendMessage, + }); + + await vi.advanceTimersByTimeAsync(10); + + expect(runAgentProcessMock).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); + expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); + expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage).toHaveBeenCalledWith( + 'shared@g.us', + 'rotated scheduled task response', + ); + }); + it('picks up newly due tasks immediately when nudged', async () => { const enqueueTask = vi.fn(); @@ -352,6 +492,7 @@ Check the run. }), expect.any(Function), expect.any(Function), + undefined, ); expect(writeTasksSnapshotMock).toHaveBeenCalledWith( 'shared-group', @@ -411,6 +552,7 @@ Check the run. }), expect.any(Function), expect.any(Function), + undefined, ); expect(writeTasksSnapshotMock).toHaveBeenCalledWith( 'shared-group', diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 3542c0a..903554b 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -1,9 +1,11 @@ import { ChildProcess } from 'child_process'; import { CronExpressionParser } from 'cron-parser'; import fs from 'fs'; +import path from 'path'; import { ASSISTANT_NAME, + DATA_DIR, SCHEDULER_POLL_INTERVAL, SERVICE_AGENT_TYPE, TIMEZONE, @@ -29,7 +31,16 @@ import { } from './group-folder.js'; import { logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; -import { detectFallbackTrigger } from './provider-fallback.js'; +import { + detectFallbackTrigger, + getActiveProvider, + getFallbackEnvOverrides, + getFallbackProviderName, + hasGroupProviderOverride, + isFallbackEnabled, + isUsageExhausted, + markPrimaryCooldown, +} from './provider-fallback.js'; import { rotateCodexToken, getCodexAccountCount, @@ -61,6 +72,25 @@ export { shouldUseTaskScopedSession, } from './task-watch-status.js'; +function isClaudeUsageExhaustedMessage(text: string): boolean { + const normalized = text + .trim() + .toLowerCase() + .replace(/[β€™β€˜`]/g, "'") + .replace(/\s+/g, ' ') + .replace(/^error:\s*/i, ''); + const looksLikeBanner = + normalized.startsWith("you're out of extra usage") || + normalized.startsWith('you are out of extra usage') || + normalized.startsWith("you've hit your limit") || + normalized.startsWith('you have hit your limit'); + const hasResetHint = + normalized.includes('resets ') || + normalized.includes('reset at ') || + normalized.includes('try again'); + return looksLikeBanner && hasResetHint && normalized.length <= 160; +} + /** * Compute the next run time for a recurring task, anchored to the * task's scheduled time rather than Date.now() to prevent cumulative @@ -249,52 +279,292 @@ async function runTask( sendTrackedMessage: deps.sendTrackedMessage, editTrackedMessage: deps.editTrackedMessage, }); + const settingsPath = path.join( + DATA_DIR, + 'sessions', + task.group_folder, + '.claude', + 'settings.json', + ); + const canFallback = + context.taskAgentType === 'claude-code' && + isFallbackEnabled() && + !hasGroupProviderOverride(settingsPath); try { await statusTracker.update('checking'); - const output = await runAgentProcess( - context.group, - { - prompt: task.prompt, - sessionId: context.sessionId, - groupFolder: task.group_folder, - chatJid: task.chat_jid, - isMain: context.isMain, - isScheduledTask: true, - runtimeTaskId: context.runtimeTaskId, - useTaskScopedSession: context.useTaskScopedSession, - assistantName: ASSISTANT_NAME, + const runTaskAttempt = async ( + provider: string, + ): Promise<{ + output: AgentOutput; + sawOutput: boolean; + streamedTriggerReason?: { + reason: string; + retryAfterMs?: number; + }; + attemptResult: string | null; + attemptError: string | null; + }> => { + let sawOutput = false; + let attemptResult: string | null = null; + let attemptError: string | null = null; + let streamedTriggerReason: + | { + reason: string; + retryAfterMs?: number; + } + | undefined; + + const output = await runAgentProcess( + context.group, + { + prompt: task.prompt, + sessionId: context.sessionId, + groupFolder: task.group_folder, + chatJid: task.chat_jid, + isMain: context.isMain, + isScheduledTask: true, + runtimeTaskId: context.runtimeTaskId, + useTaskScopedSession: context.useTaskScopedSession, + assistantName: ASSISTANT_NAME, + }, + (proc, processName) => + deps.onProcess( + context.queueJid, + proc, + processName, + context.runtimeIpcDir, + ), + async (streamedOutput: AgentOutput) => { + if (streamedOutput.phase === 'progress') { + return; + } + + if ( + canFallback && + provider === 'claude' && + !sawOutput && + streamedOutput.status === 'success' && + typeof streamedOutput.result === 'string' && + isClaudeUsageExhaustedMessage(streamedOutput.result) + ) { + if (!streamedTriggerReason) { + logger.warn( + { + taskId: task.id, + taskChatJid: task.chat_jid, + group: context.group.name, + groupFolder: task.group_folder, + resultPreview: streamedOutput.result.slice(0, 120), + }, + 'Detected Claude usage exhaustion banner during scheduled task output', + ); + } + streamedTriggerReason = { reason: 'usage-exhausted' }; + return; + } + + if (streamedOutput.result) { + sawOutput = true; + attemptResult = streamedOutput.result; + await deps.sendMessage(task.chat_jid, streamedOutput.result); + } + + if (streamedOutput.status === 'error') { + attemptError = streamedOutput.error || 'Unknown error'; + if (!sawOutput && !streamedTriggerReason) { + const trigger = detectFallbackTrigger(streamedOutput.error); + if (trigger.shouldFallback) { + streamedTriggerReason = { + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }; + } + } + } + }, + provider === 'claude' ? undefined : getFallbackEnvOverrides(), + ); + + if (output.status === 'error' && !attemptError) { + attemptError = output.error || 'Unknown error'; + } else if (output.result && !attemptResult) { + attemptResult = output.result; + } + + return { + output, + sawOutput, + streamedTriggerReason, + attemptResult, + attemptError, + }; + }; + + const shouldRotateClaudeToken = (reason: string): boolean => + reason === '429' || reason === 'usage-exhausted'; + + const runFallbackTaskAttempt = async ( + reason: string, + retryAfterMs?: number, + ): Promise => { + if (!canFallback) { + error = reason; + return; + } + + const fallbackName = getFallbackProviderName(); + markPrimaryCooldown(reason, retryAfterMs); + + logger.info( + { + taskId: task.id, + group: context.group.name, + groupFolder: task.group_folder, + reason, + retryAfterMs, + fallbackProvider: fallbackName, + }, + `Falling back to provider: ${fallbackName} for scheduled task (reason: ${reason})`, + ); + + const fallbackAttempt = await runTaskAttempt(fallbackName); + result = fallbackAttempt.attemptResult; + error = + fallbackAttempt.output.status === 'error' + ? fallbackAttempt.attemptError || 'Unknown error' + : null; + }; + + const retryClaudeTaskWithRotation = async ( + initialTrigger: { + reason: string; + retryAfterMs?: number; }, - (proc, processName) => - deps.onProcess( - context.queueJid, - proc, - processName, - context.runtimeIpcDir, - ), - async (streamedOutput: AgentOutput) => { - if (streamedOutput.phase === 'progress') { + rotationMessage?: string, + ): Promise => { + let trigger = initialTrigger; + let lastRotationMessage = rotationMessage; + + while ( + shouldRotateClaudeToken(trigger.reason) && + getTokenCount() > 1 && + rotateToken(lastRotationMessage) + ) { + logger.info( + { + taskId: task.id, + group: context.group.name, + groupFolder: task.group_folder, + reason: trigger.reason, + }, + 'Scheduled task Claude rate-limited, retrying with rotated account', + ); + + const retryAttempt = await runTaskAttempt('claude'); + result = retryAttempt.attemptResult; + error = retryAttempt.attemptError; + + if ( + retryAttempt.streamedTriggerReason && + !retryAttempt.sawOutput && + retryAttempt.output.status !== 'error' + ) { + trigger = { + reason: retryAttempt.streamedTriggerReason.reason, + retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs, + }; + lastRotationMessage = + typeof retryAttempt.output.result === 'string' + ? retryAttempt.output.result + : undefined; + continue; + } + + if (retryAttempt.output.status === 'error' && !retryAttempt.sawOutput) { + const retryTrigger = retryAttempt.streamedTriggerReason + ? { + shouldFallback: true, + reason: retryAttempt.streamedTriggerReason.reason, + retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs, + } + : detectFallbackTrigger(retryAttempt.attemptError); + if (retryTrigger.shouldFallback) { + trigger = { + reason: retryTrigger.reason, + retryAfterMs: retryTrigger.retryAfterMs, + }; + lastRotationMessage = retryAttempt.attemptError || undefined; + continue; + } + } + + if (retryAttempt.output.status === 'success') { + markTokenHealthy(); + error = null; return; } - if (streamedOutput.result) { - result = streamedOutput.result; - // Forward result to user (sendMessage handles formatting) - await deps.sendMessage(task.chat_jid, streamedOutput.result); - } - if (streamedOutput.status === 'error') { - error = streamedOutput.error || 'Unknown error'; - } - }, - ); - if (output.status === 'error') { - error = output.error || 'Unknown error'; - } else if (output.result) { - // Result was already forwarded to the user via the streaming callback above - result = output.result; + return; + } + + // Usage exhausted: don't fall back to Kimi β€” just mark cooldown and skip + if (trigger.reason === 'usage-exhausted') { + markPrimaryCooldown(trigger.reason, trigger.retryAfterMs); + logger.info( + { taskId: task.id, group: context.group.name }, + 'All Claude tokens usage-exhausted, skipping Kimi fallback for scheduled task', + ); + error = 'Claude usage exhausted'; + return; + } + + await runFallbackTaskAttempt(trigger.reason, trigger.retryAfterMs); + }; + + const provider = canFallback ? await getActiveProvider() : 'claude'; + + // Already in usage-exhausted cooldown β€” skip task instead of running on Kimi + if (provider !== 'claude' && isUsageExhausted()) { + logger.info( + { taskId: task.id, group: context.group.name, provider }, + 'Claude usage exhausted (cooldown active), skipping scheduled task', + ); + error = 'Claude usage exhausted'; + // Fall through to task completion handling below + } else { + + const attempt = await runTaskAttempt(provider); + result = attempt.attemptResult; + error = attempt.attemptError; + + if ( + provider === 'claude' && + attempt.streamedTriggerReason && + !attempt.sawOutput + ) { + await retryClaudeTaskWithRotation(attempt.streamedTriggerReason); + } else if (attempt.output.status === 'error' && provider === 'claude') { + const trigger = attempt.streamedTriggerReason + ? { + shouldFallback: true, + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + } + : detectFallbackTrigger(error); + if (trigger.shouldFallback) { + await retryClaudeTaskWithRotation({ + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }); + } + } else if (attempt.output.status === 'error') { + error = attempt.attemptError || 'Unknown error'; } + } // end else (non-exhausted path) + logger.info( { taskId: task.id, diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 2e73dbc..62e3473 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -142,21 +142,29 @@ export function getCurrentToken(): string | undefined { * Try to rotate to the next available (non-rate-limited) token. * Returns true if a fresh token was found, false if all are exhausted. */ -export function rotateToken(errorMessage?: string): boolean { +export function rotateToken( + errorMessage?: string, + opts?: { ignoreRateLimits?: boolean }, +): boolean { if (tokens.length <= 1) return false; const now = Date.now(); tokens[currentIndex].rateLimitedUntil = computeCooldownUntil(errorMessage); + const ignoreRL = opts?.ignoreRateLimits ?? false; // Find next available token for (let i = 1; i < tokens.length; i++) { const idx = (currentIndex + i) % tokens.length; const state = tokens[idx]; - if (!state.rateLimitedUntil || state.rateLimitedUntil <= now) { + if ( + ignoreRL || + !state.rateLimitedUntil || + state.rateLimitedUntil <= now + ) { state.rateLimitedUntil = null; currentIndex = idx; logger.info( - { tokenIndex: currentIndex, totalTokens: tokens.length }, + { tokenIndex: currentIndex, totalTokens: tokens.length, ignoreRL }, `Rotated to token #${currentIndex + 1}/${tokens.length}`, ); saveState(); diff --git a/src/unified-dashboard.test.ts b/src/unified-dashboard.test.ts new file mode 100644 index 0000000..5600537 --- /dev/null +++ b/src/unified-dashboard.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { ClaudeAccountUsage } from './claude-usage.js'; + +vi.mock('./claude-usage.js', () => ({ + fetchAllClaudeUsage: vi.fn(), + fetchAllClaudeProfiles: vi.fn(), + getClaudeProfile: (index: number) => + index === 0 + ? { email: 'a@example.com', planType: 'max' } + : { email: 'b@example.com', planType: 'pro' }, +})); + +vi.mock('./codex-token-rotation.js', () => ({ + getAllCodexAccounts: () => [], + updateCodexAccountUsage: vi.fn(), +})); + +import { + buildClaudeUsageRows, + mergeClaudeDashboardAccounts, +} from './unified-dashboard.js'; + +describe('unified dashboard Claude usage rows', () => { + it('keeps both Claude accounts visible when one account usage is unavailable', () => { + const rows = buildClaudeUsageRows([ + { + index: 0, + masked: 'tok-a', + isActive: true, + isRateLimited: false, + usage: { + five_hour: { + utilization: 0.4, + resets_at: '2026-03-24T04:00:00+09:00', + }, + seven_day: { + utilization: 0.7, + resets_at: '2026-03-29T04:00:00+09:00', + }, + }, + }, + { + index: 1, + masked: 'tok-b', + isActive: false, + isRateLimited: true, + usage: null, + }, + ]); + + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + name: 'Claude1* max', + h5pct: 40, + d7pct: 70, + }); + expect(rows[1]).toMatchObject({ + name: 'Claude2! pro', + h5pct: -1, + d7pct: -1, + }); + }); + + it('preserves the last successful usage per account instead of collapsing to one cache entry', () => { + const cachedAccounts: ClaudeAccountUsage[] = [ + { + index: 0, + masked: 'tok-a', + isActive: false, + isRateLimited: false, + usage: { + five_hour: { + utilization: 0.25, + resets_at: '2026-03-24T04:00:00+09:00', + }, + seven_day: { + utilization: 0.5, + resets_at: '2026-03-29T04:00:00+09:00', + }, + }, + }, + { + index: 1, + masked: 'tok-b', + isActive: true, + isRateLimited: false, + usage: { + five_hour: { + utilization: 0.6, + resets_at: '2026-03-24T06:00:00+09:00', + }, + seven_day: { + utilization: 0.8, + resets_at: '2026-03-30T04:00:00+09:00', + }, + }, + }, + ]; + const liveAccounts: ClaudeAccountUsage[] = [ + { + index: 0, + masked: 'tok-a', + isActive: true, + isRateLimited: false, + usage: null, + }, + { + index: 1, + masked: 'tok-b', + isActive: false, + isRateLimited: true, + usage: { + five_hour: { + utilization: 0.9, + resets_at: '2026-03-24T08:00:00+09:00', + }, + seven_day: { + utilization: 0.95, + resets_at: '2026-03-31T04:00:00+09:00', + }, + }, + }, + ]; + + const merged = mergeClaudeDashboardAccounts(liveAccounts, cachedAccounts); + + expect(merged).toHaveLength(2); + expect(merged[0]).toMatchObject({ + index: 0, + isActive: true, + usage: cachedAccounts[0].usage, + }); + expect(merged[1]).toMatchObject({ + index: 1, + isRateLimited: true, + usage: liveAccounts[1].usage, + }); + }); +}); diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index f26fdac..e388cec 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -5,11 +5,9 @@ import path from 'path'; import { STATUS_SHOW_ROOMS, USAGE_DASHBOARD_ENABLED } from './config.js'; import { - fetchClaudeUsage as fetchClaudeUsageApi, fetchAllClaudeUsage, fetchAllClaudeProfiles, getClaudeProfile, - type ClaudeUsageData, type ClaudeAccountUsage, } from './claude-usage.js'; import { @@ -24,7 +22,6 @@ import { renderCategorizedRoomSections, } from './dashboard-render.js'; import { getAllChats, updateRegisteredGroupName } from './db.js'; -import { readEnvFile } from './env.js'; import type { GroupQueue } from './group-queue.js'; import { logger } from './logger.js'; import { @@ -69,11 +66,8 @@ const STATUS_SNAPSHOT_MAX_AGE_MS = 60000; let statusMessageId: string | null = null; let cachedUsageContent = ''; -let cachedClaudeUsageData: ClaudeUsageData | null = null; +let cachedClaudeAccounts: ClaudeAccountUsage[] = []; let usageUpdateInProgress = false; -let usageApiBackoffUntil = 0; -let usageApi429Streak = 0; -let usageApiPollingDisabled = false; let channelMetaCache = new Map(); let channelMetaLastRefresh = 0; @@ -135,22 +129,6 @@ export async function purgeDashboardChannel( } } -function parseRetryAfterMs(retryAfter: string | null): number | null { - if (!retryAfter) return null; - - const seconds = Number(retryAfter); - if (Number.isFinite(seconds) && seconds > 0) { - return seconds * 1000; - } - - const absolute = Date.parse(retryAfter); - if (!Number.isNaN(absolute)) { - return Math.max(0, absolute - Date.now()); - } - - return null; -} - async function refreshChannelMeta( opts: UnifiedDashboardOptions, ): Promise { @@ -366,85 +344,6 @@ function buildStatusContent(): string { return `${header}\n\n${sections}`; } -async function fetchClaudeUsage(): Promise { - if (usageApiPollingDisabled) { - logger.debug('Skipping usage API call (polling disabled for this process)'); - return null; - } - if (Date.now() < usageApiBackoffUntil) { - logger.debug('Skipping usage API call (backoff active)'); - return null; - } - - const apiUsage = await fetchClaudeUsageApi(); - if (apiUsage) { - usageApi429Streak = 0; - return apiUsage; - } - - try { - const envToken = readEnvFile(['CLAUDE_CODE_OAUTH_TOKEN']); - let token = - process.env.CLAUDE_CODE_OAUTH_TOKEN || - envToken.CLAUDE_CODE_OAUTH_TOKEN || - ''; - if (!token) { - const configDir = - process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude'); - const credsPath = path.join(configDir, '.credentials.json'); - if (!fs.existsSync(credsPath)) return null; - const creds = JSON.parse(fs.readFileSync(credsPath, 'utf-8')); - token = creds?.claudeAiOauth?.accessToken || ''; - } - if (!token) return null; - - const res = await fetch('https://api.anthropic.com/api/oauth/usage', { - headers: { - Authorization: `Bearer ${token}`, - 'anthropic-beta': 'oauth-2025-04-20', - }, - }); - if (!res.ok) { - const body = await res.text().catch(() => ''); - if (res.status === 429) { - const retryAfter = res.headers.get('retry-after'); - const retryAfterMs = parseRetryAfterMs(retryAfter); - const backoffMs = Math.max(600_000, retryAfterMs ?? 0); - usageApi429Streak += 1; - usageApiBackoffUntil = Date.now() + backoffMs; - if (usageApi429Streak >= 3) { - usageApiPollingDisabled = true; - } - logger.warn( - { - status: 429, - retryAfter, - retryAfterMs, - backoffMs, - consecutive429s: usageApi429Streak, - pollingDisabled: usageApiPollingDisabled, - body: body.slice(0, 200), - }, - usageApiPollingDisabled - ? 'Usage API rate limited repeatedly (429), disabling usage polling for this process' - : 'Usage API rate limited (429), backing off', - ); - } else { - logger.warn( - { status: res.status, body: body.slice(0, 200) }, - 'Usage API returned non-OK status', - ); - } - return null; - } - usageApi429Streak = 0; - return (await res.json()) as ClaudeUsageData; - } catch (err) { - logger.debug({ err }, 'Usage API fetch failed'); - return null; - } -} - async function fetchCodexUsage( codexHomeOverride?: string, ): Promise { @@ -543,74 +442,46 @@ async function fetchCodexUsage( }); } -async function buildUsageContent(): Promise { - const activeSnapshots = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS); - const hasActiveClaudeWork = activeSnapshots.some( - (snapshot) => - snapshot.agentType === 'claude-code' && - snapshot.entries.some( - (entry) => entry.status === 'processing' || entry.status === 'waiting', - ), +type UsageRow = { + name: string; + h5pct: number; + h5reset: string; + d7pct: number; + d7reset: string; +}; + +export function mergeClaudeDashboardAccounts( + liveAccounts: ClaudeAccountUsage[] | null | undefined, + cachedAccounts: ClaudeAccountUsage[], +): ClaudeAccountUsage[] { + if (!liveAccounts) return cachedAccounts; + + const cachedByIndex = new Map( + cachedAccounts.map((account) => [account.index, account]), ); - const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED; - const [liveClaudeAccounts, codexUsage] = await Promise.all([ - shouldFetchClaudeUsage && !hasActiveClaudeWork - ? fetchAllClaudeUsage() - : Promise.resolve(null), - fetchCodexUsage(), - ]); + return liveAccounts.map((account) => ({ + ...account, + usage: account.usage || cachedByIndex.get(account.index)?.usage || null, + })); +} - // Update cache with first account's data for backward compat - const claudeUsageIsCached = - shouldFetchClaudeUsage && !liveClaudeAccounts && !!cachedClaudeUsageData; - if (shouldFetchClaudeUsage && liveClaudeAccounts?.length) { - cachedClaudeUsageData = liveClaudeAccounts[0].usage; - } +export function buildClaudeUsageRows( + claudeAccounts: ClaudeAccountUsage[], +): UsageRow[] { + const isMultiAccount = claudeAccounts.length > 1; - const lines: string[] = ['πŸ“Š *μ‚¬μš©λŸ‰*']; - const bar = (pct: number) => { - const filled = Math.round(pct / 10); - return 'β–ˆ'.repeat(filled) + 'β–‘'.repeat(10 - filled); - }; - - type UsageRow = { - name: string; - h5pct: number; - h5reset: string; - d7pct: number; - d7reset: string; - }; - const rows: UsageRow[] = []; - - const claudeAccounts = - liveClaudeAccounts || - (cachedClaudeUsageData - ? [ - { - index: 0, - masked: '', - isActive: true, - isRateLimited: false, - usage: cachedClaudeUsageData, - }, - ] - : []); - - for (const account of claudeAccounts) { - const u = account.usage; - if (!u) continue; - const h5 = u.five_hour; - const d7 = u.seven_day; + return claudeAccounts.map((account) => { + const usage = account.usage; + const h5 = usage?.five_hour; + const d7 = usage?.seven_day; const profile = getClaudeProfile(account.index); const planSuffix = profile ? ` ${profile.planType}` : ''; - const label = - claudeAccounts.length > 1 - ? `Claude${account.index + 1}${account.isActive ? '*' : ''}${account.isRateLimited ? '!' : ''}${planSuffix}` - : claudeUsageIsCached - ? `Claude*${planSuffix}` - : `Claude${planSuffix}`; - rows.push({ + const label = isMultiAccount + ? `Claude${account.index + 1}${account.isActive ? '*' : ''}${account.isRateLimited ? '!' : ''}${planSuffix}` + : `Claude${account.isActive ? '*' : ''}${account.isRateLimited ? '!' : ''}${planSuffix}`; + + return { name: label, h5pct: h5 ? h5.utilization > 1 @@ -624,7 +495,38 @@ async function buildUsageContent(): Promise { : Math.round(d7.utilization * 100) : -1, d7reset: d7 ? formatResetRemaining(d7.resets_at) : '', - }); + }; + }); +} + +async function buildUsageContent(): Promise { + const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED; + let liveClaudeAccounts: ClaudeAccountUsage[] | null = null; + + const codexUsagePromise = fetchCodexUsage(); + if (shouldFetchClaudeUsage) { + try { + liveClaudeAccounts = await fetchAllClaudeUsage(); + } catch (err) { + logger.warn({ err }, 'Failed to fetch Claude usage for dashboard'); + } + } + const codexUsage = await codexUsagePromise; + + const lines: string[] = ['πŸ“Š *μ‚¬μš©λŸ‰*']; + const bar = (pct: number) => { + const filled = Math.round(pct / 10); + return 'β–ˆ'.repeat(filled) + 'β–‘'.repeat(10 - filled); + }; + + const rows: UsageRow[] = []; + + if (shouldFetchClaudeUsage) { + cachedClaudeAccounts = mergeClaudeDashboardAccounts( + liveClaudeAccounts, + cachedClaudeAccounts, + ); + rows.push(...buildClaudeUsageRows(cachedClaudeAccounts)); } const codexAccounts = getAllCodexAccounts(); @@ -712,14 +614,6 @@ async function buildUsageContent(): Promise { lines.push('_쑰회 λΆˆκ°€_'); } - if (shouldFetchClaudeUsage && usageApiPollingDisabled) { - lines.push( - '_* Claude μ‚¬μš©λŸ‰ μ‘°νšŒλŠ” 반볡된 429둜 이번 ν”„λ‘œμ„ΈμŠ€μ—μ„œ μΌμ‹œ 쀑지_', - ); - } - if (claudeUsageIsCached) { - lines.push('_* Claude μ‚¬μš©λŸ‰μ€ μž‘μ—… 쀑일 λ•ŒλŠ” μΊμ‹œκ°’ μœ μ§€_'); - } lines.push(''); lines.push('πŸ–₯️ *μ„œλ²„*'); From 6a73421d4fdb82ae8076847ced2ca4214011b927 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 03:47:22 +0900 Subject: [PATCH 26/27] fix: use ignoreRateLimits in retryClaudeWithRotation loops Without this, token rotation was blocked by stale cooldowns even when the target token had available usage. --- src/message-agent-executor.ts | 6 ++-- src/task-scheduler.ts | 54 +++++++++++++++++------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 82b9568..055e383 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -351,7 +351,7 @@ export async function runAgentForGroup( while ( shouldRotateClaudeToken(trigger.reason) && getTokenCount() > 1 && - rotateToken(lastRotationMessage) + rotateToken(lastRotationMessage, { ignoreRateLimits: true }) ) { logger.info( { chatJid, group: group.name, runId, reason: trigger.reason }, @@ -422,7 +422,9 @@ export async function runAgentForGroup( retryAfterMs: retryAttempt.streamedTriggerReason.retryAfterMs, }; lastRotationMessage = - typeof retryOutput.result === 'string' ? retryOutput.result : undefined; + typeof retryOutput.result === 'string' + ? retryOutput.result + : undefined; continue; } diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 903554b..4db678b 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -450,7 +450,7 @@ async function runTask( while ( shouldRotateClaudeToken(trigger.reason) && getTokenCount() > 1 && - rotateToken(lastRotationMessage) + rotateToken(lastRotationMessage, { ignoreRateLimits: true }) ) { logger.info( { @@ -534,35 +534,33 @@ async function runTask( error = 'Claude usage exhausted'; // Fall through to task completion handling below } else { + const attempt = await runTaskAttempt(provider); + result = attempt.attemptResult; + error = attempt.attemptError; - const attempt = await runTaskAttempt(provider); - result = attempt.attemptResult; - error = attempt.attemptError; - - if ( - provider === 'claude' && - attempt.streamedTriggerReason && - !attempt.sawOutput - ) { - await retryClaudeTaskWithRotation(attempt.streamedTriggerReason); - } else if (attempt.output.status === 'error' && provider === 'claude') { - const trigger = attempt.streamedTriggerReason - ? { - shouldFallback: true, - reason: attempt.streamedTriggerReason.reason, - retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, - } - : detectFallbackTrigger(error); - if (trigger.shouldFallback) { - await retryClaudeTaskWithRotation({ - reason: trigger.reason, - retryAfterMs: trigger.retryAfterMs, - }); + if ( + provider === 'claude' && + attempt.streamedTriggerReason && + !attempt.sawOutput + ) { + await retryClaudeTaskWithRotation(attempt.streamedTriggerReason); + } else if (attempt.output.status === 'error' && provider === 'claude') { + const trigger = attempt.streamedTriggerReason + ? { + shouldFallback: true, + reason: attempt.streamedTriggerReason.reason, + retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, + } + : detectFallbackTrigger(error); + if (trigger.shouldFallback) { + await retryClaudeTaskWithRotation({ + reason: trigger.reason, + retryAfterMs: trigger.retryAfterMs, + }); + } + } else if (attempt.output.status === 'error') { + error = attempt.attemptError || 'Unknown error'; } - } else if (attempt.output.status === 'error') { - error = attempt.attemptError || 'Unknown error'; - } - } // end else (non-exhausted path) logger.info( From f6ba879e8aa9f0afc32d6600a21cc91b921e8917 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 24 Mar 2026 03:47:46 +0900 Subject: [PATCH 27/27] fix: usage API rate-limit guard and cleanup debug logging --- src/claude-usage.ts | 12 +++++++++--- src/provider-fallback.ts | 21 +++++++++++---------- src/task-scheduler.test.ts | 32 ++++++++++++++++---------------- src/token-rotation.ts | 6 +----- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/claude-usage.ts b/src/claude-usage.ts index 0c5def7..617f772 100644 --- a/src/claude-usage.ts +++ b/src/claude-usage.ts @@ -58,13 +58,17 @@ function loadUsageDiskCache(): void { if (fs.existsSync(USAGE_CACHE_FILE)) { usageDiskCache = JSON.parse(fs.readFileSync(USAGE_CACHE_FILE, 'utf-8')); } - } catch { /* start fresh */ } + } catch { + /* start fresh */ + } } function saveUsageDiskCache(): void { try { fs.writeFileSync(USAGE_CACHE_FILE, JSON.stringify(usageDiskCache)); - } catch { /* best effort */ } + } catch { + /* best effort */ + } } function cacheKey(token: string): string { @@ -104,7 +108,9 @@ async function fetchUsageForToken( return null; } if (res.status === 429) { - logger.warn('Claude usage API: rate limited (429), returning cached data'); + logger.warn( + 'Claude usage API: rate limited (429), returning cached data', + ); return usageDiskCache[cacheKey(token)]?.usage ?? null; } if (!res.ok) { diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts index 5140bdb..20343b7 100644 --- a/src/provider-fallback.ts +++ b/src/provider-fallback.ts @@ -48,15 +48,13 @@ interface FallbackConfig { // ── State ──────────────────────────────────────────────────────── let cooldown: CooldownState | null = null; -let lastUsageAvailabilityCheck: - | { - checkedAt: number; - result: 'available' | 'exhausted' | 'unknown'; - } - | null = null; -let usageAvailabilityCheckPromise: - | Promise<'available' | 'exhausted' | 'unknown'> - | null = null; +let lastUsageAvailabilityCheck: { + checkedAt: number; + result: 'available' | 'exhausted' | 'unknown'; +} | null = null; +let usageAvailabilityCheckPromise: Promise< + 'available' | 'exhausted' | 'unknown' +> | null = null; const USAGE_RECOVERY_RECHECK_MS = 30_000; @@ -213,7 +211,10 @@ export async function getActiveProvider(): Promise { } if (usageAvailability === 'exhausted') { // Current token exhausted β€” try rotating to another token (ignore cooldowns) - if (getTokenCount() > 1 && rotateToken(undefined, { ignoreRateLimits: true })) { + if ( + getTokenCount() > 1 && + rotateToken(undefined, { ignoreRateLimits: true }) + ) { logger.info( 'Claude current token exhausted, rotated to next token β€” retrying', ); diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index 6bf9a37..40f79cb 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -307,14 +307,14 @@ Check the run. phase: 'intermediate', result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", }); - await onOutput?.({ - status: 'success', - result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", - }); - return { - status: 'success', - result: null, - }; + await onOutput?.({ + status: 'success', + result: "You're out of extra usage Β· resets 4am (Asia/Seoul)", + }); + return { + status: 'success', + result: null, + }; }, ) .mockImplementationOnce( @@ -324,14 +324,14 @@ Check the run. _onProcess: unknown, onOutput?: (output: Record) => Promise, ) => { - await onOutput?.({ - status: 'success', - result: 'rotated scheduled task response', - }); - return { - status: 'success', - result: null, - }; + await onOutput?.({ + status: 'success', + result: 'rotated scheduled task response', + }); + return { + status: 'success', + result: null, + }; }, ); diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 62e3473..664ceb8 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -156,11 +156,7 @@ export function rotateToken( for (let i = 1; i < tokens.length; i++) { const idx = (currentIndex + i) % tokens.length; const state = tokens[idx]; - if ( - ignoreRL || - !state.rateLimitedUntil || - state.rateLimitedUntil <= now - ) { + if (ignoreRL || !state.rateLimitedUntil || state.rateLimitedUntil <= now) { state.rateLimitedUntil = null; currentIndex = idx; logger.info(