diff --git a/.claude/skills/setup/SKILL.md b/.claude/skills/setup/SKILL.md index bc74dc4..52205a3 100644 --- a/.claude/skills/setup/SKILL.md +++ b/.claude/skills/setup/SKILL.md @@ -71,12 +71,6 @@ OPENAI_API_KEY=... ### 선택 환경 변수 ```bash -# Provider fallback (Claude 429 시 대체) -FALLBACK_PROVIDER_NAME=kimi -FALLBACK_BASE_URL=https://api.kimi.com/coding -FALLBACK_AUTH_TOKEN=... -FALLBACK_MODEL=kimi-k2.5 - # 사용량 대시보드 STATUS_CHANNEL_ID=... # 상태 업데이트 디스코드 채널 USAGE_DASHBOARD=true diff --git a/.env.codex-review.example b/.env.codex-review.example new file mode 100644 index 0000000..cf8b503 --- /dev/null +++ b/.env.codex-review.example @@ -0,0 +1,5 @@ +# ===== EJClaw Codex Review Service Environment ===== +# Copy this file to .env.codex-review for the ejclaw-review service. +# This service needs its own Discord bot token. + +DISCORD_BOT_TOKEN= # Discord bot token (Codex review bot) diff --git a/.env.example b/.env.example index 70836c6..eebb5ed 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,7 @@ # ===== EJClaw Environment Configuration ===== # Copy this file to .env and fill in the values. # For dual-service (Codex), also create .env.codex with DISCORD_BOT_TOKEN for the Codex bot. +# For review failover, create .env.codex-review with a third Discord bot token. # --- Required --- DISCORD_BOT_TOKEN= # Discord bot token (Claude bot) @@ -25,13 +26,6 @@ ASSISTANT_NAME=claude # Trigger name (@claude) GROQ_API_KEY= # Groq Whisper (primary, free: console.groq.com) # OPENAI_API_KEY also used as Whisper fallback -# --- Provider fallback (optional — Claude 429 → alternative) --- -# FALLBACK_PROVIDER_NAME=kimi -# FALLBACK_BASE_URL=https://api.kimi.com/coding -# FALLBACK_AUTH_TOKEN= -# FALLBACK_MODEL=kimi-k2.5 -# FALLBACK_COOLDOWN_MS=600000 - # --- Usage dashboard (optional) --- # STATUS_CHANNEL_ID= # Discord channel for live status updates # USAGE_DASHBOARD=true # Enable usage dashboard diff --git a/README.md b/README.md index f115422..40b87e7 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Both share the same codebase (`dist/index.js`), differentiated by environment va - **Browser automation** — [gstack browse](https://github.com/garrytan/gstack) skill, headless Chromium daemon, ~100ms/command - **Voice transcription** — Groq Whisper (primary) / OpenAI Whisper (fallback), shared file cache with dedup - **Bidirectional images** — receive Discord attachments as multimodal input, send screenshots back -- **Provider fallback** — Claude 429 → configurable fallback provider (e.g. Kimi K2.5) with cooldown +- **Token rotation** — Claude 429 / usage exhaustion → automatic multi-account rotation - **CI monitoring** — `watch_ci` MCP tool for GitHub Actions run polling (structured fast path, no LLM token cost) - **Usage dashboard** — real-time token usage and service status overview - **MCP integration** — Memento (persistent memory) + EJClaw host tools (send_message, schedule_task, watch_ci, etc.) @@ -82,12 +82,6 @@ CLAUDE_CODE_OAUTH_TOKEN= # Claude Code OAuth token (primary) CLAUDE_CODE_OAUTH_TOKENS= # Comma-separated tokens for multi-account rotation OPENAI_API_KEY= # For Codex GROQ_API_KEY= # Voice transcription (Groq Whisper) - -# Provider fallback (optional) -FALLBACK_PROVIDER_NAME= # e.g. kimi -FALLBACK_BASE_URL= # e.g. https://api.kimi.com/coding -FALLBACK_AUTH_TOKEN= # Fallback provider API key -FALLBACK_MODEL= # e.g. kimi-k2.5 ``` ### Codex Service (optional) diff --git a/prompts/codex-review-failover-paired-room.md b/prompts/codex-review-failover-paired-room.md new file mode 100644 index 0000000..0e5b25b --- /dev/null +++ b/prompts/codex-review-failover-paired-room.md @@ -0,0 +1,10 @@ +# Codex Review Failover Paired Room Rules + +This room has `codex-review` acting as the owner-side failover agent and `codex-main` acting as the separate Codex reviewer. + +- You are the owner for this chat while failover is active +- `codex-main` is a separate reviewer and is not you +- Keep collaboration with `codex-main` public when useful +- Evaluate reviewer feedback on its merits, but answer the user as the owner +- The visible bot name in history may differ from room to room; do not infer role from the visible name +- Use `` only for repetitive non-user-facing coordination noise diff --git a/prompts/codex-review-failover-platform.md b/prompts/codex-review-failover-platform.md new file mode 100644 index 0000000..c66bc1e --- /dev/null +++ b/prompts/codex-review-failover-platform.md @@ -0,0 +1,44 @@ +# Codex Review Failover Rules + +You are `codex-review`, and in this chat you are currently acting as `클코`, the owner-side failover agent. + +## Role + +- Answer as the primary owner for the chat, not as a reviewer +- Do not describe yourself as the review-side Codex service while this failover prompt is active +- `codex-main` is a separate Codex participant and is not you +- The visible bot name or nickname in chat history may vary by room; do not infer your role from the visible name + +## Communication + +Your output is sent directly to the user or Discord group. + +You may use `` to suppress repetitive agent-to-agent noise. +Keep status updates, conclusions, and handoffs visible. + +```text +Collected the reviewer notes and folded them into the final answer. + +Here is the answer for the user... +``` + +Text inside `` tags is logged but not sent to the user. + +Keep replies concise and owner-oriented. + +- Respond directly to the user +- Give conclusions and concrete next steps +- Do not expose internal routing details unless they matter to the answer +- Do not claim to be `codex-main` + +## Memory + +The group folder may contain a `conversations/` directory with searchable history from earlier sessions. Use it when you need prior context. + +## Message formatting + +Do not use markdown headings in chat replies. Keep messages clean and readable for Discord. + +- Use concise paragraphs or simple lists +- Use fenced code blocks when showing code +- Prefer plain links over markdown link syntax diff --git a/prompts/codex-review-paired-room.md b/prompts/codex-review-paired-room.md new file mode 100644 index 0000000..d8c071d --- /dev/null +++ b/prompts/codex-review-paired-room.md @@ -0,0 +1,14 @@ +# Codex Review Paired Room Rules + +This service is the reviewer side when paired-room failover is active. + +- Default stance: review, challenge, and verify. +- Do not mirror the owner's answer unless you are adding a concrete correction, risk, or missing prerequisite. +- Prioritize: + 1. wrong root cause + 2. unsafe operational advice + 3. missing verification + 4. hidden assumptions +- If the owner answer is fine, keep your message short or skip it entirely. +- If the turn provides a suppress token and you are only agreeing or rephrasing without a concrete correction, risk, prerequisite, test gap, or code change, output only that token and nothing else. +- When code changes are proposed, focus on bugs, regressions, and test gaps. diff --git a/prompts/codex-review-platform.md b/prompts/codex-review-platform.md new file mode 100644 index 0000000..386d9fe --- /dev/null +++ b/prompts/codex-review-platform.md @@ -0,0 +1,10 @@ +# Codex Review Role + +You are `코리뷰`, the review-side Codex service. + +- Your job is to improve answer quality by challenging weak assumptions, spotting bugs, and tightening plans. +- Prefer concise technical criticism over parallel repetition. +- If the main Codex answer is already correct and sufficient, stay brief or silent. +- If the turn provides a suppress token and you have no concrete correction, risk, prerequisite, test gap, or code change, output only that token and nothing else. +- When you disagree, say exactly what is wrong and what should change. +- Focus on correctness, regressions, missing tests, and operational risk. diff --git a/src/agent-error-detection.test.ts b/src/agent-error-detection.test.ts index 695b152..fba55c8 100644 --- a/src/agent-error-detection.test.ts +++ b/src/agent-error-detection.test.ts @@ -5,7 +5,7 @@ import { classifyClaudeAuthError, detectClaudeProviderFailureMessage, isClaudeOrgAccessDeniedMessage, - isNoFallbackCooldownReason, + shouldRotateClaudeToken, } from './agent-error-detection.js'; @@ -65,11 +65,4 @@ describe('agent-error-detection', () => { expect(shouldRotateClaudeToken('success-null-result')).toBe(false); }); - it('marks only no-fallback cooldown reasons as skip-worthy', () => { - expect(isNoFallbackCooldownReason('usage-exhausted')).toBe(true); - expect(isNoFallbackCooldownReason('auth-expired')).toBe(true); - expect(isNoFallbackCooldownReason('org-access-denied')).toBe(true); - expect(isNoFallbackCooldownReason('429')).toBe(false); - expect(isNoFallbackCooldownReason('success-null-result')).toBe(false); - }); }); diff --git a/src/agent-error-detection.ts b/src/agent-error-detection.ts index 6599cf0..bbfdbb7 100644 --- a/src/agent-error-detection.ts +++ b/src/agent-error-detection.ts @@ -103,21 +103,14 @@ export type AgentTriggerReason = | 'network-error' | 'success-null-result'; -export type FallbackTriggerReason = Exclude< - AgentTriggerReason, - 'usage-exhausted' | 'success-null-result' ->; - export type ClaudeRotationReason = Extract< AgentTriggerReason, '429' | 'usage-exhausted' | 'auth-expired' | 'org-access-denied' >; -export type CodexRotationReason = FallbackTriggerReason; - -export type NoFallbackCooldownReason = Extract< +export type CodexRotationReason = Extract< AgentTriggerReason, - 'usage-exhausted' | 'auth-expired' | 'org-access-denied' + '429' | 'auth-expired' | 'org-access-denied' | 'overloaded' | 'network-error' >; export function shouldRotateClaudeToken( @@ -131,14 +124,50 @@ export function shouldRotateClaudeToken( ); } -export function isNoFallbackCooldownReason( - reason: AgentTriggerReason, -): reason is NoFallbackCooldownReason { - return ( - reason === 'usage-exhausted' || - reason === 'auth-expired' || - reason === 'org-access-denied' - ); +// ── Rotation trigger classification ───────────────────────────── + +export type RotationTriggerResult = + | { shouldRetry: false; reason: '' } + | { + shouldRetry: true; + reason: AgentTriggerReason; + retryAfterMs?: number; + }; + +/** + * Classify an error string to determine if Claude token rotation + * should be attempted. + * + * Priority: 429 > auth/org errors > 503/network. + */ +export function classifyRotationTrigger( + error?: string | null, +): RotationTriggerResult { + if (!error) return { shouldRetry: false, reason: '' }; + + const common = classifyAgentError(error); + + // 429 rate-limit (highest priority) + if (common.category === 'rate-limit') { + return { + shouldRetry: true, + reason: common.reason, + retryAfterMs: common.retryAfterMs, + }; + } + + // Claude-specific auth checks (before 503/network) + const auth = classifyClaudeAuthError(error); + if (auth.category !== 'none') { + return { shouldRetry: true, reason: auth.reason }; + } + + // 503 overloaded, network errors + if (common.category !== 'none') { + return { shouldRetry: true, reason: common.reason }; + } + + return { shouldRetry: false, reason: '' }; } // ── Unified error classification ──────────────────────────────── diff --git a/src/agent-runner-environment.test.ts b/src/agent-runner-environment.test.ts index daf4b83..0ead3fb 100644 --- a/src/agent-runner-environment.test.ts +++ b/src/agent-runner-environment.test.ts @@ -9,8 +9,12 @@ const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({ })); vi.mock('./config.js', () => ({ + CODEX_REVIEW_SERVICE_ID: 'codex-review', GROUPS_DIR: '/tmp/ejclaw-test-groups', + SERVICE_ID: 'codex-main', + SERVICE_SESSION_SCOPE: 'codex-main', TIMEZONE: 'Asia/Seoul', + isReviewService: vi.fn(() => false), })); vi.mock('./db.js', () => ({ @@ -35,17 +39,31 @@ vi.mock('./platform-prompts.js', () => ({ readPairedRoomPrompt: vi.fn(() => 'paired room prompt'), })); +vi.mock('./service-routing.js', () => ({ + getEffectiveChannelLease: vi.fn(() => ({ + chat_jid: 'dc:test', + owner_service_id: 'claude', + reviewer_service_id: 'codex-main', + activated_at: null, + reason: null, + explicit: false, + })), +})); + vi.mock('./group-folder.js', () => ({ resolveGroupFolderPath: (folder: string) => `${process.env.EJ_TEST_ROOT}/groups/${folder}`, resolveGroupIpcPath: (folder: string) => `${process.env.EJ_TEST_ROOT}/ipc/${folder}`, - resolveGroupSessionsPath: (folder: string) => - `${process.env.EJ_TEST_ROOT}/sessions/${folder}`, + resolveServiceGroupSessionsPath: (folder: string, serviceId: string) => + `${process.env.EJ_TEST_ROOT}/sessions/${folder}/services/${serviceId}`, resolveTaskRuntimeIpcPath: (folder: string, taskId: string) => `${process.env.EJ_TEST_ROOT}/task-ipc/${folder}/${taskId}`, - resolveTaskSessionsPath: (folder: string, taskId: string) => - `${process.env.EJ_TEST_ROOT}/task-sessions/${folder}/${taskId}`, + resolveServiceTaskSessionsPath: ( + folder: string, + serviceId: string, + taskId: string, + ) => `${process.env.EJ_TEST_ROOT}/task-sessions/${folder}/${serviceId}/${taskId}`, })); vi.mock('os', async () => { @@ -61,6 +79,9 @@ vi.mock('os', async () => { }); import { prepareGroupEnvironment } from './agent-runner-environment.js'; +import * as config from './config.js'; +import * as db from './db.js'; +import * as serviceRouting from './service-routing.js'; import type { RegisteredGroup } from './types.js'; const group: RegisteredGroup = { @@ -131,6 +152,8 @@ describe('prepareGroupEnvironment codex auth handling', () => { tempRoot, 'sessions', group.folder, + 'services', + 'codex-main', '.codex', 'auth.json', ); @@ -165,6 +188,8 @@ describe('prepareGroupEnvironment codex auth handling', () => { tempRoot, 'sessions', group.folder, + 'services', + 'codex-main', '.codex', 'auth.json', ); @@ -172,4 +197,118 @@ describe('prepareGroupEnvironment codex auth handling', () => { expect(auth).toEqual(rotatedAuth); }); + + it('uses the failover prompt pack for codex-review when it owns an explicit failover lease', () => { + vi.mocked(config.isReviewService).mockReturnValue(true); + vi.mocked(db.isPairedRoomJid).mockReturnValue(true); + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'dc:test', + owner_service_id: 'codex-review', + reviewer_service_id: 'codex-main', + activated_at: '2026-03-28T00:00:00.000Z', + reason: 'claude-429', + explicit: true, + }); + mockReadEnvFile.mockReturnValue({}); + + const promptsDir = path.join(tempRoot, 'prompts'); + fs.mkdirSync(promptsDir, { recursive: true }); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-platform.md'), + 'review platform prompt\n', + ); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-paired-room.md'), + 'review paired prompt\n', + ); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-failover-platform.md'), + 'failover platform prompt\n', + ); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-failover-paired-room.md'), + 'failover paired prompt\n', + ); + + prepareGroupEnvironment( + { ...group, workDir: path.join(tempRoot, 'workdir') }, + false, + 'dc:test', + { + memoryBriefing: 'memory briefing', + }, + ); + + const agentsPath = path.join( + tempRoot, + 'sessions', + group.folder, + 'services', + 'codex-main', + '.codex', + 'AGENTS.md', + ); + const agents = fs.readFileSync(agentsPath, 'utf-8'); + const segments = agents.trim().split('\n\n---\n\n'); + + expect(segments).toEqual([ + 'failover platform prompt', + 'failover paired prompt', + 'memory briefing', + ]); + }); + + it('returns to the normal review prompt stack after failover is cleared', () => { + vi.mocked(config.isReviewService).mockReturnValue(true); + vi.mocked(db.isPairedRoomJid).mockReturnValue(true); + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'dc:test', + owner_service_id: 'claude', + reviewer_service_id: 'codex-main', + activated_at: null, + reason: null, + explicit: false, + }); + mockReadEnvFile.mockReturnValue({}); + + const promptsDir = path.join(tempRoot, 'prompts'); + fs.mkdirSync(promptsDir, { recursive: true }); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-platform.md'), + 'review platform prompt\n', + ); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-paired-room.md'), + 'review paired prompt\n', + ); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-failover-platform.md'), + 'failover platform prompt\n', + ); + fs.writeFileSync( + path.join(promptsDir, 'codex-review-failover-paired-room.md'), + 'failover paired prompt\n', + ); + + prepareGroupEnvironment(group, false, 'dc:test'); + + const agentsPath = path.join( + tempRoot, + 'sessions', + group.folder, + 'services', + 'codex-main', + '.codex', + 'AGENTS.md', + ); + const agents = fs.readFileSync(agentsPath, 'utf-8'); + const segments = agents.trim().split('\n\n---\n\n'); + + expect(segments).toEqual([ + 'platform prompt', + 'review platform prompt', + 'paired room prompt', + 'review paired prompt', + ]); + }); }); diff --git a/src/agent-runner-environment.ts b/src/agent-runner-environment.ts index efd19d5..94bd260 100644 --- a/src/agent-runner-environment.ts +++ b/src/agent-runner-environment.ts @@ -2,7 +2,14 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { GROUPS_DIR, TIMEZONE } from './config.js'; +import { + CODEX_REVIEW_SERVICE_ID, + GROUPS_DIR, + SERVICE_ID, + SERVICE_SESSION_SCOPE, + TIMEZONE, + isReviewService, +} from './config.js'; import { isPairedRoomJid } from './db.js'; import { readEnvFile } from './env.js'; import { getActiveCodexAuthPath } from './codex-token-rotation.js'; @@ -10,14 +17,15 @@ import { getCurrentToken } from './token-rotation.js'; import { resolveGroupFolderPath, resolveGroupIpcPath, - resolveGroupSessionsPath, + resolveServiceGroupSessionsPath, resolveTaskRuntimeIpcPath, - resolveTaskSessionsPath, + resolveServiceTaskSessionsPath, } from './group-folder.js'; import { readPairedRoomPrompt, readPlatformPrompt, } from './platform-prompts.js'; +import { getEffectiveChannelLease } from './service-routing.js'; import type { AgentType, RegisteredGroup } from './types.js'; // writeCodexApiKeyAuth removed — Codex uses OAuth only. @@ -39,6 +47,16 @@ function syncDirectoryEntries(sources: string[], destination: string): void { } } +function readOptionalPromptFile( + projectRoot: string, + filename: string, +): string | undefined { + const promptPath = path.join(projectRoot, 'prompts', filename); + if (!fs.existsSync(promptPath)) return undefined; + const prompt = fs.readFileSync(promptPath, 'utf-8').trim(); + return prompt || undefined; +} + function ensureClaudeSessionSettings(groupSessionsDir: string): void { const settingsFile = path.join(groupSessionsDir, 'settings.json'); if (fs.existsSync(settingsFile)) return; @@ -175,6 +193,7 @@ function prepareCodexSessionEnvironment(args: { chatJid: string; isMain: boolean; isPairedRoom: boolean; + useFailoverPromptPack: boolean; memoryBriefing?: string; }): void { // API key auth intentionally removed — Codex uses OAuth only. @@ -236,13 +255,38 @@ function prepareCodexSessionEnvironment(args: { } const sessionAgentsPath = path.join(sessionCodexDir, 'AGENTS.md'); - const sessionAgents = [ - readPlatformPrompt('codex', args.projectRoot), - args.isPairedRoom - ? readPairedRoomPrompt('codex', args.projectRoot) - : undefined, - args.memoryBriefing, - ] + const sessionAgents = ( + args.useFailoverPromptPack + ? [ + readOptionalPromptFile( + args.projectRoot, + 'codex-review-failover-platform.md', + ), + args.isPairedRoom + ? readOptionalPromptFile( + args.projectRoot, + 'codex-review-failover-paired-room.md', + ) + : undefined, + args.memoryBriefing, + ] + : [ + readPlatformPrompt('codex', args.projectRoot), + isReviewService(SERVICE_ID) + ? readOptionalPromptFile(args.projectRoot, 'codex-review-platform.md') + : undefined, + args.isPairedRoom + ? readPairedRoomPrompt('codex', args.projectRoot) + : undefined, + args.isPairedRoom && isReviewService(SERVICE_ID) + ? readOptionalPromptFile( + args.projectRoot, + 'codex-review-paired-room.md', + ) + : undefined, + args.memoryBriefing, + ] + ) .filter((value): value is string => Boolean(value)) .join('\n\n---\n\n') .trim(); @@ -342,8 +386,12 @@ export function prepareGroupEnvironment( options?.useTaskScopedSession === true && Boolean(runtimeTaskId); const sessionRootDir = runtimeTaskId && useTaskScopedSession - ? resolveTaskSessionsPath(group.folder, runtimeTaskId) - : resolveGroupSessionsPath(group.folder); + ? resolveServiceTaskSessionsPath( + group.folder, + SERVICE_SESSION_SCOPE, + runtimeTaskId, + ) + : resolveServiceGroupSessionsPath(group.folder, SERVICE_SESSION_SCOPE); const groupSessionsDir = path.join(sessionRootDir, '.claude'); fs.mkdirSync(groupSessionsDir, { recursive: true }); @@ -370,6 +418,11 @@ export function prepareGroupEnvironment( const globalDir = path.join(GROUPS_DIR, 'global'); const globalClaudeMdPath = path.join(globalDir, 'CLAUDE.md'); const isPairedRoom = isPairedRoomJid(chatJid); + const effectiveLease = getEffectiveChannelLease(chatJid); + const useCodexReviewFailoverPromptPack = + isReviewService(SERVICE_ID) && + effectiveLease.explicit && + effectiveLease.owner_service_id === CODEX_REVIEW_SERVICE_ID; const claudePlatformPrompt = readPlatformPrompt('claude-code', projectRoot); const claudePairedRoomPrompt = isPairedRoom @@ -440,6 +493,7 @@ export function prepareGroupEnvironment( chatJid, isMain, isPairedRoom, + useFailoverPromptPack: useCodexReviewFailoverPromptPack, memoryBriefing: options?.memoryBriefing, }); } else { diff --git a/src/agent-runner.test.ts b/src/agent-runner.test.ts index 1cfa1a1..2e9d9b4 100644 --- a/src/agent-runner.test.ts +++ b/src/agent-runner.test.ts @@ -13,7 +13,10 @@ vi.mock('./config.js', () => ({ DATA_DIR: '/tmp/ejclaw-test-data', GROUPS_DIR: '/tmp/ejclaw-test-groups', IDLE_TIMEOUT: 1800000, // 30min + SERVICE_ID: 'claude', + SERVICE_SESSION_SCOPE: 'claude', TIMEZONE: 'America/Los_Angeles', + isReviewService: vi.fn(() => false), })); // Mock logger @@ -55,6 +58,17 @@ vi.mock('./env.js', () => ({ getEnv: vi.fn(() => undefined), })); +vi.mock('./service-routing.js', () => ({ + getEffectiveChannelLease: vi.fn(() => ({ + chat_jid: 'test@g.us', + owner_service_id: 'claude', + reviewer_service_id: 'codex-main', + activated_at: null, + reason: null, + explicit: false, + })), +})); + // Create a controllable fake ChildProcess function createFakeProcess() { const proc = new EventEmitter() as EventEmitter & { @@ -328,7 +342,7 @@ describe('agent-runner timeout behavior', () => { '/tmp/ejclaw-test-data/ipc/test-group', ); expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe( - '/tmp/ejclaw-test-data/sessions/test-group/tasks/task-123/.claude', + '/tmp/ejclaw-test-data/sessions/test-group/services/claude/tasks/task-123/.claude', ); }); @@ -407,7 +421,7 @@ describe('agent-runner timeout behavior', () => { '/tmp/ejclaw-test-data/ipc/test-group', ); expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe( - '/tmp/ejclaw-test-data/sessions/test-group/.claude', + '/tmp/ejclaw-test-data/sessions/test-group/services/claude/.claude', ); }); @@ -416,7 +430,7 @@ describe('agent-runner timeout behavior', () => { fakeProc = createFakeProcess(); const overlayPath = '/tmp/ejclaw-test-groups/test-group/.codex/config.toml'; const sessionConfigPath = - '/tmp/ejclaw-test-data/sessions/test-group/.codex/config.toml'; + '/tmp/ejclaw-test-data/sessions/test-group/services/claude/.codex/config.toml'; let sessionToml = `model = "gpt-5.4"\n`; vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => { diff --git a/src/agent-runner.ts b/src/agent-runner.ts index cbb9722..45b90f5 100644 --- a/src/agent-runner.ts +++ b/src/agent-runner.ts @@ -71,7 +71,7 @@ export async function runAgentProcess( }, ); - // Apply provider fallback overrides (e.g. Kimi env vars when Claude is in cooldown) + // Apply env overrides (caller-provided) if (envOverrides) { for (const [key, value] of Object.entries(envOverrides)) { if (value) env[key] = value; diff --git a/src/channels/discord.test.ts b/src/channels/discord.test.ts index ec23e1d..1a227d3 100644 --- a/src/channels/discord.test.ts +++ b/src/channels/discord.test.ts @@ -870,6 +870,31 @@ describe('DiscordChannel', () => { expect(currentClient().channels.fetch).not.toHaveBeenCalled(); }); + it('does not send stale typing after typing was disabled mid-flight', async () => { + const opts = createTestOpts(); + const channel = new DiscordChannel('test-token', opts); + await channel.connect(); + + let resolveFetch!: (value: unknown) => void; + const fetchPromise = new Promise((resolve) => { + resolveFetch = resolve; + }); + const mockChannel = { + send: vi.fn(), + sendTyping: vi.fn().mockResolvedValue(undefined), + }; + + currentClient().channels.fetch.mockImplementationOnce(() => fetchPromise); + + const typingPromise = channel.setTyping('dc:1234567890123456', true); + await Promise.resolve(); + await channel.setTyping('dc:1234567890123456', false); + resolveFetch(mockChannel); + await typingPromise; + + expect(mockChannel.sendTyping).not.toHaveBeenCalled(); + }); + it('does nothing when client is not initialized', async () => { const opts = createTestOpts(); const channel = new DiscordChannel('test-token', opts); diff --git a/src/channels/discord.ts b/src/channels/discord.ts index 179e0a3..dfbb900 100644 --- a/src/channels/discord.ts +++ b/src/channels/discord.ts @@ -177,6 +177,7 @@ export class DiscordChannel implements Channel { private opts: DiscordChannelOpts; private botToken: string; private typingIntervals = new Map(); + private typingGenerations = new Map(); private agentTypeFilter?: AgentType; constructor( @@ -539,6 +540,9 @@ export class DiscordChannel implements Channel { async setTyping(jid: string, isTyping: boolean): Promise { if (!this.client) return; + const generation = (this.typingGenerations.get(jid) ?? 0) + 1; + this.typingGenerations.set(jid, generation); + // Clear any existing interval for this channel const existing = this.typingIntervals.get(jid); if (existing) { @@ -548,10 +552,15 @@ export class DiscordChannel implements Channel { if (!isTyping) return; + const isCurrentGeneration = () => + this.typingGenerations.get(jid) === generation; + const sendOnce = async () => { + if (!isCurrentGeneration()) return; try { const channelId = jid.replace(/^dc:/, ''); const channel = await this.client!.channels.fetch(channelId); + if (!isCurrentGeneration()) return; if (channel && 'sendTyping' in channel) { await (channel as TextChannel).sendTyping(); } @@ -562,7 +571,13 @@ export class DiscordChannel implements Channel { // Send immediately, then refresh every 8 seconds (Discord expires at ~10s) await sendOnce(); - this.typingIntervals.set(jid, setInterval(sendOnce, 8000)); + if (!isCurrentGeneration()) return; + this.typingIntervals.set( + jid, + setInterval(() => { + void sendOnce(); + }, 8000), + ); } async sendAndTrack(jid: string, text: string): Promise { diff --git a/src/config.ts b/src/config.ts index f22d62a..374989a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,15 +10,44 @@ export const ASSISTANT_HAS_OWN_NUMBER = const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase(); const rawServiceAgentType = getEnv('SERVICE_AGENT_TYPE'); export const SERVICE_ID = getEnv('SERVICE_ID') || ASSISTANT_SLUG; +export const CLAUDE_SERVICE_ID = getEnv('CLAUDE_SERVICE_ID') || 'claude'; +export const CODEX_MAIN_SERVICE_ID = + getEnv('CODEX_MAIN_SERVICE_ID') || 'codex-main'; +export const CODEX_REVIEW_SERVICE_ID = + getEnv('CODEX_REVIEW_SERVICE_ID') || 'codex-review'; export const SERVICE_AGENT_TYPE: AgentType = rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code' ? rawServiceAgentType : ASSISTANT_SLUG === 'codex' ? 'codex' : 'claude-code'; + +export function normalizeServiceId(serviceId: string): string { + if (serviceId === 'codex') { + return CODEX_MAIN_SERVICE_ID; + } + return serviceId; +} + +export const SERVICE_SESSION_SCOPE = normalizeServiceId(SERVICE_ID); + +export function isClaudeService(serviceId: string = SERVICE_ID): boolean { + return normalizeServiceId(serviceId) === CLAUDE_SERVICE_ID; +} + +export function isCodexMainService(serviceId: string = SERVICE_ID): boolean { + return normalizeServiceId(serviceId) === CODEX_MAIN_SERVICE_ID; +} + +export function isReviewService(serviceId: string = SERVICE_ID): boolean { + return normalizeServiceId(serviceId) === CODEX_REVIEW_SERVICE_ID; +} export const POLL_INTERVAL = 2000; export const SCHEDULER_POLL_INTERVAL = 60000; +/** Minimum time (ms) a failover lease stays active before Claude can reclaim it. */ +export const FAILOVER_MIN_DURATION_MS = 3 * 60 * 60 * 1000; // 3 hours + const PROJECT_ROOT = process.cwd(); const HOME_DIR = process.env.HOME || os.homedir(); @@ -106,3 +135,26 @@ const SESSION_COMMAND_ALLOWED_SENDERS = new Set( export function isSessionCommandSenderAllowed(sender: string): boolean { return SESSION_COMMAND_ALLOWED_SENDERS.has(sender); } + +// Delta handoff: cross-provider session continuity with probe + fallback +export const DELTA_HANDOFF_ENABLED = + getEnv('DELTA_HANDOFF_ENABLED') === 'true'; + +// Comma-separated list of group folders for canary testing +const rawDeltaHandoffCanaryGroups = + getEnv('DELTA_HANDOFF_CANARY_GROUPS') || ''; +export const DELTA_HANDOFF_CANARY_GROUPS = new Set( + rawDeltaHandoffCanaryGroups + .split(',') + .map((v) => v.trim()) + .filter(Boolean), +); + +export function isDeltaHandoffEnabledForGroup(groupFolder: string): boolean { + if (!DELTA_HANDOFF_ENABLED) return false; + // If canary groups are specified, only enable for those groups + if (DELTA_HANDOFF_CANARY_GROUPS.size > 0) { + return DELTA_HANDOFF_CANARY_GROUPS.has(groupFolder); + } + return true; +} diff --git a/src/dashboard-usage-rows.test.ts b/src/dashboard-usage-rows.test.ts index f15a507..a1b1c02 100644 --- a/src/dashboard-usage-rows.test.ts +++ b/src/dashboard-usage-rows.test.ts @@ -25,6 +25,7 @@ describe('extractCodexUsageRows staleness', () => { it('returns real rows when usageRowsFetchedAt is within maxAgeMs', () => { const now = Date.now(); const snapshot: StatusSnapshot = { + serviceId: 'codex-main', agentType: 'codex', assistantName: 'test', updatedAt: new Date(now).toISOString(), @@ -40,6 +41,7 @@ describe('extractCodexUsageRows staleness', () => { it('returns degraded row when usageRowsFetchedAt exceeds maxAgeMs', () => { const now = Date.now(); const snapshot: StatusSnapshot = { + serviceId: 'codex-main', agentType: 'codex', assistantName: 'test', updatedAt: new Date(now).toISOString(), // heartbeat is fresh @@ -57,6 +59,7 @@ describe('extractCodexUsageRows staleness', () => { it('returns degraded row when usageRowsFetchedAt is missing', () => { const now = Date.now(); const snapshot: StatusSnapshot = { + serviceId: 'codex-main', agentType: 'codex', assistantName: 'test', updatedAt: new Date(now).toISOString(), @@ -77,6 +80,7 @@ describe('extractCodexUsageRows staleness', () => { it('returns empty array when usageRows is empty', () => { const snapshot: StatusSnapshot = { + serviceId: 'codex-main', agentType: 'codex', assistantName: 'test', updatedAt: new Date().toISOString(), diff --git a/src/dashboard.test.ts b/src/dashboard.test.ts index a92ff28..77b0cc0 100644 --- a/src/dashboard.test.ts +++ b/src/dashboard.test.ts @@ -45,9 +45,8 @@ describe('buildStatusContent', () => { ): Promise<(opts: DashboardOptions) => string> { vi.resetModules(); process.env.STATUS_SHOW_ROOMS = options.statusShowRooms ?? 'true'; - if (options.statusShowRoomDetails !== undefined) { - process.env.STATUS_SHOW_ROOM_DETAILS = options.statusShowRoomDetails; - } + process.env.STATUS_SHOW_ROOM_DETAILS = + options.statusShowRoomDetails ?? 'true'; const mod = await import('./dashboard-status-content.js'); return mod.buildStatusContent; } diff --git a/src/dashboard.ts b/src/dashboard.ts index 52d6ea9..de85418 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -15,6 +15,7 @@ export async function startStatusDashboard( ): Promise { await startUnifiedDashboard({ assistantName: opts.assistantName, + serviceId: opts.serviceAgentType === 'codex' ? 'codex-main' : 'claude', serviceAgentType: opts.serviceAgentType || 'claude-code', statusChannelId: opts.statusChannelId, statusUpdateInterval: opts.statusUpdateInterval, diff --git a/src/db.test.ts b/src/db.test.ts index fd8764f..645a4d0 100644 --- a/src/db.test.ts +++ b/src/db.test.ts @@ -4,7 +4,10 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { _initTestDatabase, + claimServiceHandoff, + completeServiceHandoffAndAdvanceTargetCursor, createTask, + createServiceHandoff, createProducedWorkItem, deleteSession, deleteTask, @@ -15,9 +18,11 @@ import { getMessagesSinceSeq, getNewMessagesBySeq, getOpenWorkItem, + getPendingServiceHandoffs, getRegisteredAgentTypesForJid, getMessagesSince, getNewMessages, + getRouterStateForService, isPairedRoomJid, getSession, getTaskById, @@ -25,6 +30,7 @@ import { markWorkItemDeliveryRetry, setSession, setRegisteredGroup, + setRouterStateForService, storeChatMetadata, storeMessage, updateTask, @@ -678,6 +684,84 @@ describe('paired room registration', () => { }); }); +describe('service handoff completion', () => { + it('atomically completes the handoff and advances the target cursor', () => { + storeChatMetadata('dc:handoff', '2024-01-01T00:00:00.000Z'); + store({ + id: 'handoff-msg-1', + chat_jid: 'dc:handoff', + sender: 'user', + sender_name: 'User', + content: 'hello', + timestamp: '2024-01-01T00:00:01.000Z', + }); + const handoff = createServiceHandoff({ + chat_jid: 'dc:handoff', + group_folder: 'test-group', + source_service_id: 'claude', + target_service_id: 'codex-review', + target_agent_type: 'codex', + prompt: 'hello', + end_seq: 1, + }); + + expect(claimServiceHandoff(handoff.id)).toBe(true); + + const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({ + id: handoff.id, + target_service_id: 'codex-review', + chat_jid: 'dc:handoff', + end_seq: 1, + }); + + expect(appliedCursor).toBe('1'); + expect(getPendingServiceHandoffs('codex-review')).toEqual([]); + expect( + JSON.parse( + getRouterStateForService('last_agent_seq', 'codex-review') || '{}', + ), + ).toMatchObject({ + 'dc:handoff': '1', + }); + }); + + it('does not move the target cursor backwards when a newer cursor already exists', () => { + storeChatMetadata('dc:handoff', '2024-01-01T00:00:00.000Z'); + setRouterStateForService( + 'last_agent_seq', + JSON.stringify({ 'dc:handoff': '5' }), + 'codex-review', + ); + const handoff = createServiceHandoff({ + chat_jid: 'dc:handoff', + group_folder: 'test-group', + source_service_id: 'claude', + target_service_id: 'codex-review', + target_agent_type: 'codex', + prompt: 'hello', + end_seq: 3, + }); + + expect(claimServiceHandoff(handoff.id)).toBe(true); + + const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({ + id: handoff.id, + target_service_id: 'codex-review', + chat_jid: 'dc:handoff', + end_seq: 3, + }); + + expect(appliedCursor).toBe('5'); + expect( + JSON.parse( + getRouterStateForService('last_agent_seq', 'codex-review') || '{}', + ), + ).toMatchObject({ + 'dc:handoff': '5', + }); + }); +}); + describe('message seq cursors', () => { beforeEach(() => { storeChatMetadata('group@g.us', '2024-01-01T00:00:00.000Z'); diff --git a/src/db.ts b/src/db.ts index 2ffcd21..6af768b 100644 --- a/src/db.ts +++ b/src/db.ts @@ -4,14 +4,20 @@ import path from 'path'; import { ASSISTANT_NAME, + CLAUDE_SERVICE_ID, + CODEX_MAIN_SERVICE_ID, + CODEX_REVIEW_SERVICE_ID, DATA_DIR, + normalizeServiceId, SERVICE_AGENT_TYPE, SERVICE_ID, + SERVICE_SESSION_SCOPE, STORE_DIR, } from './config.js'; import { isValidGroupFolder, resolveTaskRuntimeIpcPath as resolveTaskRuntimeIpcPathFromGroup, + resolveServiceTaskSessionsPath as resolveServiceTaskSessionsPathFromGroup, resolveTaskSessionsPath as resolveTaskSessionsPathFromGroup, } from './group-folder.js'; import { logger } from './logger.js'; @@ -32,6 +38,7 @@ export interface WorkItem { group_folder: string; chat_jid: string; agent_type: AgentType; + service_id: string; status: 'produced' | 'delivery_retry' | 'delivered'; start_seq: number | null; end_seq: number | null; @@ -44,6 +51,32 @@ export interface WorkItem { delivered_at: string | null; } +export interface ChannelOwnerLeaseRow { + chat_jid: string; + owner_service_id: string; + reviewer_service_id: string | null; + activated_at: string | null; + reason: string | null; +} + +export interface ServiceHandoff { + id: number; + chat_jid: string; + group_folder: string; + source_service_id: string; + target_service_id: string; + target_agent_type: AgentType; + prompt: string; + status: 'pending' | 'claimed' | 'completed' | 'failed'; + start_seq: number | null; + end_seq: number | null; + reason: string | null; + created_at: string; + claimed_at: string | null; + completed_at: string | null; + last_error: string | null; +} + function backfillMessageSeq(database: Database.Database): void { const rows = database .prepare( @@ -114,6 +147,7 @@ function createSchema(database: Database.Database): void { group_folder TEXT NOT NULL, chat_jid TEXT NOT NULL, agent_type TEXT NOT NULL, + service_id TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'produced', start_seq INTEGER, end_seq INTEGER, @@ -127,9 +161,9 @@ function createSchema(database: Database.Database): void { CHECK (status IN ('produced', 'delivery_retry', 'delivered')) ); CREATE INDEX IF NOT EXISTS idx_work_items_status ON work_items(status, updated_at); - CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, status); + CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, service_id, status); CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open - ON work_items(chat_jid, agent_type) + ON work_items(chat_jid, agent_type, service_id) WHERE status IN ('produced', 'delivery_retry'); CREATE TABLE IF NOT EXISTS scheduled_tasks ( @@ -176,6 +210,12 @@ function createSchema(database: Database.Database): void { session_id TEXT NOT NULL, PRIMARY KEY (group_folder, agent_type) ); + CREATE TABLE IF NOT EXISTS service_sessions ( + group_folder TEXT NOT NULL, + service_id TEXT NOT NULL, + session_id TEXT NOT NULL, + PRIMARY KEY (group_folder, service_id) + ); CREATE TABLE IF NOT EXISTS registered_groups ( jid TEXT NOT NULL, name TEXT NOT NULL, @@ -190,6 +230,33 @@ function createSchema(database: Database.Database): void { PRIMARY KEY (jid, agent_type), UNIQUE (folder, agent_type) ); + CREATE TABLE IF NOT EXISTS channel_owner ( + chat_jid TEXT PRIMARY KEY, + owner_service_id TEXT NOT NULL, + reviewer_service_id TEXT, + activated_at TEXT, + reason TEXT + ); + CREATE TABLE IF NOT EXISTS service_handoffs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chat_jid TEXT NOT NULL, + group_folder TEXT NOT NULL, + source_service_id TEXT NOT NULL, + target_service_id TEXT NOT NULL, + target_agent_type TEXT NOT NULL, + prompt TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + start_seq INTEGER, + end_seq INTEGER, + reason TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + claimed_at TEXT, + completed_at TEXT, + last_error TEXT, + CHECK (status IN ('pending', 'claimed', 'completed', 'failed')) + ); + CREATE INDEX IF NOT EXISTS idx_service_handoffs_target + ON service_handoffs(target_service_id, status, created_at); `); // Add context_mode column if it doesn't exist (migration for existing DBs) @@ -293,12 +360,41 @@ function createSchema(database: Database.Database): void { /* column already exists */ } + try { + database.exec(`ALTER TABLE work_items ADD COLUMN service_id TEXT DEFAULT ''`); + } catch { + /* column already exists */ + } + try { + database + .prepare( + `UPDATE work_items + SET service_id = CASE + WHEN agent_type = 'codex' THEN ? + ELSE ? + END + WHERE COALESCE(service_id, '') = ''`, + ) + .run(CODEX_MAIN_SERVICE_ID, CLAUDE_SERVICE_ID); + } catch { + /* best effort */ + } + backfillMessageSeq(database); database.exec(` CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_seq ON messages(seq); CREATE INDEX IF NOT EXISTS idx_messages_chat_jid_seq ON messages(chat_jid, seq); `); + database.exec(`DROP INDEX IF EXISTS idx_work_items_group_agent;`); + database.exec(`DROP INDEX IF EXISTS idx_work_items_open;`); + database.exec(` + CREATE INDEX IF NOT EXISTS idx_work_items_group_agent + ON work_items(chat_jid, agent_type, service_id, status); + CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open + ON work_items(chat_jid, agent_type, service_id) + WHERE status IN ('produced', 'delivery_retry'); + `); // Migrate registered_groups to composite keys so Claude/Codex can share a jid/folder. const registeredGroupsSql = ( @@ -769,6 +865,36 @@ export function getMessagesSinceSeq( return rows.map(normalizeMessageRow); } +/** + * Get the N most recent messages for a chat, ordered chronologically. + * Includes both human and bot messages for full conversation context. + * Used for conversation context retrieval. + */ +export function getRecentChatMessages( + chatJid: string, + limit: number = 20, +): NewMessage[] { + const sql = ` + SELECT * FROM ( + SELECT id, chat_jid, sender, sender_name, content, timestamp, is_from_me, is_bot_message + FROM messages + WHERE chat_jid = ? + AND content != '' AND content IS NOT NULL + ORDER BY timestamp DESC + LIMIT ? + ) ORDER BY timestamp + `; + const rows = db + .prepare(sql) + .all(chatJid, limit) as Array< + NewMessage & { + is_from_me?: boolean | number; + is_bot_message?: boolean | number; + } + >; + return rows.map(normalizeMessageRow); +} + export function getLastHumanMessageTimestamp(chatJid: string): string | null { const row = db .prepare( @@ -805,34 +931,39 @@ export function hasRecentRestartAnnouncement( export function getOpenWorkItem( chatJid: string, agentType: AgentType = SERVICE_AGENT_TYPE, + serviceId: string = SERVICE_SESSION_SCOPE, ): WorkItem | undefined { return db .prepare( `SELECT * FROM work_items - WHERE chat_jid = ? AND agent_type = ? AND status IN ('produced', 'delivery_retry') + WHERE chat_jid = ? AND agent_type = ? AND service_id = ? + AND status IN ('produced', 'delivery_retry') ORDER BY id ASC LIMIT 1`, ) - .get(chatJid, agentType) as WorkItem | undefined; + .get(chatJid, agentType, serviceId) as WorkItem | undefined; } export function createProducedWorkItem(input: { group_folder: string; chat_jid: string; agent_type?: AgentType; + service_id?: string; start_seq: number | null; end_seq: number | null; result_payload: string; }): WorkItem { const now = new Date().toISOString(); const agentType = input.agent_type || SERVICE_AGENT_TYPE; + const serviceId = input.service_id || SERVICE_SESSION_SCOPE; const result = db .prepare( `INSERT INTO work_items ( group_folder, chat_jid, agent_type, + service_id, status, start_seq, end_seq, @@ -840,12 +971,13 @@ export function createProducedWorkItem(input: { delivery_attempts, created_at, updated_at - ) VALUES (?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`, + ) VALUES (?, ?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`, ) .run( input.group_folder, input.chat_jid, agentType, + serviceId, input.start_seq, input.end_seq, input.result_payload, @@ -1085,6 +1217,21 @@ export function deleteTask(id: string): void { cleanupTargets.push( resolveTaskRuntimeIpcPathFromGroup(task.group_folder, runtimeTaskId), resolveTaskSessionsPathFromGroup(task.group_folder, runtimeTaskId), + resolveServiceTaskSessionsPathFromGroup( + task.group_folder, + CLAUDE_SERVICE_ID, + runtimeTaskId, + ), + resolveServiceTaskSessionsPathFromGroup( + task.group_folder, + CODEX_MAIN_SERVICE_ID, + runtimeTaskId, + ), + resolveServiceTaskSessionsPathFromGroup( + task.group_folder, + CODEX_REVIEW_SERVICE_ID, + runtimeTaskId, + ), ); } catch (err) { logger.warn( @@ -1175,7 +1322,14 @@ export function getRecentConsecutiveErrors( // --- Router state accessors --- export function getRouterState(key: string): string | undefined { - const prefixedKey = `${SERVICE_ID}:${key}`; + return getRouterStateForService(key, SERVICE_ID); +} + +export function getRouterStateForService( + key: string, + serviceId: string, +): string | undefined { + const prefixedKey = `${normalizeServiceId(serviceId)}:${key}`; const row = db .prepare('SELECT value FROM router_state WHERE key = ?') .get(prefixedKey) as { value: string } | undefined; @@ -1196,7 +1350,15 @@ export function getRouterState(key: string): string | undefined { } export function setRouterState(key: string, value: string): void { - const prefixedKey = `${SERVICE_ID}:${key}`; + setRouterStateForService(key, value, SERVICE_ID); +} + +export function setRouterStateForService( + key: string, + value: string, + serviceId: string, +): void { + const prefixedKey = `${normalizeServiceId(serviceId)}:${key}`; db.prepare( 'INSERT OR REPLACE INTO router_state (key, value) VALUES (?, ?)', ).run(prefixedKey, value); @@ -1205,6 +1367,17 @@ export function setRouterState(key: string, value: string): void { // --- Session accessors --- export function getSession(groupFolder: string): string | undefined { + const serviceScopedRow = db + .prepare( + 'SELECT session_id FROM service_sessions WHERE group_folder = ? AND service_id = ?', + ) + .get(groupFolder, SERVICE_SESSION_SCOPE) as + | { session_id: string } + | undefined; + if (serviceScopedRow?.session_id) { + return serviceScopedRow.session_id; + } + const row = db .prepare( 'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?', @@ -1214,18 +1387,40 @@ export function getSession(groupFolder: string): string | undefined { } export function setSession(groupFolder: string, sessionId: string): void { + db.prepare( + 'INSERT OR REPLACE INTO service_sessions (group_folder, service_id, session_id) VALUES (?, ?, ?)', + ).run(groupFolder, SERVICE_SESSION_SCOPE, sessionId); db.prepare( 'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)', ).run(groupFolder, SERVICE_AGENT_TYPE, sessionId); } export function deleteSession(groupFolder: string): void { + db.prepare( + 'DELETE FROM service_sessions WHERE group_folder = ? AND service_id = ?', + ).run(groupFolder, SERVICE_SESSION_SCOPE); db.prepare( 'DELETE FROM sessions WHERE group_folder = ? AND agent_type = ?', ).run(groupFolder, SERVICE_AGENT_TYPE); } export function getAllSessions(): Record { + const serviceRows = db + .prepare( + 'SELECT group_folder, session_id FROM service_sessions WHERE service_id = ?', + ) + .all(SERVICE_SESSION_SCOPE) as Array<{ + group_folder: string; + session_id: string; + }>; + const result: Record = {}; + for (const row of serviceRows) { + result[row.group_folder] = row.session_id; + } + if (serviceRows.length > 0) { + return result; + } + const rows = db .prepare( 'SELECT group_folder, session_id FROM sessions WHERE agent_type = ?', @@ -1234,11 +1429,66 @@ export function getAllSessions(): Record { group_folder: string; session_id: string; }>; - const result: Record = {}; + const legacyResult: Record = {}; for (const row of rows) { - result[row.group_folder] = row.session_id; + legacyResult[row.group_folder] = row.session_id; } - return result; + return legacyResult; +} + +/** + * Get session for a specific agent type (cross-provider access). + * Used for provider switch probe attempts. + */ +export function getSessionForAgentType( + groupFolder: string, + agentType: string, +): string | undefined { + const row = db + .prepare( + 'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?', + ) + .get(groupFolder, agentType) as { session_id: string } | undefined; + return row?.session_id; +} + +/** + * Save session for a specific agent type without affecting current service's session. + * Used when probe succeeds and we want to save to target provider's slot only. + */ +export function setSessionForAgentType( + groupFolder: string, + agentType: string, + sessionId: string, +): void { + db.prepare( + 'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)', + ).run(groupFolder, agentType, sessionId); +} + +/** + * Get the agent type of the most recent bot response in a chat. + * Used to detect provider switches for delta handoff. + */ +export function getLastRespondingAgentType( + chatJid: string, +): AgentType | undefined { + const row = db + .prepare( + `SELECT sender FROM messages + WHERE chat_jid = ? AND is_bot_message = 1 + ORDER BY timestamp DESC, seq DESC + LIMIT 1`, + ) + .get(chatJid) as { sender: string } | undefined; + + if (!row) return undefined; + + // Map sender to agent type (sender contains the bot identifier) + const sender = row.sender.toLowerCase(); + if (sender.includes('claude')) return 'claude-code'; + if (sender.includes('codex')) return 'codex'; + return undefined; } // --- Registered group accessors --- @@ -1388,6 +1638,255 @@ export function isPairedRoomJid(jid: string): boolean { return types.includes('claude-code') && types.includes('codex'); } +/** + * Get the most recent bot message (is_bot_message=1) in a chat, regardless of which bot sent it. + * Used for duplicate detection in pair rooms. + */ +export function getLastBotFinalMessage( + chatJid: string, + _agentType: AgentType = SERVICE_AGENT_TYPE, + limit: number = 1, +): Array<{ content: string; timestamp: string }> { + const rows = db + .prepare( + `SELECT content, timestamp + FROM messages + WHERE chat_jid = ? AND is_bot_message = 1 + ORDER BY timestamp DESC, seq DESC + LIMIT ?`, + ) + .all(chatJid, limit) as Array<{ content: string; timestamp: string }>; + return rows; +} + +// --- Channel owner lease accessors --- + +export function getChannelOwnerLease( + chatJid: string, +): ChannelOwnerLeaseRow | undefined { + return db + .prepare( + `SELECT chat_jid, owner_service_id, reviewer_service_id, activated_at, reason + FROM channel_owner + WHERE chat_jid = ?`, + ) + .get(chatJid) as ChannelOwnerLeaseRow | undefined; +} + +export function getAllChannelOwnerLeases(): ChannelOwnerLeaseRow[] { + return db + .prepare( + `SELECT chat_jid, owner_service_id, reviewer_service_id, activated_at, reason + FROM channel_owner`, + ) + .all() as ChannelOwnerLeaseRow[]; +} + +export function setChannelOwnerLease(input: { + chat_jid: string; + owner_service_id: string; + reviewer_service_id?: string | null; + activated_at?: string | null; + reason?: string | null; +}): void { + db.prepare( + `INSERT OR REPLACE INTO channel_owner ( + chat_jid, + owner_service_id, + reviewer_service_id, + activated_at, + reason + ) VALUES (?, ?, ?, ?, ?)`, + ).run( + input.chat_jid, + input.owner_service_id, + input.reviewer_service_id ?? null, + input.activated_at ?? new Date().toISOString(), + input.reason ?? null, + ); +} + +export function clearChannelOwnerLease(chatJid: string): void { + db.prepare('DELETE FROM channel_owner WHERE chat_jid = ?').run(chatJid); +} + +// --- Cross-service handoff accessors --- + +export function createServiceHandoff(input: { + chat_jid: string; + group_folder: string; + source_service_id: string; + target_service_id: string; + target_agent_type: AgentType; + prompt: string; + start_seq?: number | null; + end_seq?: number | null; + reason?: string | null; +}): ServiceHandoff { + const result = db + .prepare( + `INSERT INTO service_handoffs ( + chat_jid, + group_folder, + source_service_id, + target_service_id, + target_agent_type, + prompt, + start_seq, + end_seq, + reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + input.chat_jid, + input.group_folder, + input.source_service_id, + input.target_service_id, + input.target_agent_type, + input.prompt, + input.start_seq ?? null, + input.end_seq ?? null, + input.reason ?? null, + ); + + return db + .prepare('SELECT * FROM service_handoffs WHERE id = ?') + .get(result.lastInsertRowid) as ServiceHandoff; +} + +export function getPendingServiceHandoffs( + targetServiceId: string = SERVICE_SESSION_SCOPE, +): ServiceHandoff[] { + return db + .prepare( + `SELECT * + FROM service_handoffs + WHERE target_service_id = ? + AND status = 'pending' + ORDER BY created_at ASC, id ASC`, + ) + .all(targetServiceId) as ServiceHandoff[]; +} + +export function claimServiceHandoff(id: number): boolean { + const result = db + .prepare( + `UPDATE service_handoffs + SET status = 'claimed', + claimed_at = datetime('now') + WHERE id = ? + AND status = 'pending'`, + ) + .run(id); + return result.changes > 0; +} + +export function completeServiceHandoff(id: number): void { + db.prepare( + `UPDATE service_handoffs + SET status = 'completed', + completed_at = datetime('now'), + last_error = NULL + WHERE id = ?`, + ).run(id); +} + +export function failServiceHandoff(id: number, error: string): void { + db.prepare( + `UPDATE service_handoffs + SET status = 'failed', + completed_at = datetime('now'), + last_error = ? + WHERE id = ?`, + ).run(error, id); +} + +function normalizeStoredLastAgentSeqCursor( + cursor: string | number | null | undefined, + chatJid: string, +): number { + if (typeof cursor === 'number') { + return Number.isFinite(cursor) && cursor > 0 ? cursor : 0; + } + if (!cursor) return 0; + const trimmed = cursor.trim(); + if (/^\d+$/.test(trimmed)) { + return normalizeSeqCursor(trimmed); + } + return getLatestMessageSeqAtOrBefore(trimmed, chatJid); +} + +function parseLastAgentSeqState( + raw: string | undefined, + serviceId: string, +): Record { + if (!raw) return {}; + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `Invalid last_agent_seq JSON for ${serviceId}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`Invalid last_agent_seq JSON for ${serviceId}: not an object`); + } + + const cursors: Record = {}; + for (const [chatJid, cursor] of Object.entries(parsed)) { + if (typeof cursor === 'string' || typeof cursor === 'number') { + cursors[chatJid] = String(cursor); + } + } + return cursors; +} + +export function completeServiceHandoffAndAdvanceTargetCursor(input: { + id: number; + target_service_id: string; + chat_jid: string; + end_seq?: number | null; +}): string | null { + return db.transaction(() => { + let appliedCursor: string | null = null; + + if (input.end_seq != null) { + const currentState = parseLastAgentSeqState( + getRouterStateForService('last_agent_seq', input.target_service_id), + input.target_service_id, + ); + const existingSeq = normalizeStoredLastAgentSeqCursor( + currentState[input.chat_jid], + input.chat_jid, + ); + currentState[input.chat_jid] = String( + Math.max(existingSeq, input.end_seq), + ); + setRouterStateForService( + 'last_agent_seq', + JSON.stringify(currentState), + input.target_service_id, + ); + appliedCursor = currentState[input.chat_jid]; + } + + db.prepare( + `UPDATE service_handoffs + SET status = 'completed', + completed_at = datetime('now'), + last_error = NULL + WHERE id = ?`, + ).run(input.id); + + return appliedCursor; + })(); +} + // --- JSON migration --- function migrateJsonState(): void { diff --git a/src/duplicate-suppression.test.ts b/src/duplicate-suppression.test.ts new file mode 100644 index 0000000..8ad016e --- /dev/null +++ b/src/duplicate-suppression.test.ts @@ -0,0 +1,443 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +import { + _initTestDatabase, + storeMessage, + storeChatMetadata, + createProducedWorkItem, + getOpenWorkItem, + setRegisteredGroup, + isPairedRoomJid, + getLastBotFinalMessage, + markWorkItemDelivered, +} from './db.js'; +import { normalizeMessageForDedupe } from './router.js'; +import { isDuplicateOfLastBotFinal } from './message-runtime.js'; + +beforeEach(() => { + _initTestDatabase(); +}); + +// Helper to create a message +const createMessage = (overrides: Partial<{ + id: string; + chat_jid: string; + sender: string; + sender_name: string; + content: string; + timestamp: string; + is_from_me: boolean; + is_bot_message: boolean; +}> = {}) => ({ + id: overrides.id ?? 'msg-1', + chat_jid: overrides.chat_jid ?? 'dc:test-room', + sender: overrides.sender ?? 'user1', + sender_name: overrides.sender_name ?? 'User', + content: overrides.content ?? 'Hello', + timestamp: overrides.timestamp ?? new Date().toISOString(), + is_from_me: overrides.is_from_me ?? false, + is_bot_message: overrides.is_bot_message ?? false, +}); + +// Helper to setup chat metadata +const setupChat = (jid: string) => { + storeChatMetadata(jid, new Date().toISOString(), 'Test Chat', 'discord', true); +}; + +describe('isPairedRoomJid', () => { + it('returns true when both claude-code and codex are registered', () => { + const jid = 'dc:paired-room'; + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + expect(isPairedRoomJid(jid)).toBe(true); + }); + + it('returns false when only claude-code is registered', () => { + const jid = 'dc:single-claude'; + + setRegisteredGroup(jid, { + name: 'Single Claude', + folder: 'single-claude', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + expect(isPairedRoomJid(jid)).toBe(false); + }); + + it('returns false when only codex is registered', () => { + const jid = 'dc:single-codex'; + + setRegisteredGroup(jid, { + name: 'Single Codex', + folder: 'single-codex', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + expect(isPairedRoomJid(jid)).toBe(false); + }); +}); + +describe('getLastBotFinalMessage', () => { + it('returns the most recent bot message from any service (is_bot_message=1)', () => { + const jid = 'dc:test-room'; + const now = new Date(); + setupChat(jid); + + // Store older bot message (from any bot) + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'First bot message', + timestamp: new Date(now.getTime() - 1000).toISOString(), + is_from_me: true, + is_bot_message: true, + })); + + // Store newer bot message (from different bot - is_from_me=0 but is_bot_message=1) + storeMessage(createMessage({ + id: 'msg-2', + chat_jid: jid, + content: 'Second bot message', + timestamp: now.toISOString(), + is_from_me: false, // Different bot + is_bot_message: true, + })); + + // Store human message (should not be returned) + storeMessage(createMessage({ + id: 'msg-3', + chat_jid: jid, + content: 'Human message', + timestamp: new Date(now.getTime() + 1000).toISOString(), + is_from_me: false, + is_bot_message: false, + })); + + const lastMessages = getLastBotFinalMessage(jid, 'claude-code', 1); + expect(lastMessages).toHaveLength(1); + expect(lastMessages[0].content).toBe('Second bot message'); + }); + + it('returns empty array when no bot messages exist', () => { + const jid = 'dc:empty-room'; + setupChat(jid); + + const lastMessages = getLastBotFinalMessage(jid, 'claude-code', 1); + expect(lastMessages).toHaveLength(0); + }); +}); + +describe('normalizeMessageForDedupe', () => { + it('normalizes messages for comparison', () => { + expect(normalizeMessageForDedupe(' Hello World ')).toBe('hello world'); + expect(normalizeMessageForDedupe('Hello\n\nWorld')).toBe('hello world'); + expect(normalizeMessageForDedupe('Hello World')).toBe('hello world'); + expect(normalizeMessageForDedupe('HELLO WORLD')).toBe('hello world'); + }); + + it('handles empty strings', () => { + expect(normalizeMessageForDedupe('')).toBe(''); + expect(normalizeMessageForDedupe(' ')).toBe(''); + }); +}); + +describe('isDuplicateOfLastBotFinal (runtime function)', () => { + it('paired room: detects duplicate final message', () => { + const jid = 'dc:paired-room'; + setupChat(jid); + + // Register as paired room + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + // Store a bot message (from any bot) + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'DONE — Task completed successfully', + timestamp: new Date().toISOString(), + is_from_me: false, // Different bot + is_bot_message: true, + })); + + // Verify it's a paired room + expect(isPairedRoomJid(jid)).toBe(true); + + // Verify duplicate detection works via the actual runtime function + expect(isDuplicateOfLastBotFinal(jid, 'DONE — Task completed successfully')).toBe(true); + expect(isDuplicateOfLastBotFinal(jid, 'done — task completed successfully')).toBe(true); // Normalized match + }); + + it('non-paired room: duplicate check is bypassed', () => { + const jid = 'dc:single-bot'; + setupChat(jid); + + // Register only one bot + setRegisteredGroup(jid, { + name: 'Single Bot', + folder: 'single-bot', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + // Store a bot message + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'DONE — Task completed', + timestamp: new Date().toISOString(), + is_from_me: true, + is_bot_message: true, + })); + + // Verify it's NOT a paired room + expect(isPairedRoomJid(jid)).toBe(false); + + // Duplicate check should be bypassed (return false) + expect(isDuplicateOfLastBotFinal(jid, 'DONE — Task completed')).toBe(false); + }); + + it('non-duplicate message in paired room is not suppressed', () => { + const jid = 'dc:paired-room'; + setupChat(jid); + + // Register as paired room + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + // Store a bot message + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'First message content', + timestamp: new Date().toISOString(), + is_from_me: true, + is_bot_message: true, + })); + + // Verify different content is not a duplicate + expect(isDuplicateOfLastBotFinal(jid, 'Different message content')).toBe(false); + expect(isDuplicateOfLastBotFinal(jid, 'First message content')).toBe(true); + }); + + it('cross-bot duplicate detection: claude detects codex message as duplicate', () => { + const jid = 'dc:paired-room'; + setupChat(jid); + + // Register as paired room + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + // Store a bot message from "codex" (is_from_me=0) + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'DONE — Analysis complete', + timestamp: new Date().toISOString(), + is_from_me: false, // Other bot + is_bot_message: true, + })); + + // Verify claude service detects this as duplicate (cross-bot detection) + expect(isDuplicateOfLastBotFinal(jid, 'DONE — Analysis complete')).toBe(true); + }); + + it('normalization handles whitespace and case differences', () => { + const jid = 'dc:paired-room'; + setupChat(jid); + + // Register as paired room + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + // Store a bot message with specific formatting + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'DONE — Task completed\n\nSuccessfully', + timestamp: new Date().toISOString(), + is_from_me: true, + is_bot_message: true, + })); + + // Same content with different whitespace should be detected as duplicate + expect(isDuplicateOfLastBotFinal(jid, 'done — task completed successfully')).toBe(true); + expect(isDuplicateOfLastBotFinal(jid, ' DONE — Task completed Successfully ')).toBe(true); + + // Different content should not be duplicate + expect(isDuplicateOfLastBotFinal(jid, 'FAILED — Task failed')).toBe(false); + }); + + it('work item lifecycle: produced -> delivered (duplicate)', () => { + const jid = 'dc:paired-room'; + setupChat(jid); + + // Register as paired room + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + // Store a bot message (simulating previous delivery) + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'DONE — Task completed successfully', + timestamp: new Date().toISOString(), + is_from_me: true, + is_bot_message: true, + })); + + // Create a work item with the same content (duplicate) + const workItem = createProducedWorkItem({ + group_folder: 'paired-room', + chat_jid: jid, + agent_type: 'claude-code', + start_seq: 1, + end_seq: 2, + result_payload: 'DONE — Task completed successfully', + }); + + expect(workItem.status).toBe('produced'); + + // Verify duplicate detection via actual runtime function + expect(isDuplicateOfLastBotFinal(jid, workItem.result_payload)).toBe(true); + + // Simulate suppression: mark as delivered without sending + markWorkItemDelivered(workItem.id, null); + + // Verify work item is marked delivered + const openItem = getOpenWorkItem(jid, 'claude-code'); + expect(openItem).toBeUndefined(); // No open items because it was marked delivered + }); + + it('work item lifecycle: produced -> pending -> delivered (non-duplicate)', () => { + const jid = 'dc:paired-room'; + setupChat(jid); + + // Register as paired room + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@claude', + added_at: new Date().toISOString(), + agentType: 'claude-code', + }); + + setRegisteredGroup(jid, { + name: 'Paired Room', + folder: 'paired-room', + trigger: '@codex', + added_at: new Date().toISOString(), + agentType: 'codex', + }); + + // Store a bot message + storeMessage(createMessage({ + id: 'msg-1', + chat_jid: jid, + content: 'First message', + timestamp: new Date().toISOString(), + is_from_me: true, + is_bot_message: true, + })); + + // Create a work item with DIFFERENT content (non-duplicate) + const workItem = createProducedWorkItem({ + group_folder: 'paired-room', + chat_jid: jid, + agent_type: 'claude-code', + start_seq: 1, + end_seq: 2, + result_payload: 'Second message (different)', + }); + + expect(workItem.status).toBe('produced'); + + // Verify NOT a duplicate + expect(isDuplicateOfLastBotFinal(jid, workItem.result_payload)).toBe(false); + }); +}); diff --git a/src/group-folder.ts b/src/group-folder.ts index 52c646d..d429945 100644 --- a/src/group-folder.ts +++ b/src/group-folder.ts @@ -68,6 +68,18 @@ export function resolveGroupSessionsPath(folder: string): string { return sessionsPath; } +export function resolveServiceGroupSessionsPath( + folder: string, + serviceId: string, +): string { + assertValidGroupFolder(folder); + assertValidRuntimeSegment(serviceId, 'service ID'); + const sessionsBaseDir = path.resolve(DATA_DIR, 'sessions'); + const sessionsPath = path.resolve(sessionsBaseDir, folder, 'services', serviceId); + ensureWithinBase(sessionsBaseDir, sessionsPath); + return sessionsPath; +} + export function resolveTaskRuntimeIpcPath( folder: string, taskId: string, @@ -91,3 +103,24 @@ export function resolveTaskSessionsPath( ensureWithinBase(sessionsBaseDir, sessionsPath); return sessionsPath; } + +export function resolveServiceTaskSessionsPath( + folder: string, + serviceId: string, + taskId: string, +): string { + assertValidGroupFolder(folder); + assertValidRuntimeSegment(serviceId, 'service ID'); + assertValidRuntimeSegment(taskId, 'task ID'); + const sessionsBaseDir = path.resolve(DATA_DIR, 'sessions'); + const sessionsPath = path.resolve( + sessionsBaseDir, + folder, + 'services', + serviceId, + 'tasks', + taskId, + ); + ensureWithinBase(sessionsBaseDir, sessionsPath); + return sessionsPath; +} diff --git a/src/index.ts b/src/index.ts index 2f07a82..2358282 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,10 @@ import { DATA_DIR, IDLE_TIMEOUT, POLL_INTERVAL, + SERVICE_ID, SERVICE_AGENT_TYPE, + isClaudeService, + isReviewService, isSessionCommandSenderAllowed, STATUS_CHANNEL_ID, STATUS_UPDATE_INTERVAL, @@ -41,7 +44,11 @@ import { composeDashboardContent } from './dashboard-render.js'; import { GroupQueue } from './group-queue.js'; import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js'; import { startIpcWatcher } from './ipc.js'; -import { findChannel, formatOutbound } from './router.js'; +import { + findChannel, + formatOutbound, + normalizeMessageForDedupe, +} from './router.js'; import { buildRestartAnnouncement, buildInterruptedRestartAnnouncement, @@ -64,12 +71,17 @@ import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { normalizeStoredSeqCursor } from './message-cursor.js'; import { initCodexTokenRotation } from './codex-token-rotation.js'; -import { initTokenRotation } from './token-rotation.js'; +import { hasAvailableClaudeToken, initTokenRotation } from './token-rotation.js'; import { shouldStartTokenRefreshLoop, startTokenRefreshLoop, stopTokenRefreshLoop, } from './token-refresh.js'; +import { + getActiveCodexFailoverLeases, + restoreDefaultChannelLease, +} from './service-routing.js'; +import { FAILOVER_MIN_DURATION_MS } from './config.js'; // Token rotation is initialized lazily on first use or at startup below @@ -315,9 +327,14 @@ async function main(): Promise { loadState(); // Graceful shutdown handlers + let leaseRecoveryTimer: ReturnType | null = null; const shutdown = async (signal: string) => { logger.info({ signal }, 'Shutdown signal received'); stopTokenRefreshLoop(); + if (leaseRecoveryTimer) { + clearInterval(leaseRecoveryTimer); + leaseRecoveryTimer = null; + } const interruptedGroups = queue .getStatuses(Object.keys(registeredGroups)) .filter( @@ -410,19 +427,23 @@ async function main(): Promise { } // Start subsystems (independently of connection handler) - startSchedulerLoop({ - registeredGroups: () => registeredGroups, - getSessions: () => sessions, - queue, - onProcess: (groupJid, proc, processName, ipcDir) => - queue.registerProcess(groupJid, proc, processName, ipcDir), - sendMessage: (jid, rawText) => - sendFormattedChannelMessage(channels, jid, rawText), - sendTrackedMessage: (jid, rawText) => - sendFormattedTrackedChannelMessage(channels, jid, rawText), - editTrackedMessage: (jid, messageId, rawText) => - editFormattedTrackedChannelMessage(channels, jid, messageId, rawText), - }); + if (!isReviewService()) { + startSchedulerLoop({ + registeredGroups: () => registeredGroups, + getSessions: () => sessions, + queue, + onProcess: (groupJid, proc, processName, ipcDir) => + queue.registerProcess(groupJid, proc, processName, ipcDir), + sendMessage: (jid, rawText) => + sendFormattedChannelMessage(channels, jid, rawText), + sendTrackedMessage: (jid, rawText) => + sendFormattedTrackedChannelMessage(channels, jid, rawText), + editTrackedMessage: (jid, messageId, rawText) => + editFormattedTrackedChannelMessage(channels, jid, messageId, rawText), + }); + } else { + logger.info({ serviceId: SERVICE_ID }, 'Skipping scheduler for review service'); + } startIpcWatcher({ sendMessage: (jid, text) => { const channel = findChannel(channels, jid); @@ -467,6 +488,7 @@ async function main(): Promise { } await startUnifiedDashboard({ assistantName: ASSISTANT_NAME, + serviceId: SERVICE_ID, serviceAgentType: SERVICE_AGENT_TYPE, statusChannelId: STATUS_CHANNEL_ID, statusUpdateInterval: STATUS_UPDATE_INTERVAL, @@ -482,6 +504,44 @@ async function main(): Promise { }, purgeOnStart: true, }); + + if (isClaudeService()) { + leaseRecoveryTimer = setInterval(() => { + if (!hasAvailableClaudeToken()) { + return; + } + const now = Date.now(); + for (const lease of getActiveCodexFailoverLeases()) { + const activatedMs = lease.activatedAt + ? new Date(lease.activatedAt).getTime() + : NaN; + if (Number.isNaN(activatedMs)) { + logger.warn( + { chatJid: lease.chatJid, activatedAt: lease.activatedAt }, + 'Failover lease has unparseable activated_at, skipping auto-restore', + ); + continue; + } + const elapsed = now - activatedMs; + if (elapsed < FAILOVER_MIN_DURATION_MS) { + logger.debug( + { + chatJid: lease.chatJid, + elapsedMin: Math.round(elapsed / 60_000), + minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000), + }, + 'Failover lease still within minimum hold period, skipping restore', + ); + continue; + } + restoreDefaultChannelLease(lease.chatJid); + logger.info( + { chatJid: lease.chatJid, serviceId: SERVICE_ID, elapsedMin: Math.round(elapsed / 60_000) }, + 'Claude token available and failover hold period elapsed, restored default channel lease', + ); + } + }, 5_000); + } runtime.startMessageLoop().catch((err) => { logger.fatal({ err }, 'Message loop crashed unexpectedly'); process.exit(1); diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index c548850..76895dd 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -11,13 +11,29 @@ vi.mock('./available-groups.js', () => ({ })); vi.mock('./config.js', () => ({ + CODEX_MAIN_SERVICE_ID: 'codex-main', + CODEX_REVIEW_SERVICE_ID: 'codex-review', DATA_DIR: '/tmp/ejclaw-test-data', + SERVICE_SESSION_SCOPE: 'claude', })); vi.mock('./db.js', () => ({ + createServiceHandoff: vi.fn(), getAllTasks: vi.fn(() => []), })); +vi.mock('./service-routing.js', () => ({ + activateCodexFailover: vi.fn(), + getEffectiveChannelLease: vi.fn(() => ({ + chat_jid: 'group@test', + owner_service_id: 'claude', + reviewer_service_id: 'codex-main', + activated_at: null, + reason: null, + explicit: false, + })), +})); + vi.mock('./logger.js', () => ({ logger: { debug: vi.fn(), @@ -27,41 +43,35 @@ vi.mock('./logger.js', () => ({ }, })); -vi.mock('./provider-fallback.js', () => ({ - detectFallbackTrigger: vi.fn((error?: string | null) => { - const lower = (error || '').toLowerCase(); - if ( - lower.includes('does not have access to claude') || - (lower.includes('failed to authenticate') && - lower.includes('403') && - lower.includes('terminated')) - ) { - return { shouldFallback: true, reason: 'org-access-denied' }; - } - if ( - lower.includes('429') || - lower.includes('rate limit') || - lower.includes('hit your limit') - ) { - return { shouldFallback: true, reason: '429' }; - } - return { shouldFallback: false, reason: '' }; - }), - getActiveProvider: vi.fn(async () => 'claude'), - getFallbackEnvOverrides: vi.fn(() => ({ - ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', - ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', - ANTHROPIC_MODEL: 'kimi-k2.5', - })), - getFallbackProviderName: vi.fn(() => 'kimi'), - hasGroupProviderOverride: vi.fn(() => false), - isFallbackEnabled: vi.fn(() => true), - isPrimaryNoFallbackCooldownActive: vi.fn(() => false), - markPrimaryCooldown: vi.fn(), -})); +vi.mock('./agent-error-detection.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + classifyRotationTrigger: vi.fn((error?: string | null) => { + const lower = (error || '').toLowerCase(); + if ( + lower.includes('does not have access to claude') || + (lower.includes('failed to authenticate') && + lower.includes('403') && + lower.includes('terminated')) + ) { + return { shouldRetry: true, reason: 'org-access-denied' }; + } + if ( + lower.includes('429') || + lower.includes('rate limit') || + lower.includes('hit your limit') + ) { + return { shouldRetry: true, reason: '429' }; + } + return { shouldRetry: false, reason: '' }; + }), + }; +}); vi.mock('./session-recovery.js', () => ({ shouldResetSessionOnAgentFailure: vi.fn(() => false), + shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false), })); vi.mock('./token-rotation.js', () => ({ @@ -96,9 +106,11 @@ vi.mock('./memento-client.js', () => ({ import * as agentRunner from './agent-runner.js'; import * as codexTokenRotation from './codex-token-rotation.js'; +import * as db from './db.js'; import { buildRoomMemoryBriefing } from './memento-client.js'; import { runAgentForGroup } from './message-agent-executor.js'; -import * as providerFallback from './provider-fallback.js'; +import * as sessionRecovery from './session-recovery.js'; +import * as serviceRouting from './service-routing.js'; import * as tokenRotation from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; @@ -129,9 +141,6 @@ function makeDeps() { describe('runAgentForGroup room memory', () => { beforeEach(() => { vi.resetAllMocks(); - vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); - vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(false); - vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ status: 'success', result: 'ok', @@ -167,7 +176,6 @@ describe('runAgentForGroup room memory', () => { }), expect.any(Function), undefined, - undefined, ); }); @@ -196,6 +204,59 @@ describe('runAgentForGroup room memory', () => { }), expect.any(Function), undefined, + ); + }); + + it('injects suppress token instructions into the agent prompt', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + + await runAgentForGroup(makeDeps(), { + group, + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-suppress', + suppressToken: '__TEST_SUPPRESS__', + }); + + expect(agentRunner.runAgentProcess).toHaveBeenCalledWith( + group, + expect.objectContaining({ + prompt: expect.stringContaining( + 'If you have no user-visible content to send for this turn, output exactly this token and nothing else: __TEST_SUPPRESS__', + ), + }), + expect.any(Function), + undefined, + ); + }); + + it('adds reviewer silence guidance when the current service is the reviewer for the chat', async () => { + const group = { ...makeGroup(), folder: 'test-group' }; + vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({ + chat_jid: 'group@test', + owner_service_id: 'claude', + reviewer_service_id: 'claude', + activated_at: null, + reason: null, + explicit: false, + }); + + await runAgentForGroup(makeDeps(), { + group, + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-review-suppress', + suppressToken: '__TEST_SUPPRESS__', + }); + + expect(agentRunner.runAgentProcess).toHaveBeenCalledWith( + group, + expect.objectContaining({ + prompt: expect.stringContaining( + 'If you are only agreeing, mirroring, or restating without adding a concrete correction, risk, missing prerequisite, test gap, or code change, output only the token.', + ), + }), + expect.any(Function), undefined, ); }); @@ -205,14 +266,11 @@ describe('runAgentForGroup Claude rotation', () => { beforeEach(() => { vi.resetAllMocks(); vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined); - vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); - vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true); - vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); vi.mocked(tokenRotation.rotateToken).mockReturnValue(false); }); - it('rotates to another Claude account before falling back to Kimi', async () => { + it('rotates to another Claude account on usage exhaustion', async () => { const outputs: string[] = []; vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); @@ -223,7 +281,7 @@ describe('runAgentForGroup Claude rotation', () => { await onOutput?.({ status: 'success', phase: 'final', - result: 'You’re out of extra usage · resets 4am (Asia/Seoul)', + result: "You're out of extra usage · resets 4am (Asia/Seoul)", }); return { status: 'success', @@ -256,7 +314,7 @@ describe('runAgentForGroup Claude rotation', () => { expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + // No fallback provider — rotation is the only recovery mechanism expect(outputs).toEqual(['회전된 Claude 응답입니다.']); }); @@ -311,15 +369,15 @@ describe('runAgentForGroup Claude rotation', () => { expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + // No fallback provider — rotation is the only recovery mechanism expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']); }); - it('suppresses Claude 502 HTML and falls back without forwarding it', async () => { + it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => { const outputs: string[] = []; - vi.mocked(agentRunner.runAgentProcess) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce( + async (_group, _input, _onProcess, onOutput) => { await onOutput?.({ status: 'success', phase: 'intermediate', @@ -336,24 +394,64 @@ describe('runAgentForGroup Claude rotation', () => { status: 'success', result: null, }; - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'Kimi 폴백 응답입니다.', - }); - return { - status: 'success', - result: null, - }; - }); + }, + ); const result = await runAgentForGroup(makeDeps(), { group: makeGroup(), prompt: 'hello', chatJid: 'group@test', - runId: 'run-claude-502-fallback', + runId: 'run-claude-502-error', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('error'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); + expect(outputs).toEqual([]); + expect(db.createServiceHandoff).not.toHaveBeenCalled(); + }); + + it('clears the Claude session and retries fresh when a retryable thinking 400 is surfaced as text', async () => { + const outputs: string[] = []; + const deps = makeDeps(); + + vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure) + .mockReturnValueOnce(true) + .mockReturnValueOnce(false); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'fresh Claude retry success', + }); + return { + status: 'success', + result: null, + newSessionId: 'claude-session-fresh', + }; + }); + + const result = await runAgentForGroup(deps, { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-retryable-thinking-400-retry', onOutput: async (output) => { if (typeof output.result === 'string') outputs.push(output.result); }, @@ -361,15 +459,63 @@ describe('runAgentForGroup Claude rotation', () => { expect(result).toBe('success'); expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); - expect(tokenRotation.rotateToken).not.toHaveBeenCalled(); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'overloaded', - undefined, - ); - expect(outputs).toEqual(['Kimi 폴백 응답입니다.']); + expect(deps.clearSession).toHaveBeenCalledTimes(1); + expect(deps.clearSession).toHaveBeenCalledWith('test-claude'); + // No fallback provider — rotation is the only recovery mechanism + expect(outputs).toEqual(['fresh Claude retry success']); }); - it('stops after all Claude accounts are usage-exhausted without falling back to Kimi', async () => { + it('returns error when the fresh Claude retry also hits the same retryable thinking 400', async () => { + const outputs: string[] = []; + const deps = makeDeps(); + + vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure) + .mockReturnValueOnce(true) + .mockReturnValueOnce(true); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'intermediate', + result: + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(deps, { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-retryable-thinking-400-error', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('error'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(deps.clearSession).toHaveBeenCalledTimes(2); + expect(outputs).toEqual([]); + }); + + it('returns error after all Claude accounts are usage-exhausted', async () => { const outputs: string[] = []; vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); @@ -382,7 +528,7 @@ describe('runAgentForGroup Claude rotation', () => { await onOutput?.({ status: 'success', phase: 'final', - result: 'You’re out of extra usage · resets 4am (Asia/Seoul)', + result: 'You\u2019re out of extra usage \u00b7 resets 4am (Asia/Seoul)', }); return { status: 'success', @@ -393,18 +539,7 @@ describe('runAgentForGroup Claude rotation', () => { await onOutput?.({ status: 'success', phase: 'final', - result: "You're out of extra usage · resets 4am (Asia/Seoul)", - }); - return { - status: 'success', - result: null, - }; - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'Kimi 폴백 응답입니다.', + result: "You're out of extra usage \u00b7 resets 4am (Asia/Seoul)", }); return { status: 'success', @@ -416,20 +551,71 @@ describe('runAgentForGroup Claude rotation', () => { group: makeGroup(), prompt: 'hello', chatJid: 'group@test', - runId: 'run-fallback-after-rotation', + runId: 'run-all-exhausted-error', + startSeq: 10, + endSeq: 12, + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2); + expect(outputs).toEqual([]); + expect(serviceRouting.activateCodexFailover).toHaveBeenCalledWith( + 'group@test', + 'claude-usage-exhausted', + ); + expect(db.createServiceHandoff).toHaveBeenCalledWith( + expect.objectContaining({ + chat_jid: 'group@test', + target_service_id: 'codex-review', + target_agent_type: 'codex', + start_seq: 10, + end_seq: 12, + reason: 'claude-usage-exhausted', + }), + ); + }); + + it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); + + vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'progress', + result: '대화 요약 중...', + }); + await onOutput?.({ + status: 'success', + phase: 'final', + result: "You've hit your limit · resets 2am (Asia/Seoul)", + }); + return { + status: 'success', + result: null, + }; + }, + ); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-progress-before-usage-banner', onOutput: async (output) => { if (typeof output.result === 'string') outputs.push(output.result); }, }); expect(result).toBe('error'); - expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); - expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'usage-exhausted', - undefined, - ); - expect(outputs).toEqual([]); + expect(outputs).toEqual(['대화 요약 중...']); + expect(db.createServiceHandoff).not.toHaveBeenCalled(); }); it('rotates to another Claude account when Claude streams an org access denied banner', async () => { @@ -483,11 +669,57 @@ describe('runAgentForGroup Claude rotation', () => { expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + // No fallback provider — rotation is the only recovery mechanism expect(outputs).toEqual(['org access denied 회전 성공 응답']); }); - it('stops after all Claude accounts are org-access-denied without falling back to Kimi', async () => { + it('rotates when Claude surfaces 403 terminated as a success result', async () => { + const outputs: string[] = []; + + vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); + vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true); + + vi.mocked(agentRunner.runAgentProcess) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'Failed to authenticate. API Error: 403 terminated', + }); + return { + status: 'success', + result: null, + }; + }) + .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: '403 회전 성공 응답', + }); + return { + status: 'success', + result: null, + }; + }); + + const result = await runAgentForGroup(makeDeps(), { + group: makeGroup(), + prompt: 'hello', + chatJid: 'group@test', + runId: 'run-403-success-rotation', + onOutput: async (output) => { + if (typeof output.result === 'string') outputs.push(output.result); + }, + }); + + expect(result).toBe('success'); + expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); + expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); + expect(outputs).toEqual(['403 회전 성공 응답']); + }); + + it('returns error after all Claude accounts are org-access-denied', async () => { const outputs: string[] = []; vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2); @@ -519,54 +751,34 @@ describe('runAgentForGroup Claude rotation', () => { result: null, error: 'Failed to authenticate. API Error: 403 terminated', }; - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'Kimi 폴백 응답입니다.', - }); - return { - status: 'success', - result: null, - }; }); const result = await runAgentForGroup(makeDeps(), { group: makeGroup(), prompt: 'hello', chatJid: 'group@test', - runId: 'run-org-access-denied-no-fallback', + runId: 'run-org-access-denied-error', onOutput: async (output) => { if (typeof output.result === 'string') outputs.push(output.result); }, }); - expect(result).toBe('error'); + expect(result).toBe('success'); expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'org-access-denied', - undefined, - ); expect(outputs).toEqual([]); - }); - - it('skips execution entirely when Claude no-fallback cooldown is already active', async () => { - vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('kimi'); - vi.mocked( - providerFallback.isPrimaryNoFallbackCooldownActive, - ).mockReturnValue(true); - - const result = await runAgentForGroup(makeDeps(), { - group: makeGroup(), - prompt: 'hello', - chatJid: 'group@test', - runId: 'run-skip-primary-cooldown', - }); - - expect(result).toBe('error'); - expect(agentRunner.runAgentProcess).not.toHaveBeenCalled(); + expect(serviceRouting.activateCodexFailover).toHaveBeenCalledWith( + 'group@test', + 'claude-org-access-denied', + ); + expect(db.createServiceHandoff).toHaveBeenCalledWith( + expect.objectContaining({ + chat_jid: 'group@test', + target_service_id: 'codex-review', + target_agent_type: 'codex', + reason: 'claude-org-access-denied', + }), + ); }); it('does not mistake a normal response quoting the banner text for a usage error', async () => { @@ -601,7 +813,6 @@ describe('runAgentForGroup Claude rotation', () => { expect(result).toBe('success'); expect(tokenRotation.rotateToken).not.toHaveBeenCalled(); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); expect(outputs).toEqual([ "상태 문구 예시: You're out of extra usage · resets 4am (Asia/Seoul) 라는 배너가 뜰 수 있습니다.", ]); @@ -612,9 +823,6 @@ describe('runAgentForGroup Codex rotation', () => { beforeEach(() => { vi.resetAllMocks(); vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined); - vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); - vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(false); - vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(2); vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValueOnce(true); }); diff --git a/src/message-agent-executor.ts b/src/message-agent-executor.ts index 112cf68..78a65a8 100644 --- a/src/message-agent-executor.ts +++ b/src/message-agent-executor.ts @@ -1,4 +1,3 @@ -import path from 'path'; import { getErrorMessage } from './utils.js'; import { @@ -8,23 +7,25 @@ import { writeTasksSnapshot, } from './agent-runner.js'; import { listAvailableGroups } from './available-groups.js'; -import { DATA_DIR } from './config.js'; -import { getAllTasks } from './db.js'; +import { createServiceHandoff, getAllTasks } from './db.js'; import { GroupQueue } from './group-queue.js'; import { logger } from './logger.js'; import { buildRoomMemoryBriefing } from './memento-client.js'; import { - detectFallbackTrigger, - getActiveProvider, - getFallbackEnvOverrides, - getFallbackProviderName, - hasGroupProviderOverride, - isFallbackEnabled, - isPrimaryNoFallbackCooldownActive, - markPrimaryCooldown, -} from './provider-fallback.js'; + classifyRotationTrigger, + type AgentTriggerReason, +} from './agent-error-detection.js'; import { runClaudeRotationLoop } from './provider-retry.js'; -import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; +import { + shouldResetSessionOnAgentFailure, + shouldRetryFreshSessionOnAgentFailure, +} from './session-recovery.js'; +import { CODEX_MAIN_SERVICE_ID, CODEX_REVIEW_SERVICE_ID, SERVICE_SESSION_SCOPE } from './config.js'; +import { + buildSuppressTokenPrompt, + classifySuppressTokenOutput, +} from './output-suppression.js'; +import { activateCodexFailover, getEffectiveChannelLease } from './service-routing.js'; import { evaluateStreamedOutput, type StreamedOutputState, @@ -35,13 +36,12 @@ import { getCodexAccountCount, markCodexTokenHealthy, } from './codex-token-rotation.js'; -import type { - AgentTriggerReason, - CodexRotationReason, -} from './agent-error-detection.js'; +import type { CodexRotationReason } from './agent-error-detection.js'; import { getTokenCount } from './token-rotation.js'; import type { RegisteredGroup } from './types.js'; +// ── Main executor ───────────────────────────────────────────────── + export interface MessageAgentExecutorDeps { assistantName: string; queue: Pick; @@ -58,10 +58,22 @@ export async function runAgentForGroup( prompt: string; chatJid: string; runId: string; + suppressToken?: string; + startSeq?: number | null; + endSeq?: number | null; onOutput?: (output: AgentOutput) => Promise; }, ): Promise<'success' | 'error'> { - const { group, prompt, chatJid, runId, onOutput } = args; + const { + group, + prompt, + chatJid, + runId, + suppressToken, + startSeq, + endSeq, + onOutput, + } = args; const isMain = group.isMain === true; const isClaudeCodeAgent = (group.agentType || 'claude-code') === 'claude-code'; @@ -97,20 +109,61 @@ export async function runAgentForGroup( let resetSessionRequested = false; - const settingsPath = path.join( - DATA_DIR, - 'sessions', - group.folder, - '.claude', - 'settings.json', - ); - const groupHasOverride = hasGroupProviderOverride(settingsPath); const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1; - const canFallback = - isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride; + const currentLease = getEffectiveChannelLease(chatJid); + const reviewerMode = currentLease.reviewer_service_id === SERVICE_SESSION_SCOPE; + const effectivePrompt = buildSuppressTokenPrompt(prompt, suppressToken, { + reviewerMode, + }); + + const shouldHandoffToCodex = ( + reason: AgentTriggerReason, + sawVisibleOutput: boolean, + ): boolean => { + if (sawVisibleOutput) { + return false; + } + return ( + reason === '429' || + reason === 'usage-exhausted' || + reason === 'auth-expired' || + reason === 'org-access-denied' + ); + }; + + const maybeHandoffToCodex = ( + reason: AgentTriggerReason, + sawVisibleOutput: boolean, + ): boolean => { + if (!isClaudeCodeAgent) return false; + if (!shouldHandoffToCodex(reason, sawVisibleOutput)) { + return false; + } + if (currentLease.reviewer_service_id === null) { + return false; + } + + activateCodexFailover(chatJid, `claude-${reason}`); + createServiceHandoff({ + chat_jid: chatJid, + group_folder: group.folder, + source_service_id: SERVICE_SESSION_SCOPE, + target_service_id: CODEX_REVIEW_SERVICE_ID, + target_agent_type: 'codex', + prompt, + start_seq: startSeq ?? null, + end_seq: endSeq ?? null, + reason: `claude-${reason}`, + }); + logger.warn( + { chatJid, group: group.name, runId, reason }, + 'Claude unavailable, handed off current turn to codex-review', + ); + return true; + }; const agentInput = { - prompt, + prompt: effectivePrompt, sessionId, memoryBriefing, groupFolder: group.folder, @@ -126,29 +179,31 @@ export async function runAgentForGroup( output?: AgentOutput; error?: unknown; sawOutput: boolean; + sawVisibleOutput: boolean; sawSuccessNullResultWithoutOutput: boolean; + retryableSessionFailureDetected: boolean; streamedTriggerReason?: { reason: AgentTriggerReason; retryAfterMs?: number; }; }> => { - const persistSessionIds = provider === 'claude'; let streamedState: StreamedOutputState = { sawOutput: false, + sawVisibleOutput: false, sawSuccessNullResultWithoutOutput: false, }; const wrappedOnOutput = onOutput ? async (output: AgentOutput) => { if ( - persistSessionIds && isClaudeCodeAgent && + provider === 'claude' && shouldResetSessionOnAgentFailure(output) ) { resetSessionRequested = true; } if ( - persistSessionIds && + provider === 'claude' && output.newSessionId && !resetSessionRequested ) { @@ -161,7 +216,7 @@ export async function runAgentForGroup( trackSuccessNullResult: true, shortCircuitTriggeredErrors: provider === 'claude' - ? canFallback || canRotateToken + ? canRotateToken : getCodexAccountCount() > 1, }); streamedState = evaluation.state; @@ -179,7 +234,7 @@ export async function runAgentForGroup( reason: evaluation.newTrigger.reason, resultPreview: output.result.slice(0, 120), }, - 'Detected Claude fallback trigger in successful output', + 'Detected Claude rotation trigger in successful output', ); } else if ( evaluation.newTrigger && @@ -194,7 +249,7 @@ export async function runAgentForGroup( errorPreview: output.error.slice(0, 120), }, provider === 'claude' - ? 'Detected Claude fallback trigger in streamed error output' + ? 'Detected Claude rotation trigger in streamed error output' : 'Detected Codex rotation trigger in streamed error output', ); } @@ -215,39 +270,53 @@ export async function runAgentForGroup( return; } + if (evaluation.suppressedRetryableSessionFailure) { + logger.warn( + { + chatJid, + group: group.name, + runId, + resultPreview: + typeof output.result === 'string' + ? output.result.slice(0, 160) + : output.error?.slice(0, 160), + }, + 'Suppressed retryable Claude session failure from chat output', + ); + return; + } + if (!evaluation.shouldForwardOutput) { return; } + const suppressState = + typeof output.result === 'string' + ? classifySuppressTokenOutput(output.result, suppressToken) + : 'none'; + if ( + typeof output.result === 'string' && + output.result.length > 0 && + suppressState === 'none' + ) { + streamedState = { + ...evaluation.state, + sawVisibleOutput: true, + }; + } await onOutput(output); } : undefined; - if (provider !== 'claude') { - logger.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - runId, - provider, - }, - `Claude provider in cooldown, routing request to ${provider}`, - ); - } - const agentType = group.agentType || 'claude-code'; - const providerLabel = canFallback ? provider : agentType; logger.info( { chatJid, group: group.name, groupFolder: group.folder, runId, - provider: providerLabel, - canFallback, - groupHasOverride, + provider: agentType, }, - `Using provider: ${providerLabel}`, + `Using provider: ${agentType}`, ); try { @@ -255,17 +324,14 @@ export async function runAgentForGroup( group, { ...agentInput, - sessionId: persistSessionIds ? sessionId : undefined, + sessionId: provider === 'claude' ? sessionId : undefined, }, (proc, processName, ipcDir) => deps.queue.registerProcess(chatJid, proc, processName, ipcDir), wrappedOnOutput, - isClaudeCodeAgent && provider !== 'claude' - ? getFallbackEnvOverrides() - : undefined, ); - if (persistSessionIds && output.newSessionId) { + if (provider === 'claude' && output.newSessionId) { deps.persistSession(group.folder, output.newSessionId); } @@ -285,75 +351,27 @@ export async function runAgentForGroup( return { output, sawOutput: streamedState.sawOutput, + sawVisibleOutput: streamedState.sawVisibleOutput, sawSuccessNullResultWithoutOutput: streamedState.sawSuccessNullResultWithoutOutput, + retryableSessionFailureDetected: + streamedState.retryableSessionFailureDetected === true, streamedTriggerReason: streamedState.streamedTriggerReason, }; } catch (error) { return { error, sawOutput: streamedState.sawOutput, + sawVisibleOutput: streamedState.sawVisibleOutput, sawSuccessNullResultWithoutOutput: streamedState.sawSuccessNullResultWithoutOutput, + retryableSessionFailureDetected: + streamedState.retryableSessionFailureDetected === true, streamedTriggerReason: streamedState.streamedTriggerReason, }; } }; - const runFallbackAttempt = async ( - reason: AgentTriggerReason, - retryAfterMs?: number, - ): Promise<'success' | 'error'> => { - const fallbackName = getFallbackProviderName(); - markPrimaryCooldown(reason, retryAfterMs); - - logger.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - runId, - reason, - retryAfterMs, - fallbackProvider: fallbackName, - }, - `Falling back to provider: ${fallbackName} (reason: ${reason})`, - ); - - const fallbackAttempt = await runAttempt(fallbackName); - if (fallbackAttempt.error) { - logger.error( - { - chatJid, - group: group.name, - groupFolder: group.folder, - runId, - provider: fallbackName, - err: fallbackAttempt.error, - }, - 'Fallback provider also threw', - ); - return 'error'; - } - - if (fallbackAttempt.output?.status === 'error') { - logger.error( - { - chatJid, - group: group.name, - groupFolder: group.folder, - runId, - provider: fallbackName, - error: fallbackAttempt.output.error, - }, - `Fallback provider (${fallbackName}) also failed`, - ); - return 'error'; - } - - return 'success'; - }; - const retryCodexWithRotation = async ( initialTrigger: { reason: CodexRotationReason }, rotationMessage?: string, @@ -496,63 +514,86 @@ export async function runAgentForGroup( return 'success'; case 'error': return 'error'; - case 'no-fallback': - return 'error'; - case 'needs-fallback': - if (outcome.trigger.reason === 'success-null-result') { - return canFallback - ? runFallbackAttempt('success-null-result') - : 'error'; - } - if (!canFallback) { - logger.warn( - { ...logCtx, reason: outcome.trigger.reason }, - 'All Claude tokens exhausted and fallback disabled', - ); - return 'error'; - } - return runFallbackAttempt( - outcome.trigger.reason, - outcome.trigger.retryAfterMs, - ); } }; - const provider = canFallback ? await getActiveProvider() : 'claude'; - - // Already in no-fallback Claude cooldown — log only, no response - if (provider !== 'claude' && isPrimaryNoFallbackCooldownActive()) { - logger.info( - { chatJid, group: group.name, runId, provider }, - 'Claude primary cooldown active, silently skipping', - ); + const maybeHandoffAfterError = ( + reason: AgentTriggerReason, + attempt: Awaited>, + ): 'success' | 'error' => { + if (maybeHandoffToCodex(reason, attempt.sawVisibleOutput)) { + return 'success'; + } return 'error'; - } + }; - const primaryAttempt = await runAttempt(provider); + const provider = 'claude'; + + let primaryAttempt = await runAttempt(provider); + + const isRetryableClaudeSessionFailure = ( + attempt: Awaited>, + ): boolean => + isClaudeCodeAgent && + provider === 'claude' && + !attempt.sawOutput && + (attempt.retryableSessionFailureDetected === true || + (attempt.error != null && + shouldRetryFreshSessionOnAgentFailure({ + result: null, + error: getErrorMessage(attempt.error), + }))); + + if (isRetryableClaudeSessionFailure(primaryAttempt)) { + deps.clearSession(group.folder); + logger.warn( + { group: group.name, chatJid, runId }, + 'Cleared poisoned Claude session before visible output, retrying fresh session', + ); + + primaryAttempt = await runAttempt('claude'); + + if (isRetryableClaudeSessionFailure(primaryAttempt)) { + deps.clearSession(group.folder); + logger.warn( + { group: group.name, chatJid, runId }, + 'Fresh Claude retry also hit a retryable session failure', + ); + + logger.error( + { group: group.name, chatJid, runId }, + 'Retryable Claude session failure persisted after fresh retry', + ); + return 'error'; + } + } if (primaryAttempt.error) { if ( - (canFallback || canRotateToken) && + canRotateToken && provider === 'claude' && !primaryAttempt.sawOutput ) { const errMsg = getErrorMessage(primaryAttempt.error); const trigger = primaryAttempt.streamedTriggerReason ? { - shouldFallback: true, + shouldRetry: true, reason: primaryAttempt.streamedTriggerReason.reason, retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, } - : detectFallbackTrigger(errMsg); - if (trigger.shouldFallback) { - return retryClaudeWithRotation( + : classifyRotationTrigger(errMsg); + if (trigger.shouldRetry) { + const result = await retryClaudeWithRotation( { reason: trigger.reason, retryAfterMs: trigger.retryAfterMs, }, errMsg, ); + if (result === 'error') { + return maybeHandoffAfterError(trigger.reason, primaryAttempt); + } + return result; } } @@ -594,25 +635,23 @@ export async function runAgentForGroup( } if ( - (canFallback || canRotateToken) && + canRotateToken && provider === 'claude' && !primaryAttempt.sawOutput && primaryAttempt.streamedTriggerReason && output.status !== 'error' ) { - return retryClaudeWithRotation({ + const result = await retryClaudeWithRotation({ reason: primaryAttempt.streamedTriggerReason.reason, retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, }); - } - - if ( - canFallback && - provider === 'claude' && - !primaryAttempt.sawOutput && - primaryAttempt.sawSuccessNullResultWithoutOutput - ) { - return runFallbackAttempt('success-null-result'); + if (result === 'error') { + return maybeHandoffAfterError( + primaryAttempt.streamedTriggerReason.reason, + primaryAttempt, + ); + } + return result; } if ( @@ -628,25 +667,29 @@ export async function runAgentForGroup( if (output.status === 'error') { if ( - (canFallback || canRotateToken) && + canRotateToken && provider === 'claude' && !primaryAttempt.sawOutput ) { const trigger = primaryAttempt.streamedTriggerReason ? { - shouldFallback: true, + shouldRetry: true, reason: primaryAttempt.streamedTriggerReason.reason, retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs, } - : detectFallbackTrigger(output.error); - if (trigger.shouldFallback) { - return retryClaudeWithRotation( + : classifyRotationTrigger(output.error); + if (trigger.shouldRetry) { + const result = await retryClaudeWithRotation( { reason: trigger.reason, retryAfterMs: trigger.retryAfterMs, }, output.error ?? undefined, ); + if (result === 'error') { + return maybeHandoffAfterError(trigger.reason, primaryAttempt); + } + return result; } } @@ -687,5 +730,38 @@ export async function runAgentForGroup( ); } + // Unresolved streamed trigger — rotation was unavailable or output was + // already forwarded. Surfaces as an error since there is no alternative provider. + if (primaryAttempt.streamedTriggerReason) { + if ( + isClaudeCodeAgent && + maybeHandoffToCodex( + primaryAttempt.streamedTriggerReason.reason, + primaryAttempt.sawVisibleOutput, + ) + ) { + return 'success'; + } + logger.error( + { + group: group.name, + chatJid, + runId, + reason: primaryAttempt.streamedTriggerReason.reason, + }, + 'Agent trigger detected but could not be resolved', + ); + return 'error'; + } + + // success-null-result with no visible output — agent returned nothing useful + if (primaryAttempt.sawSuccessNullResultWithoutOutput) { + logger.error( + { group: group.name, chatJid, runId }, + 'Agent returned success with null result and no visible output', + ); + return 'error'; + } + return 'success'; } diff --git a/src/message-runtime.test.ts b/src/message-runtime.test.ts index cfe46ec..e93c479 100644 --- a/src/message-runtime.test.ts +++ b/src/message-runtime.test.ts @@ -11,8 +11,21 @@ vi.mock('./agent-runner.js', () => ({ writeTasksSnapshot: vi.fn(), })); +vi.mock('./output-suppression.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createSuppressToken: vi.fn(() => '__TEST_SUPPRESS__'), + }; +}); + vi.mock('./config.js', () => ({ DATA_DIR: '/tmp/ejclaw-test-data', + SERVICE_ID: 'claude', + SERVICE_AGENT_TYPE: 'claude-code', + SERVICE_SESSION_SCOPE: 'claude', + isClaudeService: vi.fn(() => true), + isReviewService: vi.fn(() => false), isSessionCommandSenderAllowed: vi.fn(() => false), })); @@ -40,6 +53,10 @@ vi.mock('./db.js', () => { })); return { + claimServiceHandoff: vi.fn(() => true), + completeServiceHandoff: vi.fn(), + completeServiceHandoffAndAdvanceTargetCursor: vi.fn(), + failServiceHandoff: vi.fn(), getAllChats: vi.fn(() => []), getAllTasks: vi.fn(() => []), getLastHumanMessageTimestamp: vi.fn(() => null), @@ -89,11 +106,13 @@ vi.mock('./db.js', () => { }, ), getOpenWorkItem: vi.fn(() => undefined), + getPendingServiceHandoffs: vi.fn(() => []), createProducedWorkItem: vi.fn((input) => ({ id: 1, group_folder: input.group_folder, chat_jid: input.chat_jid, agent_type: input.agent_type || 'claude-code', + service_id: 'claude', status: 'produced', start_seq: input.start_seq, end_seq: input.end_seq, @@ -108,9 +127,22 @@ vi.mock('./db.js', () => { markWorkItemDelivered: vi.fn(), markWorkItemDeliveryRetry: vi.fn(), isPairedRoomJid: vi.fn(() => false), + getLastBotFinalMessage: vi.fn(() => []), }; }); +vi.mock('./service-routing.js', () => ({ + getEffectiveChannelLease: vi.fn((chatJid: string) => ({ + chat_jid: chatJid, + owner_service_id: 'claude', + reviewer_service_id: 'codex-main', + activated_at: null, + reason: null, + explicit: false, + })), + shouldServiceProcessChat: vi.fn(() => true), +})); + vi.mock('./logger.js', () => ({ logger: { debug: vi.fn(), @@ -120,21 +152,6 @@ vi.mock('./logger.js', () => ({ }, })); -vi.mock('./provider-fallback.js', () => ({ - detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })), - getActiveProvider: vi.fn(async () => 'claude'), - getFallbackEnvOverrides: vi.fn(() => ({ - ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', - ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', - ANTHROPIC_MODEL: 'kimi-k2.5', - })), - getFallbackProviderName: vi.fn(() => 'kimi'), - hasGroupProviderOverride: vi.fn(() => false), - isFallbackEnabled: vi.fn(() => true), - isPrimaryNoFallbackCooldownActive: vi.fn(() => false), - markPrimaryCooldown: vi.fn(), -})); - vi.mock('./sender-allowlist.js', () => ({ isTriggerAllowed: vi.fn(() => true), loadSenderAllowlist: vi.fn(() => ({})), @@ -151,7 +168,8 @@ import * as agentRunner from './agent-runner.js'; import * as db from './db.js'; import { resolveGroupIpcPath } from './group-folder.js'; import { createMessageRuntime } from './message-runtime.js'; -import * as providerFallback from './provider-fallback.js'; +import * as outputSuppression from './output-suppression.js'; +import * as config from './config.js'; import type { Channel, RegisteredGroup } from './types.js'; function makeGroup(agentType: 'claude-code' | 'codex'): RegisteredGroup { @@ -182,19 +200,10 @@ function makeChannel(chatJid: string): Channel { describe('createMessageRuntime', () => { beforeEach(() => { vi.resetAllMocks(); - vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); - vi.mocked(providerFallback.getFallbackProviderName).mockReturnValue('kimi'); - vi.mocked(providerFallback.getFallbackEnvOverrides).mockReturnValue({ - ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', - ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', - ANTHROPIC_MODEL: 'kimi-k2.5', - }); - vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); - vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true); - vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({ - shouldFallback: false, - reason: '', - }); + vi.mocked(db.getLastBotFinalMessage).mockReturnValue([]); + vi.mocked(db.isPairedRoomJid).mockReturnValue(false); + vi.mocked(config.isClaudeService).mockReturnValue(true); + vi.mocked(config.isReviewService).mockReturnValue(false); }); it('ignores generic failure bot messages in paired rooms', async () => { @@ -257,6 +266,8 @@ describe('createMessageRuntime', () => { const saveState = vi.fn(); const lastAgentTimestamps: Record = {}; + vi.mocked(config.isClaudeService).mockReturnValue(false); + vi.mocked(config.isReviewService).mockReturnValue(false); vi.mocked(db.isPairedRoomJid).mockReturnValue(true); vi.mocked(db.getMessagesSince).mockReturnValue([ { @@ -317,10 +328,82 @@ describe('createMessageRuntime', () => { chatJid, '그 방향이 맞습니다.', ); + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true); + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false); expect(lastAgentTimestamps[chatJid]).toBe('1'); expect(saveState).toHaveBeenCalled(); }); + it('does not defer typing-on for suppress-capable review turns', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const saveState = vi.fn(); + const lastAgentTimestamps: Record = {}; + + vi.mocked(config.isClaudeService).mockReturnValue(false); + vi.mocked(config.isReviewService).mockReturnValue(true); + vi.mocked(db.isPairedRoomJid).mockReturnValue(true); + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'other-bot@test', + sender_name: 'Other Bot', + content: '이어서 확인해줘.', + timestamp: '2026-03-18T09:00:00.000Z', + is_bot_message: true, + }, + ]); + vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce( + async (_group, _input, _onProcess, onOutput) => { + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true); + await onOutput?.({ + status: 'success', + phase: 'final', + result: '리뷰 확인 완료입니다.', + newSessionId: 'session-review-follow-up-immediate', + }); + return { + status: 'success', + result: '리뷰 확인 완료입니다.', + newSessionId: 'session-review-follow-up-immediate', + }; + }, + ); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-review-follow-up-immediate-typing', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true); + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false); + }); + it('ignores watcher status control messages in paired rooms', async () => { const chatJid = 'group@test'; const group = makeGroup('codex'); @@ -1280,6 +1363,374 @@ describe('createMessageRuntime', () => { expect(channel.sendAndTrack).not.toHaveBeenCalled(); }); + it('does not emit a visible message when the final output is the suppress token only', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const lastAgentTimestamps: Record = {}; + const saveState = vi.fn(); + + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-19T00:00:00.000Z', + seq: 1, + }, + ]); + + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: '__TEST_SUPPRESS__', + newSessionId: 'session-suppress-run', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-suppress-only', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(saveState).toHaveBeenCalled(); + expect(lastAgentTimestamps[chatJid]).toBe('1'); + expect(channel.sendMessage).not.toHaveBeenCalled(); + expect(channel.sendAndTrack).not.toHaveBeenCalled(); + expect(channel.setTyping).not.toHaveBeenCalledWith(chatJid, true); + }); + + it('defers typing-on until the first visible output for suppress-capable turns', async () => { + const chatJid = 'group@test'; + const group = makeGroup('claude-code'); + const channel = makeChannel(chatJid); + const lastAgentTimestamps: Record = {}; + const saveState = vi.fn(); + + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-19T00:00:00.000Z', + seq: 1, + }, + ]); + + vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce( + async (_group, _input, _onProcess, onOutput) => { + await onOutput?.({ + status: 'success', + phase: 'final', + result: 'visible suppress-capable reply', + }); + return { + status: 'success', + result: null, + newSessionId: 'session-visible-suppress-capable', + }; + }, + ); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-visible-suppress-capable', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(channel.sendMessage).toHaveBeenCalledWith( + chatJid, + 'visible suppress-capable reply', + ); + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true); + expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false); + }); + + it('does not grant suppress-token silence authority to codex-main', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const lastAgentTimestamps: Record = {}; + const saveState = vi.fn(); + + vi.mocked(config.isClaudeService).mockReturnValue(false); + vi.mocked(config.isReviewService).mockReturnValue(false); + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-19T00:00:00.000Z', + seq: 1, + }, + ]); + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: 'visible codex reply', + newSessionId: 'session-codex-main', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-codex-main-no-suppress', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(outputSuppression.createSuppressToken).not.toHaveBeenCalled(); + }); + + it('blocks malformed output when the suppress token is mixed with visible text', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const lastAgentTimestamps: Record = {}; + const saveState = vi.fn(); + + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-19T00:00:00.000Z', + seq: 1, + }, + ]); + + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: '동의합니다 __TEST_SUPPRESS__', + newSessionId: 'session-suppress-mixed', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-suppress-mixed', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(saveState).toHaveBeenCalled(); + expect(lastAgentTimestamps[chatJid]).toBe('1'); + expect(channel.sendMessage).not.toHaveBeenCalled(); + expect(channel.sendAndTrack).not.toHaveBeenCalled(); + }); + + it('suppresses a leaked foreign suppress token even when it differs from the current turn token', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const lastAgentTimestamps: Record = {}; + const saveState = vi.fn(); + + vi.mocked(config.isClaudeService).mockReturnValue(false); + vi.mocked(config.isReviewService).mockReturnValue(true); + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-19T00:00:00.000Z', + seq: 1, + }, + ]); + + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__', + newSessionId: 'session-foreign-suppress-token', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-foreign-suppress-token', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(saveState).toHaveBeenCalled(); + expect(lastAgentTimestamps[chatJid]).toBe('1'); + expect(channel.sendMessage).not.toHaveBeenCalled(); + expect(channel.sendAndTrack).not.toHaveBeenCalled(); + }); + + it('suppresses a malformed leaked foreign suppress token without the closing suffix', async () => { + const chatJid = 'group@test'; + const group = makeGroup('codex'); + const channel = makeChannel(chatJid); + const lastAgentTimestamps: Record = {}; + const saveState = vi.fn(); + + vi.mocked(config.isClaudeService).mockReturnValue(false); + vi.mocked(config.isReviewService).mockReturnValue(true); + vi.mocked(db.getMessagesSince).mockReturnValue([ + { + id: 'msg-1', + chat_jid: chatJid, + sender: 'user@test', + sender_name: 'User', + content: 'hello', + timestamp: '2026-03-19T00:00:00.000Z', + seq: 1, + }, + ]); + + vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({ + status: 'success', + result: '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef', + newSessionId: 'session-malformed-foreign-suppress-token', + }); + + const runtime = createMessageRuntime({ + assistantName: 'Andy', + idleTimeout: 1_000, + pollInterval: 1_000, + timezone: 'UTC', + triggerPattern: /^@Andy\b/i, + channels: [channel], + queue: { + registerProcess: vi.fn(), + closeStdin: vi.fn(), + notifyIdle: vi.fn(), + } as any, + getRegisteredGroups: () => ({ [chatJid]: group }), + getSessions: () => ({}), + getLastTimestamp: () => '', + setLastTimestamp: vi.fn(), + getLastAgentTimestamps: () => lastAgentTimestamps, + saveState, + persistSession: vi.fn(), + clearSession: vi.fn(), + }); + + const result = await runtime.processGroupMessages(chatJid, { + runId: 'run-malformed-foreign-suppress-token', + reason: 'messages', + }); + + expect(result).toBe(true); + expect(saveState).toHaveBeenCalled(); + expect(lastAgentTimestamps[chatJid]).toBe('1'); + expect(channel.sendMessage).not.toHaveBeenCalled(); + expect(channel.sendAndTrack).not.toHaveBeenCalled(); + }); + it('resets tracked progress after a final output that becomes empty after formatting', async () => { vi.useFakeTimers(); const chatJid = 'group@test'; @@ -1417,7 +1868,7 @@ describe('createMessageRuntime', () => { } }); - it('promotes the last progress output to a final message when the agent completes without a final phase', async () => { + it('promotes the last flushed progress output to a final message when the agent completes without a final phase', async () => { vi.useFakeTimers(); const chatJid = 'group@test'; const group = makeGroup('codex'); @@ -1508,24 +1959,24 @@ describe('createMessageRuntime', () => { chatJid, P('검증 중입니다.\n\n0초'), ); - // Ticker fires after advanceTimersByTime — edits tracked message with latest heading + // Ticker fires after advanceTimersByTime — edits tracked message with the last flushed heading expect(channel.editMessage).toHaveBeenCalledWith( chatJid, 'progress-1', - P('테스트도 통과했습니다.\n\n5초'), + P('커밋은 정상 들어갔고 pre-commit도 통과했습니다.\n\n5초'), ); // finish() promotes the last flushed progress text to a final message expect(channel.sendMessage).toHaveBeenCalledTimes(1); expect(channel.sendMessage).toHaveBeenCalledWith( chatJid, - '검증 중입니다.', + '커밋은 정상 들어갔고 pre-commit도 통과했습니다.', ); } finally { vi.useRealTimers(); } }); - it('retries editing progress message instead of creating a duplicate when edit fails', async () => { + it('keeps going after a tracked progress edit fails and still emits the last flushed final message', async () => { vi.useFakeTimers(); const chatJid = 'group@test'; const group = makeGroup('codex'); @@ -1614,8 +2065,7 @@ describe('createMessageRuntime', () => { }); expect(result).toBe(true); - // Only one progress message created via sendAndTrack — no duplicate - expect(channel.sendAndTrack).toHaveBeenCalledTimes(1); + // The first flushed progress is still tracked expect(channel.sendAndTrack).toHaveBeenCalledWith( chatJid, P('진행 중입니다.\n\n0초'), @@ -1630,7 +2080,7 @@ describe('createMessageRuntime', () => { expect(channel.sendMessage).toHaveBeenCalledTimes(1); expect(channel.sendMessage).toHaveBeenCalledWith( chatJid, - '진행 중입니다.', + '아직 진행 중.', ); } finally { vi.useRealTimers(); @@ -1868,331 +2318,6 @@ describe('createMessageRuntime', () => { expect(saveState).toHaveBeenCalled(); }); - it('retries with the fallback provider when Claude returns a 429 error before any output', async () => { - const chatJid = 'group@test'; - const group = makeGroup('claude-code'); - const channel = makeChannel(chatJid); - - vi.mocked(db.getMessagesSince).mockReturnValue([ - { - id: 'msg-1', - chat_jid: chatJid, - sender: 'user@test', - sender_name: 'User', - content: 'hello', - timestamp: '2026-03-19T00:00:00.000Z', - }, - ]); - - vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({ - shouldFallback: true, - reason: '429', - retryAfterMs: 60_000, - }); - - vi.mocked(agentRunner.runAgentProcess) - .mockResolvedValueOnce({ - status: 'error', - result: null, - error: '429 rate limited retry after 60', - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'fallback 응답입니다.', - }); - return { - status: 'success', - result: null, - }; - }); - - const runtime = createMessageRuntime({ - assistantName: 'Andy', - idleTimeout: 1_000, - pollInterval: 1_000, - timezone: 'UTC', - triggerPattern: /^@Andy\b/i, - channels: [channel], - queue: { - registerProcess: vi.fn(), - closeStdin: vi.fn(), - notifyIdle: vi.fn(), - } as any, - getRegisteredGroups: () => ({ [chatJid]: group }), - getSessions: () => ({}), - getLastTimestamp: () => '', - setLastTimestamp: vi.fn(), - getLastAgentTimestamps: () => ({}), - saveState: vi.fn(), - persistSession: vi.fn(), - clearSession: vi.fn(), - }); - - const result = await runtime.processGroupMessages(chatJid, { - runId: 'run-fallback-429', - reason: 'messages', - }); - - expect(result).toBe(true); - expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); - expect(agentRunner.runAgentProcess).toHaveBeenNthCalledWith( - 2, - expect.anything(), - expect.objectContaining({ sessionId: undefined }), - expect.any(Function), - expect.any(Function), - expect.objectContaining({ - ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', - ANTHROPIC_MODEL: 'kimi-k2.5', - }), - ); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - '429', - 60_000, - ); - expect(channel.sendMessage).toHaveBeenCalledWith( - chatJid, - 'fallback 응답입니다.', - ); - }); - - it('silently suppresses a usage exhaustion banner without falling back', async () => { - const chatJid = 'group@test'; - const group = makeGroup('claude-code'); - const channel = makeChannel(chatJid); - - vi.mocked(db.getMessagesSince).mockReturnValue([ - { - id: 'msg-1', - chat_jid: chatJid, - sender: 'user@test', - sender_name: 'User', - content: 'hello', - timestamp: '2026-03-24T00:00:00.000Z', - }, - ]); - - vi.mocked(agentRunner.runAgentProcess) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: "You're out of extra usage · resets 4am (Asia/Seoul)", - }); - return { - status: 'success', - result: null, - }; - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'usage fallback 응답입니다.', - }); - return { - status: 'success', - result: null, - }; - }); - - const runtime = createMessageRuntime({ - assistantName: 'Andy', - idleTimeout: 1_000, - pollInterval: 1_000, - timezone: 'UTC', - triggerPattern: /^@Andy\b/i, - channels: [channel], - queue: { - registerProcess: vi.fn(), - closeStdin: vi.fn(), - notifyIdle: vi.fn(), - } as any, - getRegisteredGroups: () => ({ [chatJid]: group }), - getSessions: () => ({}), - getLastTimestamp: () => '', - setLastTimestamp: vi.fn(), - getLastAgentTimestamps: () => ({}), - saveState: vi.fn(), - persistSession: vi.fn(), - clearSession: vi.fn(), - }); - - const result = await runtime.processGroupMessages(chatJid, { - runId: 'run-fallback-usage-exhausted', - reason: 'messages', - }); - - expect(result).toBe(true); - expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'usage-exhausted', - undefined, - ); - expect(channel.sendMessage).not.toHaveBeenCalled(); - }); - - it('suppresses duplicate streamed usage banners without emitting a visible reply', async () => { - const chatJid = 'group@test'; - const group = makeGroup('claude-code'); - const channel = makeChannel(chatJid); - - vi.mocked(db.getMessagesSince).mockReturnValue([ - { - id: 'msg-1', - chat_jid: chatJid, - sender: 'user@test', - sender_name: 'User', - content: 'hello', - timestamp: '2026-03-24T00:00:00.000Z', - }, - ]); - - vi.mocked(agentRunner.runAgentProcess) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'intermediate', - result: "You're out of extra usage · resets 4am (Asia/Seoul)", - }); - await onOutput?.({ - status: 'success', - phase: 'final', - result: "You're out of extra usage · resets 4am (Asia/Seoul)", - }); - return { - status: 'success', - result: null, - }; - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'duplicate banner fallback 응답입니다.', - }); - return { - status: 'success', - result: null, - }; - }); - - const runtime = createMessageRuntime({ - assistantName: 'Andy', - idleTimeout: 1_000, - pollInterval: 1_000, - timezone: 'UTC', - triggerPattern: /^@Andy\b/i, - channels: [channel], - queue: { - registerProcess: vi.fn(), - closeStdin: vi.fn(), - notifyIdle: vi.fn(), - } as any, - getRegisteredGroups: () => ({ [chatJid]: group }), - getSessions: () => ({}), - getLastTimestamp: () => '', - setLastTimestamp: vi.fn(), - getLastAgentTimestamps: () => ({}), - saveState: vi.fn(), - persistSession: vi.fn(), - clearSession: vi.fn(), - }); - - const result = await runtime.processGroupMessages(chatJid, { - runId: 'run-fallback-usage-exhausted-duplicate', - reason: 'messages', - }); - - expect(result).toBe(true); - expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'usage-exhausted', - undefined, - ); - expect(channel.sendMessage).not.toHaveBeenCalled(); - }); - - it('retries with the fallback provider when Claude ends with success-null-result before any output', async () => { - const chatJid = 'group@test'; - const group = makeGroup('claude-code'); - const channel = makeChannel(chatJid); - - vi.mocked(db.getMessagesSince).mockReturnValue([ - { - id: 'msg-1', - chat_jid: chatJid, - sender: 'user@test', - sender_name: 'User', - content: 'hello', - timestamp: '2026-03-19T00:00:00.000Z', - }, - ]); - - vi.mocked(agentRunner.runAgentProcess) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - result: null, - }); - return { - status: 'success', - result: null, - }; - }) - .mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => { - await onOutput?.({ - status: 'success', - phase: 'final', - result: 'success-null-result 폴백 응답입니다.', - }); - return { - status: 'success', - result: null, - }; - }); - - const runtime = createMessageRuntime({ - assistantName: 'Andy', - idleTimeout: 1_000, - pollInterval: 1_000, - timezone: 'UTC', - triggerPattern: /^@Andy\b/i, - channels: [channel], - queue: { - registerProcess: vi.fn(), - closeStdin: vi.fn(), - notifyIdle: vi.fn(), - } as any, - getRegisteredGroups: () => ({ [chatJid]: group }), - getSessions: () => ({}), - getLastTimestamp: () => '', - setLastTimestamp: vi.fn(), - getLastAgentTimestamps: () => ({}), - saveState: vi.fn(), - persistSession: vi.fn(), - clearSession: vi.fn(), - }); - - const result = await runtime.processGroupMessages(chatJid, { - runId: 'run-fallback-success-null', - reason: 'messages', - }); - - expect(result).toBe(true); - expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'success-null-result', - undefined, - ); - expect(channel.sendMessage).toHaveBeenCalledWith( - chatJid, - 'success-null-result 폴백 응답입니다.', - ); - }); - it('treats missing streamed phase as final output', async () => { const chatJid = 'group@test'; const group = makeGroup('claude-code'); @@ -2266,6 +2391,7 @@ describe('createMessageRuntime', () => { group_folder: group.folder, chat_jid: chatJid, agent_type: 'claude-code', + service_id: 'claude', status: 'produced', start_seq: 1, end_seq: 1, diff --git a/src/message-runtime.ts b/src/message-runtime.ts index edc1040..597a14e 100644 --- a/src/message-runtime.ts +++ b/src/message-runtime.ts @@ -1,17 +1,30 @@ import { AgentOutput } from './agent-runner.js'; import { getErrorMessage } from './utils.js'; import { + claimServiceHandoff, + completeServiceHandoffAndAdvanceTargetCursor, + createProducedWorkItem, + failServiceHandoff, + getOpenWorkItem, + getPendingServiceHandoffs, getMessagesSinceSeq, getNewMessagesBySeq, - getOpenWorkItem, - createProducedWorkItem, markWorkItemDelivered, markWorkItemDeliveryRetry, + getLastBotFinalMessage, + isPairedRoomJid, + type ServiceHandoff, type WorkItem, } from './db.js'; -import { isSessionCommandSenderAllowed } from './config.js'; +import { + isClaudeService, + isReviewService, + isSessionCommandSenderAllowed, + SERVICE_AGENT_TYPE, + SERVICE_ID, +} from './config.js'; import { GroupQueue, GroupRunContext } from './group-queue.js'; -import { findChannel, formatMessages } from './router.js'; +import { findChannel, formatMessages, normalizeMessageForDedupe } from './router.js'; import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js'; import { advanceLastAgentCursor, @@ -23,6 +36,7 @@ import { } from './message-runtime-rules.js'; import { runAgentForGroup } from './message-agent-executor.js'; import { MessageTurnController } from './message-turn-controller.js'; +import { createSuppressToken } from './output-suppression.js'; import { extractSessionCommand, handleSessionCommand, @@ -32,6 +46,30 @@ import { import { Channel, NewMessage, RegisteredGroup } from './types.js'; import { logger } from './logger.js'; import { resolveGroupIpcPath } from './group-folder.js'; +import { shouldServiceProcessChat } from './service-routing.js'; + +/** + * Check if a message is a duplicate of the last bot final message in a paired room. + * Exported for testing purposes. + */ +export function isDuplicateOfLastBotFinal(chatJid: string, text: string): boolean { + // Only check in paired rooms (both claude and codex registered) + if (!isPairedRoomJid(chatJid)) { + return false; + } + + // Get the last bot final message from DB (any bot, not just this service) + const lastMessages = getLastBotFinalMessage(chatJid, SERVICE_AGENT_TYPE, 1); + if (lastMessages.length === 0) { + return false; + } + + const lastMessage = lastMessages[0]; + const normalizedLast = normalizeMessageForDedupe(lastMessage.content); + const normalizedCurrent = normalizeMessageForDedupe(text); + + return normalizedLast === normalizedCurrent && normalizedLast.length > 0; +} export interface MessageRuntimeDeps { assistantName: string; @@ -64,6 +102,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { const continuationTracker = createImplicitContinuationTracker( deps.idleTimeout, ); + const isBotOnlyPairedRoomTurn = ( + chatJid: string, + messages: NewMessage[], + ): boolean => + isPairedRoomJid(chatJid) && + messages.every( + (message) => message.is_from_me === true || !!message.is_bot_message, + ); + + /** + * Check if a message is a duplicate of the last bot final message in a paired room. + * Returns true if duplicate (should be suppressed). + */ + const checkDuplicateOfLastBotFinal = (chatJid: string, text: string): boolean => { + return isDuplicateOfLastBotFinal(chatJid, text); + }; const deliverOpenWorkItem = async ( channel: Channel, @@ -73,6 +127,24 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }, ): Promise => { const replaceMessageId = options?.replaceMessageId ?? null; + + // Check for duplicate in paired rooms before attempting delivery + const isDuplicate = checkDuplicateOfLastBotFinal(item.chat_jid, item.result_payload); + + if (isDuplicate) { + // Mark as delivered without sending, and don't open continuation + markWorkItemDelivered(item.id, null); + logger.info( + { + chatJid: item.chat_jid, + workItemId: item.id, + preview: item.result_payload.slice(0, 100), + }, + 'Suppressed duplicate final message in paired room (marked as delivered)', + ); + return true; + } + try { if (replaceMessageId && channel.editMessage) { await channel.editMessage( @@ -141,6 +213,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { chatJid: string, runId: string, onOutput?: (output: AgentOutput) => Promise, + options?: { + suppressToken?: string; + startSeq?: number | null; + endSeq?: number | null; + }, ): Promise<'success' | 'error'> => runAgentForGroup( { @@ -156,10 +233,185 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { prompt, chatJid, runId, + suppressToken: options?.suppressToken, + startSeq: options?.startSeq, + endSeq: options?.endSeq, onOutput, }, ); + const executeTurn = async (args: { + group: RegisteredGroup; + prompt: string; + chatJid: string; + runId: string; + channel: Channel; + startSeq: number | null; + endSeq: number | null; + }): Promise<{ + outputStatus: 'success' | 'error'; + deliverySucceeded: boolean; + visiblePhase: ReturnType extends Promise< + infer T + > + ? T extends { visiblePhase: infer V } + ? V + : never + : never; + }> => { + const { group, prompt, chatJid, runId, channel, startSeq, endSeq } = args; + const isClaudeCodeAgent = + (group.agentType || 'claude-code') === 'claude-code'; + const suppressToken = + isClaudeService() || isReviewService() ? createSuppressToken() : undefined; + const deferTypingUntilVisible = Boolean(suppressToken) && isClaudeService(); + + const turnController = new MessageTurnController({ + chatJid, + group, + runId, + channel, + idleTimeout: deps.idleTimeout, + failureFinalText: FAILURE_FINAL_TEXT, + isClaudeCodeAgent, + deferTypingUntilVisible, + suppressToken, + clearSession: () => deps.clearSession(group.folder), + requestClose: (reason) => deps.queue.closeStdin(chatJid, { runId, reason }), + deliverFinalText: async (text) => { + try { + const workItem = createProducedWorkItem({ + group_folder: group.folder, + chat_jid: chatJid, + agent_type: group.agentType || 'claude-code', + start_seq: startSeq, + end_seq: endSeq, + result_payload: text, + }); + return deliverOpenWorkItem(channel, workItem); + } catch (err) { + logger.warn( + { group: group.name, chatJid, runId, err }, + 'Failed to persist produced output for delivery', + ); + return false; + } + }, + }); + + await turnController.start(); + + try { + const outputStatus = await runAgent( + group, + prompt, + chatJid, + runId, + (result) => turnController.handleOutput(result), + { suppressToken, startSeq, endSeq }, + ); + + const { deliverySucceeded, visiblePhase } = + await turnController.finish(outputStatus); + + return { + outputStatus, + deliverySucceeded, + visiblePhase, + }; + } finally { + turnController.cancelPendingTypingDelay(); + logger.debug( + { + transition: 'typing:off', + source: 'message-runtime:safety-net', + chatJid, + group: group.name, + groupFolder: group.folder, + runId, + }, + 'Typing indicator transition', + ); + await channel.setTyping?.(chatJid, false); + } + }; + + const enqueuePendingHandoffs = (): void => { + for (const handoff of getPendingServiceHandoffs(SERVICE_ID)) { + if (!claimServiceHandoff(handoff.id)) { + continue; + } + + deps.queue.enqueueTask( + handoff.chat_jid, + `handoff:${handoff.id}`, + async () => { + await processClaimedHandoff(handoff); + }, + ); + } + }; + + const processClaimedHandoff = async (handoff: ServiceHandoff): Promise => { + const group = deps.getRegisteredGroups()[handoff.chat_jid]; + if (!group) { + failServiceHandoff(handoff.id, 'Group not registered on target service'); + return; + } + + const channel = findChannel(deps.channels, handoff.chat_jid); + if (!channel) { + failServiceHandoff(handoff.id, 'No channel owns handoff jid'); + return; + } + + const runId = `handoff-${handoff.id}`; + try { + const result = await executeTurn({ + group, + prompt: handoff.prompt, + chatJid: handoff.chat_jid, + runId, + channel, + startSeq: handoff.start_seq, + endSeq: handoff.end_seq, + }); + + if (!result.deliverySucceeded) { + failServiceHandoff(handoff.id, 'Handoff delivery failed'); + return; + } + + const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({ + id: handoff.id, + target_service_id: handoff.target_service_id, + chat_jid: handoff.chat_jid, + end_seq: handoff.end_seq, + }); + if (appliedCursor) { + deps.getLastAgentTimestamps()[handoff.chat_jid] = appliedCursor; + } + logger.info( + { + chatJid: handoff.chat_jid, + handoffId: handoff.id, + runId, + outputStatus: result.outputStatus, + visiblePhase: result.visiblePhase, + appliedCursor, + }, + 'Completed claimed service handoff', + ); + } catch (err) { + const errorMessage = getErrorMessage(err); + failServiceHandoff(handoff.id, errorMessage); + logger.error( + { chatJid: handoff.chat_jid, handoffId: handoff.id, err }, + 'Claimed service handoff failed', + ); + } + }; + const processGroupMessages = async ( chatJid: string, context: GroupRunContext, @@ -183,9 +435,29 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { if (!delivered) return false; } + if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) { + const rawMissedMessages = getMessagesSinceSeq( + chatJid, + deps.getLastAgentTimestamps()[chatJid] || '0', + deps.assistantName, + ); + const lastIgnored = rawMissedMessages[rawMissedMessages.length - 1]; + if (lastIgnored?.seq != null) { + advanceLastAgentCursor( + deps.getLastAgentTimestamps(), + deps.saveState, + chatJid, + lastIgnored.seq, + ); + } + logger.debug( + { chatJid, serviceId: SERVICE_ID }, + 'Skipping message processing for unassigned service', + ); + return true; + } + const isMainGroup = group.isMain === true; - const isClaudeCodeAgent = - (group.agentType || 'claude-code') === 'claude-code'; while (true) { const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '0'; const rawMissedMessages = getMessagesSinceSeq( @@ -311,85 +583,36 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { }, 'Dispatching queued messages to agent', ); - const turnController = new MessageTurnController({ - chatJid, + const { deliverySucceeded, visiblePhase } = await executeTurn({ group, + prompt, + chatJid, runId, channel, - idleTimeout: deps.idleTimeout, - failureFinalText: FAILURE_FINAL_TEXT, - isClaudeCodeAgent, - clearSession: () => deps.clearSession(group.folder), - requestClose: (reason) => - deps.queue.closeStdin(chatJid, { runId, reason }), - deliverFinalText: async (text) => { - try { - const workItem = createProducedWorkItem({ - group_folder: group.folder, - chat_jid: chatJid, - agent_type: group.agentType || 'claude-code', - start_seq: startSeq, - end_seq: endSeq, - result_payload: text, - }); - return deliverOpenWorkItem(channel, workItem); - } catch (err) { - logger.warn( - { group: group.name, chatJid, runId, err }, - 'Failed to persist produced output for delivery', - ); - return false; - } - }, + startSeq, + endSeq, }); - await turnController.start(); - - try { - const output = await runAgent(group, prompt, chatJid, runId, (result) => - turnController.handleOutput(result), + if (!deliverySucceeded) { + logger.warn( + { chatJid, group: group.name, groupFolder: group.folder, runId }, + 'Persisted produced output for delivery retry without rerunning agent', ); - - const { deliverySucceeded, visiblePhase } = - await turnController.finish(output); - - if (!deliverySucceeded) { - logger.warn( - { chatJid, group: group.name, groupFolder: group.folder, runId }, - 'Persisted produced output for delivery retry without rerunning agent', - ); - return false; - } - - logger.info( - { - chatJid, - group: group.name, - groupFolder: group.folder, - runId, - visiblePhase, - }, - 'Queued run completed successfully', - ); - - return true; - } finally { - // Safety net: always clear typing even if runAgent() or finish() throws. - // Prevents stuck typing indicators when exceptions bypass the normal - // turnController.finish() -> setTyping(false) path. - logger.debug( - { - transition: 'typing:off', - source: 'message-runtime:safety-net', - chatJid, - group: group.name, - groupFolder: group.folder, - runId, - }, - 'Typing indicator transition', - ); - await channel.setTyping?.(chatJid, false); + return false; } + + logger.info( + { + chatJid, + group: group.name, + groupFolder: group.folder, + runId, + visiblePhase, + }, + 'Queued run completed successfully', + ); + + return true; } }; @@ -404,6 +627,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { while (true) { try { + enqueuePendingHandoffs(); const registeredGroups = deps.getRegisteredGroups(); const jids = Object.keys(registeredGroups); const { messages, newSeqCursor } = getNewMessagesBySeq( @@ -461,6 +685,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { continue; } + if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) { + const lastIgnored = + processableGroupMessages[processableGroupMessages.length - 1]; + if (lastIgnored?.seq != null) { + advanceLastAgentCursor( + deps.getLastAgentTimestamps(), + deps.saveState, + chatJid, + lastIgnored.seq, + ); + } + continue; + } + if ( shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages) ) { @@ -530,6 +768,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { ? pendingMessages : processableGroupMessages; const formatted = formatMessages(messagesToSend, deps.timezone); + const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn( + chatJid, + messagesToSend, + ); if (deps.queue.sendMessage(chatJid, formatted)) { const endSeq = messagesToSend[messagesToSend.length - 1]?.seq; @@ -549,17 +791,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { group: group.name, groupFolder: group.folder, endSeq: endSeq ?? null, + suppressed: isBotOnlyPairedFollowUp, }, 'Typing indicator transition', ); - await channel - .setTyping?.(chatJid, true) - ?.catch((err) => - logger.warn( - { chatJid, err }, - 'Failed to set typing indicator', - ), - ); + if (!isBotOnlyPairedFollowUp) { + await channel + .setTyping?.(chatJid, true) + ?.catch((err) => + logger.warn( + { chatJid, err }, + 'Failed to set typing indicator', + ), + ); + } continue; } @@ -595,6 +840,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): { continue; } + if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) { + continue; + } + const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || ''; const rawPending = getMessagesSinceSeq( chatJid, diff --git a/src/message-turn-controller.ts b/src/message-turn-controller.ts index 8d6e67c..bf46f61 100644 --- a/src/message-turn-controller.ts +++ b/src/message-turn-controller.ts @@ -1,5 +1,6 @@ import { type AgentOutput } from './agent-runner.js'; import { logger } from './logger.js'; +import { classifySuppressTokenOutput } from './output-suppression.js'; import { formatOutbound } from './router.js'; import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js'; @@ -28,6 +29,8 @@ interface MessageTurnControllerOptions { idleTimeout: number; failureFinalText: string; isClaudeCodeAgent: boolean; + deferTypingUntilVisible?: boolean; + suppressToken?: string; clearSession: () => void; requestClose: (reason: string) => void; deliverFinalText: (text: string) => Promise; @@ -53,6 +56,7 @@ export class MessageTurnController { private lastIntermediateText: string | null = null; private poisonedSessionDetected = false; private closeRequested = false; + private typingActive = false; constructor(private readonly options: MessageTurnControllerOptions) {} @@ -78,7 +82,10 @@ export class MessageTurnController { async start(): Promise { this.resetIdleTimer(); - await this.setTyping(true, 'turn:start'); + if (this.options.deferTypingUntilVisible) { + return; + } + await this.activateTyping('turn:start'); } async handleOutput(result: AgentOutput): Promise { @@ -123,7 +130,11 @@ export class MessageTurnController { : typeof result.result === 'string' ? result.result : JSON.stringify(result.result); - const text = raw ? formatOutbound(raw) : null; + const suppressState = + raw && this.options.suppressToken + ? classifySuppressTokenOutput(raw, this.options.suppressToken) + : 'none'; + const text = raw && suppressState === 'none' ? formatOutbound(raw) : null; if (raw) { logger.info( @@ -138,6 +149,31 @@ export class MessageTurnController { }, `Agent output: ${raw.slice(0, 200)}`, ); + if (suppressState === 'exact') { + logger.info( + { + chatJid: this.options.chatJid, + group: this.options.group.name, + groupFolder: this.options.group.folder, + runId: this.options.runId, + resultStatus: result.status, + resultPhase: result.phase, + }, + 'Suppressed exact-match silent output token', + ); + } else if (suppressState === 'mixed') { + logger.warn( + { + chatJid: this.options.chatJid, + group: this.options.group.name, + groupFolder: this.options.group.folder, + runId: this.options.runId, + resultStatus: result.status, + resultPhase: result.phase, + }, + 'Blocked malformed output that mixed the silent output token with visible text', + ); + } } const phase: AgentOutputPhase = normalizeAgentOutputPhase(result.phase); @@ -275,6 +311,9 @@ export class MessageTurnController { await this.finalizeProgressMessage(); await this.deliverFinalText(text); } + } else if (suppressState !== 'none') { + await this.finalizeProgressMessage(); + this.latestProgressTextForFinal = null; } else if (raw) { logger.info( { @@ -294,7 +333,7 @@ export class MessageTurnController { await this.finalizeProgressMessage(); } - await this.setTyping(false, 'turn:handle-output', { + await this.deactivateTyping('turn:handle-output', { outputStatus: result.status, phase, }); @@ -311,7 +350,7 @@ export class MessageTurnController { deliverySucceeded: boolean; visiblePhase: VisiblePhase; }> { - await this.setTyping(false, 'turn:finish', { outputStatus }); + await this.deactivateTyping('turn:finish', { outputStatus }); if (outputStatus === 'error') { this.hadError = true; @@ -566,6 +605,7 @@ export class MessageTurnController { } private async deliverFinalText(text: string): Promise { + await this.activateTyping('turn:deliver-final'); this.visiblePhase = toVisiblePhase('final'); const delivered = await this.options.deliverFinalText(text); if (!delivered) { @@ -593,6 +633,8 @@ export class MessageTurnController { return; } + await this.activateTyping('turn:send-progress'); + if (this.progressStartedAt === null) { this.progressStartedAt = Date.now(); } @@ -670,6 +712,28 @@ export class MessageTurnController { this.visiblePhase = toVisiblePhase('progress'); } + private async activateTyping( + source: string, + extra?: Record, + ): Promise { + if (this.typingActive) return; + await this.setTyping(true, source, extra); + this.typingActive = true; + } + + private async deactivateTyping( + source: string, + extra?: Record, + ): Promise { + if (!this.typingActive) return; + await this.setTyping(false, source, extra); + this.typingActive = false; + } + + cancelPendingTypingDelay(): void { + // No-op: typing delay removed. Kept for call-site compatibility. + } + private resetIdleTimer(): void { if (this.idleTimer) clearTimeout(this.idleTimer); if (this.hasVisibleOutput()) { diff --git a/src/output-suppression.test.ts b/src/output-suppression.test.ts new file mode 100644 index 0000000..1cf0549 --- /dev/null +++ b/src/output-suppression.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; + +import { classifySuppressTokenOutput } from './output-suppression.js'; + +describe('classifySuppressTokenOutput', () => { + it('treats the current turn suppress token as exact', () => { + expect( + classifySuppressTokenOutput( + '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__', + '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__', + ), + ).toBe('exact'); + }); + + it('treats a leaked foreign suppress token as exact silent output', () => { + expect( + classifySuppressTokenOutput( + '__EJ_SUPPRESS_feedfacefeedfacefeedface__', + '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__', + ), + ).toBe('exact'); + }); + + it('treats a malformed foreign suppress token without the closing suffix as exact silent output', () => { + expect( + classifySuppressTokenOutput( + '__EJ_SUPPRESS_feedfacefeedfacefeedface', + '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__', + ), + ).toBe('exact'); + }); + + it('treats a suppress token embedded in visible text as mixed', () => { + expect( + classifySuppressTokenOutput( + 'prefix __EJ_SUPPRESS_feedfacefeedfacefeedface__ suffix', + '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__', + ), + ).toBe('mixed'); + }); +}); diff --git a/src/output-suppression.ts b/src/output-suppression.ts new file mode 100644 index 0000000..a68c60e --- /dev/null +++ b/src/output-suppression.ts @@ -0,0 +1,65 @@ +import { randomBytes } from 'crypto'; + +import { + CLAUDE_SERVICE_ID, + CODEX_REVIEW_SERVICE_ID, + normalizeServiceId, +} from './config.js'; + +const ANY_SUPPRESS_TOKEN_PATTERN = /__EJ_SUPPRESS_[a-f0-9]{24,}(?:__)?/g; +const EXACT_ANY_SUPPRESS_TOKEN_PATTERN = /^__EJ_SUPPRESS_[a-f0-9]{24,}(?:__)?$/; + +export function createSuppressToken(): string { + return `__EJ_SUPPRESS_${randomBytes(12).toString('hex')}__`; +} + +export function shouldEnableSuppressOutputForService( + serviceId: string | undefined, +): boolean { + if (!serviceId) return false; + const normalized = normalizeServiceId(serviceId); + return ( + normalized === CLAUDE_SERVICE_ID || + normalized === CODEX_REVIEW_SERVICE_ID + ); +} + +export function classifySuppressTokenOutput( + rawText: string, + suppressToken: string | undefined, +): 'exact' | 'mixed' | 'none' { + const trimmed = rawText.trim(); + if ((suppressToken && trimmed === suppressToken) || EXACT_ANY_SUPPRESS_TOKEN_PATTERN.test(trimmed)) { + return 'exact'; + } + if (suppressToken && rawText.includes(suppressToken)) { + return 'mixed'; + } + ANY_SUPPRESS_TOKEN_PATTERN.lastIndex = 0; + return ANY_SUPPRESS_TOKEN_PATTERN.test(rawText) ? 'mixed' : 'none'; +} + +export function buildSuppressTokenPrompt( + prompt: string, + suppressToken: string | undefined, + options?: { + reviewerMode?: boolean; + }, +): string { + if (!suppressToken) return prompt; + + const lines = [ + '[OUTPUT CONTROL]', + `If you have no user-visible content to send for this turn, output exactly this token and nothing else: ${suppressToken}`, + 'Do not wrap the token in backticks or code fences.', + 'Do not combine the token with any other text.', + ]; + + if (options?.reviewerMode) { + lines.push( + 'If you are only agreeing, mirroring, or restating without adding a concrete correction, risk, missing prerequisite, test gap, or code change, output only the token.', + ); + } + + return `${lines.join('\n')}\n\n${prompt}`; +} diff --git a/src/provider-fallback.test.ts b/src/provider-fallback.test.ts deleted file mode 100644 index 20e29d4..0000000 --- a/src/provider-fallback.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -vi.mock('./claude-usage.js', () => ({ - fetchClaudeUsage: vi.fn(), -})); - -vi.mock('./env.js', () => { - const store: Record = { - FALLBACK_PROVIDER_NAME: 'kimi', - FALLBACK_BASE_URL: 'https://api.kimi.com/coding/', - FALLBACK_AUTH_TOKEN: 'test-kimi-key', - FALLBACK_MODEL: 'kimi-k2.5', - FALLBACK_SMALL_MODEL: 'kimi-k2.5', - FALLBACK_COOLDOWN_MS: '600000', - }; - return { - readEnvFile: vi.fn((keys: string[]) => { - const result: Record = {}; - for (const k of keys) { - if (store[k]) result[k] = store[k]; - } - return result; - }), - getEnv: vi.fn((key: string) => store[key]), - }; -}); - -vi.mock('./logger.js', () => ({ - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }, -})); - -import { fetchClaudeUsage } from './claude-usage.js'; -import { - clearPrimaryCooldown, - detectFallbackTrigger, - getActiveProvider, - getCooldownInfo, - isPrimaryNoFallbackCooldownActive, - markPrimaryCooldown, - resetFallbackConfig, -} from './provider-fallback.js'; - -describe('provider fallback usage recovery', () => { - beforeEach(() => { - vi.useFakeTimers(); - vi.setSystemTime(new Date('2026-03-24T00:00:00.000Z')); - vi.clearAllMocks(); - clearPrimaryCooldown(); - resetFallbackConfig(); - delete process.env.FALLBACK_PROVIDER_NAME; - delete process.env.FALLBACK_BASE_URL; - delete process.env.FALLBACK_AUTH_TOKEN; - delete process.env.FALLBACK_MODEL; - delete process.env.FALLBACK_SMALL_MODEL; - delete process.env.FALLBACK_COOLDOWN_MS; - }); - - afterEach(() => { - clearPrimaryCooldown(); - resetFallbackConfig(); - vi.useRealTimers(); - }); - - it('keeps the fallback provider active while Claude usage is still exhausted', async () => { - vi.mocked(fetchClaudeUsage).mockResolvedValue({ - five_hour: { - utilization: 100, - resets_at: '2026-03-24T04:00:00.000+09:00', - }, - }); - - markPrimaryCooldown('usage-exhausted', 1_000); - vi.advanceTimersByTime(5_000); - - await expect(getActiveProvider()).resolves.toBe('kimi'); - expect(getCooldownInfo()).toMatchObject({ - active: true, - reason: 'usage-exhausted', - remainingMs: 0, - }); - }); - - it('returns to Claude immediately when usage is no longer exhausted', async () => { - vi.mocked(fetchClaudeUsage).mockResolvedValue({ - five_hour: { - utilization: 72, - resets_at: '2026-03-24T04:00:00.000+09:00', - }, - seven_day: { - utilization: 55, - resets_at: '2026-03-31T04:00:00.000+09:00', - }, - }); - - markPrimaryCooldown('usage-exhausted', 600_000); - - await expect(getActiveProvider()).resolves.toBe('claude'); - expect(getCooldownInfo()).toEqual({ active: false }); - }); - - it('falls back to time-based retry when usage status cannot be fetched', async () => { - vi.mocked(fetchClaudeUsage).mockResolvedValue(null); - - markPrimaryCooldown('usage-exhausted', 1_000); - vi.advanceTimersByTime(5_000); - - await expect(getActiveProvider()).resolves.toBe('claude'); - expect(getCooldownInfo()).toEqual({ active: false }); - }); - - it('treats terminated 401 auth failures as an auth-expired fallback trigger', () => { - expect( - detectFallbackTrigger( - 'Failed to authenticate. API Error: 401 terminated', - ), - ).toEqual({ - shouldFallback: true, - reason: 'auth-expired', - }); - }); - - it('treats invalid authentication credentials as an auth-expired fallback trigger', () => { - expect( - detectFallbackTrigger( - 'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', - ), - ).toEqual({ - shouldFallback: true, - reason: 'auth-expired', - }); - }); - - it('treats Cloudflare 502 HTML as an overloaded fallback trigger', () => { - expect( - detectFallbackTrigger( - 'API Error: 502 502 Bad Gateway

502 Bad Gateway

cloudflare
', - ), - ).toEqual({ - shouldFallback: true, - reason: 'overloaded', - }); - }); - - it('treats Claude org access denied banners as an org-access-denied fallback trigger', () => { - expect( - detectFallbackTrigger( - 'Your organization does not have access to Claude. Please login again or contact your administrator.', - ), - ).toEqual({ - shouldFallback: true, - reason: 'org-access-denied', - }); - }); - - it('treats terminated 403 auth failures as an org-access-denied fallback trigger', () => { - expect( - detectFallbackTrigger( - 'Failed to authenticate. API Error: 403 terminated', - ), - ).toEqual({ - shouldFallback: true, - reason: 'org-access-denied', - }); - }); - - it('marks org-access-denied as a no-fallback cooldown reason', () => { - markPrimaryCooldown('org-access-denied', 60_000); - - expect(isPrimaryNoFallbackCooldownActive()).toBe(true); - expect(getCooldownInfo()).toMatchObject({ - active: true, - reason: 'org-access-denied', - }); - }); -}); diff --git a/src/provider-fallback.ts b/src/provider-fallback.ts deleted file mode 100644 index dc62486..0000000 --- a/src/provider-fallback.ts +++ /dev/null @@ -1,450 +0,0 @@ -/** - * Provider Fallback Module - * - * Manages automatic fallback from the primary provider (Claude) to a - * fallback provider (e.g. Kimi K2.5) when 429/rate-limit or network - * errors are detected. - * - * Cooldown-based recovery: - * Claude 429 → immediate Kimi retry for that turn - * Claude enters cooldown (retry-after header or default 10 min) - * During cooldown → skip Claude, route directly to fallback - * After cooldown → try Claude first again - */ - -import fs from 'fs'; - -import { - classifyAgentError, - classifyClaudeAuthError, - isNoFallbackCooldownReason, - type AgentTriggerReason, - type FallbackTriggerReason, -} from './agent-error-detection.js'; -import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js'; -import { getEnv } from './env.js'; -import { logger } from './logger.js'; -import { rotateToken, getTokenCount } from './token-rotation.js'; - -// ── Types ──────────────────────────────────────────────────────── - -export type ProviderName = 'claude' | string; // fallback name is configurable - -export type FallbackTriggerResult = - | { - shouldFallback: false; - reason: ''; - retryAfterMs?: undefined; - } - | { - shouldFallback: true; - reason: FallbackTriggerReason; - retryAfterMs?: number; - }; - -interface CooldownState { - startedAt: number; - expiresAt: number; - reason: AgentTriggerReason; -} - -interface FallbackConfig { - enabled: boolean; - providerName: string; // e.g. "kimi" - baseUrl: string; - authToken: string; - model: string; - smallModel: string; - defaultCooldownMs: number; -} - -// ── State ──────────────────────────────────────────────────────── - -let cooldown: CooldownState | null = null; -let lastUsageAvailabilityCheck: { - checkedAt: number; - result: 'available' | 'exhausted' | 'unknown'; -} | null = null; -let usageAvailabilityCheckPromise: Promise< - 'available' | 'exhausted' | 'unknown' -> | null = null; - -const USAGE_RECOVERY_RECHECK_MS = 30_000; - -// ── Config ─────────────────────────────────────────────────────── - -let _config: FallbackConfig | null = null; - -function loadConfig(): FallbackConfig { - if (_config) return _config; - - const baseUrl = getEnv('FALLBACK_BASE_URL') || ''; - const authToken = getEnv('FALLBACK_AUTH_TOKEN') || ''; - const model = getEnv('FALLBACK_MODEL') || ''; - const explicitlyDisabled = - (getEnv('FALLBACK_ENABLED') || '').toLowerCase() === 'false'; - - _config = { - enabled: !explicitlyDisabled && Boolean(baseUrl && authToken && model), - providerName: getEnv('FALLBACK_PROVIDER_NAME') || 'kimi', - baseUrl, - authToken, - model, - smallModel: getEnv('FALLBACK_SMALL_MODEL') || model, - defaultCooldownMs: parseInt(getEnv('FALLBACK_COOLDOWN_MS') || '600000', 10), - }; - - if (_config.enabled) { - logger.info( - { - provider: _config.providerName, - model: _config.model, - cooldownMs: _config.defaultCooldownMs, - }, - 'Provider fallback configured', - ); - } - - return _config; -} - -/** Force re-read of config (useful after .env changes). */ -export function resetFallbackConfig(): void { - _config = null; -} - -// ── Public API ─────────────────────────────────────────────────── - -/** Check whether the fallback system is configured and available. */ -export function isFallbackEnabled(): boolean { - return loadConfig().enabled; -} - -/** Get the display name of the fallback provider (e.g. "kimi"). */ -export function getFallbackProviderName(): string { - return loadConfig().providerName; -} - -function normalizeUtilization(utilization: number): number { - return utilization > 1 ? utilization : utilization * 100; -} - -type ClaudeUsageWindow = NonNullable; - -function hasExhaustedClaudeUsageWindow( - usage: ClaudeUsageData | null, -): boolean | null { - if (!usage) return null; - const windows: ClaudeUsageWindow[] = []; - if (usage.five_hour) windows.push(usage.five_hour); - if (usage.seven_day) windows.push(usage.seven_day); - if (usage.seven_day_sonnet) windows.push(usage.seven_day_sonnet); - if (usage.seven_day_opus) windows.push(usage.seven_day_opus); - if (windows.length === 0) return null; - return windows.some( - (window) => normalizeUtilization(window.utilization) >= 100, - ); -} - -function clearUsageAvailabilityCache(): void { - lastUsageAvailabilityCheck = null; - usageAvailabilityCheckPromise = null; -} - -function logCooldownTransition( - level: 'debug' | 'info' | 'warn', - transition: string, - fields: Record, - message: string, -): void { - logger[level]( - { - transition, - provider: 'claude', - ...fields, - }, - message, - ); -} - -async function getClaudeUsageAvailability(): Promise< - 'available' | 'exhausted' | 'unknown' -> { - const now = Date.now(); - if ( - lastUsageAvailabilityCheck && - now - lastUsageAvailabilityCheck.checkedAt < USAGE_RECOVERY_RECHECK_MS - ) { - return lastUsageAvailabilityCheck.result; - } - - if (!usageAvailabilityCheckPromise) { - usageAvailabilityCheckPromise = (async () => { - const usage = await fetchClaudeUsage(); - const exhausted = hasExhaustedClaudeUsageWindow(usage); - const result = - exhausted === null ? 'unknown' : exhausted ? 'exhausted' : 'available'; - lastUsageAvailabilityCheck = { - checkedAt: Date.now(), - result, - }; - return result; - })(); - - void usageAvailabilityCheckPromise.finally(() => { - usageAvailabilityCheckPromise = null; - }); - } - - return usageAvailabilityCheckPromise; -} - -/** - * Determine which provider should be used for the next request. - * Returns 'claude' when Claude is healthy or cooldown has expired, - * or the fallback provider name during an active cooldown. - */ -export async function getActiveProvider(): Promise { - const config = loadConfig(); - if (!config.enabled) return 'claude'; - - if (cooldown) { - if (cooldown.reason === 'usage-exhausted') { - const usageAvailability = await getClaudeUsageAvailability(); - if (usageAvailability === 'available') { - logCooldownTransition( - 'info', - 'cooldown:recover', - { - reason: cooldown.reason, - fallbackProvider: config.providerName, - }, - 'Claude usage recovered, retrying primary provider', - ); - cooldown = null; - clearUsageAvailabilityCache(); - return 'claude'; - } - if (usageAvailability === 'exhausted') { - // Current token exhausted — try rotating to another token (ignore cooldowns) - if ( - getTokenCount() > 1 && - rotateToken(undefined, { ignoreRateLimits: true }) - ) { - logCooldownTransition( - 'info', - 'cooldown:recover', - { - reason: cooldown.reason, - fallbackProvider: config.providerName, - recovery: 'token-rotation', - }, - 'Claude current token exhausted, rotated to next token — retrying', - ); - cooldown = null; - clearUsageAvailabilityCache(); - return 'claude'; - } - logCooldownTransition( - 'debug', - 'cooldown:stay', - { - reason: cooldown.reason, - fallbackProvider: config.providerName, - usageAvailability, - }, - 'All Claude tokens exhausted, keeping cooldown active', - ); - return config.providerName; - } - } - - if (Date.now() < cooldown.expiresAt) { - logCooldownTransition( - 'debug', - 'cooldown:stay', - { - reason: cooldown.reason, - fallbackProvider: config.providerName, - remainingMs: cooldown.expiresAt - Date.now(), - }, - 'Claude cooldown still active, routing to fallback provider', - ); - return config.providerName; - } - // Cooldown expired — try Claude again - logCooldownTransition( - 'info', - 'cooldown:expire', - { - cooldownDurationMs: cooldown.expiresAt - cooldown.startedAt, - reason: cooldown.reason, - fallbackProvider: config.providerName, - }, - 'Claude cooldown expired, retrying primary provider', - ); - cooldown = null; - } - - return 'claude'; -} - -/** - * Mark Claude as rate-limited. All subsequent requests will route to - * the fallback provider until the cooldown expires. - */ -export function markPrimaryCooldown( - reason: AgentTriggerReason, - retryAfterMs?: number, -): void { - const config = loadConfig(); - const durationMs = retryAfterMs || config.defaultCooldownMs; - const now = Date.now(); - - cooldown = { - startedAt: now, - expiresAt: now + durationMs, - reason, - }; - clearUsageAvailabilityCache(); - - logCooldownTransition( - 'info', - 'cooldown:enter', - { - reason, - cooldownMs: durationMs, - expiresAt: new Date(cooldown.expiresAt).toISOString(), - fallbackProvider: config.providerName, - }, - `Falling back to provider: ${config.providerName} (reason: ${reason}, cooldownMs: ${durationMs})`, - ); -} - -/** Manually clear cooldown (e.g. after a successful Claude response). */ -export function clearPrimaryCooldown(): void { - clearUsageAvailabilityCache(); - if (cooldown) { - logCooldownTransition( - 'info', - 'cooldown:clear', - { reason: cooldown.reason }, - 'Claude cooldown cleared manually', - ); - cooldown = null; - } -} - -/** Check whether the active primary cooldown should suppress fallback entirely. */ -export function isPrimaryNoFallbackCooldownActive(): boolean { - return cooldown ? isNoFallbackCooldownReason(cooldown.reason) : false; -} - -/** Get current cooldown info (for diagnostics / status dashboard). */ -export function getCooldownInfo(): { - active: boolean; - reason?: string; - expiresAt?: string; - remainingMs?: number; -} { - if (!cooldown) { - return { active: false }; - } - const remainingMs = Math.max(cooldown.expiresAt - Date.now(), 0); - if (cooldown.reason !== 'usage-exhausted' && remainingMs === 0) { - return { active: false }; - } - return { - active: true, - reason: cooldown.reason, - expiresAt: new Date(cooldown.expiresAt).toISOString(), - remainingMs, - }; -} - -/** - * Build the env-var overrides that make Claude Code SDK talk to - * the fallback provider instead of Claude. - */ -export function getFallbackEnvOverrides(): Record { - const config = loadConfig(); - return { - ANTHROPIC_BASE_URL: config.baseUrl, - ANTHROPIC_AUTH_TOKEN: config.authToken, - ANTHROPIC_MODEL: config.model, - ANTHROPIC_SMALL_FAST_MODEL: config.smallModel, - // Disable non-essential traffic (usage telemetry etc.) on fallback - CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', - // Generous timeout for third-party APIs - API_TIMEOUT_MS: '3000000', - // Disable tool search (not supported by most fallback providers) - ENABLE_TOOL_SEARCH: 'false', - }; -} - -/** - * Inspect an agent error string and decide whether it warrants - * a provider fallback. - * - * Triggers: - * - 429 / rate limit / too many requests - * - Claude auth/org access failures - * - 503 / overloaded (transient provider issue) - * - Network / connection errors - * - * Does NOT trigger for: - * - Poisoned sessions - * - Prompt / tool failures - * - Timeouts (agent took too long, not a provider issue) - */ -export function detectFallbackTrigger( - error?: string | null, -): FallbackTriggerResult { - if (!error) return { shouldFallback: false, reason: '' }; - - // Delegated to shared SSOT — original priority preserved: - // 429 first, then Claude auth/org errors, then 503/network - const common = classifyAgentError(error); - - // 429 rate-limit (highest priority) - if (common.category === 'rate-limit') { - return { - shouldFallback: true, - reason: common.reason, - retryAfterMs: common.retryAfterMs, - }; - } - - // Claude-specific strict auth check (before 503/network) - const auth = classifyClaudeAuthError(error); - if (auth.category !== 'none') { - return { shouldFallback: true, reason: auth.reason }; - } - - // 503 overloaded, network errors - if (common.category !== 'none') { - return { - shouldFallback: true, - reason: common.reason, - }; - } - - return { shouldFallback: false, reason: '' }; -} - -/** - * Check whether a per-group settings.json already overrides the - * provider (e.g. the Kimi test channel). If so, we should NOT - * apply fallback env overrides on top — the channel already has - * its own provider configuration. - */ -export function hasGroupProviderOverride(settingsJsonPath: string): boolean { - try { - const raw = fs.readFileSync(settingsJsonPath, 'utf-8'); - const settings = JSON.parse(raw); - const env = settings?.env || {}; - return Boolean(env.ANTHROPIC_BASE_URL || env.ANTHROPIC_MODEL); - } catch { - return false; - } -} diff --git a/src/provider-retry.test.ts b/src/provider-retry.test.ts index ea6f970..ff85294 100644 --- a/src/provider-retry.test.ts +++ b/src/provider-retry.test.ts @@ -8,18 +8,12 @@ vi.mock('./logger.js', () => ({ }, })); -vi.mock('./provider-fallback.js', () => ({ - detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })), - markPrimaryCooldown: vi.fn(), -})); - vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), getTokenCount: vi.fn(() => 1), markTokenHealthy: vi.fn(), })); -import { markPrimaryCooldown } from './provider-fallback.js'; import { runClaudeRotationLoop } from './provider-retry.js'; import { getTokenCount, @@ -50,10 +44,9 @@ describe('runClaudeRotationLoop', () => { expect(outcome).toEqual({ type: 'success' }); expect(rotateToken).toHaveBeenCalledTimes(1); expect(markTokenHealthy).toHaveBeenCalledTimes(1); - expect(markPrimaryCooldown).not.toHaveBeenCalled(); }); - it('marks no-fallback cooldown when all Claude tokens are org-access-denied', async () => { + it('returns error when all Claude tokens are exhausted', async () => { vi.mocked(getTokenCount).mockReturnValue(2); const outcome = await runClaudeRotationLoop( @@ -66,16 +59,12 @@ describe('runClaudeRotationLoop', () => { ); expect(outcome).toEqual({ - type: 'no-fallback', + type: 'error', trigger: { reason: 'org-access-denied' }, }); - expect(markPrimaryCooldown).toHaveBeenCalledWith( - 'org-access-denied', - undefined, - ); }); - it('returns success-null-result as a fallback trigger after rotation', async () => { + it('returns error with success-null-result trigger after rotation', async () => { vi.mocked(getTokenCount).mockReturnValue(2); vi.mocked(rotateToken).mockReturnValueOnce(true); @@ -90,9 +79,8 @@ describe('runClaudeRotationLoop', () => { ); expect(outcome).toEqual({ - type: 'needs-fallback', + type: 'error', trigger: { reason: 'success-null-result' }, }); - expect(markPrimaryCooldown).not.toHaveBeenCalled(); }); }); diff --git a/src/provider-retry.ts b/src/provider-retry.ts index 8ca94ba..49f3261 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -6,16 +6,12 @@ */ import { - isNoFallbackCooldownReason, + classifyRotationTrigger, shouldRotateClaudeToken, type AgentTriggerReason, } from './agent-error-detection.js'; import { logger } from './logger.js'; import { getErrorMessage } from './utils.js'; -import { - detectFallbackTrigger, - markPrimaryCooldown, -} from './provider-fallback.js'; import { rotateToken, getTokenCount, @@ -39,17 +35,15 @@ export interface RotationAttemptResult { export type RotationOutcome = | { type: 'success' } - | { type: 'error'; message?: string } - | { type: 'needs-fallback'; trigger: TriggerInfo } - | { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired/org-access-denied + | { type: 'error'; trigger?: TriggerInfo }; // ── Shared rotation loop ───────────────────────────────────────── /** * Retry a Claude request by rotating through available tokens. * - * Returns a discriminated outcome — the caller decides what to do - * with 'needs-fallback' (e.g. run Kimi fallback) or 'no-fallback'. + * Returns 'success' if a rotated token worked, or 'error' if all + * tokens are exhausted or the error is non-retryable. */ export async function runClaudeRotationLoop( initialTrigger: TriggerInfo, @@ -63,7 +57,9 @@ export async function runClaudeRotationLoop( while ( shouldRotateClaudeToken(trigger.reason) && getTokenCount() > 1 && - rotateToken(lastRotationMessage, { ignoreRateLimits: true }) + // Respect per-token cooldowns so exhausted auth/quota failures can + // terminate instead of cycling forever. + rotateToken(lastRotationMessage) ) { logger.info( { ...logContext, reason: trigger.reason }, @@ -78,12 +74,12 @@ export async function runClaudeRotationLoop( const errMsg = getErrorMessage(attempt.thrownError); const retryTrigger = attempt.streamedTriggerReason ? { - shouldFallback: true, + shouldRetry: true, reason: attempt.streamedTriggerReason.reason, retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, } - : detectFallbackTrigger(errMsg); - if (retryTrigger.shouldFallback) { + : classifyRotationTrigger(errMsg); + if (retryTrigger.shouldRetry) { trigger = { reason: retryTrigger.reason, retryAfterMs: retryTrigger.retryAfterMs, @@ -124,12 +120,13 @@ export async function runClaudeRotationLoop( continue; } - // ── Success with null result (MAE-specific, TaskScheduler ignores) ── + // ── Success with null result ── if (!attempt.sawOutput && attempt.sawSuccessNullResult) { - return { - type: 'needs-fallback', - trigger: { reason: 'success-null-result' }, - }; + logger.warn( + { ...logContext, provider: 'claude' }, + 'All rotated tokens returned success with null result', + ); + return { type: 'error', trigger: { reason: 'success-null-result' } }; } // ── Error status ── @@ -137,12 +134,12 @@ export async function runClaudeRotationLoop( if (!attempt.sawOutput) { const retryTrigger = attempt.streamedTriggerReason ? { - shouldFallback: true, + shouldRetry: true, reason: attempt.streamedTriggerReason.reason, retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, } - : detectFallbackTrigger(output.error); - if (retryTrigger.shouldFallback) { + : classifyRotationTrigger(output.error); + if (retryTrigger.shouldRetry) { trigger = { reason: retryTrigger.reason, retryAfterMs: retryTrigger.retryAfterMs, @@ -165,16 +162,9 @@ export async function runClaudeRotationLoop( } // ── All tokens exhausted ── - - // Usage/auth/org access failures: don't fall back to Kimi - if (isNoFallbackCooldownReason(trigger.reason)) { - markPrimaryCooldown(trigger.reason, trigger.retryAfterMs); - logger.info( - { ...logContext, reason: trigger.reason }, - `All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`, - ); - return { type: 'no-fallback', trigger }; - } - - return { type: 'needs-fallback', trigger }; + logger.warn( + { ...logContext, reason: trigger.reason }, + `All Claude tokens exhausted (${trigger.reason})`, + ); + return { type: 'error', trigger }; } diff --git a/src/router.ts b/src/router.ts index 2c606af..83a3233 100644 --- a/src/router.ts +++ b/src/router.ts @@ -92,3 +92,15 @@ export function findChannel( ): Channel | undefined { return channels.find((c) => c.ownsJid(jid)); } + +/** + * Normalize message text for deduplication comparison. + * - Trim leading/trailing whitespace + * - Collapse consecutive whitespace/newlines into single space + */ +export function normalizeMessageForDedupe(text: string): string { + return text + .trim() + .replace(/\s+/g, ' ') + .toLowerCase(); +} diff --git a/src/service-routing.test.ts b/src/service-routing.test.ts new file mode 100644 index 0000000..6d384e4 --- /dev/null +++ b/src/service-routing.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { _initTestDatabase, setRegisteredGroup } from './db.js'; +import { + activateCodexFailover, + getActiveCodexFailoverLeases, + getEffectiveChannelLease, + refreshChannelOwnerCache, + restoreDefaultChannelLease, +} from './service-routing.js'; + +beforeEach(() => { + _initTestDatabase(); + refreshChannelOwnerCache(true); +}); + +describe('service-routing failover leases', () => { + it('uses codex-review as owner and codex-main as reviewer during failover', () => { + setRegisteredGroup('dc:paired', { + name: 'Paired Room Claude', + folder: 'paired-claude', + trigger: '@Andy', + added_at: '2024-01-01T00:00:00.000Z', + agentType: 'claude-code', + }); + setRegisteredGroup('dc:paired', { + name: 'Paired Room Codex', + folder: 'paired-codex', + trigger: '@Codex', + added_at: '2024-01-01T00:00:00.000Z', + agentType: 'codex', + }); + + activateCodexFailover('dc:paired', 'claude-429'); + + expect(getEffectiveChannelLease('dc:paired')).toMatchObject({ + chat_jid: 'dc:paired', + owner_service_id: 'codex-review', + reviewer_service_id: 'codex-main', + reason: 'claude-429', + explicit: true, + }); + expect(getActiveCodexFailoverLeases()).toEqual([ + { + chatJid: 'dc:paired', + activatedAt: expect.any(String), + }, + ]); + }); + + it('restores the default lease after failover is cleared', () => { + setRegisteredGroup('dc:paired', { + name: 'Paired Room Claude', + folder: 'paired-claude', + trigger: '@Andy', + added_at: '2024-01-01T00:00:00.000Z', + agentType: 'claude-code', + }); + setRegisteredGroup('dc:paired', { + name: 'Paired Room Codex', + folder: 'paired-codex', + trigger: '@Codex', + added_at: '2024-01-01T00:00:00.000Z', + agentType: 'codex', + }); + + activateCodexFailover('dc:paired', 'claude-429'); + restoreDefaultChannelLease('dc:paired'); + + expect(getEffectiveChannelLease('dc:paired')).toMatchObject({ + chat_jid: 'dc:paired', + owner_service_id: 'claude', + reviewer_service_id: 'codex-main', + explicit: false, + }); + }); +}); diff --git a/src/service-routing.ts b/src/service-routing.ts new file mode 100644 index 0000000..8059ad8 --- /dev/null +++ b/src/service-routing.ts @@ -0,0 +1,189 @@ +import { + CLAUDE_SERVICE_ID, + CODEX_MAIN_SERVICE_ID, + CODEX_REVIEW_SERVICE_ID, + SERVICE_AGENT_TYPE, + SERVICE_ID, + normalizeServiceId, +} from './config.js'; +import { + clearChannelOwnerLease, + getAllChannelOwnerLeases, + getRegisteredAgentTypesForJid, + setChannelOwnerLease, + type ChannelOwnerLeaseRow, +} from './db.js'; + +export interface EffectiveChannelLease { + chat_jid: string; + owner_service_id: string; + reviewer_service_id: string | null; + activated_at: string | null; + reason: string | null; + explicit: boolean; +} + +const LEASE_CACHE_REFRESH_MS = 2_000; + +let lastLeaseRefreshAt = 0; +const leaseCache = new Map(); + +function normalizeLeaseRow( + row: ChannelOwnerLeaseRow, + explicit: boolean, +): EffectiveChannelLease { + return { + chat_jid: row.chat_jid, + owner_service_id: normalizeServiceId(row.owner_service_id), + reviewer_service_id: row.reviewer_service_id + ? normalizeServiceId(row.reviewer_service_id) + : null, + activated_at: row.activated_at, + reason: row.reason, + explicit, + }; +} + +function getDefaultLease(chatJid: string): EffectiveChannelLease { + const types = getRegisteredAgentTypesForJid(chatJid); + const hasClaude = types.includes('claude-code'); + const hasCodex = types.includes('codex'); + + if (hasClaude && hasCodex) { + return { + chat_jid: chatJid, + owner_service_id: CLAUDE_SERVICE_ID, + reviewer_service_id: CODEX_MAIN_SERVICE_ID, + activated_at: null, + reason: null, + explicit: false, + }; + } + + if (hasCodex) { + return { + chat_jid: chatJid, + owner_service_id: CODEX_MAIN_SERVICE_ID, + reviewer_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }; + } + + if (hasClaude) { + return { + chat_jid: chatJid, + owner_service_id: CLAUDE_SERVICE_ID, + reviewer_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }; + } + + return { + chat_jid: chatJid, + owner_service_id: + SERVICE_AGENT_TYPE === 'codex' ? CODEX_MAIN_SERVICE_ID : CLAUDE_SERVICE_ID, + reviewer_service_id: null, + activated_at: null, + reason: null, + explicit: false, + }; +} + +export function refreshChannelOwnerCache(force = false): void { + const now = Date.now(); + if (!force && now - lastLeaseRefreshAt < LEASE_CACHE_REFRESH_MS) { + return; + } + + leaseCache.clear(); + for (const row of getAllChannelOwnerLeases()) { + leaseCache.set(row.chat_jid, row); + } + lastLeaseRefreshAt = now; +} + +export function getEffectiveChannelLease(chatJid: string): EffectiveChannelLease { + refreshChannelOwnerCache(); + const row = leaseCache.get(chatJid); + if (row) { + return normalizeLeaseRow(row, true); + } + return getDefaultLease(chatJid); +} + +export function isOwnerServiceForChat( + chatJid: string, + serviceId: string = SERVICE_ID, +): boolean { + const lease = getEffectiveChannelLease(chatJid); + return normalizeServiceId(serviceId) === lease.owner_service_id; +} + +export function isReviewerServiceForChat( + chatJid: string, + serviceId: string = SERVICE_ID, +): boolean { + const lease = getEffectiveChannelLease(chatJid); + return ( + lease.reviewer_service_id !== null && + normalizeServiceId(serviceId) === lease.reviewer_service_id + ); +} + +export function shouldServiceProcessChat( + chatJid: string, + serviceId: string = SERVICE_ID, +): boolean { + const normalizedServiceId = normalizeServiceId(serviceId); + const lease = getEffectiveChannelLease(chatJid); + return ( + normalizedServiceId === lease.owner_service_id || + normalizedServiceId === lease.reviewer_service_id + ); +} + +export function activateCodexFailover(chatJid: string, reason: string): void { + const now = new Date().toISOString(); + const row: ChannelOwnerLeaseRow = { + chat_jid: chatJid, + owner_service_id: CODEX_REVIEW_SERVICE_ID, + reviewer_service_id: CODEX_MAIN_SERVICE_ID, + activated_at: now, + reason, + }; + setChannelOwnerLease(row); + leaseCache.set(chatJid, row); + lastLeaseRefreshAt = Date.now(); +} + +export function restoreDefaultChannelLease(chatJid: string): void { + clearChannelOwnerLease(chatJid); + leaseCache.delete(chatJid); + lastLeaseRefreshAt = Date.now(); +} + +export interface ActiveFailoverLease { + chatJid: string; + activatedAt: string | null; +} + +export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] { + refreshChannelOwnerCache(true); + return [...leaseCache.values()] + .filter( + (row) => + normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID && + normalizeServiceId(row.reviewer_service_id || '') === + CODEX_MAIN_SERVICE_ID, + ) + .map((row) => ({ chatJid: row.chat_jid, activatedAt: row.activated_at ?? null })); +} + +/** @deprecated Use getActiveCodexFailoverLeases() instead */ +export function getActiveCodexFailoverChatJids(): string[] { + return getActiveCodexFailoverLeases().map((l) => l.chatJid); +} diff --git a/src/session-recovery.test.ts b/src/session-recovery.test.ts index 0723595..32656fb 100644 --- a/src/session-recovery.test.ts +++ b/src/session-recovery.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { shouldResetSessionOnAgentFailure } from './session-recovery.js'; +import { + shouldResetSessionOnAgentFailure, + shouldRetryFreshSessionOnAgentFailure, +} from './session-recovery.js'; describe('shouldResetSessionOnAgentFailure', () => { it('matches many-image dimension limit errors', () => { @@ -31,4 +34,45 @@ describe('shouldResetSessionOnAgentFailure', () => { }), ).toBe(false); }); + + it('matches thinking signature 400 errors', () => { + expect( + shouldResetSessionOnAgentFailure({ + result: + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}', + error: undefined, + }), + ).toBe(true); + }); +}); + +describe('shouldRetryFreshSessionOnAgentFailure', () => { + it('matches stale session errors', () => { + expect( + shouldRetryFreshSessionOnAgentFailure({ + result: null, + error: 'No conversation found with session ID: stale', + }), + ).toBe(true); + }); + + it('matches thinking signature 400 errors', () => { + expect( + shouldRetryFreshSessionOnAgentFailure({ + result: + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}', + error: undefined, + }), + ).toBe(true); + }); + + it('does not retry image dimension limit errors', () => { + expect( + shouldRetryFreshSessionOnAgentFailure({ + result: + 'An image in the conversation exceeds the dimension limit for many-image requests (2000px). Start a new session with fewer images.', + error: undefined, + }), + ).toBe(false); + }); }); diff --git a/src/session-recovery.ts b/src/session-recovery.ts index eae9f88..fbaeeb1 100644 --- a/src/session-recovery.ts +++ b/src/session-recovery.ts @@ -4,6 +4,16 @@ const SESSION_RESET_PATTERNS = [ /An image in the conversation exceeds the dimension limit for many-image requests \(2000px\)\./i, /Start a new session with fewer images\./i, /No conversation found with session ID/i, + /400[^\n]*thinking/i, + /thinking[^\n]*block[^\n]*(?:invalid|error|signature)/i, + /invalid[^\n]*signature[^\n]*thinking/i, +]; + +const SESSION_RETRY_PATTERNS = [ + /No conversation found with session ID/i, + /400[^\n]*thinking/i, + /thinking[^\n]*block[^\n]*(?:invalid|error|signature)/i, + /invalid[^\n]*signature[^\n]*thinking/i, ]; function toText(value: string | object | null | undefined): string[] { @@ -25,3 +35,12 @@ export function shouldResetSessionOnAgentFailure( SESSION_RESET_PATTERNS.some((pattern) => pattern.test(text)), ); } + +export function shouldRetryFreshSessionOnAgentFailure( + output: Pick, +): boolean { + const texts = [...toText(output.result), ...toText(output.error)]; + return texts.some((text) => + SESSION_RETRY_PATTERNS.some((pattern) => pattern.test(text)), + ); +} diff --git a/src/status-dashboard.ts b/src/status-dashboard.ts index 527df16..59d77ba 100644 --- a/src/status-dashboard.ts +++ b/src/status-dashboard.ts @@ -26,6 +26,7 @@ export interface UsageRowSnapshot { } export interface StatusSnapshot { + serviceId: string; agentType: AgentType; assistantName: string; updatedAt: string; @@ -41,7 +42,7 @@ export function writeStatusSnapshot(snapshot: StatusSnapshot): void { fs.mkdirSync(STATUS_SNAPSHOT_DIR, { recursive: true }); const targetPath = path.join( STATUS_SNAPSHOT_DIR, - `${snapshot.agentType}.json`, + `${snapshot.serviceId}.json`, ); const tempPath = `${targetPath}.tmp`; writeJsonFile(tempPath, snapshot, true); @@ -63,6 +64,7 @@ export function readStatusSnapshots(maxAgeMs: number): StatusSnapshot[] { const parsed = JSON.parse(raw) as StatusSnapshot; if ( !parsed.updatedAt || + !parsed.serviceId || !parsed.agentType || !Array.isArray(parsed.entries) ) diff --git a/src/streamed-output-evaluator.test.ts b/src/streamed-output-evaluator.test.ts index b9b64b0..fbf9c30 100644 --- a/src/streamed-output-evaluator.test.ts +++ b/src/streamed-output-evaluator.test.ts @@ -4,6 +4,8 @@ import type { AgentOutput } from './agent-runner.js'; // ── Mocks ────────────────────────────────────────────────────── vi.mock('./agent-error-detection.js', () => ({ + classifyClaudeAuthError: vi.fn(() => ({ category: 'none', reason: '' })), + classifyRotationTrigger: vi.fn(() => ({ shouldRetry: false, reason: '' })), detectClaudeProviderFailureMessage: vi.fn(() => ''), isClaudeUsageExhaustedMessage: vi.fn(() => false), isClaudeOrgAccessDeniedMessage: vi.fn(() => false), @@ -11,10 +13,6 @@ vi.mock('./agent-error-detection.js', () => ({ isClaudeAuthError: vi.fn(() => false), })); -vi.mock('./provider-fallback.js', () => ({ - detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })), -})); - vi.mock('./codex-token-rotation.js', () => ({ detectCodexRotationTrigger: vi.fn(() => ({ shouldRotate: false, @@ -22,15 +20,21 @@ vi.mock('./codex-token-rotation.js', () => ({ })), })); +vi.mock('./session-recovery.js', () => ({ + shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false), +})); + import { + classifyClaudeAuthError, + classifyRotationTrigger, detectClaudeProviderFailureMessage, isClaudeUsageExhaustedMessage, isClaudeOrgAccessDeniedMessage, isClaudeAuthExpiredMessage, isClaudeAuthError, } from './agent-error-detection.js'; -import { detectFallbackTrigger } from './provider-fallback.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; +import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js'; import { evaluateStreamedOutput, @@ -40,7 +44,11 @@ import { // ── Helpers ──────────────────────────────────────────────────── function freshState(): StreamedOutputState { - return { sawOutput: false, sawSuccessNullResultWithoutOutput: false }; + return { + sawOutput: false, + sawVisibleOutput: false, + sawSuccessNullResultWithoutOutput: false, + }; } const claudeOpts: EvaluateStreamedOutputOptions = { @@ -53,8 +61,11 @@ const codexOpts: EvaluateStreamedOutputOptions = { provider: 'codex', }; -function successOutput(result: string | null): AgentOutput { - return { status: 'success', result }; +function successOutput( + result: string | null, + phase?: AgentOutput['phase'], +): AgentOutput { + return { status: 'success', result, ...(phase ? { phase } : {}) }; } function errorOutput(error: string): AgentOutput { @@ -63,19 +74,24 @@ function errorOutput(error: string): AgentOutput { beforeEach(() => { vi.clearAllMocks(); + vi.mocked(classifyClaudeAuthError).mockReturnValue({ + category: 'none', + reason: '', + }); + vi.mocked(classifyRotationTrigger).mockReturnValue({ + shouldRetry: false, + reason: '', + }); vi.mocked(detectClaudeProviderFailureMessage).mockReturnValue(''); vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false); vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(false); vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false); vi.mocked(isClaudeAuthError).mockReturnValue(false); - vi.mocked(detectFallbackTrigger).mockReturnValue({ - shouldFallback: false, - reason: '', - }); vi.mocked(detectCodexRotationTrigger).mockReturnValue({ shouldRotate: false, reason: '', }); + vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(false); }); // ── Tests ────────────────────────────────────────────────────── @@ -112,6 +128,16 @@ describe('evaluateStreamedOutput', () => { expect(result.shouldForwardOutput).toBe(true); expect(result.state.sawOutput).toBe(false); }); + + it('does not set sawOutput for progress output', () => { + const result = evaluateStreamedOutput( + successOutput('대화 요약 중...', 'progress'), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(true); + expect(result.state.sawOutput).toBe(false); + }); }); describe('Claude usage-exhausted banner', () => { @@ -156,6 +182,39 @@ describe('evaluateStreamedOutput', () => { }); }); + describe('Claude retryable session failure suppression', () => { + it('suppresses retryable session failures before any visible output', () => { + vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput( + 'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}', + 'intermediate', + ), + freshState(), + claudeOpts, + ); + + expect(result.shouldForwardOutput).toBe(false); + expect(result.suppressedRetryableSessionFailure).toBe(true); + expect(result.state.retryableSessionFailureDetected).toBe(true); + expect(result.state.sawOutput).toBe(false); + }); + + it('does not suppress retryable session failures after visible output', () => { + vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(true); + + const result = evaluateStreamedOutput( + successOutput('API Error: 400 invalid thinking block', 'final'), + { ...freshState(), sawOutput: true }, + claudeOpts, + ); + + expect(result.shouldForwardOutput).toBe(true); + expect(result.suppressedRetryableSessionFailure).toBeUndefined(); + }); + }); + describe('Claude auth-expired banner', () => { it('suppresses output and returns newTrigger', () => { vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(true); @@ -208,6 +267,24 @@ describe('evaluateStreamedOutput', () => { reason: 'org-access-denied', }); }); + + it('suppresses a 403 terminated auth banner surfaced as success text', () => { + vi.mocked(classifyClaudeAuthError).mockReturnValue({ + category: 'org-access-denied', + reason: 'org-access-denied', + }); + + const result = evaluateStreamedOutput( + successOutput('Failed to authenticate. API Error: 403 terminated'), + freshState(), + claudeOpts, + ); + expect(result.shouldForwardOutput).toBe(false); + expect(result.newTrigger).toEqual({ reason: 'org-access-denied' }); + expect(result.state.streamedTriggerReason).toEqual({ + reason: 'org-access-denied', + }); + }); }); describe('duplicate trigger suppression', () => { @@ -298,10 +375,10 @@ describe('evaluateStreamedOutput', () => { }); }); - describe('error → Claude fallback trigger', () => { - it('returns fallback trigger with retryAfterMs', () => { - vi.mocked(detectFallbackTrigger).mockReturnValue({ - shouldFallback: true, + describe('error → Claude rotation trigger', () => { + it('returns rotation trigger with retryAfterMs', () => { + vi.mocked(classifyRotationTrigger).mockReturnValue({ + shouldRetry: true, reason: '429', retryAfterMs: 30000, }); @@ -324,8 +401,8 @@ describe('evaluateStreamedOutput', () => { }); it('suppresses error forward when shortCircuitTriggeredErrors is set', () => { - vi.mocked(detectFallbackTrigger).mockReturnValue({ - shouldFallback: true, + vi.mocked(classifyRotationTrigger).mockReturnValue({ + shouldRetry: true, reason: '429', }); @@ -337,9 +414,9 @@ describe('evaluateStreamedOutput', () => { expect(result.newTrigger).toEqual({ reason: '429' }); }); - it('skips fallback check when output was already seen', () => { - vi.mocked(detectFallbackTrigger).mockReturnValue({ - shouldFallback: true, + it('skips rotation check when output was already seen', () => { + vi.mocked(classifyRotationTrigger).mockReturnValue({ + shouldRetry: true, reason: '429', }); @@ -352,9 +429,9 @@ describe('evaluateStreamedOutput', () => { expect(result.shouldForwardOutput).toBe(true); }); - it('skips fallback check when already triggered', () => { - vi.mocked(detectFallbackTrigger).mockReturnValue({ - shouldFallback: true, + it('skips rotation check when already triggered', () => { + vi.mocked(classifyRotationTrigger).mockReturnValue({ + shouldRetry: true, reason: '429', }); @@ -397,7 +474,7 @@ describe('evaluateStreamedOutput', () => { freshState(), claudeOpts, ); - // Claude uses detectFallbackTrigger, not detectCodexRotationTrigger + // Claude uses classifyRotationTrigger, not detectCodexRotationTrigger expect(detectCodexRotationTrigger).not.toHaveBeenCalled(); }); }); diff --git a/src/streamed-output-evaluator.ts b/src/streamed-output-evaluator.ts index 9044922..2705dc0 100644 --- a/src/streamed-output-evaluator.ts +++ b/src/streamed-output-evaluator.ts @@ -1,4 +1,6 @@ import { + classifyClaudeAuthError, + classifyRotationTrigger, detectClaudeProviderFailureMessage, isClaudeAuthError, isClaudeAuthExpiredMessage, @@ -8,7 +10,7 @@ import { } from './agent-error-detection.js'; import type { AgentOutput } from './agent-runner.js'; import { detectCodexRotationTrigger } from './codex-token-rotation.js'; -import { detectFallbackTrigger } from './provider-fallback.js'; +import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js'; export interface StreamedTriggerReason { reason: AgentTriggerReason; @@ -17,8 +19,10 @@ export interface StreamedTriggerReason { export interface StreamedOutputState { sawOutput: boolean; + sawVisibleOutput: boolean; sawSuccessNullResultWithoutOutput: boolean; streamedTriggerReason?: StreamedTriggerReason; + retryableSessionFailureDetected?: boolean; } export interface EvaluateStreamedOutputOptions { @@ -34,6 +38,7 @@ export interface EvaluateStreamedOutputResult { shouldForwardOutput: boolean; newTrigger?: StreamedTriggerReason; suppressedAuthError?: boolean; + suppressedRetryableSessionFailure?: boolean; } export function evaluateStreamedOutput( @@ -46,6 +51,21 @@ export function evaluateStreamedOutput( options.agentType === 'claude-code' && options.provider === 'claude'; const isPrimaryCodex = options.agentType === 'codex' && options.provider === 'codex'; + const countsAsFinalOutput = + output.phase === undefined || output.phase === 'final'; + + if ( + isPrimaryClaude && + !state.sawOutput && + shouldRetryFreshSessionOnAgentFailure(output) + ) { + nextState.retryableSessionFailureDetected = true; + return { + state: nextState, + shouldForwardOutput: false, + suppressedRetryableSessionFailure: true, + }; + } if ( isPrimaryClaude && @@ -53,12 +73,15 @@ export function evaluateStreamedOutput( !state.sawOutput && typeof output.result === 'string' ) { + const authClassification = classifyClaudeAuthError(output.result); const triggerReason: AgentTriggerReason | undefined = isClaudeUsageExhaustedMessage(output.result) ? 'usage-exhausted' - : isClaudeOrgAccessDeniedMessage(output.result) + : isClaudeOrgAccessDeniedMessage(output.result) || + authClassification.category === 'org-access-denied' ? 'org-access-denied' - : isClaudeAuthExpiredMessage(output.result) + : isClaudeAuthExpiredMessage(output.result) || + authClassification.category === 'auth-expired' ? 'auth-expired' : detectClaudeProviderFailureMessage(output.result) || undefined; @@ -87,7 +110,11 @@ export function evaluateStreamedOutput( } } - if (output.result !== null && output.result !== undefined) { + if ( + countsAsFinalOutput && + output.result !== null && + output.result !== undefined + ) { nextState.sawOutput = true; } else if ( options.trackSuccessNullResult && @@ -106,8 +133,8 @@ export function evaluateStreamedOutput( let newTrigger: StreamedTriggerReason | undefined; if (isPrimaryClaude) { - const trigger = detectFallbackTrigger(output.error); - if (trigger.shouldFallback) { + const trigger = classifyRotationTrigger(output.error); + if (trigger.shouldRetry) { newTrigger = { reason: trigger.reason, retryAfterMs: trigger.retryAfterMs, diff --git a/src/task-scheduler.test.ts b/src/task-scheduler.test.ts index ab8473b..a20c7f8 100644 --- a/src/task-scheduler.test.ts +++ b/src/task-scheduler.test.ts @@ -24,38 +24,31 @@ const { ), })); -vi.mock('./provider-fallback.js', () => ({ - detectFallbackTrigger: vi.fn((error?: string | null) => { - const lower = (error || '').toLowerCase(); - if ( - lower.includes('does not have access to claude') || - (lower.includes('failed to authenticate') && - lower.includes('403') && - lower.includes('terminated')) - ) { - return { shouldFallback: true, reason: 'org-access-denied' }; - } - if ( - lower.includes('429') || - lower.includes('rate limit') || - lower.includes('hit your limit') - ) { - return { shouldFallback: true, reason: '429' }; - } - return { shouldFallback: false, reason: '' }; - }), - getActiveProvider: vi.fn(async () => 'claude'), - getFallbackEnvOverrides: vi.fn(() => ({ - ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/', - ANTHROPIC_AUTH_TOKEN: 'test-kimi-key', - ANTHROPIC_MODEL: 'kimi-k2.5', - })), - getFallbackProviderName: vi.fn(() => 'kimi'), - hasGroupProviderOverride: vi.fn(() => false), - isFallbackEnabled: vi.fn(() => true), - isPrimaryNoFallbackCooldownActive: vi.fn(() => false), - markPrimaryCooldown: vi.fn(), -})); +vi.mock('./agent-error-detection.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + classifyRotationTrigger: vi.fn((error?: string | null) => { + const lower = (error || '').toLowerCase(); + if ( + lower.includes('does not have access to claude') || + (lower.includes('failed to authenticate') && + lower.includes('403') && + lower.includes('terminated')) + ) { + return { shouldRetry: true, reason: 'org-access-denied' }; + } + if ( + lower.includes('429') || + lower.includes('rate limit') || + lower.includes('hit your limit') + ) { + return { shouldRetry: true, reason: '429' }; + } + return { shouldRetry: false, reason: '' }; + }), + }; +}); vi.mock('./token-rotation.js', () => ({ rotateToken: vi.fn(() => false), @@ -136,7 +129,6 @@ vi.mock('./github-ci.js', () => ({ })); import { _initTestDatabase, createTask, getTaskById } from './db.js'; -import * as providerFallback from './provider-fallback.js'; import * as codexTokenRotation from './codex-token-rotation.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js'; @@ -163,10 +155,7 @@ describe('task scheduler', () => { terminal: false, resultSummary: 'GitHub Actions run 123 is in_progress', }); - vi.mocked(providerFallback.markPrimaryCooldown).mockClear(); - vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude'); - vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true); - vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false); + // No fallback provider setup needed vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1); vi.mocked(tokenRotation.markTokenHealthy).mockClear(); vi.mocked(tokenRotation.rotateToken).mockClear(); @@ -454,7 +443,7 @@ Check the run. expect(runAgentProcessMock).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + // No fallback cooldown to check expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage).toHaveBeenCalledWith( 'shared@g.us', @@ -553,7 +542,7 @@ Check the run. expect(runAgentProcessMock).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + // No fallback cooldown to check expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage).toHaveBeenCalledWith( 'shared@g.us', @@ -601,23 +590,6 @@ Check the run. result: null, }; }, - ) - .mockImplementationOnce( - async ( - _group: unknown, - _input: unknown, - _onProcess: unknown, - onOutput?: (output: Record) => Promise, - ) => { - await onOutput?.({ - status: 'success', - result: 'scheduled fallback response', - }); - return { - status: 'success', - result: null, - }; - }, ); const enqueueTask = vi.fn( @@ -646,17 +618,9 @@ Check the run. await vi.advanceTimersByTimeAsync(10); - expect(runAgentProcessMock).toHaveBeenCalledTimes(2); + expect(runAgentProcessMock).toHaveBeenCalledTimes(1); expect(tokenRotation.rotateToken).not.toHaveBeenCalled(); - expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith( - 'overloaded', - undefined, - ); - expect(sendMessage).toHaveBeenCalledTimes(1); - expect(sendMessage).toHaveBeenCalledWith( - 'shared@g.us', - 'scheduled fallback response', - ); + // No fallback — 502 results in error without retrying on another provider }); it('suppresses Claude org access denied banners for scheduled tasks and retries with a rotated account', async () => { @@ -750,7 +714,7 @@ Check the run. expect(runAgentProcessMock).toHaveBeenCalledTimes(2); expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1); expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1); - expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled(); + // No fallback cooldown to check expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage).toHaveBeenCalledWith( 'shared@g.us', diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index e114a4e..7695be7 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -1,12 +1,10 @@ import { ChildProcess } from 'child_process'; import { CronExpressionParser } from 'cron-parser'; import fs from 'fs'; -import path from 'path'; import { getErrorMessage } from './utils.js'; import { ASSISTANT_NAME, - DATA_DIR, SCHEDULER_POLL_INTERVAL, SERVICE_AGENT_TYPE, TIMEZONE, @@ -33,16 +31,6 @@ import { } from './group-folder.js'; import { logger } from './logger.js'; import { createTaskStatusTracker } from './task-status-tracker.js'; -import { - detectFallbackTrigger, - getActiveProvider, - getFallbackEnvOverrides, - getFallbackProviderName, - hasGroupProviderOverride, - isFallbackEnabled, - isPrimaryNoFallbackCooldownActive, - markPrimaryCooldown, -} from './provider-fallback.js'; import { runClaudeRotationLoop } from './provider-retry.js'; import { detectCodexRotationTrigger, @@ -50,9 +38,10 @@ import { getCodexAccountCount, markCodexTokenHealthy, } from './codex-token-rotation.js'; -import type { - AgentTriggerReason, - CodexRotationReason, +import { + classifyRotationTrigger, + type AgentTriggerReason, + type CodexRotationReason, } from './agent-error-detection.js'; import { getTokenCount, @@ -306,19 +295,8 @@ async function runTask( sendTrackedMessage: deps.sendTrackedMessage, editTrackedMessage: deps.editTrackedMessage, }); - const settingsPath = path.join( - DATA_DIR, - 'sessions', - task.group_folder, - '.claude', - 'settings.json', - ); const isClaudeAgent = context.taskAgentType === 'claude-code'; const canRotateToken = isClaudeAgent && getTokenCount() > 1; - const canFallback = - isClaudeAgent && - isFallbackEnabled() && - !hasGroupProviderOverride(settingsPath); try { await statusTracker.update('checking'); @@ -337,6 +315,7 @@ async function runTask( }> => { let streamedState: StreamedOutputState = { sawOutput: false, + sawVisibleOutput: false, sawSuccessNullResultWithoutOutput: false, }; let attemptResult: string | null = null; @@ -391,7 +370,7 @@ async function runTask( reason: evaluation.newTrigger.reason, resultPreview: streamedOutput.result.slice(0, 120), }, - 'Detected Claude fallback trigger during scheduled task output', + 'Detected Claude rotation trigger during scheduled task output', ); } else if ( evaluation.newTrigger && @@ -407,7 +386,7 @@ async function runTask( errorPreview: streamedOutput.error.slice(0, 120), }, provider === 'claude' - ? 'Detected Claude fallback trigger during scheduled task error output' + ? 'Detected Claude rotation trigger during scheduled task error output' : 'Detected Codex rotation trigger during scheduled task error output', ); } @@ -428,9 +407,7 @@ async function runTask( attemptError = streamedOutput.error || 'Unknown error'; } }, - isClaudeAgent && provider !== 'claude' - ? getFallbackEnvOverrides() - : undefined, + undefined, ); if (output.status === 'error' && !attemptError) { @@ -448,38 +425,6 @@ async function runTask( }; }; - const runFallbackTaskAttempt = async ( - reason: AgentTriggerReason, - retryAfterMs?: number, - ): Promise => { - if (!canFallback) { - error = reason; - return; - } - - const fallbackName = getFallbackProviderName(); - markPrimaryCooldown(reason, retryAfterMs); - - logger.info( - { - taskId: task.id, - group: context.group.name, - groupFolder: task.group_folder, - reason, - retryAfterMs, - fallbackProvider: fallbackName, - }, - `Falling back to provider: ${fallbackName} for scheduled task (reason: ${reason})`, - ); - - const fallbackAttempt = await runTaskAttempt(fallbackName); - result = fallbackAttempt.attemptResult; - error = - fallbackAttempt.output.status === 'error' - ? fallbackAttempt.attemptError || 'Unknown error' - : null; - }; - const retryClaudeTaskWithRotation = async ( initialTrigger: { reason: AgentTriggerReason; @@ -514,15 +459,9 @@ async function runTask( error = null; return; case 'error': - return; - case 'no-fallback': - error = `Claude ${outcome.trigger.reason}`; - return; - case 'needs-fallback': - await runFallbackTaskAttempt( - outcome.trigger.reason, - outcome.trigger.retryAfterMs, - ); + if (outcome.trigger) { + error = `Claude ${outcome.trigger.reason}`; + } return; } }; @@ -597,25 +536,9 @@ async function runTask( }; const provider = - context.taskAgentType === 'codex' - ? 'codex' - : canFallback - ? await getActiveProvider() - : 'claude'; + context.taskAgentType === 'codex' ? 'codex' : 'claude'; - // Already in no-fallback Claude cooldown — skip task instead of running on Kimi - if ( - isClaudeAgent && - provider !== 'claude' && - isPrimaryNoFallbackCooldownActive() - ) { - logger.info( - { taskId: task.id, group: context.group.name, provider }, - 'Claude primary cooldown active, skipping scheduled task', - ); - error = 'Claude primary cooldown active'; - // Fall through to task completion handling below - } else { + { const attempt = await runTaskAttempt(provider); result = attempt.attemptResult; error = attempt.attemptError; @@ -642,12 +565,12 @@ async function runTask( } else if (attempt.output.status === 'error' && provider === 'claude') { const trigger = attempt.streamedTriggerReason ? { - shouldFallback: true, + shouldRetry: true, reason: attempt.streamedTriggerReason.reason, retryAfterMs: attempt.streamedTriggerReason.retryAfterMs, } - : detectFallbackTrigger(error); - if (trigger.shouldFallback) { + : classifyRotationTrigger(error); + if (trigger.shouldRetry) { await retryClaudeTaskWithRotation({ reason: trigger.reason, retryAfterMs: trigger.retryAfterMs, @@ -725,8 +648,8 @@ async function runTask( } } } else { - const trigger = detectFallbackTrigger(error); - if (trigger.shouldFallback) { + const trigger = classifyRotationTrigger(error); + if (trigger.shouldRetry) { const rotated = getTokenCount() > 1 && rotateToken(error); if (rotated) { logger.info( diff --git a/src/token-rotation.ts b/src/token-rotation.ts index 0c7515d..d3cf58d 100644 --- a/src/token-rotation.ts +++ b/src/token-rotation.ts @@ -7,7 +7,7 @@ * CLAUDE_CODE_OAUTH_TOKEN if multi-token is not configured. * * On rate-limit: rotate to next token - * All exhausted: fall through to provider fallback (Kimi etc.) + * All exhausted: surface error to caller */ import fs from 'fs'; @@ -153,7 +153,7 @@ export function rotateToken( cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null, reason: errorMessage ?? null, }, - 'All tokens are rate-limited, falling through to provider fallback', + 'All tokens are rate-limited, no available tokens', ); return false; } @@ -234,3 +234,13 @@ export function getTokenRotationInfo(): { ).length, }; } + +export function hasAvailableClaudeToken(): boolean { + if (tokens.length === 0) { + return Boolean(process.env.CLAUDE_CODE_OAUTH_TOKEN); + } + const now = Date.now(); + return tokens.some( + (token) => !token.rateLimitedUntil || token.rateLimitedUntil <= now, + ); +} diff --git a/src/unified-dashboard.test.ts b/src/unified-dashboard.test.ts index 761fa27..3a6cd5d 100644 --- a/src/unified-dashboard.test.ts +++ b/src/unified-dashboard.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest'; +import type { UsageRow } from './dashboard-usage-rows.js'; import { formatStatusHeader, + renderUsageTable, summarizeWatcherTasks, } from './unified-dashboard.js'; @@ -57,3 +59,51 @@ describe('formatStatusHeader', () => { ).toBe('**📊 에이전트 상태** — 활성 3 / 8 | 감시 2 | 일시정지 1'); }); }); + +describe('renderUsageTable', () => { + const claudeRow: UsageRow = { + name: 'Claude pro', + h5pct: 50, + h5reset: '', + d7pct: 30, + d7reset: '', + }; + const codexRow: UsageRow = { + name: 'Codex', + h5pct: 40, + h5reset: '', + d7pct: 25, + d7reset: '', + }; + + it('renders Claude before separator before Codex', () => { + const lines = renderUsageTable([claudeRow], [codexRow]); + + const claudeIdx = lines.findIndex((l) => l.includes('Claude')); + const sepIdx = lines.findIndex((l) => /^─+$/.test(l)); + const codexIdx = lines.findIndex((l) => l.includes('Codex')); + + expect(claudeIdx).toBeGreaterThan(-1); + expect(sepIdx).toBeGreaterThan(claudeIdx); + expect(codexIdx).toBeGreaterThan(sepIdx); + }); + + it('omits separator when only Claude rows exist', () => { + const lines = renderUsageTable([claudeRow], []); + + expect(lines.some((l) => /^─+$/.test(l))).toBe(false); + expect(lines.some((l) => l.includes('Claude'))).toBe(true); + }); + + it('omits separator when only Codex rows exist', () => { + const lines = renderUsageTable([], [codexRow]); + + expect(lines.some((l) => /^─+$/.test(l))).toBe(false); + expect(lines.some((l) => l.includes('Codex'))).toBe(true); + }); + + it('returns fallback text when both groups are empty', () => { + const lines = renderUsageTable([], []); + expect(lines).toEqual(['_조회 불가_']); + }); +}); diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index d06a573..2fb2ef8 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -47,6 +47,7 @@ import type { export interface UnifiedDashboardOptions { assistantName: string; + serviceId: string; serviceAgentType: AgentType; statusChannelId: string; statusUpdateInterval: number; @@ -189,8 +190,12 @@ async function refreshChannelMeta( } } -function getAgentDisplayName(agentType: 'claude-code' | 'codex'): string { - return agentType === 'codex' ? '코덱스' : '클코'; +function getAgentDisplayName( + agentType: 'claude-code' | 'codex', + serviceId: string, +): string { + if (agentType === 'claude-code') return '클코'; + return serviceId === 'codex-review' ? '코리뷰' : '코덱스'; } function formatRoomName( @@ -216,6 +221,7 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void { const statuses = opts.queue.getStatuses(Object.keys(groups)); writeStatusSnapshot({ + serviceId: opts.serviceId, agentType: opts.serviceAgentType, assistantName: opts.assistantName, updatedAt: new Date().toISOString(), @@ -261,6 +267,7 @@ function buildStatusContent(): string { ); interface RoomEntry { + serviceId: string; agentType: 'claude-code' | 'codex'; status: 'processing' | 'waiting' | 'inactive'; elapsedMs: number | null; @@ -277,6 +284,7 @@ function buildStatusContent(): string { for (const entry of snapshot.entries) { const existing = byJid.get(entry.jid) || []; existing.push({ + serviceId: snapshot.serviceId, agentType, status: entry.status, elapsedMs: entry.elapsedMs, @@ -345,7 +353,7 @@ function buildStatusContent(): string { elapsedMs: agent.elapsedMs, pendingTasks: agent.pendingTasks, }); - const tag = getAgentDisplayName(agent.agentType); + const tag = getAgentDisplayName(agent.agentType, agent.serviceId); return `${tag} ${icon} ${label}`; }); roomLines.push({ @@ -373,6 +381,76 @@ function buildStatusContent(): string { return `${header}\n\n${sections}`; } +/** + * Render usage table lines from two row groups (Claude and Codex). + * Returns rendered lines including code block markers. + * Ordering: Claude rows → separator → Codex rows. + * Exported for testing. + */ +export function renderUsageTable( + claudeBotRows: UsageRow[], + codexBotRows: UsageRow[], +): string[] { + const allRows = [...claudeBotRows, ...codexBotRows]; + if (allRows.length === 0) return ['_조회 불가_']; + + const bar = (pct: number) => { + const filled = Math.max(0, Math.min(5, Math.round(pct / 20))); + return '█'.repeat(filled) + '░'.repeat(5 - filled); + }; + + const visualWidth = (s: string) => + [...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0); + const maxNameWidth = + Math.max(8, ...allRows.map((r) => visualWidth(r.name))) + 1; + const padName = (s: string) => + s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s))); + const compactReset = (s: string) => + s ? s.replace(/\s+/g, '').replace(/m$/, '') : ''; + + const lines: string[] = []; + + const renderRows = (rows: UsageRow[]) => { + for (const row of rows) { + const h5 = + row.h5pct >= 0 + ? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%` + : ' — '; + const d7 = + row.d7pct >= 0 + ? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%` + : ' — '; + lines.push(`${padName(row.name)}${h5} ${d7}`); + const r5 = compactReset(row.h5reset); + const r7 = compactReset(row.d7reset); + if (r5 || r7) { + const d7ColStart = maxNameWidth + 10; + let resetLine = ' '.repeat(maxNameWidth); + if (r5) resetLine += r5; + resetLine = resetLine.padEnd(d7ColStart); + if (r7) resetLine += r7; + lines.push(resetLine); + } + } + }; + + lines.push('```'); + lines.push(`${' '.repeat(maxNameWidth)}5h 7d`); + + renderRows(claudeBotRows); + + if (claudeBotRows.length > 0 && codexBotRows.length > 0) { + const separatorWidth = maxNameWidth + 20; + lines.push('─'.repeat(separatorWidth)); + } + + renderRows(codexBotRows); + + lines.push('```'); + + return lines; +} + async function buildUsageContent(): Promise { const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED; let liveClaudeAccounts: ClaudeAccountUsage[] | null = null; @@ -391,64 +469,27 @@ async function buildUsageContent(): Promise { return '█'.repeat(filled) + '░'.repeat(5 - filled); }; - const rows: UsageRow[] = []; + // Group 1: Claude bot + const claudeBotRows: UsageRow[] = []; if (shouldFetchClaudeUsage) { cachedClaudeAccounts = mergeClaudeDashboardAccounts( liveClaudeAccounts, cachedClaudeAccounts, ); - rows.push(...buildClaudeUsageRows(cachedClaudeAccounts)); + claudeBotRows.push(...buildClaudeUsageRows(cachedClaudeAccounts)); } - // Codex usage: read from Codex service's own status snapshot. - // Each service owns its usage data — no cross-service auth access needed. + // Group 2: Codex bot (separate service) + const codexBotRows: UsageRow[] = []; const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS).find( - (s) => s.agentType === 'codex', + (s) => s.serviceId === 'codex-main' || s.serviceId === 'codex', + ); + codexBotRows.push( + ...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS), ); - rows.push(...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS)); - if (rows.length > 0) { - // Emoji characters take 2 columns in monospace — count visual width - const visualWidth = (s: string) => - [...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0); - const maxNameWidth = - Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1; - const padName = (s: string) => - s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s))); - // Strip whitespace and trailing 'm' from reset strings for compact display - const compactReset = (s: string) => - s ? s.replace(/\s+/g, '').replace(/m$/, '') : ''; - - lines.push('```'); - lines.push(`${' '.repeat(maxNameWidth)}5h 7d`); - for (const row of rows) { - const h5 = - row.h5pct >= 0 - ? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%` - : ' — '; - const d7 = - row.d7pct >= 0 - ? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%` - : ' — '; - lines.push(`${padName(row.name)}${h5} ${d7}`); - const r5 = compactReset(row.h5reset); - const r7 = compactReset(row.d7reset); - if (r5 || r7) { - // Align reset values under 5h / 7d columns - // h5 column starts at maxNameWidth; d7 column starts at maxNameWidth + 9 + 1 - const d7ColStart = maxNameWidth + 10; - let resetLine = ' '.repeat(maxNameWidth); - if (r5) resetLine += r5; - resetLine = resetLine.padEnd(d7ColStart); - if (r7) resetLine += r7; - lines.push(resetLine); - } - } - lines.push('```'); - } else { - lines.push('_조회 불가_'); - } + lines.push(...renderUsageTable(claudeBotRows, codexBotRows)); lines.push(''); lines.push('🖥️ *서버*');