feat: add codex review failover and suppress output hardening

This commit is contained in:
Eyejoker
2026-03-28 05:40:38 +09:00
parent d1c693fb17
commit ba9f6871b6
49 changed files with 3884 additions and 1763 deletions

View File

@@ -71,12 +71,6 @@ OPENAI_API_KEY=...
### 선택 환경 변수
```bash
# Provider fallback (Claude 429 시 대체)
FALLBACK_PROVIDER_NAME=kimi
FALLBACK_BASE_URL=https://api.kimi.com/coding
FALLBACK_AUTH_TOKEN=...
FALLBACK_MODEL=kimi-k2.5
# 사용량 대시보드
STATUS_CHANNEL_ID=... # 상태 업데이트 디스코드 채널
USAGE_DASHBOARD=true

View File

@@ -0,0 +1,5 @@
# ===== EJClaw Codex Review Service Environment =====
# Copy this file to .env.codex-review for the ejclaw-review service.
# This service needs its own Discord bot token.
DISCORD_BOT_TOKEN= # Discord bot token (Codex review bot)

View File

@@ -1,6 +1,7 @@
# ===== EJClaw Environment Configuration =====
# Copy this file to .env and fill in the values.
# For dual-service (Codex), also create .env.codex with DISCORD_BOT_TOKEN for the Codex bot.
# For review failover, create .env.codex-review with a third Discord bot token.
# --- Required ---
DISCORD_BOT_TOKEN= # Discord bot token (Claude bot)
@@ -25,13 +26,6 @@ ASSISTANT_NAME=claude # Trigger name (@claude)
GROQ_API_KEY= # Groq Whisper (primary, free: console.groq.com)
# OPENAI_API_KEY also used as Whisper fallback
# --- Provider fallback (optional — Claude 429 → alternative) ---
# FALLBACK_PROVIDER_NAME=kimi
# FALLBACK_BASE_URL=https://api.kimi.com/coding
# FALLBACK_AUTH_TOKEN=
# FALLBACK_MODEL=kimi-k2.5
# FALLBACK_COOLDOWN_MS=600000
# --- Usage dashboard (optional) ---
# STATUS_CHANNEL_ID= # Discord channel for live status updates
# USAGE_DASHBOARD=true # Enable usage dashboard

View File

@@ -25,7 +25,7 @@ Both share the same codebase (`dist/index.js`), differentiated by environment va
- **Browser automation** — [gstack browse](https://github.com/garrytan/gstack) skill, headless Chromium daemon, ~100ms/command
- **Voice transcription** — Groq Whisper (primary) / OpenAI Whisper (fallback), shared file cache with dedup
- **Bidirectional images** — receive Discord attachments as multimodal input, send screenshots back
- **Provider fallback** — Claude 429 → configurable fallback provider (e.g. Kimi K2.5) with cooldown
- **Token rotation** — Claude 429 / usage exhaustion → automatic multi-account rotation
- **CI monitoring** — `watch_ci` MCP tool for GitHub Actions run polling (structured fast path, no LLM token cost)
- **Usage dashboard** — real-time token usage and service status overview
- **MCP integration** — Memento (persistent memory) + EJClaw host tools (send_message, schedule_task, watch_ci, etc.)
@@ -82,12 +82,6 @@ CLAUDE_CODE_OAUTH_TOKEN= # Claude Code OAuth token (primary)
CLAUDE_CODE_OAUTH_TOKENS= # Comma-separated tokens for multi-account rotation
OPENAI_API_KEY= # For Codex
GROQ_API_KEY= # Voice transcription (Groq Whisper)
# Provider fallback (optional)
FALLBACK_PROVIDER_NAME= # e.g. kimi
FALLBACK_BASE_URL= # e.g. https://api.kimi.com/coding
FALLBACK_AUTH_TOKEN= # Fallback provider API key
FALLBACK_MODEL= # e.g. kimi-k2.5
```
### Codex Service (optional)

View File

@@ -0,0 +1,10 @@
# Codex Review Failover Paired Room Rules
This room has `codex-review` acting as the owner-side failover agent and `codex-main` acting as the separate Codex reviewer.
- You are the owner for this chat while failover is active
- `codex-main` is a separate reviewer and is not you
- Keep collaboration with `codex-main` public when useful
- Evaluate reviewer feedback on its merits, but answer the user as the owner
- The visible bot name in history may differ from room to room; do not infer role from the visible name
- Use `<internal>` only for repetitive non-user-facing coordination noise

View File

@@ -0,0 +1,44 @@
# Codex Review Failover Rules
You are `codex-review`, and in this chat you are currently acting as `클코`, the owner-side failover agent.
## Role
- Answer as the primary owner for the chat, not as a reviewer
- Do not describe yourself as the review-side Codex service while this failover prompt is active
- `codex-main` is a separate Codex participant and is not you
- The visible bot name or nickname in chat history may vary by room; do not infer your role from the visible name
## Communication
Your output is sent directly to the user or Discord group.
You may use `<internal>` to suppress repetitive agent-to-agent noise.
Keep status updates, conclusions, and handoffs visible.
```text
<internal>Collected the reviewer notes and folded them into the final answer.</internal>
Here is the answer for the user...
```
Text inside `<internal>` tags is logged but not sent to the user.
Keep replies concise and owner-oriented.
- Respond directly to the user
- Give conclusions and concrete next steps
- Do not expose internal routing details unless they matter to the answer
- Do not claim to be `codex-main`
## Memory
The group folder may contain a `conversations/` directory with searchable history from earlier sessions. Use it when you need prior context.
## Message formatting
Do not use markdown headings in chat replies. Keep messages clean and readable for Discord.
- Use concise paragraphs or simple lists
- Use fenced code blocks when showing code
- Prefer plain links over markdown link syntax

View File

@@ -0,0 +1,14 @@
# Codex Review Paired Room Rules
This service is the reviewer side when paired-room failover is active.
- Default stance: review, challenge, and verify.
- Do not mirror the owner's answer unless you are adding a concrete correction, risk, or missing prerequisite.
- Prioritize:
1. wrong root cause
2. unsafe operational advice
3. missing verification
4. hidden assumptions
- If the owner answer is fine, keep your message short or skip it entirely.
- If the turn provides a suppress token and you are only agreeing or rephrasing without a concrete correction, risk, prerequisite, test gap, or code change, output only that token and nothing else.
- When code changes are proposed, focus on bugs, regressions, and test gaps.

View File

@@ -0,0 +1,10 @@
# Codex Review Role
You are `코리뷰`, the review-side Codex service.
- Your job is to improve answer quality by challenging weak assumptions, spotting bugs, and tightening plans.
- Prefer concise technical criticism over parallel repetition.
- If the main Codex answer is already correct and sufficient, stay brief or silent.
- If the turn provides a suppress token and you have no concrete correction, risk, prerequisite, test gap, or code change, output only that token and nothing else.
- When you disagree, say exactly what is wrong and what should change.
- Focus on correctness, regressions, missing tests, and operational risk.

View File

@@ -5,7 +5,7 @@ import {
classifyClaudeAuthError,
detectClaudeProviderFailureMessage,
isClaudeOrgAccessDeniedMessage,
isNoFallbackCooldownReason,
shouldRotateClaudeToken,
} from './agent-error-detection.js';
@@ -65,11 +65,4 @@ describe('agent-error-detection', () => {
expect(shouldRotateClaudeToken('success-null-result')).toBe(false);
});
it('marks only no-fallback cooldown reasons as skip-worthy', () => {
expect(isNoFallbackCooldownReason('usage-exhausted')).toBe(true);
expect(isNoFallbackCooldownReason('auth-expired')).toBe(true);
expect(isNoFallbackCooldownReason('org-access-denied')).toBe(true);
expect(isNoFallbackCooldownReason('429')).toBe(false);
expect(isNoFallbackCooldownReason('success-null-result')).toBe(false);
});
});

View File

@@ -103,21 +103,14 @@ export type AgentTriggerReason =
| 'network-error'
| 'success-null-result';
export type FallbackTriggerReason = Exclude<
AgentTriggerReason,
'usage-exhausted' | 'success-null-result'
>;
export type ClaudeRotationReason = Extract<
AgentTriggerReason,
'429' | 'usage-exhausted' | 'auth-expired' | 'org-access-denied'
>;
export type CodexRotationReason = FallbackTriggerReason;
export type NoFallbackCooldownReason = Extract<
export type CodexRotationReason = Extract<
AgentTriggerReason,
'usage-exhausted' | 'auth-expired' | 'org-access-denied'
'429' | 'auth-expired' | 'org-access-denied' | 'overloaded' | 'network-error'
>;
export function shouldRotateClaudeToken(
@@ -131,14 +124,50 @@ export function shouldRotateClaudeToken(
);
}
export function isNoFallbackCooldownReason(
reason: AgentTriggerReason,
): reason is NoFallbackCooldownReason {
return (
reason === 'usage-exhausted' ||
reason === 'auth-expired' ||
reason === 'org-access-denied'
);
// ── Rotation trigger classification ─────────────────────────────
export type RotationTriggerResult =
| { shouldRetry: false; reason: '' }
| {
shouldRetry: true;
reason: AgentTriggerReason;
retryAfterMs?: number;
};
/**
* Classify an error string to determine if Claude token rotation
* should be attempted.
*
* Priority: 429 > auth/org errors > 503/network.
*/
export function classifyRotationTrigger(
error?: string | null,
): RotationTriggerResult {
if (!error) return { shouldRetry: false, reason: '' };
const common = classifyAgentError(error);
// 429 rate-limit (highest priority)
if (common.category === 'rate-limit') {
return {
shouldRetry: true,
reason: common.reason,
retryAfterMs: common.retryAfterMs,
};
}
// Claude-specific auth checks (before 503/network)
const auth = classifyClaudeAuthError(error);
if (auth.category !== 'none') {
return { shouldRetry: true, reason: auth.reason };
}
// 503 overloaded, network errors
if (common.category !== 'none') {
return { shouldRetry: true, reason: common.reason };
}
return { shouldRetry: false, reason: '' };
}
// ── Unified error classification ────────────────────────────────

View File

@@ -9,8 +9,12 @@ const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({
}));
vi.mock('./config.js', () => ({
CODEX_REVIEW_SERVICE_ID: 'codex-review',
GROUPS_DIR: '/tmp/ejclaw-test-groups',
SERVICE_ID: 'codex-main',
SERVICE_SESSION_SCOPE: 'codex-main',
TIMEZONE: 'Asia/Seoul',
isReviewService: vi.fn(() => false),
}));
vi.mock('./db.js', () => ({
@@ -35,17 +39,31 @@ vi.mock('./platform-prompts.js', () => ({
readPairedRoomPrompt: vi.fn(() => 'paired room prompt'),
}));
vi.mock('./service-routing.js', () => ({
getEffectiveChannelLease: vi.fn(() => ({
chat_jid: 'dc:test',
owner_service_id: 'claude',
reviewer_service_id: 'codex-main',
activated_at: null,
reason: null,
explicit: false,
})),
}));
vi.mock('./group-folder.js', () => ({
resolveGroupFolderPath: (folder: string) =>
`${process.env.EJ_TEST_ROOT}/groups/${folder}`,
resolveGroupIpcPath: (folder: string) =>
`${process.env.EJ_TEST_ROOT}/ipc/${folder}`,
resolveGroupSessionsPath: (folder: string) =>
`${process.env.EJ_TEST_ROOT}/sessions/${folder}`,
resolveServiceGroupSessionsPath: (folder: string, serviceId: string) =>
`${process.env.EJ_TEST_ROOT}/sessions/${folder}/services/${serviceId}`,
resolveTaskRuntimeIpcPath: (folder: string, taskId: string) =>
`${process.env.EJ_TEST_ROOT}/task-ipc/${folder}/${taskId}`,
resolveTaskSessionsPath: (folder: string, taskId: string) =>
`${process.env.EJ_TEST_ROOT}/task-sessions/${folder}/${taskId}`,
resolveServiceTaskSessionsPath: (
folder: string,
serviceId: string,
taskId: string,
) => `${process.env.EJ_TEST_ROOT}/task-sessions/${folder}/${serviceId}/${taskId}`,
}));
vi.mock('os', async () => {
@@ -61,6 +79,9 @@ vi.mock('os', async () => {
});
import { prepareGroupEnvironment } from './agent-runner-environment.js';
import * as config from './config.js';
import * as db from './db.js';
import * as serviceRouting from './service-routing.js';
import type { RegisteredGroup } from './types.js';
const group: RegisteredGroup = {
@@ -131,6 +152,8 @@ describe('prepareGroupEnvironment codex auth handling', () => {
tempRoot,
'sessions',
group.folder,
'services',
'codex-main',
'.codex',
'auth.json',
);
@@ -165,6 +188,8 @@ describe('prepareGroupEnvironment codex auth handling', () => {
tempRoot,
'sessions',
group.folder,
'services',
'codex-main',
'.codex',
'auth.json',
);
@@ -172,4 +197,118 @@ describe('prepareGroupEnvironment codex auth handling', () => {
expect(auth).toEqual(rotatedAuth);
});
it('uses the failover prompt pack for codex-review when it owns an explicit failover lease', () => {
vi.mocked(config.isReviewService).mockReturnValue(true);
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
chat_jid: 'dc:test',
owner_service_id: 'codex-review',
reviewer_service_id: 'codex-main',
activated_at: '2026-03-28T00:00:00.000Z',
reason: 'claude-429',
explicit: true,
});
mockReadEnvFile.mockReturnValue({});
const promptsDir = path.join(tempRoot, 'prompts');
fs.mkdirSync(promptsDir, { recursive: true });
fs.writeFileSync(
path.join(promptsDir, 'codex-review-platform.md'),
'review platform prompt\n',
);
fs.writeFileSync(
path.join(promptsDir, 'codex-review-paired-room.md'),
'review paired prompt\n',
);
fs.writeFileSync(
path.join(promptsDir, 'codex-review-failover-platform.md'),
'failover platform prompt\n',
);
fs.writeFileSync(
path.join(promptsDir, 'codex-review-failover-paired-room.md'),
'failover paired prompt\n',
);
prepareGroupEnvironment(
{ ...group, workDir: path.join(tempRoot, 'workdir') },
false,
'dc:test',
{
memoryBriefing: 'memory briefing',
},
);
const agentsPath = path.join(
tempRoot,
'sessions',
group.folder,
'services',
'codex-main',
'.codex',
'AGENTS.md',
);
const agents = fs.readFileSync(agentsPath, 'utf-8');
const segments = agents.trim().split('\n\n---\n\n');
expect(segments).toEqual([
'failover platform prompt',
'failover paired prompt',
'memory briefing',
]);
});
it('returns to the normal review prompt stack after failover is cleared', () => {
vi.mocked(config.isReviewService).mockReturnValue(true);
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
chat_jid: 'dc:test',
owner_service_id: 'claude',
reviewer_service_id: 'codex-main',
activated_at: null,
reason: null,
explicit: false,
});
mockReadEnvFile.mockReturnValue({});
const promptsDir = path.join(tempRoot, 'prompts');
fs.mkdirSync(promptsDir, { recursive: true });
fs.writeFileSync(
path.join(promptsDir, 'codex-review-platform.md'),
'review platform prompt\n',
);
fs.writeFileSync(
path.join(promptsDir, 'codex-review-paired-room.md'),
'review paired prompt\n',
);
fs.writeFileSync(
path.join(promptsDir, 'codex-review-failover-platform.md'),
'failover platform prompt\n',
);
fs.writeFileSync(
path.join(promptsDir, 'codex-review-failover-paired-room.md'),
'failover paired prompt\n',
);
prepareGroupEnvironment(group, false, 'dc:test');
const agentsPath = path.join(
tempRoot,
'sessions',
group.folder,
'services',
'codex-main',
'.codex',
'AGENTS.md',
);
const agents = fs.readFileSync(agentsPath, 'utf-8');
const segments = agents.trim().split('\n\n---\n\n');
expect(segments).toEqual([
'platform prompt',
'review platform prompt',
'paired room prompt',
'review paired prompt',
]);
});
});

View File

@@ -2,7 +2,14 @@ import fs from 'fs';
import os from 'os';
import path from 'path';
import { GROUPS_DIR, TIMEZONE } from './config.js';
import {
CODEX_REVIEW_SERVICE_ID,
GROUPS_DIR,
SERVICE_ID,
SERVICE_SESSION_SCOPE,
TIMEZONE,
isReviewService,
} from './config.js';
import { isPairedRoomJid } from './db.js';
import { readEnvFile } from './env.js';
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
@@ -10,14 +17,15 @@ import { getCurrentToken } from './token-rotation.js';
import {
resolveGroupFolderPath,
resolveGroupIpcPath,
resolveGroupSessionsPath,
resolveServiceGroupSessionsPath,
resolveTaskRuntimeIpcPath,
resolveTaskSessionsPath,
resolveServiceTaskSessionsPath,
} from './group-folder.js';
import {
readPairedRoomPrompt,
readPlatformPrompt,
} from './platform-prompts.js';
import { getEffectiveChannelLease } from './service-routing.js';
import type { AgentType, RegisteredGroup } from './types.js';
// writeCodexApiKeyAuth removed — Codex uses OAuth only.
@@ -39,6 +47,16 @@ function syncDirectoryEntries(sources: string[], destination: string): void {
}
}
function readOptionalPromptFile(
projectRoot: string,
filename: string,
): string | undefined {
const promptPath = path.join(projectRoot, 'prompts', filename);
if (!fs.existsSync(promptPath)) return undefined;
const prompt = fs.readFileSync(promptPath, 'utf-8').trim();
return prompt || undefined;
}
function ensureClaudeSessionSettings(groupSessionsDir: string): void {
const settingsFile = path.join(groupSessionsDir, 'settings.json');
if (fs.existsSync(settingsFile)) return;
@@ -175,6 +193,7 @@ function prepareCodexSessionEnvironment(args: {
chatJid: string;
isMain: boolean;
isPairedRoom: boolean;
useFailoverPromptPack: boolean;
memoryBriefing?: string;
}): void {
// API key auth intentionally removed — Codex uses OAuth only.
@@ -236,13 +255,38 @@ function prepareCodexSessionEnvironment(args: {
}
const sessionAgentsPath = path.join(sessionCodexDir, 'AGENTS.md');
const sessionAgents = [
readPlatformPrompt('codex', args.projectRoot),
args.isPairedRoom
? readPairedRoomPrompt('codex', args.projectRoot)
: undefined,
args.memoryBriefing,
]
const sessionAgents = (
args.useFailoverPromptPack
? [
readOptionalPromptFile(
args.projectRoot,
'codex-review-failover-platform.md',
),
args.isPairedRoom
? readOptionalPromptFile(
args.projectRoot,
'codex-review-failover-paired-room.md',
)
: undefined,
args.memoryBriefing,
]
: [
readPlatformPrompt('codex', args.projectRoot),
isReviewService(SERVICE_ID)
? readOptionalPromptFile(args.projectRoot, 'codex-review-platform.md')
: undefined,
args.isPairedRoom
? readPairedRoomPrompt('codex', args.projectRoot)
: undefined,
args.isPairedRoom && isReviewService(SERVICE_ID)
? readOptionalPromptFile(
args.projectRoot,
'codex-review-paired-room.md',
)
: undefined,
args.memoryBriefing,
]
)
.filter((value): value is string => Boolean(value))
.join('\n\n---\n\n')
.trim();
@@ -342,8 +386,12 @@ export function prepareGroupEnvironment(
options?.useTaskScopedSession === true && Boolean(runtimeTaskId);
const sessionRootDir =
runtimeTaskId && useTaskScopedSession
? resolveTaskSessionsPath(group.folder, runtimeTaskId)
: resolveGroupSessionsPath(group.folder);
? resolveServiceTaskSessionsPath(
group.folder,
SERVICE_SESSION_SCOPE,
runtimeTaskId,
)
: resolveServiceGroupSessionsPath(group.folder, SERVICE_SESSION_SCOPE);
const groupSessionsDir = path.join(sessionRootDir, '.claude');
fs.mkdirSync(groupSessionsDir, { recursive: true });
@@ -370,6 +418,11 @@ export function prepareGroupEnvironment(
const globalDir = path.join(GROUPS_DIR, 'global');
const globalClaudeMdPath = path.join(globalDir, 'CLAUDE.md');
const isPairedRoom = isPairedRoomJid(chatJid);
const effectiveLease = getEffectiveChannelLease(chatJid);
const useCodexReviewFailoverPromptPack =
isReviewService(SERVICE_ID) &&
effectiveLease.explicit &&
effectiveLease.owner_service_id === CODEX_REVIEW_SERVICE_ID;
const claudePlatformPrompt = readPlatformPrompt('claude-code', projectRoot);
const claudePairedRoomPrompt = isPairedRoom
@@ -440,6 +493,7 @@ export function prepareGroupEnvironment(
chatJid,
isMain,
isPairedRoom,
useFailoverPromptPack: useCodexReviewFailoverPromptPack,
memoryBriefing: options?.memoryBriefing,
});
} else {

View File

@@ -13,7 +13,10 @@ vi.mock('./config.js', () => ({
DATA_DIR: '/tmp/ejclaw-test-data',
GROUPS_DIR: '/tmp/ejclaw-test-groups',
IDLE_TIMEOUT: 1800000, // 30min
SERVICE_ID: 'claude',
SERVICE_SESSION_SCOPE: 'claude',
TIMEZONE: 'America/Los_Angeles',
isReviewService: vi.fn(() => false),
}));
// Mock logger
@@ -55,6 +58,17 @@ vi.mock('./env.js', () => ({
getEnv: vi.fn(() => undefined),
}));
vi.mock('./service-routing.js', () => ({
getEffectiveChannelLease: vi.fn(() => ({
chat_jid: 'test@g.us',
owner_service_id: 'claude',
reviewer_service_id: 'codex-main',
activated_at: null,
reason: null,
explicit: false,
})),
}));
// Create a controllable fake ChildProcess
function createFakeProcess() {
const proc = new EventEmitter() as EventEmitter & {
@@ -328,7 +342,7 @@ describe('agent-runner timeout behavior', () => {
'/tmp/ejclaw-test-data/ipc/test-group',
);
expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe(
'/tmp/ejclaw-test-data/sessions/test-group/tasks/task-123/.claude',
'/tmp/ejclaw-test-data/sessions/test-group/services/claude/tasks/task-123/.claude',
);
});
@@ -407,7 +421,7 @@ describe('agent-runner timeout behavior', () => {
'/tmp/ejclaw-test-data/ipc/test-group',
);
expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe(
'/tmp/ejclaw-test-data/sessions/test-group/.claude',
'/tmp/ejclaw-test-data/sessions/test-group/services/claude/.claude',
);
});
@@ -416,7 +430,7 @@ describe('agent-runner timeout behavior', () => {
fakeProc = createFakeProcess();
const overlayPath = '/tmp/ejclaw-test-groups/test-group/.codex/config.toml';
const sessionConfigPath =
'/tmp/ejclaw-test-data/sessions/test-group/.codex/config.toml';
'/tmp/ejclaw-test-data/sessions/test-group/services/claude/.codex/config.toml';
let sessionToml = `model = "gpt-5.4"\n`;
vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => {

View File

@@ -71,7 +71,7 @@ export async function runAgentProcess(
},
);
// Apply provider fallback overrides (e.g. Kimi env vars when Claude is in cooldown)
// Apply env overrides (caller-provided)
if (envOverrides) {
for (const [key, value] of Object.entries(envOverrides)) {
if (value) env[key] = value;

View File

@@ -870,6 +870,31 @@ describe('DiscordChannel', () => {
expect(currentClient().channels.fetch).not.toHaveBeenCalled();
});
it('does not send stale typing after typing was disabled mid-flight', async () => {
const opts = createTestOpts();
const channel = new DiscordChannel('test-token', opts);
await channel.connect();
let resolveFetch!: (value: unknown) => void;
const fetchPromise = new Promise((resolve) => {
resolveFetch = resolve;
});
const mockChannel = {
send: vi.fn(),
sendTyping: vi.fn().mockResolvedValue(undefined),
};
currentClient().channels.fetch.mockImplementationOnce(() => fetchPromise);
const typingPromise = channel.setTyping('dc:1234567890123456', true);
await Promise.resolve();
await channel.setTyping('dc:1234567890123456', false);
resolveFetch(mockChannel);
await typingPromise;
expect(mockChannel.sendTyping).not.toHaveBeenCalled();
});
it('does nothing when client is not initialized', async () => {
const opts = createTestOpts();
const channel = new DiscordChannel('test-token', opts);

View File

@@ -177,6 +177,7 @@ export class DiscordChannel implements Channel {
private opts: DiscordChannelOpts;
private botToken: string;
private typingIntervals = new Map<string, NodeJS.Timeout>();
private typingGenerations = new Map<string, number>();
private agentTypeFilter?: AgentType;
constructor(
@@ -539,6 +540,9 @@ export class DiscordChannel implements Channel {
async setTyping(jid: string, isTyping: boolean): Promise<void> {
if (!this.client) return;
const generation = (this.typingGenerations.get(jid) ?? 0) + 1;
this.typingGenerations.set(jid, generation);
// Clear any existing interval for this channel
const existing = this.typingIntervals.get(jid);
if (existing) {
@@ -548,10 +552,15 @@ export class DiscordChannel implements Channel {
if (!isTyping) return;
const isCurrentGeneration = () =>
this.typingGenerations.get(jid) === generation;
const sendOnce = async () => {
if (!isCurrentGeneration()) return;
try {
const channelId = jid.replace(/^dc:/, '');
const channel = await this.client!.channels.fetch(channelId);
if (!isCurrentGeneration()) return;
if (channel && 'sendTyping' in channel) {
await (channel as TextChannel).sendTyping();
}
@@ -562,7 +571,13 @@ export class DiscordChannel implements Channel {
// Send immediately, then refresh every 8 seconds (Discord expires at ~10s)
await sendOnce();
this.typingIntervals.set(jid, setInterval(sendOnce, 8000));
if (!isCurrentGeneration()) return;
this.typingIntervals.set(
jid,
setInterval(() => {
void sendOnce();
}, 8000),
);
}
async sendAndTrack(jid: string, text: string): Promise<string | null> {

View File

@@ -10,15 +10,44 @@ export const ASSISTANT_HAS_OWN_NUMBER =
const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase();
const rawServiceAgentType = getEnv('SERVICE_AGENT_TYPE');
export const SERVICE_ID = getEnv('SERVICE_ID') || ASSISTANT_SLUG;
export const CLAUDE_SERVICE_ID = getEnv('CLAUDE_SERVICE_ID') || 'claude';
export const CODEX_MAIN_SERVICE_ID =
getEnv('CODEX_MAIN_SERVICE_ID') || 'codex-main';
export const CODEX_REVIEW_SERVICE_ID =
getEnv('CODEX_REVIEW_SERVICE_ID') || 'codex-review';
export const SERVICE_AGENT_TYPE: AgentType =
rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code'
? rawServiceAgentType
: ASSISTANT_SLUG === 'codex'
? 'codex'
: 'claude-code';
export function normalizeServiceId(serviceId: string): string {
if (serviceId === 'codex') {
return CODEX_MAIN_SERVICE_ID;
}
return serviceId;
}
export const SERVICE_SESSION_SCOPE = normalizeServiceId(SERVICE_ID);
export function isClaudeService(serviceId: string = SERVICE_ID): boolean {
return normalizeServiceId(serviceId) === CLAUDE_SERVICE_ID;
}
export function isCodexMainService(serviceId: string = SERVICE_ID): boolean {
return normalizeServiceId(serviceId) === CODEX_MAIN_SERVICE_ID;
}
export function isReviewService(serviceId: string = SERVICE_ID): boolean {
return normalizeServiceId(serviceId) === CODEX_REVIEW_SERVICE_ID;
}
export const POLL_INTERVAL = 2000;
export const SCHEDULER_POLL_INTERVAL = 60000;
/** Minimum time (ms) a failover lease stays active before Claude can reclaim it. */
export const FAILOVER_MIN_DURATION_MS = 3 * 60 * 60 * 1000; // 3 hours
const PROJECT_ROOT = process.cwd();
const HOME_DIR = process.env.HOME || os.homedir();
@@ -106,3 +135,26 @@ const SESSION_COMMAND_ALLOWED_SENDERS = new Set(
export function isSessionCommandSenderAllowed(sender: string): boolean {
return SESSION_COMMAND_ALLOWED_SENDERS.has(sender);
}
// Delta handoff: cross-provider session continuity with probe + fallback
export const DELTA_HANDOFF_ENABLED =
getEnv('DELTA_HANDOFF_ENABLED') === 'true';
// Comma-separated list of group folders for canary testing
const rawDeltaHandoffCanaryGroups =
getEnv('DELTA_HANDOFF_CANARY_GROUPS') || '';
export const DELTA_HANDOFF_CANARY_GROUPS = new Set(
rawDeltaHandoffCanaryGroups
.split(',')
.map((v) => v.trim())
.filter(Boolean),
);
export function isDeltaHandoffEnabledForGroup(groupFolder: string): boolean {
if (!DELTA_HANDOFF_ENABLED) return false;
// If canary groups are specified, only enable for those groups
if (DELTA_HANDOFF_CANARY_GROUPS.size > 0) {
return DELTA_HANDOFF_CANARY_GROUPS.has(groupFolder);
}
return true;
}

View File

@@ -25,6 +25,7 @@ describe('extractCodexUsageRows staleness', () => {
it('returns real rows when usageRowsFetchedAt is within maxAgeMs', () => {
const now = Date.now();
const snapshot: StatusSnapshot = {
serviceId: 'codex-main',
agentType: 'codex',
assistantName: 'test',
updatedAt: new Date(now).toISOString(),
@@ -40,6 +41,7 @@ describe('extractCodexUsageRows staleness', () => {
it('returns degraded row when usageRowsFetchedAt exceeds maxAgeMs', () => {
const now = Date.now();
const snapshot: StatusSnapshot = {
serviceId: 'codex-main',
agentType: 'codex',
assistantName: 'test',
updatedAt: new Date(now).toISOString(), // heartbeat is fresh
@@ -57,6 +59,7 @@ describe('extractCodexUsageRows staleness', () => {
it('returns degraded row when usageRowsFetchedAt is missing', () => {
const now = Date.now();
const snapshot: StatusSnapshot = {
serviceId: 'codex-main',
agentType: 'codex',
assistantName: 'test',
updatedAt: new Date(now).toISOString(),
@@ -77,6 +80,7 @@ describe('extractCodexUsageRows staleness', () => {
it('returns empty array when usageRows is empty', () => {
const snapshot: StatusSnapshot = {
serviceId: 'codex-main',
agentType: 'codex',
assistantName: 'test',
updatedAt: new Date().toISOString(),

View File

@@ -45,9 +45,8 @@ describe('buildStatusContent', () => {
): Promise<(opts: DashboardOptions) => string> {
vi.resetModules();
process.env.STATUS_SHOW_ROOMS = options.statusShowRooms ?? 'true';
if (options.statusShowRoomDetails !== undefined) {
process.env.STATUS_SHOW_ROOM_DETAILS = options.statusShowRoomDetails;
}
process.env.STATUS_SHOW_ROOM_DETAILS =
options.statusShowRoomDetails ?? 'true';
const mod = await import('./dashboard-status-content.js');
return mod.buildStatusContent;
}

View File

@@ -15,6 +15,7 @@ export async function startStatusDashboard(
): Promise<void> {
await startUnifiedDashboard({
assistantName: opts.assistantName,
serviceId: opts.serviceAgentType === 'codex' ? 'codex-main' : 'claude',
serviceAgentType: opts.serviceAgentType || 'claude-code',
statusChannelId: opts.statusChannelId,
statusUpdateInterval: opts.statusUpdateInterval,

View File

@@ -4,7 +4,10 @@ import { describe, it, expect, beforeEach } from 'vitest';
import {
_initTestDatabase,
claimServiceHandoff,
completeServiceHandoffAndAdvanceTargetCursor,
createTask,
createServiceHandoff,
createProducedWorkItem,
deleteSession,
deleteTask,
@@ -15,9 +18,11 @@ import {
getMessagesSinceSeq,
getNewMessagesBySeq,
getOpenWorkItem,
getPendingServiceHandoffs,
getRegisteredAgentTypesForJid,
getMessagesSince,
getNewMessages,
getRouterStateForService,
isPairedRoomJid,
getSession,
getTaskById,
@@ -25,6 +30,7 @@ import {
markWorkItemDeliveryRetry,
setSession,
setRegisteredGroup,
setRouterStateForService,
storeChatMetadata,
storeMessage,
updateTask,
@@ -678,6 +684,84 @@ describe('paired room registration', () => {
});
});
describe('service handoff completion', () => {
it('atomically completes the handoff and advances the target cursor', () => {
storeChatMetadata('dc:handoff', '2024-01-01T00:00:00.000Z');
store({
id: 'handoff-msg-1',
chat_jid: 'dc:handoff',
sender: 'user',
sender_name: 'User',
content: 'hello',
timestamp: '2024-01-01T00:00:01.000Z',
});
const handoff = createServiceHandoff({
chat_jid: 'dc:handoff',
group_folder: 'test-group',
source_service_id: 'claude',
target_service_id: 'codex-review',
target_agent_type: 'codex',
prompt: 'hello',
end_seq: 1,
});
expect(claimServiceHandoff(handoff.id)).toBe(true);
const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({
id: handoff.id,
target_service_id: 'codex-review',
chat_jid: 'dc:handoff',
end_seq: 1,
});
expect(appliedCursor).toBe('1');
expect(getPendingServiceHandoffs('codex-review')).toEqual([]);
expect(
JSON.parse(
getRouterStateForService('last_agent_seq', 'codex-review') || '{}',
),
).toMatchObject({
'dc:handoff': '1',
});
});
it('does not move the target cursor backwards when a newer cursor already exists', () => {
storeChatMetadata('dc:handoff', '2024-01-01T00:00:00.000Z');
setRouterStateForService(
'last_agent_seq',
JSON.stringify({ 'dc:handoff': '5' }),
'codex-review',
);
const handoff = createServiceHandoff({
chat_jid: 'dc:handoff',
group_folder: 'test-group',
source_service_id: 'claude',
target_service_id: 'codex-review',
target_agent_type: 'codex',
prompt: 'hello',
end_seq: 3,
});
expect(claimServiceHandoff(handoff.id)).toBe(true);
const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({
id: handoff.id,
target_service_id: 'codex-review',
chat_jid: 'dc:handoff',
end_seq: 3,
});
expect(appliedCursor).toBe('5');
expect(
JSON.parse(
getRouterStateForService('last_agent_seq', 'codex-review') || '{}',
),
).toMatchObject({
'dc:handoff': '5',
});
});
});
describe('message seq cursors', () => {
beforeEach(() => {
storeChatMetadata('group@g.us', '2024-01-01T00:00:00.000Z');

519
src/db.ts
View File

@@ -4,14 +4,20 @@ import path from 'path';
import {
ASSISTANT_NAME,
CLAUDE_SERVICE_ID,
CODEX_MAIN_SERVICE_ID,
CODEX_REVIEW_SERVICE_ID,
DATA_DIR,
normalizeServiceId,
SERVICE_AGENT_TYPE,
SERVICE_ID,
SERVICE_SESSION_SCOPE,
STORE_DIR,
} from './config.js';
import {
isValidGroupFolder,
resolveTaskRuntimeIpcPath as resolveTaskRuntimeIpcPathFromGroup,
resolveServiceTaskSessionsPath as resolveServiceTaskSessionsPathFromGroup,
resolveTaskSessionsPath as resolveTaskSessionsPathFromGroup,
} from './group-folder.js';
import { logger } from './logger.js';
@@ -32,6 +38,7 @@ export interface WorkItem {
group_folder: string;
chat_jid: string;
agent_type: AgentType;
service_id: string;
status: 'produced' | 'delivery_retry' | 'delivered';
start_seq: number | null;
end_seq: number | null;
@@ -44,6 +51,32 @@ export interface WorkItem {
delivered_at: string | null;
}
export interface ChannelOwnerLeaseRow {
chat_jid: string;
owner_service_id: string;
reviewer_service_id: string | null;
activated_at: string | null;
reason: string | null;
}
export interface ServiceHandoff {
id: number;
chat_jid: string;
group_folder: string;
source_service_id: string;
target_service_id: string;
target_agent_type: AgentType;
prompt: string;
status: 'pending' | 'claimed' | 'completed' | 'failed';
start_seq: number | null;
end_seq: number | null;
reason: string | null;
created_at: string;
claimed_at: string | null;
completed_at: string | null;
last_error: string | null;
}
function backfillMessageSeq(database: Database.Database): void {
const rows = database
.prepare(
@@ -114,6 +147,7 @@ function createSchema(database: Database.Database): void {
group_folder TEXT NOT NULL,
chat_jid TEXT NOT NULL,
agent_type TEXT NOT NULL,
service_id TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'produced',
start_seq INTEGER,
end_seq INTEGER,
@@ -127,9 +161,9 @@ function createSchema(database: Database.Database): void {
CHECK (status IN ('produced', 'delivery_retry', 'delivered'))
);
CREATE INDEX IF NOT EXISTS idx_work_items_status ON work_items(status, updated_at);
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, status);
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, service_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open
ON work_items(chat_jid, agent_type)
ON work_items(chat_jid, agent_type, service_id)
WHERE status IN ('produced', 'delivery_retry');
CREATE TABLE IF NOT EXISTS scheduled_tasks (
@@ -176,6 +210,12 @@ function createSchema(database: Database.Database): void {
session_id TEXT NOT NULL,
PRIMARY KEY (group_folder, agent_type)
);
CREATE TABLE IF NOT EXISTS service_sessions (
group_folder TEXT NOT NULL,
service_id TEXT NOT NULL,
session_id TEXT NOT NULL,
PRIMARY KEY (group_folder, service_id)
);
CREATE TABLE IF NOT EXISTS registered_groups (
jid TEXT NOT NULL,
name TEXT NOT NULL,
@@ -190,6 +230,33 @@ function createSchema(database: Database.Database): void {
PRIMARY KEY (jid, agent_type),
UNIQUE (folder, agent_type)
);
CREATE TABLE IF NOT EXISTS channel_owner (
chat_jid TEXT PRIMARY KEY,
owner_service_id TEXT NOT NULL,
reviewer_service_id TEXT,
activated_at TEXT,
reason TEXT
);
CREATE TABLE IF NOT EXISTS service_handoffs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
chat_jid TEXT NOT NULL,
group_folder TEXT NOT NULL,
source_service_id TEXT NOT NULL,
target_service_id TEXT NOT NULL,
target_agent_type TEXT NOT NULL,
prompt TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
start_seq INTEGER,
end_seq INTEGER,
reason TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
claimed_at TEXT,
completed_at TEXT,
last_error TEXT,
CHECK (status IN ('pending', 'claimed', 'completed', 'failed'))
);
CREATE INDEX IF NOT EXISTS idx_service_handoffs_target
ON service_handoffs(target_service_id, status, created_at);
`);
// Add context_mode column if it doesn't exist (migration for existing DBs)
@@ -293,12 +360,41 @@ function createSchema(database: Database.Database): void {
/* column already exists */
}
try {
database.exec(`ALTER TABLE work_items ADD COLUMN service_id TEXT DEFAULT ''`);
} catch {
/* column already exists */
}
try {
database
.prepare(
`UPDATE work_items
SET service_id = CASE
WHEN agent_type = 'codex' THEN ?
ELSE ?
END
WHERE COALESCE(service_id, '') = ''`,
)
.run(CODEX_MAIN_SERVICE_ID, CLAUDE_SERVICE_ID);
} catch {
/* best effort */
}
backfillMessageSeq(database);
database.exec(`
CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_seq ON messages(seq);
CREATE INDEX IF NOT EXISTS idx_messages_chat_jid_seq ON messages(chat_jid, seq);
`);
database.exec(`DROP INDEX IF EXISTS idx_work_items_group_agent;`);
database.exec(`DROP INDEX IF EXISTS idx_work_items_open;`);
database.exec(`
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent
ON work_items(chat_jid, agent_type, service_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open
ON work_items(chat_jid, agent_type, service_id)
WHERE status IN ('produced', 'delivery_retry');
`);
// Migrate registered_groups to composite keys so Claude/Codex can share a jid/folder.
const registeredGroupsSql = (
@@ -769,6 +865,36 @@ export function getMessagesSinceSeq(
return rows.map(normalizeMessageRow);
}
/**
* Get the N most recent messages for a chat, ordered chronologically.
* Includes both human and bot messages for full conversation context.
* Used for conversation context retrieval.
*/
export function getRecentChatMessages(
chatJid: string,
limit: number = 20,
): NewMessage[] {
const sql = `
SELECT * FROM (
SELECT id, chat_jid, sender, sender_name, content, timestamp, is_from_me, is_bot_message
FROM messages
WHERE chat_jid = ?
AND content != '' AND content IS NOT NULL
ORDER BY timestamp DESC
LIMIT ?
) ORDER BY timestamp
`;
const rows = db
.prepare(sql)
.all(chatJid, limit) as Array<
NewMessage & {
is_from_me?: boolean | number;
is_bot_message?: boolean | number;
}
>;
return rows.map(normalizeMessageRow);
}
export function getLastHumanMessageTimestamp(chatJid: string): string | null {
const row = db
.prepare(
@@ -805,34 +931,39 @@ export function hasRecentRestartAnnouncement(
export function getOpenWorkItem(
chatJid: string,
agentType: AgentType = SERVICE_AGENT_TYPE,
serviceId: string = SERVICE_SESSION_SCOPE,
): WorkItem | undefined {
return db
.prepare(
`SELECT *
FROM work_items
WHERE chat_jid = ? AND agent_type = ? AND status IN ('produced', 'delivery_retry')
WHERE chat_jid = ? AND agent_type = ? AND service_id = ?
AND status IN ('produced', 'delivery_retry')
ORDER BY id ASC
LIMIT 1`,
)
.get(chatJid, agentType) as WorkItem | undefined;
.get(chatJid, agentType, serviceId) as WorkItem | undefined;
}
export function createProducedWorkItem(input: {
group_folder: string;
chat_jid: string;
agent_type?: AgentType;
service_id?: string;
start_seq: number | null;
end_seq: number | null;
result_payload: string;
}): WorkItem {
const now = new Date().toISOString();
const agentType = input.agent_type || SERVICE_AGENT_TYPE;
const serviceId = input.service_id || SERVICE_SESSION_SCOPE;
const result = db
.prepare(
`INSERT INTO work_items (
group_folder,
chat_jid,
agent_type,
service_id,
status,
start_seq,
end_seq,
@@ -840,12 +971,13 @@ export function createProducedWorkItem(input: {
delivery_attempts,
created_at,
updated_at
) VALUES (?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`,
) VALUES (?, ?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`,
)
.run(
input.group_folder,
input.chat_jid,
agentType,
serviceId,
input.start_seq,
input.end_seq,
input.result_payload,
@@ -1085,6 +1217,21 @@ export function deleteTask(id: string): void {
cleanupTargets.push(
resolveTaskRuntimeIpcPathFromGroup(task.group_folder, runtimeTaskId),
resolveTaskSessionsPathFromGroup(task.group_folder, runtimeTaskId),
resolveServiceTaskSessionsPathFromGroup(
task.group_folder,
CLAUDE_SERVICE_ID,
runtimeTaskId,
),
resolveServiceTaskSessionsPathFromGroup(
task.group_folder,
CODEX_MAIN_SERVICE_ID,
runtimeTaskId,
),
resolveServiceTaskSessionsPathFromGroup(
task.group_folder,
CODEX_REVIEW_SERVICE_ID,
runtimeTaskId,
),
);
} catch (err) {
logger.warn(
@@ -1175,7 +1322,14 @@ export function getRecentConsecutiveErrors(
// --- Router state accessors ---
export function getRouterState(key: string): string | undefined {
const prefixedKey = `${SERVICE_ID}:${key}`;
return getRouterStateForService(key, SERVICE_ID);
}
export function getRouterStateForService(
key: string,
serviceId: string,
): string | undefined {
const prefixedKey = `${normalizeServiceId(serviceId)}:${key}`;
const row = db
.prepare('SELECT value FROM router_state WHERE key = ?')
.get(prefixedKey) as { value: string } | undefined;
@@ -1196,7 +1350,15 @@ export function getRouterState(key: string): string | undefined {
}
export function setRouterState(key: string, value: string): void {
const prefixedKey = `${SERVICE_ID}:${key}`;
setRouterStateForService(key, value, SERVICE_ID);
}
export function setRouterStateForService(
key: string,
value: string,
serviceId: string,
): void {
const prefixedKey = `${normalizeServiceId(serviceId)}:${key}`;
db.prepare(
'INSERT OR REPLACE INTO router_state (key, value) VALUES (?, ?)',
).run(prefixedKey, value);
@@ -1205,6 +1367,17 @@ export function setRouterState(key: string, value: string): void {
// --- Session accessors ---
export function getSession(groupFolder: string): string | undefined {
const serviceScopedRow = db
.prepare(
'SELECT session_id FROM service_sessions WHERE group_folder = ? AND service_id = ?',
)
.get(groupFolder, SERVICE_SESSION_SCOPE) as
| { session_id: string }
| undefined;
if (serviceScopedRow?.session_id) {
return serviceScopedRow.session_id;
}
const row = db
.prepare(
'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?',
@@ -1214,18 +1387,40 @@ export function getSession(groupFolder: string): string | undefined {
}
export function setSession(groupFolder: string, sessionId: string): void {
db.prepare(
'INSERT OR REPLACE INTO service_sessions (group_folder, service_id, session_id) VALUES (?, ?, ?)',
).run(groupFolder, SERVICE_SESSION_SCOPE, sessionId);
db.prepare(
'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)',
).run(groupFolder, SERVICE_AGENT_TYPE, sessionId);
}
export function deleteSession(groupFolder: string): void {
db.prepare(
'DELETE FROM service_sessions WHERE group_folder = ? AND service_id = ?',
).run(groupFolder, SERVICE_SESSION_SCOPE);
db.prepare(
'DELETE FROM sessions WHERE group_folder = ? AND agent_type = ?',
).run(groupFolder, SERVICE_AGENT_TYPE);
}
export function getAllSessions(): Record<string, string> {
const serviceRows = db
.prepare(
'SELECT group_folder, session_id FROM service_sessions WHERE service_id = ?',
)
.all(SERVICE_SESSION_SCOPE) as Array<{
group_folder: string;
session_id: string;
}>;
const result: Record<string, string> = {};
for (const row of serviceRows) {
result[row.group_folder] = row.session_id;
}
if (serviceRows.length > 0) {
return result;
}
const rows = db
.prepare(
'SELECT group_folder, session_id FROM sessions WHERE agent_type = ?',
@@ -1234,11 +1429,66 @@ export function getAllSessions(): Record<string, string> {
group_folder: string;
session_id: string;
}>;
const result: Record<string, string> = {};
const legacyResult: Record<string, string> = {};
for (const row of rows) {
result[row.group_folder] = row.session_id;
legacyResult[row.group_folder] = row.session_id;
}
return result;
return legacyResult;
}
/**
* Get session for a specific agent type (cross-provider access).
* Used for provider switch probe attempts.
*/
export function getSessionForAgentType(
groupFolder: string,
agentType: string,
): string | undefined {
const row = db
.prepare(
'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?',
)
.get(groupFolder, agentType) as { session_id: string } | undefined;
return row?.session_id;
}
/**
* Save session for a specific agent type without affecting current service's session.
* Used when probe succeeds and we want to save to target provider's slot only.
*/
export function setSessionForAgentType(
groupFolder: string,
agentType: string,
sessionId: string,
): void {
db.prepare(
'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)',
).run(groupFolder, agentType, sessionId);
}
/**
* Get the agent type of the most recent bot response in a chat.
* Used to detect provider switches for delta handoff.
*/
export function getLastRespondingAgentType(
chatJid: string,
): AgentType | undefined {
const row = db
.prepare(
`SELECT sender FROM messages
WHERE chat_jid = ? AND is_bot_message = 1
ORDER BY timestamp DESC, seq DESC
LIMIT 1`,
)
.get(chatJid) as { sender: string } | undefined;
if (!row) return undefined;
// Map sender to agent type (sender contains the bot identifier)
const sender = row.sender.toLowerCase();
if (sender.includes('claude')) return 'claude-code';
if (sender.includes('codex')) return 'codex';
return undefined;
}
// --- Registered group accessors ---
@@ -1388,6 +1638,255 @@ export function isPairedRoomJid(jid: string): boolean {
return types.includes('claude-code') && types.includes('codex');
}
/**
* Get the most recent bot message (is_bot_message=1) in a chat, regardless of which bot sent it.
* Used for duplicate detection in pair rooms.
*/
export function getLastBotFinalMessage(
chatJid: string,
_agentType: AgentType = SERVICE_AGENT_TYPE,
limit: number = 1,
): Array<{ content: string; timestamp: string }> {
const rows = db
.prepare(
`SELECT content, timestamp
FROM messages
WHERE chat_jid = ? AND is_bot_message = 1
ORDER BY timestamp DESC, seq DESC
LIMIT ?`,
)
.all(chatJid, limit) as Array<{ content: string; timestamp: string }>;
return rows;
}
// --- Channel owner lease accessors ---
export function getChannelOwnerLease(
chatJid: string,
): ChannelOwnerLeaseRow | undefined {
return db
.prepare(
`SELECT chat_jid, owner_service_id, reviewer_service_id, activated_at, reason
FROM channel_owner
WHERE chat_jid = ?`,
)
.get(chatJid) as ChannelOwnerLeaseRow | undefined;
}
export function getAllChannelOwnerLeases(): ChannelOwnerLeaseRow[] {
return db
.prepare(
`SELECT chat_jid, owner_service_id, reviewer_service_id, activated_at, reason
FROM channel_owner`,
)
.all() as ChannelOwnerLeaseRow[];
}
export function setChannelOwnerLease(input: {
chat_jid: string;
owner_service_id: string;
reviewer_service_id?: string | null;
activated_at?: string | null;
reason?: string | null;
}): void {
db.prepare(
`INSERT OR REPLACE INTO channel_owner (
chat_jid,
owner_service_id,
reviewer_service_id,
activated_at,
reason
) VALUES (?, ?, ?, ?, ?)`,
).run(
input.chat_jid,
input.owner_service_id,
input.reviewer_service_id ?? null,
input.activated_at ?? new Date().toISOString(),
input.reason ?? null,
);
}
export function clearChannelOwnerLease(chatJid: string): void {
db.prepare('DELETE FROM channel_owner WHERE chat_jid = ?').run(chatJid);
}
// --- Cross-service handoff accessors ---
export function createServiceHandoff(input: {
chat_jid: string;
group_folder: string;
source_service_id: string;
target_service_id: string;
target_agent_type: AgentType;
prompt: string;
start_seq?: number | null;
end_seq?: number | null;
reason?: string | null;
}): ServiceHandoff {
const result = db
.prepare(
`INSERT INTO service_handoffs (
chat_jid,
group_folder,
source_service_id,
target_service_id,
target_agent_type,
prompt,
start_seq,
end_seq,
reason
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(
input.chat_jid,
input.group_folder,
input.source_service_id,
input.target_service_id,
input.target_agent_type,
input.prompt,
input.start_seq ?? null,
input.end_seq ?? null,
input.reason ?? null,
);
return db
.prepare('SELECT * FROM service_handoffs WHERE id = ?')
.get(result.lastInsertRowid) as ServiceHandoff;
}
export function getPendingServiceHandoffs(
targetServiceId: string = SERVICE_SESSION_SCOPE,
): ServiceHandoff[] {
return db
.prepare(
`SELECT *
FROM service_handoffs
WHERE target_service_id = ?
AND status = 'pending'
ORDER BY created_at ASC, id ASC`,
)
.all(targetServiceId) as ServiceHandoff[];
}
export function claimServiceHandoff(id: number): boolean {
const result = db
.prepare(
`UPDATE service_handoffs
SET status = 'claimed',
claimed_at = datetime('now')
WHERE id = ?
AND status = 'pending'`,
)
.run(id);
return result.changes > 0;
}
export function completeServiceHandoff(id: number): void {
db.prepare(
`UPDATE service_handoffs
SET status = 'completed',
completed_at = datetime('now'),
last_error = NULL
WHERE id = ?`,
).run(id);
}
export function failServiceHandoff(id: number, error: string): void {
db.prepare(
`UPDATE service_handoffs
SET status = 'failed',
completed_at = datetime('now'),
last_error = ?
WHERE id = ?`,
).run(error, id);
}
function normalizeStoredLastAgentSeqCursor(
cursor: string | number | null | undefined,
chatJid: string,
): number {
if (typeof cursor === 'number') {
return Number.isFinite(cursor) && cursor > 0 ? cursor : 0;
}
if (!cursor) return 0;
const trimmed = cursor.trim();
if (/^\d+$/.test(trimmed)) {
return normalizeSeqCursor(trimmed);
}
return getLatestMessageSeqAtOrBefore(trimmed, chatJid);
}
function parseLastAgentSeqState(
raw: string | undefined,
serviceId: string,
): Record<string, string> {
if (!raw) return {};
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch (err) {
throw new Error(
`Invalid last_agent_seq JSON for ${serviceId}: ${
err instanceof Error ? err.message : String(err)
}`,
);
}
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error(`Invalid last_agent_seq JSON for ${serviceId}: not an object`);
}
const cursors: Record<string, string> = {};
for (const [chatJid, cursor] of Object.entries(parsed)) {
if (typeof cursor === 'string' || typeof cursor === 'number') {
cursors[chatJid] = String(cursor);
}
}
return cursors;
}
export function completeServiceHandoffAndAdvanceTargetCursor(input: {
id: number;
target_service_id: string;
chat_jid: string;
end_seq?: number | null;
}): string | null {
return db.transaction(() => {
let appliedCursor: string | null = null;
if (input.end_seq != null) {
const currentState = parseLastAgentSeqState(
getRouterStateForService('last_agent_seq', input.target_service_id),
input.target_service_id,
);
const existingSeq = normalizeStoredLastAgentSeqCursor(
currentState[input.chat_jid],
input.chat_jid,
);
currentState[input.chat_jid] = String(
Math.max(existingSeq, input.end_seq),
);
setRouterStateForService(
'last_agent_seq',
JSON.stringify(currentState),
input.target_service_id,
);
appliedCursor = currentState[input.chat_jid];
}
db.prepare(
`UPDATE service_handoffs
SET status = 'completed',
completed_at = datetime('now'),
last_error = NULL
WHERE id = ?`,
).run(input.id);
return appliedCursor;
})();
}
// --- JSON migration ---
function migrateJsonState(): void {

View File

@@ -0,0 +1,443 @@
import { describe, it, expect, beforeEach, vi } from 'vitest';
import {
_initTestDatabase,
storeMessage,
storeChatMetadata,
createProducedWorkItem,
getOpenWorkItem,
setRegisteredGroup,
isPairedRoomJid,
getLastBotFinalMessage,
markWorkItemDelivered,
} from './db.js';
import { normalizeMessageForDedupe } from './router.js';
import { isDuplicateOfLastBotFinal } from './message-runtime.js';
beforeEach(() => {
_initTestDatabase();
});
// Helper to create a message
const createMessage = (overrides: Partial<{
id: string;
chat_jid: string;
sender: string;
sender_name: string;
content: string;
timestamp: string;
is_from_me: boolean;
is_bot_message: boolean;
}> = {}) => ({
id: overrides.id ?? 'msg-1',
chat_jid: overrides.chat_jid ?? 'dc:test-room',
sender: overrides.sender ?? 'user1',
sender_name: overrides.sender_name ?? 'User',
content: overrides.content ?? 'Hello',
timestamp: overrides.timestamp ?? new Date().toISOString(),
is_from_me: overrides.is_from_me ?? false,
is_bot_message: overrides.is_bot_message ?? false,
});
// Helper to setup chat metadata
const setupChat = (jid: string) => {
storeChatMetadata(jid, new Date().toISOString(), 'Test Chat', 'discord', true);
};
describe('isPairedRoomJid', () => {
it('returns true when both claude-code and codex are registered', () => {
const jid = 'dc:paired-room';
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
expect(isPairedRoomJid(jid)).toBe(true);
});
it('returns false when only claude-code is registered', () => {
const jid = 'dc:single-claude';
setRegisteredGroup(jid, {
name: 'Single Claude',
folder: 'single-claude',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
expect(isPairedRoomJid(jid)).toBe(false);
});
it('returns false when only codex is registered', () => {
const jid = 'dc:single-codex';
setRegisteredGroup(jid, {
name: 'Single Codex',
folder: 'single-codex',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
expect(isPairedRoomJid(jid)).toBe(false);
});
});
describe('getLastBotFinalMessage', () => {
it('returns the most recent bot message from any service (is_bot_message=1)', () => {
const jid = 'dc:test-room';
const now = new Date();
setupChat(jid);
// Store older bot message (from any bot)
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'First bot message',
timestamp: new Date(now.getTime() - 1000).toISOString(),
is_from_me: true,
is_bot_message: true,
}));
// Store newer bot message (from different bot - is_from_me=0 but is_bot_message=1)
storeMessage(createMessage({
id: 'msg-2',
chat_jid: jid,
content: 'Second bot message',
timestamp: now.toISOString(),
is_from_me: false, // Different bot
is_bot_message: true,
}));
// Store human message (should not be returned)
storeMessage(createMessage({
id: 'msg-3',
chat_jid: jid,
content: 'Human message',
timestamp: new Date(now.getTime() + 1000).toISOString(),
is_from_me: false,
is_bot_message: false,
}));
const lastMessages = getLastBotFinalMessage(jid, 'claude-code', 1);
expect(lastMessages).toHaveLength(1);
expect(lastMessages[0].content).toBe('Second bot message');
});
it('returns empty array when no bot messages exist', () => {
const jid = 'dc:empty-room';
setupChat(jid);
const lastMessages = getLastBotFinalMessage(jid, 'claude-code', 1);
expect(lastMessages).toHaveLength(0);
});
});
describe('normalizeMessageForDedupe', () => {
it('normalizes messages for comparison', () => {
expect(normalizeMessageForDedupe(' Hello World ')).toBe('hello world');
expect(normalizeMessageForDedupe('Hello\n\nWorld')).toBe('hello world');
expect(normalizeMessageForDedupe('Hello World')).toBe('hello world');
expect(normalizeMessageForDedupe('HELLO WORLD')).toBe('hello world');
});
it('handles empty strings', () => {
expect(normalizeMessageForDedupe('')).toBe('');
expect(normalizeMessageForDedupe(' ')).toBe('');
});
});
describe('isDuplicateOfLastBotFinal (runtime function)', () => {
it('paired room: detects duplicate final message', () => {
const jid = 'dc:paired-room';
setupChat(jid);
// Register as paired room
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
// Store a bot message (from any bot)
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'DONE — Task completed successfully',
timestamp: new Date().toISOString(),
is_from_me: false, // Different bot
is_bot_message: true,
}));
// Verify it's a paired room
expect(isPairedRoomJid(jid)).toBe(true);
// Verify duplicate detection works via the actual runtime function
expect(isDuplicateOfLastBotFinal(jid, 'DONE — Task completed successfully')).toBe(true);
expect(isDuplicateOfLastBotFinal(jid, 'done — task completed successfully')).toBe(true); // Normalized match
});
it('non-paired room: duplicate check is bypassed', () => {
const jid = 'dc:single-bot';
setupChat(jid);
// Register only one bot
setRegisteredGroup(jid, {
name: 'Single Bot',
folder: 'single-bot',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
// Store a bot message
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'DONE — Task completed',
timestamp: new Date().toISOString(),
is_from_me: true,
is_bot_message: true,
}));
// Verify it's NOT a paired room
expect(isPairedRoomJid(jid)).toBe(false);
// Duplicate check should be bypassed (return false)
expect(isDuplicateOfLastBotFinal(jid, 'DONE — Task completed')).toBe(false);
});
it('non-duplicate message in paired room is not suppressed', () => {
const jid = 'dc:paired-room';
setupChat(jid);
// Register as paired room
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
// Store a bot message
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'First message content',
timestamp: new Date().toISOString(),
is_from_me: true,
is_bot_message: true,
}));
// Verify different content is not a duplicate
expect(isDuplicateOfLastBotFinal(jid, 'Different message content')).toBe(false);
expect(isDuplicateOfLastBotFinal(jid, 'First message content')).toBe(true);
});
it('cross-bot duplicate detection: claude detects codex message as duplicate', () => {
const jid = 'dc:paired-room';
setupChat(jid);
// Register as paired room
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
// Store a bot message from "codex" (is_from_me=0)
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'DONE — Analysis complete',
timestamp: new Date().toISOString(),
is_from_me: false, // Other bot
is_bot_message: true,
}));
// Verify claude service detects this as duplicate (cross-bot detection)
expect(isDuplicateOfLastBotFinal(jid, 'DONE — Analysis complete')).toBe(true);
});
it('normalization handles whitespace and case differences', () => {
const jid = 'dc:paired-room';
setupChat(jid);
// Register as paired room
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
// Store a bot message with specific formatting
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'DONE — Task completed\n\nSuccessfully',
timestamp: new Date().toISOString(),
is_from_me: true,
is_bot_message: true,
}));
// Same content with different whitespace should be detected as duplicate
expect(isDuplicateOfLastBotFinal(jid, 'done — task completed successfully')).toBe(true);
expect(isDuplicateOfLastBotFinal(jid, ' DONE — Task completed Successfully ')).toBe(true);
// Different content should not be duplicate
expect(isDuplicateOfLastBotFinal(jid, 'FAILED — Task failed')).toBe(false);
});
it('work item lifecycle: produced -> delivered (duplicate)', () => {
const jid = 'dc:paired-room';
setupChat(jid);
// Register as paired room
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
// Store a bot message (simulating previous delivery)
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'DONE — Task completed successfully',
timestamp: new Date().toISOString(),
is_from_me: true,
is_bot_message: true,
}));
// Create a work item with the same content (duplicate)
const workItem = createProducedWorkItem({
group_folder: 'paired-room',
chat_jid: jid,
agent_type: 'claude-code',
start_seq: 1,
end_seq: 2,
result_payload: 'DONE — Task completed successfully',
});
expect(workItem.status).toBe('produced');
// Verify duplicate detection via actual runtime function
expect(isDuplicateOfLastBotFinal(jid, workItem.result_payload)).toBe(true);
// Simulate suppression: mark as delivered without sending
markWorkItemDelivered(workItem.id, null);
// Verify work item is marked delivered
const openItem = getOpenWorkItem(jid, 'claude-code');
expect(openItem).toBeUndefined(); // No open items because it was marked delivered
});
it('work item lifecycle: produced -> pending -> delivered (non-duplicate)', () => {
const jid = 'dc:paired-room';
setupChat(jid);
// Register as paired room
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@claude',
added_at: new Date().toISOString(),
agentType: 'claude-code',
});
setRegisteredGroup(jid, {
name: 'Paired Room',
folder: 'paired-room',
trigger: '@codex',
added_at: new Date().toISOString(),
agentType: 'codex',
});
// Store a bot message
storeMessage(createMessage({
id: 'msg-1',
chat_jid: jid,
content: 'First message',
timestamp: new Date().toISOString(),
is_from_me: true,
is_bot_message: true,
}));
// Create a work item with DIFFERENT content (non-duplicate)
const workItem = createProducedWorkItem({
group_folder: 'paired-room',
chat_jid: jid,
agent_type: 'claude-code',
start_seq: 1,
end_seq: 2,
result_payload: 'Second message (different)',
});
expect(workItem.status).toBe('produced');
// Verify NOT a duplicate
expect(isDuplicateOfLastBotFinal(jid, workItem.result_payload)).toBe(false);
});
});

View File

@@ -68,6 +68,18 @@ export function resolveGroupSessionsPath(folder: string): string {
return sessionsPath;
}
export function resolveServiceGroupSessionsPath(
folder: string,
serviceId: string,
): string {
assertValidGroupFolder(folder);
assertValidRuntimeSegment(serviceId, 'service ID');
const sessionsBaseDir = path.resolve(DATA_DIR, 'sessions');
const sessionsPath = path.resolve(sessionsBaseDir, folder, 'services', serviceId);
ensureWithinBase(sessionsBaseDir, sessionsPath);
return sessionsPath;
}
export function resolveTaskRuntimeIpcPath(
folder: string,
taskId: string,
@@ -91,3 +103,24 @@ export function resolveTaskSessionsPath(
ensureWithinBase(sessionsBaseDir, sessionsPath);
return sessionsPath;
}
export function resolveServiceTaskSessionsPath(
folder: string,
serviceId: string,
taskId: string,
): string {
assertValidGroupFolder(folder);
assertValidRuntimeSegment(serviceId, 'service ID');
assertValidRuntimeSegment(taskId, 'task ID');
const sessionsBaseDir = path.resolve(DATA_DIR, 'sessions');
const sessionsPath = path.resolve(
sessionsBaseDir,
folder,
'services',
serviceId,
'tasks',
taskId,
);
ensureWithinBase(sessionsBaseDir, sessionsPath);
return sessionsPath;
}

View File

@@ -6,7 +6,10 @@ import {
DATA_DIR,
IDLE_TIMEOUT,
POLL_INTERVAL,
SERVICE_ID,
SERVICE_AGENT_TYPE,
isClaudeService,
isReviewService,
isSessionCommandSenderAllowed,
STATUS_CHANNEL_ID,
STATUS_UPDATE_INTERVAL,
@@ -41,7 +44,11 @@ import { composeDashboardContent } from './dashboard-render.js';
import { GroupQueue } from './group-queue.js';
import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js';
import { startIpcWatcher } from './ipc.js';
import { findChannel, formatOutbound } from './router.js';
import {
findChannel,
formatOutbound,
normalizeMessageForDedupe,
} from './router.js';
import {
buildRestartAnnouncement,
buildInterruptedRestartAnnouncement,
@@ -64,12 +71,17 @@ import { Channel, NewMessage, RegisteredGroup } from './types.js';
import { logger } from './logger.js';
import { normalizeStoredSeqCursor } from './message-cursor.js';
import { initCodexTokenRotation } from './codex-token-rotation.js';
import { initTokenRotation } from './token-rotation.js';
import { hasAvailableClaudeToken, initTokenRotation } from './token-rotation.js';
import {
shouldStartTokenRefreshLoop,
startTokenRefreshLoop,
stopTokenRefreshLoop,
} from './token-refresh.js';
import {
getActiveCodexFailoverLeases,
restoreDefaultChannelLease,
} from './service-routing.js';
import { FAILOVER_MIN_DURATION_MS } from './config.js';
// Token rotation is initialized lazily on first use or at startup below
@@ -315,9 +327,14 @@ async function main(): Promise<void> {
loadState();
// Graceful shutdown handlers
let leaseRecoveryTimer: ReturnType<typeof setInterval> | null = null;
const shutdown = async (signal: string) => {
logger.info({ signal }, 'Shutdown signal received');
stopTokenRefreshLoop();
if (leaseRecoveryTimer) {
clearInterval(leaseRecoveryTimer);
leaseRecoveryTimer = null;
}
const interruptedGroups = queue
.getStatuses(Object.keys(registeredGroups))
.filter(
@@ -410,19 +427,23 @@ async function main(): Promise<void> {
}
// Start subsystems (independently of connection handler)
startSchedulerLoop({
registeredGroups: () => registeredGroups,
getSessions: () => sessions,
queue,
onProcess: (groupJid, proc, processName, ipcDir) =>
queue.registerProcess(groupJid, proc, processName, ipcDir),
sendMessage: (jid, rawText) =>
sendFormattedChannelMessage(channels, jid, rawText),
sendTrackedMessage: (jid, rawText) =>
sendFormattedTrackedChannelMessage(channels, jid, rawText),
editTrackedMessage: (jid, messageId, rawText) =>
editFormattedTrackedChannelMessage(channels, jid, messageId, rawText),
});
if (!isReviewService()) {
startSchedulerLoop({
registeredGroups: () => registeredGroups,
getSessions: () => sessions,
queue,
onProcess: (groupJid, proc, processName, ipcDir) =>
queue.registerProcess(groupJid, proc, processName, ipcDir),
sendMessage: (jid, rawText) =>
sendFormattedChannelMessage(channels, jid, rawText),
sendTrackedMessage: (jid, rawText) =>
sendFormattedTrackedChannelMessage(channels, jid, rawText),
editTrackedMessage: (jid, messageId, rawText) =>
editFormattedTrackedChannelMessage(channels, jid, messageId, rawText),
});
} else {
logger.info({ serviceId: SERVICE_ID }, 'Skipping scheduler for review service');
}
startIpcWatcher({
sendMessage: (jid, text) => {
const channel = findChannel(channels, jid);
@@ -467,6 +488,7 @@ async function main(): Promise<void> {
}
await startUnifiedDashboard({
assistantName: ASSISTANT_NAME,
serviceId: SERVICE_ID,
serviceAgentType: SERVICE_AGENT_TYPE,
statusChannelId: STATUS_CHANNEL_ID,
statusUpdateInterval: STATUS_UPDATE_INTERVAL,
@@ -482,6 +504,44 @@ async function main(): Promise<void> {
},
purgeOnStart: true,
});
if (isClaudeService()) {
leaseRecoveryTimer = setInterval(() => {
if (!hasAvailableClaudeToken()) {
return;
}
const now = Date.now();
for (const lease of getActiveCodexFailoverLeases()) {
const activatedMs = lease.activatedAt
? new Date(lease.activatedAt).getTime()
: NaN;
if (Number.isNaN(activatedMs)) {
logger.warn(
{ chatJid: lease.chatJid, activatedAt: lease.activatedAt },
'Failover lease has unparseable activated_at, skipping auto-restore',
);
continue;
}
const elapsed = now - activatedMs;
if (elapsed < FAILOVER_MIN_DURATION_MS) {
logger.debug(
{
chatJid: lease.chatJid,
elapsedMin: Math.round(elapsed / 60_000),
minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000),
},
'Failover lease still within minimum hold period, skipping restore',
);
continue;
}
restoreDefaultChannelLease(lease.chatJid);
logger.info(
{ chatJid: lease.chatJid, serviceId: SERVICE_ID, elapsedMin: Math.round(elapsed / 60_000) },
'Claude token available and failover hold period elapsed, restored default channel lease',
);
}
}, 5_000);
}
runtime.startMessageLoop().catch((err) => {
logger.fatal({ err }, 'Message loop crashed unexpectedly');
process.exit(1);

View File

@@ -11,13 +11,29 @@ vi.mock('./available-groups.js', () => ({
}));
vi.mock('./config.js', () => ({
CODEX_MAIN_SERVICE_ID: 'codex-main',
CODEX_REVIEW_SERVICE_ID: 'codex-review',
DATA_DIR: '/tmp/ejclaw-test-data',
SERVICE_SESSION_SCOPE: 'claude',
}));
vi.mock('./db.js', () => ({
createServiceHandoff: vi.fn(),
getAllTasks: vi.fn(() => []),
}));
vi.mock('./service-routing.js', () => ({
activateCodexFailover: vi.fn(),
getEffectiveChannelLease: vi.fn(() => ({
chat_jid: 'group@test',
owner_service_id: 'claude',
reviewer_service_id: 'codex-main',
activated_at: null,
reason: null,
explicit: false,
})),
}));
vi.mock('./logger.js', () => ({
logger: {
debug: vi.fn(),
@@ -27,41 +43,35 @@ vi.mock('./logger.js', () => ({
},
}));
vi.mock('./provider-fallback.js', () => ({
detectFallbackTrigger: vi.fn((error?: string | null) => {
const lower = (error || '').toLowerCase();
if (
lower.includes('does not have access to claude') ||
(lower.includes('failed to authenticate') &&
lower.includes('403') &&
lower.includes('terminated'))
) {
return { shouldFallback: true, reason: 'org-access-denied' };
}
if (
lower.includes('429') ||
lower.includes('rate limit') ||
lower.includes('hit your limit')
) {
return { shouldFallback: true, reason: '429' };
}
return { shouldFallback: false, reason: '' };
}),
getActiveProvider: vi.fn(async () => 'claude'),
getFallbackEnvOverrides: vi.fn(() => ({
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
ANTHROPIC_MODEL: 'kimi-k2.5',
})),
getFallbackProviderName: vi.fn(() => 'kimi'),
hasGroupProviderOverride: vi.fn(() => false),
isFallbackEnabled: vi.fn(() => true),
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
markPrimaryCooldown: vi.fn(),
}));
vi.mock('./agent-error-detection.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('./agent-error-detection.js')>();
return {
...actual,
classifyRotationTrigger: vi.fn((error?: string | null) => {
const lower = (error || '').toLowerCase();
if (
lower.includes('does not have access to claude') ||
(lower.includes('failed to authenticate') &&
lower.includes('403') &&
lower.includes('terminated'))
) {
return { shouldRetry: true, reason: 'org-access-denied' };
}
if (
lower.includes('429') ||
lower.includes('rate limit') ||
lower.includes('hit your limit')
) {
return { shouldRetry: true, reason: '429' };
}
return { shouldRetry: false, reason: '' };
}),
};
});
vi.mock('./session-recovery.js', () => ({
shouldResetSessionOnAgentFailure: vi.fn(() => false),
shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false),
}));
vi.mock('./token-rotation.js', () => ({
@@ -96,9 +106,11 @@ vi.mock('./memento-client.js', () => ({
import * as agentRunner from './agent-runner.js';
import * as codexTokenRotation from './codex-token-rotation.js';
import * as db from './db.js';
import { buildRoomMemoryBriefing } from './memento-client.js';
import { runAgentForGroup } from './message-agent-executor.js';
import * as providerFallback from './provider-fallback.js';
import * as sessionRecovery from './session-recovery.js';
import * as serviceRouting from './service-routing.js';
import * as tokenRotation from './token-rotation.js';
import type { RegisteredGroup } from './types.js';
@@ -129,9 +141,6 @@ function makeDeps() {
describe('runAgentForGroup room memory', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(false);
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
status: 'success',
result: 'ok',
@@ -167,7 +176,6 @@ describe('runAgentForGroup room memory', () => {
}),
expect.any(Function),
undefined,
undefined,
);
});
@@ -196,6 +204,59 @@ describe('runAgentForGroup room memory', () => {
}),
expect.any(Function),
undefined,
);
});
it('injects suppress token instructions into the agent prompt', async () => {
const group = { ...makeGroup(), folder: 'test-group' };
await runAgentForGroup(makeDeps(), {
group,
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-suppress',
suppressToken: '__TEST_SUPPRESS__',
});
expect(agentRunner.runAgentProcess).toHaveBeenCalledWith(
group,
expect.objectContaining({
prompt: expect.stringContaining(
'If you have no user-visible content to send for this turn, output exactly this token and nothing else: __TEST_SUPPRESS__',
),
}),
expect.any(Function),
undefined,
);
});
it('adds reviewer silence guidance when the current service is the reviewer for the chat', async () => {
const group = { ...makeGroup(), folder: 'test-group' };
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
chat_jid: 'group@test',
owner_service_id: 'claude',
reviewer_service_id: 'claude',
activated_at: null,
reason: null,
explicit: false,
});
await runAgentForGroup(makeDeps(), {
group,
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-review-suppress',
suppressToken: '__TEST_SUPPRESS__',
});
expect(agentRunner.runAgentProcess).toHaveBeenCalledWith(
group,
expect.objectContaining({
prompt: expect.stringContaining(
'If you are only agreeing, mirroring, or restating without adding a concrete correction, risk, missing prerequisite, test gap, or code change, output only the token.',
),
}),
expect.any(Function),
undefined,
);
});
@@ -205,14 +266,11 @@ describe('runAgentForGroup Claude rotation', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
vi.mocked(tokenRotation.rotateToken).mockReturnValue(false);
});
it('rotates to another Claude account before falling back to Kimi', async () => {
it('rotates to another Claude account on usage exhaustion', async () => {
const outputs: string[] = [];
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
@@ -223,7 +281,7 @@ describe('runAgentForGroup Claude rotation', () => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'Youre out of extra usage · resets 4am (Asia/Seoul)',
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
});
return {
status: 'success',
@@ -256,7 +314,7 @@ describe('runAgentForGroup Claude rotation', () => {
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
// No fallback provider — rotation is the only recovery mechanism
expect(outputs).toEqual(['회전된 Claude 응답입니다.']);
});
@@ -311,15 +369,15 @@ describe('runAgentForGroup Claude rotation', () => {
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
// No fallback provider — rotation is the only recovery mechanism
expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']);
});
it('suppresses Claude 502 HTML and falls back without forwarding it', async () => {
it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => {
const outputs: string[] = [];
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'intermediate',
@@ -336,24 +394,64 @@ describe('runAgentForGroup Claude rotation', () => {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'Kimi 폴백 응답입니다.',
});
return {
status: 'success',
result: null,
};
});
},
);
const result = await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-claude-502-fallback',
runId: 'run-claude-502-error',
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
});
expect(result).toBe('error');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
expect(outputs).toEqual([]);
expect(db.createServiceHandoff).not.toHaveBeenCalled();
});
it('clears the Claude session and retries fresh when a retryable thinking 400 is surfaced as text', async () => {
const outputs: string[] = [];
const deps = makeDeps();
vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure)
.mockReturnValueOnce(true)
.mockReturnValueOnce(false);
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'intermediate',
result:
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'fresh Claude retry success',
});
return {
status: 'success',
result: null,
newSessionId: 'claude-session-fresh',
};
});
const result = await runAgentForGroup(deps, {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-retryable-thinking-400-retry',
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
@@ -361,15 +459,63 @@ describe('runAgentForGroup Claude rotation', () => {
expect(result).toBe('success');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'overloaded',
undefined,
);
expect(outputs).toEqual(['Kimi 폴백 응답입니다.']);
expect(deps.clearSession).toHaveBeenCalledTimes(1);
expect(deps.clearSession).toHaveBeenCalledWith('test-claude');
// No fallback provider — rotation is the only recovery mechanism
expect(outputs).toEqual(['fresh Claude retry success']);
});
it('stops after all Claude accounts are usage-exhausted without falling back to Kimi', async () => {
it('returns error when the fresh Claude retry also hits the same retryable thinking 400', async () => {
const outputs: string[] = [];
const deps = makeDeps();
vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure)
.mockReturnValueOnce(true)
.mockReturnValueOnce(true);
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'intermediate',
result:
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'intermediate',
result:
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
});
return {
status: 'success',
result: null,
};
});
const result = await runAgentForGroup(deps, {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-retryable-thinking-400-error',
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
});
expect(result).toBe('error');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(deps.clearSession).toHaveBeenCalledTimes(2);
expect(outputs).toEqual([]);
});
it('returns error after all Claude accounts are usage-exhausted', async () => {
const outputs: string[] = [];
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
@@ -382,7 +528,7 @@ describe('runAgentForGroup Claude rotation', () => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'Youre out of extra usage · resets 4am (Asia/Seoul)',
result: 'You\u2019re out of extra usage \u00b7 resets 4am (Asia/Seoul)',
});
return {
status: 'success',
@@ -393,18 +539,7 @@ describe('runAgentForGroup Claude rotation', () => {
await onOutput?.({
status: 'success',
phase: 'final',
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'Kimi 폴백 응답입니다.',
result: "You're out of extra usage \u00b7 resets 4am (Asia/Seoul)",
});
return {
status: 'success',
@@ -416,20 +551,71 @@ describe('runAgentForGroup Claude rotation', () => {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-fallback-after-rotation',
runId: 'run-all-exhausted-error',
startSeq: 10,
endSeq: 12,
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
});
expect(result).toBe('success');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
expect(outputs).toEqual([]);
expect(serviceRouting.activateCodexFailover).toHaveBeenCalledWith(
'group@test',
'claude-usage-exhausted',
);
expect(db.createServiceHandoff).toHaveBeenCalledWith(
expect.objectContaining({
chat_jid: 'group@test',
target_service_id: 'codex-review',
target_agent_type: 'codex',
start_seq: 10,
end_seq: 12,
reason: 'claude-usage-exhausted',
}),
);
});
it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => {
const outputs: string[] = [];
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'progress',
result: '대화 요약 중...',
});
await onOutput?.({
status: 'success',
phase: 'final',
result: "You've hit your limit · resets 2am (Asia/Seoul)",
});
return {
status: 'success',
result: null,
};
},
);
const result = await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-progress-before-usage-banner',
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
});
expect(result).toBe('error');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'usage-exhausted',
undefined,
);
expect(outputs).toEqual([]);
expect(outputs).toEqual(['대화 요약 중...']);
expect(db.createServiceHandoff).not.toHaveBeenCalled();
});
it('rotates to another Claude account when Claude streams an org access denied banner', async () => {
@@ -483,11 +669,57 @@ describe('runAgentForGroup Claude rotation', () => {
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
// No fallback provider — rotation is the only recovery mechanism
expect(outputs).toEqual(['org access denied 회전 성공 응답']);
});
it('stops after all Claude accounts are org-access-denied without falling back to Kimi', async () => {
it('rotates when Claude surfaces 403 terminated as a success result', async () => {
const outputs: string[] = [];
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true);
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'Failed to authenticate. API Error: 403 terminated',
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: '403 회전 성공 응답',
});
return {
status: 'success',
result: null,
};
});
const result = await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-403-success-rotation',
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
});
expect(result).toBe('success');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(outputs).toEqual(['403 회전 성공 응답']);
});
it('returns error after all Claude accounts are org-access-denied', async () => {
const outputs: string[] = [];
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
@@ -519,54 +751,34 @@ describe('runAgentForGroup Claude rotation', () => {
result: null,
error: 'Failed to authenticate. API Error: 403 terminated',
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'Kimi 폴백 응답입니다.',
});
return {
status: 'success',
result: null,
};
});
const result = await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-org-access-denied-no-fallback',
runId: 'run-org-access-denied-error',
onOutput: async (output) => {
if (typeof output.result === 'string') outputs.push(output.result);
},
});
expect(result).toBe('error');
expect(result).toBe('success');
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'org-access-denied',
undefined,
);
expect(outputs).toEqual([]);
});
it('skips execution entirely when Claude no-fallback cooldown is already active', async () => {
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('kimi');
vi.mocked(
providerFallback.isPrimaryNoFallbackCooldownActive,
).mockReturnValue(true);
const result = await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-skip-primary-cooldown',
});
expect(result).toBe('error');
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
expect(serviceRouting.activateCodexFailover).toHaveBeenCalledWith(
'group@test',
'claude-org-access-denied',
);
expect(db.createServiceHandoff).toHaveBeenCalledWith(
expect.objectContaining({
chat_jid: 'group@test',
target_service_id: 'codex-review',
target_agent_type: 'codex',
reason: 'claude-org-access-denied',
}),
);
});
it('does not mistake a normal response quoting the banner text for a usage error', async () => {
@@ -601,7 +813,6 @@ describe('runAgentForGroup Claude rotation', () => {
expect(result).toBe('success');
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
expect(outputs).toEqual([
"상태 문구 예시: You're out of extra usage · resets 4am (Asia/Seoul) 라는 배너가 뜰 수 있습니다.",
]);
@@ -612,9 +823,6 @@ describe('runAgentForGroup Codex rotation', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(false);
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(2);
vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValueOnce(true);
});

View File

@@ -1,4 +1,3 @@
import path from 'path';
import { getErrorMessage } from './utils.js';
import {
@@ -8,23 +7,25 @@ import {
writeTasksSnapshot,
} from './agent-runner.js';
import { listAvailableGroups } from './available-groups.js';
import { DATA_DIR } from './config.js';
import { getAllTasks } from './db.js';
import { createServiceHandoff, getAllTasks } from './db.js';
import { GroupQueue } from './group-queue.js';
import { logger } from './logger.js';
import { buildRoomMemoryBriefing } from './memento-client.js';
import {
detectFallbackTrigger,
getActiveProvider,
getFallbackEnvOverrides,
getFallbackProviderName,
hasGroupProviderOverride,
isFallbackEnabled,
isPrimaryNoFallbackCooldownActive,
markPrimaryCooldown,
} from './provider-fallback.js';
classifyRotationTrigger,
type AgentTriggerReason,
} from './agent-error-detection.js';
import { runClaudeRotationLoop } from './provider-retry.js';
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
import {
shouldResetSessionOnAgentFailure,
shouldRetryFreshSessionOnAgentFailure,
} from './session-recovery.js';
import { CODEX_MAIN_SERVICE_ID, CODEX_REVIEW_SERVICE_ID, SERVICE_SESSION_SCOPE } from './config.js';
import {
buildSuppressTokenPrompt,
classifySuppressTokenOutput,
} from './output-suppression.js';
import { activateCodexFailover, getEffectiveChannelLease } from './service-routing.js';
import {
evaluateStreamedOutput,
type StreamedOutputState,
@@ -35,13 +36,12 @@ import {
getCodexAccountCount,
markCodexTokenHealthy,
} from './codex-token-rotation.js';
import type {
AgentTriggerReason,
CodexRotationReason,
} from './agent-error-detection.js';
import type { CodexRotationReason } from './agent-error-detection.js';
import { getTokenCount } from './token-rotation.js';
import type { RegisteredGroup } from './types.js';
// ── Main executor ─────────────────────────────────────────────────
export interface MessageAgentExecutorDeps {
assistantName: string;
queue: Pick<GroupQueue, 'registerProcess'>;
@@ -58,10 +58,22 @@ export async function runAgentForGroup(
prompt: string;
chatJid: string;
runId: string;
suppressToken?: string;
startSeq?: number | null;
endSeq?: number | null;
onOutput?: (output: AgentOutput) => Promise<void>;
},
): Promise<'success' | 'error'> {
const { group, prompt, chatJid, runId, onOutput } = args;
const {
group,
prompt,
chatJid,
runId,
suppressToken,
startSeq,
endSeq,
onOutput,
} = args;
const isMain = group.isMain === true;
const isClaudeCodeAgent =
(group.agentType || 'claude-code') === 'claude-code';
@@ -97,20 +109,61 @@ export async function runAgentForGroup(
let resetSessionRequested = false;
const settingsPath = path.join(
DATA_DIR,
'sessions',
group.folder,
'.claude',
'settings.json',
);
const groupHasOverride = hasGroupProviderOverride(settingsPath);
const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1;
const canFallback =
isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride;
const currentLease = getEffectiveChannelLease(chatJid);
const reviewerMode = currentLease.reviewer_service_id === SERVICE_SESSION_SCOPE;
const effectivePrompt = buildSuppressTokenPrompt(prompt, suppressToken, {
reviewerMode,
});
const shouldHandoffToCodex = (
reason: AgentTriggerReason,
sawVisibleOutput: boolean,
): boolean => {
if (sawVisibleOutput) {
return false;
}
return (
reason === '429' ||
reason === 'usage-exhausted' ||
reason === 'auth-expired' ||
reason === 'org-access-denied'
);
};
const maybeHandoffToCodex = (
reason: AgentTriggerReason,
sawVisibleOutput: boolean,
): boolean => {
if (!isClaudeCodeAgent) return false;
if (!shouldHandoffToCodex(reason, sawVisibleOutput)) {
return false;
}
if (currentLease.reviewer_service_id === null) {
return false;
}
activateCodexFailover(chatJid, `claude-${reason}`);
createServiceHandoff({
chat_jid: chatJid,
group_folder: group.folder,
source_service_id: SERVICE_SESSION_SCOPE,
target_service_id: CODEX_REVIEW_SERVICE_ID,
target_agent_type: 'codex',
prompt,
start_seq: startSeq ?? null,
end_seq: endSeq ?? null,
reason: `claude-${reason}`,
});
logger.warn(
{ chatJid, group: group.name, runId, reason },
'Claude unavailable, handed off current turn to codex-review',
);
return true;
};
const agentInput = {
prompt,
prompt: effectivePrompt,
sessionId,
memoryBriefing,
groupFolder: group.folder,
@@ -126,29 +179,31 @@ export async function runAgentForGroup(
output?: AgentOutput;
error?: unknown;
sawOutput: boolean;
sawVisibleOutput: boolean;
sawSuccessNullResultWithoutOutput: boolean;
retryableSessionFailureDetected: boolean;
streamedTriggerReason?: {
reason: AgentTriggerReason;
retryAfterMs?: number;
};
}> => {
const persistSessionIds = provider === 'claude';
let streamedState: StreamedOutputState = {
sawOutput: false,
sawVisibleOutput: false,
sawSuccessNullResultWithoutOutput: false,
};
const wrappedOnOutput = onOutput
? async (output: AgentOutput) => {
if (
persistSessionIds &&
isClaudeCodeAgent &&
provider === 'claude' &&
shouldResetSessionOnAgentFailure(output)
) {
resetSessionRequested = true;
}
if (
persistSessionIds &&
provider === 'claude' &&
output.newSessionId &&
!resetSessionRequested
) {
@@ -161,7 +216,7 @@ export async function runAgentForGroup(
trackSuccessNullResult: true,
shortCircuitTriggeredErrors:
provider === 'claude'
? canFallback || canRotateToken
? canRotateToken
: getCodexAccountCount() > 1,
});
streamedState = evaluation.state;
@@ -179,7 +234,7 @@ export async function runAgentForGroup(
reason: evaluation.newTrigger.reason,
resultPreview: output.result.slice(0, 120),
},
'Detected Claude fallback trigger in successful output',
'Detected Claude rotation trigger in successful output',
);
} else if (
evaluation.newTrigger &&
@@ -194,7 +249,7 @@ export async function runAgentForGroup(
errorPreview: output.error.slice(0, 120),
},
provider === 'claude'
? 'Detected Claude fallback trigger in streamed error output'
? 'Detected Claude rotation trigger in streamed error output'
: 'Detected Codex rotation trigger in streamed error output',
);
}
@@ -215,39 +270,53 @@ export async function runAgentForGroup(
return;
}
if (evaluation.suppressedRetryableSessionFailure) {
logger.warn(
{
chatJid,
group: group.name,
runId,
resultPreview:
typeof output.result === 'string'
? output.result.slice(0, 160)
: output.error?.slice(0, 160),
},
'Suppressed retryable Claude session failure from chat output',
);
return;
}
if (!evaluation.shouldForwardOutput) {
return;
}
const suppressState =
typeof output.result === 'string'
? classifySuppressTokenOutput(output.result, suppressToken)
: 'none';
if (
typeof output.result === 'string' &&
output.result.length > 0 &&
suppressState === 'none'
) {
streamedState = {
...evaluation.state,
sawVisibleOutput: true,
};
}
await onOutput(output);
}
: undefined;
if (provider !== 'claude') {
logger.info(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
provider,
},
`Claude provider in cooldown, routing request to ${provider}`,
);
}
const agentType = group.agentType || 'claude-code';
const providerLabel = canFallback ? provider : agentType;
logger.info(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
provider: providerLabel,
canFallback,
groupHasOverride,
provider: agentType,
},
`Using provider: ${providerLabel}`,
`Using provider: ${agentType}`,
);
try {
@@ -255,17 +324,14 @@ export async function runAgentForGroup(
group,
{
...agentInput,
sessionId: persistSessionIds ? sessionId : undefined,
sessionId: provider === 'claude' ? sessionId : undefined,
},
(proc, processName, ipcDir) =>
deps.queue.registerProcess(chatJid, proc, processName, ipcDir),
wrappedOnOutput,
isClaudeCodeAgent && provider !== 'claude'
? getFallbackEnvOverrides()
: undefined,
);
if (persistSessionIds && output.newSessionId) {
if (provider === 'claude' && output.newSessionId) {
deps.persistSession(group.folder, output.newSessionId);
}
@@ -285,75 +351,27 @@ export async function runAgentForGroup(
return {
output,
sawOutput: streamedState.sawOutput,
sawVisibleOutput: streamedState.sawVisibleOutput,
sawSuccessNullResultWithoutOutput:
streamedState.sawSuccessNullResultWithoutOutput,
retryableSessionFailureDetected:
streamedState.retryableSessionFailureDetected === true,
streamedTriggerReason: streamedState.streamedTriggerReason,
};
} catch (error) {
return {
error,
sawOutput: streamedState.sawOutput,
sawVisibleOutput: streamedState.sawVisibleOutput,
sawSuccessNullResultWithoutOutput:
streamedState.sawSuccessNullResultWithoutOutput,
retryableSessionFailureDetected:
streamedState.retryableSessionFailureDetected === true,
streamedTriggerReason: streamedState.streamedTriggerReason,
};
}
};
const runFallbackAttempt = async (
reason: AgentTriggerReason,
retryAfterMs?: number,
): Promise<'success' | 'error'> => {
const fallbackName = getFallbackProviderName();
markPrimaryCooldown(reason, retryAfterMs);
logger.info(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
reason,
retryAfterMs,
fallbackProvider: fallbackName,
},
`Falling back to provider: ${fallbackName} (reason: ${reason})`,
);
const fallbackAttempt = await runAttempt(fallbackName);
if (fallbackAttempt.error) {
logger.error(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
provider: fallbackName,
err: fallbackAttempt.error,
},
'Fallback provider also threw',
);
return 'error';
}
if (fallbackAttempt.output?.status === 'error') {
logger.error(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
provider: fallbackName,
error: fallbackAttempt.output.error,
},
`Fallback provider (${fallbackName}) also failed`,
);
return 'error';
}
return 'success';
};
const retryCodexWithRotation = async (
initialTrigger: { reason: CodexRotationReason },
rotationMessage?: string,
@@ -496,63 +514,86 @@ export async function runAgentForGroup(
return 'success';
case 'error':
return 'error';
case 'no-fallback':
return 'error';
case 'needs-fallback':
if (outcome.trigger.reason === 'success-null-result') {
return canFallback
? runFallbackAttempt('success-null-result')
: 'error';
}
if (!canFallback) {
logger.warn(
{ ...logCtx, reason: outcome.trigger.reason },
'All Claude tokens exhausted and fallback disabled',
);
return 'error';
}
return runFallbackAttempt(
outcome.trigger.reason,
outcome.trigger.retryAfterMs,
);
}
};
const provider = canFallback ? await getActiveProvider() : 'claude';
// Already in no-fallback Claude cooldown — log only, no response
if (provider !== 'claude' && isPrimaryNoFallbackCooldownActive()) {
logger.info(
{ chatJid, group: group.name, runId, provider },
'Claude primary cooldown active, silently skipping',
);
const maybeHandoffAfterError = (
reason: AgentTriggerReason,
attempt: Awaited<ReturnType<typeof runAttempt>>,
): 'success' | 'error' => {
if (maybeHandoffToCodex(reason, attempt.sawVisibleOutput)) {
return 'success';
}
return 'error';
}
};
const primaryAttempt = await runAttempt(provider);
const provider = 'claude';
let primaryAttempt = await runAttempt(provider);
const isRetryableClaudeSessionFailure = (
attempt: Awaited<ReturnType<typeof runAttempt>>,
): boolean =>
isClaudeCodeAgent &&
provider === 'claude' &&
!attempt.sawOutput &&
(attempt.retryableSessionFailureDetected === true ||
(attempt.error != null &&
shouldRetryFreshSessionOnAgentFailure({
result: null,
error: getErrorMessage(attempt.error),
})));
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
deps.clearSession(group.folder);
logger.warn(
{ group: group.name, chatJid, runId },
'Cleared poisoned Claude session before visible output, retrying fresh session',
);
primaryAttempt = await runAttempt('claude');
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
deps.clearSession(group.folder);
logger.warn(
{ group: group.name, chatJid, runId },
'Fresh Claude retry also hit a retryable session failure',
);
logger.error(
{ group: group.name, chatJid, runId },
'Retryable Claude session failure persisted after fresh retry',
);
return 'error';
}
}
if (primaryAttempt.error) {
if (
(canFallback || canRotateToken) &&
canRotateToken &&
provider === 'claude' &&
!primaryAttempt.sawOutput
) {
const errMsg = getErrorMessage(primaryAttempt.error);
const trigger = primaryAttempt.streamedTriggerReason
? {
shouldFallback: true,
shouldRetry: true,
reason: primaryAttempt.streamedTriggerReason.reason,
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
}
: detectFallbackTrigger(errMsg);
if (trigger.shouldFallback) {
return retryClaudeWithRotation(
: classifyRotationTrigger(errMsg);
if (trigger.shouldRetry) {
const result = await retryClaudeWithRotation(
{
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,
},
errMsg,
);
if (result === 'error') {
return maybeHandoffAfterError(trigger.reason, primaryAttempt);
}
return result;
}
}
@@ -594,25 +635,23 @@ export async function runAgentForGroup(
}
if (
(canFallback || canRotateToken) &&
canRotateToken &&
provider === 'claude' &&
!primaryAttempt.sawOutput &&
primaryAttempt.streamedTriggerReason &&
output.status !== 'error'
) {
return retryClaudeWithRotation({
const result = await retryClaudeWithRotation({
reason: primaryAttempt.streamedTriggerReason.reason,
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
});
}
if (
canFallback &&
provider === 'claude' &&
!primaryAttempt.sawOutput &&
primaryAttempt.sawSuccessNullResultWithoutOutput
) {
return runFallbackAttempt('success-null-result');
if (result === 'error') {
return maybeHandoffAfterError(
primaryAttempt.streamedTriggerReason.reason,
primaryAttempt,
);
}
return result;
}
if (
@@ -628,25 +667,29 @@ export async function runAgentForGroup(
if (output.status === 'error') {
if (
(canFallback || canRotateToken) &&
canRotateToken &&
provider === 'claude' &&
!primaryAttempt.sawOutput
) {
const trigger = primaryAttempt.streamedTriggerReason
? {
shouldFallback: true,
shouldRetry: true,
reason: primaryAttempt.streamedTriggerReason.reason,
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
}
: detectFallbackTrigger(output.error);
if (trigger.shouldFallback) {
return retryClaudeWithRotation(
: classifyRotationTrigger(output.error);
if (trigger.shouldRetry) {
const result = await retryClaudeWithRotation(
{
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,
},
output.error ?? undefined,
);
if (result === 'error') {
return maybeHandoffAfterError(trigger.reason, primaryAttempt);
}
return result;
}
}
@@ -687,5 +730,38 @@ export async function runAgentForGroup(
);
}
// Unresolved streamed trigger — rotation was unavailable or output was
// already forwarded. Surfaces as an error since there is no alternative provider.
if (primaryAttempt.streamedTriggerReason) {
if (
isClaudeCodeAgent &&
maybeHandoffToCodex(
primaryAttempt.streamedTriggerReason.reason,
primaryAttempt.sawVisibleOutput,
)
) {
return 'success';
}
logger.error(
{
group: group.name,
chatJid,
runId,
reason: primaryAttempt.streamedTriggerReason.reason,
},
'Agent trigger detected but could not be resolved',
);
return 'error';
}
// success-null-result with no visible output — agent returned nothing useful
if (primaryAttempt.sawSuccessNullResultWithoutOutput) {
logger.error(
{ group: group.name, chatJid, runId },
'Agent returned success with null result and no visible output',
);
return 'error';
}
return 'success';
}

View File

@@ -11,8 +11,21 @@ vi.mock('./agent-runner.js', () => ({
writeTasksSnapshot: vi.fn(),
}));
vi.mock('./output-suppression.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('./output-suppression.js')>();
return {
...actual,
createSuppressToken: vi.fn(() => '__TEST_SUPPRESS__'),
};
});
vi.mock('./config.js', () => ({
DATA_DIR: '/tmp/ejclaw-test-data',
SERVICE_ID: 'claude',
SERVICE_AGENT_TYPE: 'claude-code',
SERVICE_SESSION_SCOPE: 'claude',
isClaudeService: vi.fn(() => true),
isReviewService: vi.fn(() => false),
isSessionCommandSenderAllowed: vi.fn(() => false),
}));
@@ -40,6 +53,10 @@ vi.mock('./db.js', () => {
}));
return {
claimServiceHandoff: vi.fn(() => true),
completeServiceHandoff: vi.fn(),
completeServiceHandoffAndAdvanceTargetCursor: vi.fn(),
failServiceHandoff: vi.fn(),
getAllChats: vi.fn(() => []),
getAllTasks: vi.fn(() => []),
getLastHumanMessageTimestamp: vi.fn(() => null),
@@ -89,11 +106,13 @@ vi.mock('./db.js', () => {
},
),
getOpenWorkItem: vi.fn(() => undefined),
getPendingServiceHandoffs: vi.fn(() => []),
createProducedWorkItem: vi.fn((input) => ({
id: 1,
group_folder: input.group_folder,
chat_jid: input.chat_jid,
agent_type: input.agent_type || 'claude-code',
service_id: 'claude',
status: 'produced',
start_seq: input.start_seq,
end_seq: input.end_seq,
@@ -108,9 +127,22 @@ vi.mock('./db.js', () => {
markWorkItemDelivered: vi.fn(),
markWorkItemDeliveryRetry: vi.fn(),
isPairedRoomJid: vi.fn(() => false),
getLastBotFinalMessage: vi.fn(() => []),
};
});
vi.mock('./service-routing.js', () => ({
getEffectiveChannelLease: vi.fn((chatJid: string) => ({
chat_jid: chatJid,
owner_service_id: 'claude',
reviewer_service_id: 'codex-main',
activated_at: null,
reason: null,
explicit: false,
})),
shouldServiceProcessChat: vi.fn(() => true),
}));
vi.mock('./logger.js', () => ({
logger: {
debug: vi.fn(),
@@ -120,21 +152,6 @@ vi.mock('./logger.js', () => ({
},
}));
vi.mock('./provider-fallback.js', () => ({
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
getActiveProvider: vi.fn(async () => 'claude'),
getFallbackEnvOverrides: vi.fn(() => ({
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
ANTHROPIC_MODEL: 'kimi-k2.5',
})),
getFallbackProviderName: vi.fn(() => 'kimi'),
hasGroupProviderOverride: vi.fn(() => false),
isFallbackEnabled: vi.fn(() => true),
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
markPrimaryCooldown: vi.fn(),
}));
vi.mock('./sender-allowlist.js', () => ({
isTriggerAllowed: vi.fn(() => true),
loadSenderAllowlist: vi.fn(() => ({})),
@@ -151,7 +168,8 @@ import * as agentRunner from './agent-runner.js';
import * as db from './db.js';
import { resolveGroupIpcPath } from './group-folder.js';
import { createMessageRuntime } from './message-runtime.js';
import * as providerFallback from './provider-fallback.js';
import * as outputSuppression from './output-suppression.js';
import * as config from './config.js';
import type { Channel, RegisteredGroup } from './types.js';
function makeGroup(agentType: 'claude-code' | 'codex'): RegisteredGroup {
@@ -182,19 +200,10 @@ function makeChannel(chatJid: string): Channel {
describe('createMessageRuntime', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
vi.mocked(providerFallback.getFallbackProviderName).mockReturnValue('kimi');
vi.mocked(providerFallback.getFallbackEnvOverrides).mockReturnValue({
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
ANTHROPIC_MODEL: 'kimi-k2.5',
});
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({
shouldFallback: false,
reason: '',
});
vi.mocked(db.getLastBotFinalMessage).mockReturnValue([]);
vi.mocked(db.isPairedRoomJid).mockReturnValue(false);
vi.mocked(config.isClaudeService).mockReturnValue(true);
vi.mocked(config.isReviewService).mockReturnValue(false);
});
it('ignores generic failure bot messages in paired rooms', async () => {
@@ -257,6 +266,8 @@ describe('createMessageRuntime', () => {
const saveState = vi.fn();
const lastAgentTimestamps: Record<string, string> = {};
vi.mocked(config.isClaudeService).mockReturnValue(false);
vi.mocked(config.isReviewService).mockReturnValue(false);
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
@@ -317,10 +328,82 @@ describe('createMessageRuntime', () => {
chatJid,
'그 방향이 맞습니다.',
);
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false);
expect(lastAgentTimestamps[chatJid]).toBe('1');
expect(saveState).toHaveBeenCalled();
});
it('does not defer typing-on for suppress-capable review turns', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
const channel = makeChannel(chatJid);
const saveState = vi.fn();
const lastAgentTimestamps: Record<string, string> = {};
vi.mocked(config.isClaudeService).mockReturnValue(false);
vi.mocked(config.isReviewService).mockReturnValue(true);
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'other-bot@test',
sender_name: 'Other Bot',
content: '이어서 확인해줘.',
timestamp: '2026-03-18T09:00:00.000Z',
is_bot_message: true,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
await onOutput?.({
status: 'success',
phase: 'final',
result: '리뷰 확인 완료입니다.',
newSessionId: 'session-review-follow-up-immediate',
});
return {
status: 'success',
result: '리뷰 확인 완료입니다.',
newSessionId: 'session-review-follow-up-immediate',
};
},
);
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-review-follow-up-immediate-typing',
reason: 'messages',
});
expect(result).toBe(true);
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false);
});
it('ignores watcher status control messages in paired rooms', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
@@ -1280,6 +1363,374 @@ describe('createMessageRuntime', () => {
expect(channel.sendAndTrack).not.toHaveBeenCalled();
});
it('does not emit a visible message when the final output is the suppress token only', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
const channel = makeChannel(chatJid);
const lastAgentTimestamps: Record<string, string> = {};
const saveState = vi.fn();
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
seq: 1,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
status: 'success',
result: '__TEST_SUPPRESS__',
newSessionId: 'session-suppress-run',
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-suppress-only',
reason: 'messages',
});
expect(result).toBe(true);
expect(saveState).toHaveBeenCalled();
expect(lastAgentTimestamps[chatJid]).toBe('1');
expect(channel.sendMessage).not.toHaveBeenCalled();
expect(channel.sendAndTrack).not.toHaveBeenCalled();
expect(channel.setTyping).not.toHaveBeenCalledWith(chatJid, true);
});
it('defers typing-on until the first visible output for suppress-capable turns', async () => {
const chatJid = 'group@test';
const group = makeGroup('claude-code');
const channel = makeChannel(chatJid);
const lastAgentTimestamps: Record<string, string> = {};
const saveState = vi.fn();
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
seq: 1,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'visible suppress-capable reply',
});
return {
status: 'success',
result: null,
newSessionId: 'session-visible-suppress-capable',
};
},
);
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-visible-suppress-capable',
reason: 'messages',
});
expect(result).toBe(true);
expect(channel.sendMessage).toHaveBeenCalledWith(
chatJid,
'visible suppress-capable reply',
);
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false);
});
it('does not grant suppress-token silence authority to codex-main', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
const channel = makeChannel(chatJid);
const lastAgentTimestamps: Record<string, string> = {};
const saveState = vi.fn();
vi.mocked(config.isClaudeService).mockReturnValue(false);
vi.mocked(config.isReviewService).mockReturnValue(false);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
seq: 1,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
status: 'success',
result: 'visible codex reply',
newSessionId: 'session-codex-main',
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-codex-main-no-suppress',
reason: 'messages',
});
expect(result).toBe(true);
expect(outputSuppression.createSuppressToken).not.toHaveBeenCalled();
});
it('blocks malformed output when the suppress token is mixed with visible text', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
const channel = makeChannel(chatJid);
const lastAgentTimestamps: Record<string, string> = {};
const saveState = vi.fn();
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
seq: 1,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
status: 'success',
result: '동의합니다 __TEST_SUPPRESS__',
newSessionId: 'session-suppress-mixed',
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-suppress-mixed',
reason: 'messages',
});
expect(result).toBe(true);
expect(saveState).toHaveBeenCalled();
expect(lastAgentTimestamps[chatJid]).toBe('1');
expect(channel.sendMessage).not.toHaveBeenCalled();
expect(channel.sendAndTrack).not.toHaveBeenCalled();
});
it('suppresses a leaked foreign suppress token even when it differs from the current turn token', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
const channel = makeChannel(chatJid);
const lastAgentTimestamps: Record<string, string> = {};
const saveState = vi.fn();
vi.mocked(config.isClaudeService).mockReturnValue(false);
vi.mocked(config.isReviewService).mockReturnValue(true);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
seq: 1,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
status: 'success',
result: '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
newSessionId: 'session-foreign-suppress-token',
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-foreign-suppress-token',
reason: 'messages',
});
expect(result).toBe(true);
expect(saveState).toHaveBeenCalled();
expect(lastAgentTimestamps[chatJid]).toBe('1');
expect(channel.sendMessage).not.toHaveBeenCalled();
expect(channel.sendAndTrack).not.toHaveBeenCalled();
});
it('suppresses a malformed leaked foreign suppress token without the closing suffix', async () => {
const chatJid = 'group@test';
const group = makeGroup('codex');
const channel = makeChannel(chatJid);
const lastAgentTimestamps: Record<string, string> = {};
const saveState = vi.fn();
vi.mocked(config.isClaudeService).mockReturnValue(false);
vi.mocked(config.isReviewService).mockReturnValue(true);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
seq: 1,
},
]);
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
status: 'success',
result: '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef',
newSessionId: 'session-malformed-foreign-suppress-token',
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => lastAgentTimestamps,
saveState,
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-malformed-foreign-suppress-token',
reason: 'messages',
});
expect(result).toBe(true);
expect(saveState).toHaveBeenCalled();
expect(lastAgentTimestamps[chatJid]).toBe('1');
expect(channel.sendMessage).not.toHaveBeenCalled();
expect(channel.sendAndTrack).not.toHaveBeenCalled();
});
it('resets tracked progress after a final output that becomes empty after formatting', async () => {
vi.useFakeTimers();
const chatJid = 'group@test';
@@ -1417,7 +1868,7 @@ describe('createMessageRuntime', () => {
}
});
it('promotes the last progress output to a final message when the agent completes without a final phase', async () => {
it('promotes the last flushed progress output to a final message when the agent completes without a final phase', async () => {
vi.useFakeTimers();
const chatJid = 'group@test';
const group = makeGroup('codex');
@@ -1508,24 +1959,24 @@ describe('createMessageRuntime', () => {
chatJid,
P('검증 중입니다.\n\n0초'),
);
// Ticker fires after advanceTimersByTime — edits tracked message with latest heading
// Ticker fires after advanceTimersByTime — edits tracked message with the last flushed heading
expect(channel.editMessage).toHaveBeenCalledWith(
chatJid,
'progress-1',
P('테스트도 통과했습니다.\n\n5초'),
P('커밋은 정상 들어갔고 pre-commit도 통과했습니다.\n\n5초'),
);
// finish() promotes the last flushed progress text to a final message
expect(channel.sendMessage).toHaveBeenCalledTimes(1);
expect(channel.sendMessage).toHaveBeenCalledWith(
chatJid,
'검증 중입니다.',
'커밋은 정상 들어갔고 pre-commit도 통과했습니다.',
);
} finally {
vi.useRealTimers();
}
});
it('retries editing progress message instead of creating a duplicate when edit fails', async () => {
it('keeps going after a tracked progress edit fails and still emits the last flushed final message', async () => {
vi.useFakeTimers();
const chatJid = 'group@test';
const group = makeGroup('codex');
@@ -1614,8 +2065,7 @@ describe('createMessageRuntime', () => {
});
expect(result).toBe(true);
// Only one progress message created via sendAndTrack — no duplicate
expect(channel.sendAndTrack).toHaveBeenCalledTimes(1);
// The first flushed progress is still tracked
expect(channel.sendAndTrack).toHaveBeenCalledWith(
chatJid,
P('진행 중입니다.\n\n0초'),
@@ -1630,7 +2080,7 @@ describe('createMessageRuntime', () => {
expect(channel.sendMessage).toHaveBeenCalledTimes(1);
expect(channel.sendMessage).toHaveBeenCalledWith(
chatJid,
'진행 중입니다.',
'아직 진행 중.',
);
} finally {
vi.useRealTimers();
@@ -1868,331 +2318,6 @@ describe('createMessageRuntime', () => {
expect(saveState).toHaveBeenCalled();
});
it('retries with the fallback provider when Claude returns a 429 error before any output', async () => {
const chatJid = 'group@test';
const group = makeGroup('claude-code');
const channel = makeChannel(chatJid);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
},
]);
vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({
shouldFallback: true,
reason: '429',
retryAfterMs: 60_000,
});
vi.mocked(agentRunner.runAgentProcess)
.mockResolvedValueOnce({
status: 'error',
result: null,
error: '429 rate limited retry after 60',
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'fallback 응답입니다.',
});
return {
status: 'success',
result: null,
};
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => ({}),
saveState: vi.fn(),
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-fallback-429',
reason: 'messages',
});
expect(result).toBe(true);
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(agentRunner.runAgentProcess).toHaveBeenNthCalledWith(
2,
expect.anything(),
expect.objectContaining({ sessionId: undefined }),
expect.any(Function),
expect.any(Function),
expect.objectContaining({
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
ANTHROPIC_MODEL: 'kimi-k2.5',
}),
);
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'429',
60_000,
);
expect(channel.sendMessage).toHaveBeenCalledWith(
chatJid,
'fallback 응답입니다.',
);
});
it('silently suppresses a usage exhaustion banner without falling back', async () => {
const chatJid = 'group@test';
const group = makeGroup('claude-code');
const channel = makeChannel(chatJid);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-24T00:00:00.000Z',
},
]);
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'usage fallback 응답입니다.',
});
return {
status: 'success',
result: null,
};
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => ({}),
saveState: vi.fn(),
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-fallback-usage-exhausted',
reason: 'messages',
});
expect(result).toBe(true);
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'usage-exhausted',
undefined,
);
expect(channel.sendMessage).not.toHaveBeenCalled();
});
it('suppresses duplicate streamed usage banners without emitting a visible reply', async () => {
const chatJid = 'group@test';
const group = makeGroup('claude-code');
const channel = makeChannel(chatJid);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-24T00:00:00.000Z',
},
]);
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'intermediate',
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
});
await onOutput?.({
status: 'success',
phase: 'final',
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'duplicate banner fallback 응답입니다.',
});
return {
status: 'success',
result: null,
};
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => ({}),
saveState: vi.fn(),
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-fallback-usage-exhausted-duplicate',
reason: 'messages',
});
expect(result).toBe(true);
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'usage-exhausted',
undefined,
);
expect(channel.sendMessage).not.toHaveBeenCalled();
});
it('retries with the fallback provider when Claude ends with success-null-result before any output', async () => {
const chatJid = 'group@test';
const group = makeGroup('claude-code');
const channel = makeChannel(chatJid);
vi.mocked(db.getMessagesSince).mockReturnValue([
{
id: 'msg-1',
chat_jid: chatJid,
sender: 'user@test',
sender_name: 'User',
content: 'hello',
timestamp: '2026-03-19T00:00:00.000Z',
},
]);
vi.mocked(agentRunner.runAgentProcess)
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
result: null,
});
return {
status: 'success',
result: null,
};
})
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result: 'success-null-result 폴백 응답입니다.',
});
return {
status: 'success',
result: null,
};
});
const runtime = createMessageRuntime({
assistantName: 'Andy',
idleTimeout: 1_000,
pollInterval: 1_000,
timezone: 'UTC',
triggerPattern: /^@Andy\b/i,
channels: [channel],
queue: {
registerProcess: vi.fn(),
closeStdin: vi.fn(),
notifyIdle: vi.fn(),
} as any,
getRegisteredGroups: () => ({ [chatJid]: group }),
getSessions: () => ({}),
getLastTimestamp: () => '',
setLastTimestamp: vi.fn(),
getLastAgentTimestamps: () => ({}),
saveState: vi.fn(),
persistSession: vi.fn(),
clearSession: vi.fn(),
});
const result = await runtime.processGroupMessages(chatJid, {
runId: 'run-fallback-success-null',
reason: 'messages',
});
expect(result).toBe(true);
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'success-null-result',
undefined,
);
expect(channel.sendMessage).toHaveBeenCalledWith(
chatJid,
'success-null-result 폴백 응답입니다.',
);
});
it('treats missing streamed phase as final output', async () => {
const chatJid = 'group@test';
const group = makeGroup('claude-code');
@@ -2266,6 +2391,7 @@ describe('createMessageRuntime', () => {
group_folder: group.folder,
chat_jid: chatJid,
agent_type: 'claude-code',
service_id: 'claude',
status: 'produced',
start_seq: 1,
end_seq: 1,

View File

@@ -1,17 +1,30 @@
import { AgentOutput } from './agent-runner.js';
import { getErrorMessage } from './utils.js';
import {
claimServiceHandoff,
completeServiceHandoffAndAdvanceTargetCursor,
createProducedWorkItem,
failServiceHandoff,
getOpenWorkItem,
getPendingServiceHandoffs,
getMessagesSinceSeq,
getNewMessagesBySeq,
getOpenWorkItem,
createProducedWorkItem,
markWorkItemDelivered,
markWorkItemDeliveryRetry,
getLastBotFinalMessage,
isPairedRoomJid,
type ServiceHandoff,
type WorkItem,
} from './db.js';
import { isSessionCommandSenderAllowed } from './config.js';
import {
isClaudeService,
isReviewService,
isSessionCommandSenderAllowed,
SERVICE_AGENT_TYPE,
SERVICE_ID,
} from './config.js';
import { GroupQueue, GroupRunContext } from './group-queue.js';
import { findChannel, formatMessages } from './router.js';
import { findChannel, formatMessages, normalizeMessageForDedupe } from './router.js';
import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js';
import {
advanceLastAgentCursor,
@@ -23,6 +36,7 @@ import {
} from './message-runtime-rules.js';
import { runAgentForGroup } from './message-agent-executor.js';
import { MessageTurnController } from './message-turn-controller.js';
import { createSuppressToken } from './output-suppression.js';
import {
extractSessionCommand,
handleSessionCommand,
@@ -32,6 +46,30 @@ import {
import { Channel, NewMessage, RegisteredGroup } from './types.js';
import { logger } from './logger.js';
import { resolveGroupIpcPath } from './group-folder.js';
import { shouldServiceProcessChat } from './service-routing.js';
/**
* Check if a message is a duplicate of the last bot final message in a paired room.
* Exported for testing purposes.
*/
export function isDuplicateOfLastBotFinal(chatJid: string, text: string): boolean {
// Only check in paired rooms (both claude and codex registered)
if (!isPairedRoomJid(chatJid)) {
return false;
}
// Get the last bot final message from DB (any bot, not just this service)
const lastMessages = getLastBotFinalMessage(chatJid, SERVICE_AGENT_TYPE, 1);
if (lastMessages.length === 0) {
return false;
}
const lastMessage = lastMessages[0];
const normalizedLast = normalizeMessageForDedupe(lastMessage.content);
const normalizedCurrent = normalizeMessageForDedupe(text);
return normalizedLast === normalizedCurrent && normalizedLast.length > 0;
}
export interface MessageRuntimeDeps {
assistantName: string;
@@ -64,6 +102,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
const continuationTracker = createImplicitContinuationTracker(
deps.idleTimeout,
);
const isBotOnlyPairedRoomTurn = (
chatJid: string,
messages: NewMessage[],
): boolean =>
isPairedRoomJid(chatJid) &&
messages.every(
(message) => message.is_from_me === true || !!message.is_bot_message,
);
/**
* Check if a message is a duplicate of the last bot final message in a paired room.
* Returns true if duplicate (should be suppressed).
*/
const checkDuplicateOfLastBotFinal = (chatJid: string, text: string): boolean => {
return isDuplicateOfLastBotFinal(chatJid, text);
};
const deliverOpenWorkItem = async (
channel: Channel,
@@ -73,6 +127,24 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
},
): Promise<boolean> => {
const replaceMessageId = options?.replaceMessageId ?? null;
// Check for duplicate in paired rooms before attempting delivery
const isDuplicate = checkDuplicateOfLastBotFinal(item.chat_jid, item.result_payload);
if (isDuplicate) {
// Mark as delivered without sending, and don't open continuation
markWorkItemDelivered(item.id, null);
logger.info(
{
chatJid: item.chat_jid,
workItemId: item.id,
preview: item.result_payload.slice(0, 100),
},
'Suppressed duplicate final message in paired room (marked as delivered)',
);
return true;
}
try {
if (replaceMessageId && channel.editMessage) {
await channel.editMessage(
@@ -141,6 +213,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
chatJid: string,
runId: string,
onOutput?: (output: AgentOutput) => Promise<void>,
options?: {
suppressToken?: string;
startSeq?: number | null;
endSeq?: number | null;
},
): Promise<'success' | 'error'> =>
runAgentForGroup(
{
@@ -156,10 +233,185 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
prompt,
chatJid,
runId,
suppressToken: options?.suppressToken,
startSeq: options?.startSeq,
endSeq: options?.endSeq,
onOutput,
},
);
const executeTurn = async (args: {
group: RegisteredGroup;
prompt: string;
chatJid: string;
runId: string;
channel: Channel;
startSeq: number | null;
endSeq: number | null;
}): Promise<{
outputStatus: 'success' | 'error';
deliverySucceeded: boolean;
visiblePhase: ReturnType<MessageTurnController['finish']> extends Promise<
infer T
>
? T extends { visiblePhase: infer V }
? V
: never
: never;
}> => {
const { group, prompt, chatJid, runId, channel, startSeq, endSeq } = args;
const isClaudeCodeAgent =
(group.agentType || 'claude-code') === 'claude-code';
const suppressToken =
isClaudeService() || isReviewService() ? createSuppressToken() : undefined;
const deferTypingUntilVisible = Boolean(suppressToken) && isClaudeService();
const turnController = new MessageTurnController({
chatJid,
group,
runId,
channel,
idleTimeout: deps.idleTimeout,
failureFinalText: FAILURE_FINAL_TEXT,
isClaudeCodeAgent,
deferTypingUntilVisible,
suppressToken,
clearSession: () => deps.clearSession(group.folder),
requestClose: (reason) => deps.queue.closeStdin(chatJid, { runId, reason }),
deliverFinalText: async (text) => {
try {
const workItem = createProducedWorkItem({
group_folder: group.folder,
chat_jid: chatJid,
agent_type: group.agentType || 'claude-code',
start_seq: startSeq,
end_seq: endSeq,
result_payload: text,
});
return deliverOpenWorkItem(channel, workItem);
} catch (err) {
logger.warn(
{ group: group.name, chatJid, runId, err },
'Failed to persist produced output for delivery',
);
return false;
}
},
});
await turnController.start();
try {
const outputStatus = await runAgent(
group,
prompt,
chatJid,
runId,
(result) => turnController.handleOutput(result),
{ suppressToken, startSeq, endSeq },
);
const { deliverySucceeded, visiblePhase } =
await turnController.finish(outputStatus);
return {
outputStatus,
deliverySucceeded,
visiblePhase,
};
} finally {
turnController.cancelPendingTypingDelay();
logger.debug(
{
transition: 'typing:off',
source: 'message-runtime:safety-net',
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
},
'Typing indicator transition',
);
await channel.setTyping?.(chatJid, false);
}
};
const enqueuePendingHandoffs = (): void => {
for (const handoff of getPendingServiceHandoffs(SERVICE_ID)) {
if (!claimServiceHandoff(handoff.id)) {
continue;
}
deps.queue.enqueueTask(
handoff.chat_jid,
`handoff:${handoff.id}`,
async () => {
await processClaimedHandoff(handoff);
},
);
}
};
const processClaimedHandoff = async (handoff: ServiceHandoff): Promise<void> => {
const group = deps.getRegisteredGroups()[handoff.chat_jid];
if (!group) {
failServiceHandoff(handoff.id, 'Group not registered on target service');
return;
}
const channel = findChannel(deps.channels, handoff.chat_jid);
if (!channel) {
failServiceHandoff(handoff.id, 'No channel owns handoff jid');
return;
}
const runId = `handoff-${handoff.id}`;
try {
const result = await executeTurn({
group,
prompt: handoff.prompt,
chatJid: handoff.chat_jid,
runId,
channel,
startSeq: handoff.start_seq,
endSeq: handoff.end_seq,
});
if (!result.deliverySucceeded) {
failServiceHandoff(handoff.id, 'Handoff delivery failed');
return;
}
const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({
id: handoff.id,
target_service_id: handoff.target_service_id,
chat_jid: handoff.chat_jid,
end_seq: handoff.end_seq,
});
if (appliedCursor) {
deps.getLastAgentTimestamps()[handoff.chat_jid] = appliedCursor;
}
logger.info(
{
chatJid: handoff.chat_jid,
handoffId: handoff.id,
runId,
outputStatus: result.outputStatus,
visiblePhase: result.visiblePhase,
appliedCursor,
},
'Completed claimed service handoff',
);
} catch (err) {
const errorMessage = getErrorMessage(err);
failServiceHandoff(handoff.id, errorMessage);
logger.error(
{ chatJid: handoff.chat_jid, handoffId: handoff.id, err },
'Claimed service handoff failed',
);
}
};
const processGroupMessages = async (
chatJid: string,
context: GroupRunContext,
@@ -183,9 +435,29 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
if (!delivered) return false;
}
if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) {
const rawMissedMessages = getMessagesSinceSeq(
chatJid,
deps.getLastAgentTimestamps()[chatJid] || '0',
deps.assistantName,
);
const lastIgnored = rawMissedMessages[rawMissedMessages.length - 1];
if (lastIgnored?.seq != null) {
advanceLastAgentCursor(
deps.getLastAgentTimestamps(),
deps.saveState,
chatJid,
lastIgnored.seq,
);
}
logger.debug(
{ chatJid, serviceId: SERVICE_ID },
'Skipping message processing for unassigned service',
);
return true;
}
const isMainGroup = group.isMain === true;
const isClaudeCodeAgent =
(group.agentType || 'claude-code') === 'claude-code';
while (true) {
const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '0';
const rawMissedMessages = getMessagesSinceSeq(
@@ -311,85 +583,36 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
},
'Dispatching queued messages to agent',
);
const turnController = new MessageTurnController({
chatJid,
const { deliverySucceeded, visiblePhase } = await executeTurn({
group,
prompt,
chatJid,
runId,
channel,
idleTimeout: deps.idleTimeout,
failureFinalText: FAILURE_FINAL_TEXT,
isClaudeCodeAgent,
clearSession: () => deps.clearSession(group.folder),
requestClose: (reason) =>
deps.queue.closeStdin(chatJid, { runId, reason }),
deliverFinalText: async (text) => {
try {
const workItem = createProducedWorkItem({
group_folder: group.folder,
chat_jid: chatJid,
agent_type: group.agentType || 'claude-code',
start_seq: startSeq,
end_seq: endSeq,
result_payload: text,
});
return deliverOpenWorkItem(channel, workItem);
} catch (err) {
logger.warn(
{ group: group.name, chatJid, runId, err },
'Failed to persist produced output for delivery',
);
return false;
}
},
startSeq,
endSeq,
});
await turnController.start();
try {
const output = await runAgent(group, prompt, chatJid, runId, (result) =>
turnController.handleOutput(result),
if (!deliverySucceeded) {
logger.warn(
{ chatJid, group: group.name, groupFolder: group.folder, runId },
'Persisted produced output for delivery retry without rerunning agent',
);
const { deliverySucceeded, visiblePhase } =
await turnController.finish(output);
if (!deliverySucceeded) {
logger.warn(
{ chatJid, group: group.name, groupFolder: group.folder, runId },
'Persisted produced output for delivery retry without rerunning agent',
);
return false;
}
logger.info(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
visiblePhase,
},
'Queued run completed successfully',
);
return true;
} finally {
// Safety net: always clear typing even if runAgent() or finish() throws.
// Prevents stuck typing indicators when exceptions bypass the normal
// turnController.finish() -> setTyping(false) path.
logger.debug(
{
transition: 'typing:off',
source: 'message-runtime:safety-net',
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
},
'Typing indicator transition',
);
await channel.setTyping?.(chatJid, false);
return false;
}
logger.info(
{
chatJid,
group: group.name,
groupFolder: group.folder,
runId,
visiblePhase,
},
'Queued run completed successfully',
);
return true;
}
};
@@ -404,6 +627,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
while (true) {
try {
enqueuePendingHandoffs();
const registeredGroups = deps.getRegisteredGroups();
const jids = Object.keys(registeredGroups);
const { messages, newSeqCursor } = getNewMessagesBySeq(
@@ -461,6 +685,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
continue;
}
if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) {
const lastIgnored =
processableGroupMessages[processableGroupMessages.length - 1];
if (lastIgnored?.seq != null) {
advanceLastAgentCursor(
deps.getLastAgentTimestamps(),
deps.saveState,
chatJid,
lastIgnored.seq,
);
}
continue;
}
if (
shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)
) {
@@ -530,6 +768,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
? pendingMessages
: processableGroupMessages;
const formatted = formatMessages(messagesToSend, deps.timezone);
const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn(
chatJid,
messagesToSend,
);
if (deps.queue.sendMessage(chatJid, formatted)) {
const endSeq = messagesToSend[messagesToSend.length - 1]?.seq;
@@ -549,17 +791,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
group: group.name,
groupFolder: group.folder,
endSeq: endSeq ?? null,
suppressed: isBotOnlyPairedFollowUp,
},
'Typing indicator transition',
);
await channel
.setTyping?.(chatJid, true)
?.catch((err) =>
logger.warn(
{ chatJid, err },
'Failed to set typing indicator',
),
);
if (!isBotOnlyPairedFollowUp) {
await channel
.setTyping?.(chatJid, true)
?.catch((err) =>
logger.warn(
{ chatJid, err },
'Failed to set typing indicator',
),
);
}
continue;
}
@@ -595,6 +840,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
continue;
}
if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) {
continue;
}
const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '';
const rawPending = getMessagesSinceSeq(
chatJid,

View File

@@ -1,5 +1,6 @@
import { type AgentOutput } from './agent-runner.js';
import { logger } from './logger.js';
import { classifySuppressTokenOutput } from './output-suppression.js';
import { formatOutbound } from './router.js';
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
@@ -28,6 +29,8 @@ interface MessageTurnControllerOptions {
idleTimeout: number;
failureFinalText: string;
isClaudeCodeAgent: boolean;
deferTypingUntilVisible?: boolean;
suppressToken?: string;
clearSession: () => void;
requestClose: (reason: string) => void;
deliverFinalText: (text: string) => Promise<boolean>;
@@ -53,6 +56,7 @@ export class MessageTurnController {
private lastIntermediateText: string | null = null;
private poisonedSessionDetected = false;
private closeRequested = false;
private typingActive = false;
constructor(private readonly options: MessageTurnControllerOptions) {}
@@ -78,7 +82,10 @@ export class MessageTurnController {
async start(): Promise<void> {
this.resetIdleTimer();
await this.setTyping(true, 'turn:start');
if (this.options.deferTypingUntilVisible) {
return;
}
await this.activateTyping('turn:start');
}
async handleOutput(result: AgentOutput): Promise<void> {
@@ -123,7 +130,11 @@ export class MessageTurnController {
: typeof result.result === 'string'
? result.result
: JSON.stringify(result.result);
const text = raw ? formatOutbound(raw) : null;
const suppressState =
raw && this.options.suppressToken
? classifySuppressTokenOutput(raw, this.options.suppressToken)
: 'none';
const text = raw && suppressState === 'none' ? formatOutbound(raw) : null;
if (raw) {
logger.info(
@@ -138,6 +149,31 @@ export class MessageTurnController {
},
`Agent output: ${raw.slice(0, 200)}`,
);
if (suppressState === 'exact') {
logger.info(
{
chatJid: this.options.chatJid,
group: this.options.group.name,
groupFolder: this.options.group.folder,
runId: this.options.runId,
resultStatus: result.status,
resultPhase: result.phase,
},
'Suppressed exact-match silent output token',
);
} else if (suppressState === 'mixed') {
logger.warn(
{
chatJid: this.options.chatJid,
group: this.options.group.name,
groupFolder: this.options.group.folder,
runId: this.options.runId,
resultStatus: result.status,
resultPhase: result.phase,
},
'Blocked malformed output that mixed the silent output token with visible text',
);
}
}
const phase: AgentOutputPhase = normalizeAgentOutputPhase(result.phase);
@@ -275,6 +311,9 @@ export class MessageTurnController {
await this.finalizeProgressMessage();
await this.deliverFinalText(text);
}
} else if (suppressState !== 'none') {
await this.finalizeProgressMessage();
this.latestProgressTextForFinal = null;
} else if (raw) {
logger.info(
{
@@ -294,7 +333,7 @@ export class MessageTurnController {
await this.finalizeProgressMessage();
}
await this.setTyping(false, 'turn:handle-output', {
await this.deactivateTyping('turn:handle-output', {
outputStatus: result.status,
phase,
});
@@ -311,7 +350,7 @@ export class MessageTurnController {
deliverySucceeded: boolean;
visiblePhase: VisiblePhase;
}> {
await this.setTyping(false, 'turn:finish', { outputStatus });
await this.deactivateTyping('turn:finish', { outputStatus });
if (outputStatus === 'error') {
this.hadError = true;
@@ -566,6 +605,7 @@ export class MessageTurnController {
}
private async deliverFinalText(text: string): Promise<void> {
await this.activateTyping('turn:deliver-final');
this.visiblePhase = toVisiblePhase('final');
const delivered = await this.options.deliverFinalText(text);
if (!delivered) {
@@ -593,6 +633,8 @@ export class MessageTurnController {
return;
}
await this.activateTyping('turn:send-progress');
if (this.progressStartedAt === null) {
this.progressStartedAt = Date.now();
}
@@ -670,6 +712,28 @@ export class MessageTurnController {
this.visiblePhase = toVisiblePhase('progress');
}
private async activateTyping(
source: string,
extra?: Record<string, unknown>,
): Promise<void> {
if (this.typingActive) return;
await this.setTyping(true, source, extra);
this.typingActive = true;
}
private async deactivateTyping(
source: string,
extra?: Record<string, unknown>,
): Promise<void> {
if (!this.typingActive) return;
await this.setTyping(false, source, extra);
this.typingActive = false;
}
cancelPendingTypingDelay(): void {
// No-op: typing delay removed. Kept for call-site compatibility.
}
private resetIdleTimer(): void {
if (this.idleTimer) clearTimeout(this.idleTimer);
if (this.hasVisibleOutput()) {

View File

@@ -0,0 +1,41 @@
import { describe, expect, it } from 'vitest';
import { classifySuppressTokenOutput } from './output-suppression.js';
describe('classifySuppressTokenOutput', () => {
it('treats the current turn suppress token as exact', () => {
expect(
classifySuppressTokenOutput(
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
),
).toBe('exact');
});
it('treats a leaked foreign suppress token as exact silent output', () => {
expect(
classifySuppressTokenOutput(
'__EJ_SUPPRESS_feedfacefeedfacefeedface__',
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
),
).toBe('exact');
});
it('treats a malformed foreign suppress token without the closing suffix as exact silent output', () => {
expect(
classifySuppressTokenOutput(
'__EJ_SUPPRESS_feedfacefeedfacefeedface',
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
),
).toBe('exact');
});
it('treats a suppress token embedded in visible text as mixed', () => {
expect(
classifySuppressTokenOutput(
'prefix __EJ_SUPPRESS_feedfacefeedfacefeedface__ suffix',
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
),
).toBe('mixed');
});
});

65
src/output-suppression.ts Normal file
View File

@@ -0,0 +1,65 @@
import { randomBytes } from 'crypto';
import {
CLAUDE_SERVICE_ID,
CODEX_REVIEW_SERVICE_ID,
normalizeServiceId,
} from './config.js';
const ANY_SUPPRESS_TOKEN_PATTERN = /__EJ_SUPPRESS_[a-f0-9]{24,}(?:__)?/g;
const EXACT_ANY_SUPPRESS_TOKEN_PATTERN = /^__EJ_SUPPRESS_[a-f0-9]{24,}(?:__)?$/;
export function createSuppressToken(): string {
return `__EJ_SUPPRESS_${randomBytes(12).toString('hex')}__`;
}
export function shouldEnableSuppressOutputForService(
serviceId: string | undefined,
): boolean {
if (!serviceId) return false;
const normalized = normalizeServiceId(serviceId);
return (
normalized === CLAUDE_SERVICE_ID ||
normalized === CODEX_REVIEW_SERVICE_ID
);
}
export function classifySuppressTokenOutput(
rawText: string,
suppressToken: string | undefined,
): 'exact' | 'mixed' | 'none' {
const trimmed = rawText.trim();
if ((suppressToken && trimmed === suppressToken) || EXACT_ANY_SUPPRESS_TOKEN_PATTERN.test(trimmed)) {
return 'exact';
}
if (suppressToken && rawText.includes(suppressToken)) {
return 'mixed';
}
ANY_SUPPRESS_TOKEN_PATTERN.lastIndex = 0;
return ANY_SUPPRESS_TOKEN_PATTERN.test(rawText) ? 'mixed' : 'none';
}
export function buildSuppressTokenPrompt(
prompt: string,
suppressToken: string | undefined,
options?: {
reviewerMode?: boolean;
},
): string {
if (!suppressToken) return prompt;
const lines = [
'[OUTPUT CONTROL]',
`If you have no user-visible content to send for this turn, output exactly this token and nothing else: ${suppressToken}`,
'Do not wrap the token in backticks or code fences.',
'Do not combine the token with any other text.',
];
if (options?.reviewerMode) {
lines.push(
'If you are only agreeing, mirroring, or restating without adding a concrete correction, risk, missing prerequisite, test gap, or code change, output only the token.',
);
}
return `${lines.join('\n')}\n\n${prompt}`;
}

View File

@@ -1,180 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('./claude-usage.js', () => ({
fetchClaudeUsage: vi.fn(),
}));
vi.mock('./env.js', () => {
const store: Record<string, string> = {
FALLBACK_PROVIDER_NAME: 'kimi',
FALLBACK_BASE_URL: 'https://api.kimi.com/coding/',
FALLBACK_AUTH_TOKEN: 'test-kimi-key',
FALLBACK_MODEL: 'kimi-k2.5',
FALLBACK_SMALL_MODEL: 'kimi-k2.5',
FALLBACK_COOLDOWN_MS: '600000',
};
return {
readEnvFile: vi.fn((keys: string[]) => {
const result: Record<string, string> = {};
for (const k of keys) {
if (store[k]) result[k] = store[k];
}
return result;
}),
getEnv: vi.fn((key: string) => store[key]),
};
});
vi.mock('./logger.js', () => ({
logger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
},
}));
import { fetchClaudeUsage } from './claude-usage.js';
import {
clearPrimaryCooldown,
detectFallbackTrigger,
getActiveProvider,
getCooldownInfo,
isPrimaryNoFallbackCooldownActive,
markPrimaryCooldown,
resetFallbackConfig,
} from './provider-fallback.js';
describe('provider fallback usage recovery', () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-03-24T00:00:00.000Z'));
vi.clearAllMocks();
clearPrimaryCooldown();
resetFallbackConfig();
delete process.env.FALLBACK_PROVIDER_NAME;
delete process.env.FALLBACK_BASE_URL;
delete process.env.FALLBACK_AUTH_TOKEN;
delete process.env.FALLBACK_MODEL;
delete process.env.FALLBACK_SMALL_MODEL;
delete process.env.FALLBACK_COOLDOWN_MS;
});
afterEach(() => {
clearPrimaryCooldown();
resetFallbackConfig();
vi.useRealTimers();
});
it('keeps the fallback provider active while Claude usage is still exhausted', async () => {
vi.mocked(fetchClaudeUsage).mockResolvedValue({
five_hour: {
utilization: 100,
resets_at: '2026-03-24T04:00:00.000+09:00',
},
});
markPrimaryCooldown('usage-exhausted', 1_000);
vi.advanceTimersByTime(5_000);
await expect(getActiveProvider()).resolves.toBe('kimi');
expect(getCooldownInfo()).toMatchObject({
active: true,
reason: 'usage-exhausted',
remainingMs: 0,
});
});
it('returns to Claude immediately when usage is no longer exhausted', async () => {
vi.mocked(fetchClaudeUsage).mockResolvedValue({
five_hour: {
utilization: 72,
resets_at: '2026-03-24T04:00:00.000+09:00',
},
seven_day: {
utilization: 55,
resets_at: '2026-03-31T04:00:00.000+09:00',
},
});
markPrimaryCooldown('usage-exhausted', 600_000);
await expect(getActiveProvider()).resolves.toBe('claude');
expect(getCooldownInfo()).toEqual({ active: false });
});
it('falls back to time-based retry when usage status cannot be fetched', async () => {
vi.mocked(fetchClaudeUsage).mockResolvedValue(null);
markPrimaryCooldown('usage-exhausted', 1_000);
vi.advanceTimersByTime(5_000);
await expect(getActiveProvider()).resolves.toBe('claude');
expect(getCooldownInfo()).toEqual({ active: false });
});
it('treats terminated 401 auth failures as an auth-expired fallback trigger', () => {
expect(
detectFallbackTrigger(
'Failed to authenticate. API Error: 401 terminated',
),
).toEqual({
shouldFallback: true,
reason: 'auth-expired',
});
});
it('treats invalid authentication credentials as an auth-expired fallback trigger', () => {
expect(
detectFallbackTrigger(
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
),
).toEqual({
shouldFallback: true,
reason: 'auth-expired',
});
});
it('treats Cloudflare 502 HTML as an overloaded fallback trigger', () => {
expect(
detectFallbackTrigger(
'API Error: 502 <html><head><title>502 Bad Gateway</title></head><body><center><h1>502 Bad Gateway</h1></center><hr><center>cloudflare</center></body></html>',
),
).toEqual({
shouldFallback: true,
reason: 'overloaded',
});
});
it('treats Claude org access denied banners as an org-access-denied fallback trigger', () => {
expect(
detectFallbackTrigger(
'Your organization does not have access to Claude. Please login again or contact your administrator.',
),
).toEqual({
shouldFallback: true,
reason: 'org-access-denied',
});
});
it('treats terminated 403 auth failures as an org-access-denied fallback trigger', () => {
expect(
detectFallbackTrigger(
'Failed to authenticate. API Error: 403 terminated',
),
).toEqual({
shouldFallback: true,
reason: 'org-access-denied',
});
});
it('marks org-access-denied as a no-fallback cooldown reason', () => {
markPrimaryCooldown('org-access-denied', 60_000);
expect(isPrimaryNoFallbackCooldownActive()).toBe(true);
expect(getCooldownInfo()).toMatchObject({
active: true,
reason: 'org-access-denied',
});
});
});

View File

@@ -1,450 +0,0 @@
/**
* Provider Fallback Module
*
* Manages automatic fallback from the primary provider (Claude) to a
* fallback provider (e.g. Kimi K2.5) when 429/rate-limit or network
* errors are detected.
*
* Cooldown-based recovery:
* Claude 429 → immediate Kimi retry for that turn
* Claude enters cooldown (retry-after header or default 10 min)
* During cooldown → skip Claude, route directly to fallback
* After cooldown → try Claude first again
*/
import fs from 'fs';
import {
classifyAgentError,
classifyClaudeAuthError,
isNoFallbackCooldownReason,
type AgentTriggerReason,
type FallbackTriggerReason,
} from './agent-error-detection.js';
import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js';
import { getEnv } from './env.js';
import { logger } from './logger.js';
import { rotateToken, getTokenCount } from './token-rotation.js';
// ── Types ────────────────────────────────────────────────────────
export type ProviderName = 'claude' | string; // fallback name is configurable
export type FallbackTriggerResult =
| {
shouldFallback: false;
reason: '';
retryAfterMs?: undefined;
}
| {
shouldFallback: true;
reason: FallbackTriggerReason;
retryAfterMs?: number;
};
interface CooldownState {
startedAt: number;
expiresAt: number;
reason: AgentTriggerReason;
}
interface FallbackConfig {
enabled: boolean;
providerName: string; // e.g. "kimi"
baseUrl: string;
authToken: string;
model: string;
smallModel: string;
defaultCooldownMs: number;
}
// ── State ────────────────────────────────────────────────────────
let cooldown: CooldownState | null = null;
let lastUsageAvailabilityCheck: {
checkedAt: number;
result: 'available' | 'exhausted' | 'unknown';
} | null = null;
let usageAvailabilityCheckPromise: Promise<
'available' | 'exhausted' | 'unknown'
> | null = null;
const USAGE_RECOVERY_RECHECK_MS = 30_000;
// ── Config ───────────────────────────────────────────────────────
let _config: FallbackConfig | null = null;
function loadConfig(): FallbackConfig {
if (_config) return _config;
const baseUrl = getEnv('FALLBACK_BASE_URL') || '';
const authToken = getEnv('FALLBACK_AUTH_TOKEN') || '';
const model = getEnv('FALLBACK_MODEL') || '';
const explicitlyDisabled =
(getEnv('FALLBACK_ENABLED') || '').toLowerCase() === 'false';
_config = {
enabled: !explicitlyDisabled && Boolean(baseUrl && authToken && model),
providerName: getEnv('FALLBACK_PROVIDER_NAME') || 'kimi',
baseUrl,
authToken,
model,
smallModel: getEnv('FALLBACK_SMALL_MODEL') || model,
defaultCooldownMs: parseInt(getEnv('FALLBACK_COOLDOWN_MS') || '600000', 10),
};
if (_config.enabled) {
logger.info(
{
provider: _config.providerName,
model: _config.model,
cooldownMs: _config.defaultCooldownMs,
},
'Provider fallback configured',
);
}
return _config;
}
/** Force re-read of config (useful after .env changes). */
export function resetFallbackConfig(): void {
_config = null;
}
// ── Public API ───────────────────────────────────────────────────
/** Check whether the fallback system is configured and available. */
export function isFallbackEnabled(): boolean {
return loadConfig().enabled;
}
/** Get the display name of the fallback provider (e.g. "kimi"). */
export function getFallbackProviderName(): string {
return loadConfig().providerName;
}
function normalizeUtilization(utilization: number): number {
return utilization > 1 ? utilization : utilization * 100;
}
type ClaudeUsageWindow = NonNullable<ClaudeUsageData[keyof ClaudeUsageData]>;
function hasExhaustedClaudeUsageWindow(
usage: ClaudeUsageData | null,
): boolean | null {
if (!usage) return null;
const windows: ClaudeUsageWindow[] = [];
if (usage.five_hour) windows.push(usage.five_hour);
if (usage.seven_day) windows.push(usage.seven_day);
if (usage.seven_day_sonnet) windows.push(usage.seven_day_sonnet);
if (usage.seven_day_opus) windows.push(usage.seven_day_opus);
if (windows.length === 0) return null;
return windows.some(
(window) => normalizeUtilization(window.utilization) >= 100,
);
}
function clearUsageAvailabilityCache(): void {
lastUsageAvailabilityCheck = null;
usageAvailabilityCheckPromise = null;
}
function logCooldownTransition(
level: 'debug' | 'info' | 'warn',
transition: string,
fields: Record<string, unknown>,
message: string,
): void {
logger[level](
{
transition,
provider: 'claude',
...fields,
},
message,
);
}
async function getClaudeUsageAvailability(): Promise<
'available' | 'exhausted' | 'unknown'
> {
const now = Date.now();
if (
lastUsageAvailabilityCheck &&
now - lastUsageAvailabilityCheck.checkedAt < USAGE_RECOVERY_RECHECK_MS
) {
return lastUsageAvailabilityCheck.result;
}
if (!usageAvailabilityCheckPromise) {
usageAvailabilityCheckPromise = (async () => {
const usage = await fetchClaudeUsage();
const exhausted = hasExhaustedClaudeUsageWindow(usage);
const result =
exhausted === null ? 'unknown' : exhausted ? 'exhausted' : 'available';
lastUsageAvailabilityCheck = {
checkedAt: Date.now(),
result,
};
return result;
})();
void usageAvailabilityCheckPromise.finally(() => {
usageAvailabilityCheckPromise = null;
});
}
return usageAvailabilityCheckPromise;
}
/**
* Determine which provider should be used for the next request.
* Returns 'claude' when Claude is healthy or cooldown has expired,
* or the fallback provider name during an active cooldown.
*/
export async function getActiveProvider(): Promise<string> {
const config = loadConfig();
if (!config.enabled) return 'claude';
if (cooldown) {
if (cooldown.reason === 'usage-exhausted') {
const usageAvailability = await getClaudeUsageAvailability();
if (usageAvailability === 'available') {
logCooldownTransition(
'info',
'cooldown:recover',
{
reason: cooldown.reason,
fallbackProvider: config.providerName,
},
'Claude usage recovered, retrying primary provider',
);
cooldown = null;
clearUsageAvailabilityCache();
return 'claude';
}
if (usageAvailability === 'exhausted') {
// Current token exhausted — try rotating to another token (ignore cooldowns)
if (
getTokenCount() > 1 &&
rotateToken(undefined, { ignoreRateLimits: true })
) {
logCooldownTransition(
'info',
'cooldown:recover',
{
reason: cooldown.reason,
fallbackProvider: config.providerName,
recovery: 'token-rotation',
},
'Claude current token exhausted, rotated to next token — retrying',
);
cooldown = null;
clearUsageAvailabilityCache();
return 'claude';
}
logCooldownTransition(
'debug',
'cooldown:stay',
{
reason: cooldown.reason,
fallbackProvider: config.providerName,
usageAvailability,
},
'All Claude tokens exhausted, keeping cooldown active',
);
return config.providerName;
}
}
if (Date.now() < cooldown.expiresAt) {
logCooldownTransition(
'debug',
'cooldown:stay',
{
reason: cooldown.reason,
fallbackProvider: config.providerName,
remainingMs: cooldown.expiresAt - Date.now(),
},
'Claude cooldown still active, routing to fallback provider',
);
return config.providerName;
}
// Cooldown expired — try Claude again
logCooldownTransition(
'info',
'cooldown:expire',
{
cooldownDurationMs: cooldown.expiresAt - cooldown.startedAt,
reason: cooldown.reason,
fallbackProvider: config.providerName,
},
'Claude cooldown expired, retrying primary provider',
);
cooldown = null;
}
return 'claude';
}
/**
* Mark Claude as rate-limited. All subsequent requests will route to
* the fallback provider until the cooldown expires.
*/
export function markPrimaryCooldown(
reason: AgentTriggerReason,
retryAfterMs?: number,
): void {
const config = loadConfig();
const durationMs = retryAfterMs || config.defaultCooldownMs;
const now = Date.now();
cooldown = {
startedAt: now,
expiresAt: now + durationMs,
reason,
};
clearUsageAvailabilityCache();
logCooldownTransition(
'info',
'cooldown:enter',
{
reason,
cooldownMs: durationMs,
expiresAt: new Date(cooldown.expiresAt).toISOString(),
fallbackProvider: config.providerName,
},
`Falling back to provider: ${config.providerName} (reason: ${reason}, cooldownMs: ${durationMs})`,
);
}
/** Manually clear cooldown (e.g. after a successful Claude response). */
export function clearPrimaryCooldown(): void {
clearUsageAvailabilityCache();
if (cooldown) {
logCooldownTransition(
'info',
'cooldown:clear',
{ reason: cooldown.reason },
'Claude cooldown cleared manually',
);
cooldown = null;
}
}
/** Check whether the active primary cooldown should suppress fallback entirely. */
export function isPrimaryNoFallbackCooldownActive(): boolean {
return cooldown ? isNoFallbackCooldownReason(cooldown.reason) : false;
}
/** Get current cooldown info (for diagnostics / status dashboard). */
export function getCooldownInfo(): {
active: boolean;
reason?: string;
expiresAt?: string;
remainingMs?: number;
} {
if (!cooldown) {
return { active: false };
}
const remainingMs = Math.max(cooldown.expiresAt - Date.now(), 0);
if (cooldown.reason !== 'usage-exhausted' && remainingMs === 0) {
return { active: false };
}
return {
active: true,
reason: cooldown.reason,
expiresAt: new Date(cooldown.expiresAt).toISOString(),
remainingMs,
};
}
/**
* Build the env-var overrides that make Claude Code SDK talk to
* the fallback provider instead of Claude.
*/
export function getFallbackEnvOverrides(): Record<string, string> {
const config = loadConfig();
return {
ANTHROPIC_BASE_URL: config.baseUrl,
ANTHROPIC_AUTH_TOKEN: config.authToken,
ANTHROPIC_MODEL: config.model,
ANTHROPIC_SMALL_FAST_MODEL: config.smallModel,
// Disable non-essential traffic (usage telemetry etc.) on fallback
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
// Generous timeout for third-party APIs
API_TIMEOUT_MS: '3000000',
// Disable tool search (not supported by most fallback providers)
ENABLE_TOOL_SEARCH: 'false',
};
}
/**
* Inspect an agent error string and decide whether it warrants
* a provider fallback.
*
* Triggers:
* - 429 / rate limit / too many requests
* - Claude auth/org access failures
* - 503 / overloaded (transient provider issue)
* - Network / connection errors
*
* Does NOT trigger for:
* - Poisoned sessions
* - Prompt / tool failures
* - Timeouts (agent took too long, not a provider issue)
*/
export function detectFallbackTrigger(
error?: string | null,
): FallbackTriggerResult {
if (!error) return { shouldFallback: false, reason: '' };
// Delegated to shared SSOT — original priority preserved:
// 429 first, then Claude auth/org errors, then 503/network
const common = classifyAgentError(error);
// 429 rate-limit (highest priority)
if (common.category === 'rate-limit') {
return {
shouldFallback: true,
reason: common.reason,
retryAfterMs: common.retryAfterMs,
};
}
// Claude-specific strict auth check (before 503/network)
const auth = classifyClaudeAuthError(error);
if (auth.category !== 'none') {
return { shouldFallback: true, reason: auth.reason };
}
// 503 overloaded, network errors
if (common.category !== 'none') {
return {
shouldFallback: true,
reason: common.reason,
};
}
return { shouldFallback: false, reason: '' };
}
/**
* Check whether a per-group settings.json already overrides the
* provider (e.g. the Kimi test channel). If so, we should NOT
* apply fallback env overrides on top — the channel already has
* its own provider configuration.
*/
export function hasGroupProviderOverride(settingsJsonPath: string): boolean {
try {
const raw = fs.readFileSync(settingsJsonPath, 'utf-8');
const settings = JSON.parse(raw);
const env = settings?.env || {};
return Boolean(env.ANTHROPIC_BASE_URL || env.ANTHROPIC_MODEL);
} catch {
return false;
}
}

View File

@@ -8,18 +8,12 @@ vi.mock('./logger.js', () => ({
},
}));
vi.mock('./provider-fallback.js', () => ({
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
markPrimaryCooldown: vi.fn(),
}));
vi.mock('./token-rotation.js', () => ({
rotateToken: vi.fn(() => false),
getTokenCount: vi.fn(() => 1),
markTokenHealthy: vi.fn(),
}));
import { markPrimaryCooldown } from './provider-fallback.js';
import { runClaudeRotationLoop } from './provider-retry.js';
import {
getTokenCount,
@@ -50,10 +44,9 @@ describe('runClaudeRotationLoop', () => {
expect(outcome).toEqual({ type: 'success' });
expect(rotateToken).toHaveBeenCalledTimes(1);
expect(markTokenHealthy).toHaveBeenCalledTimes(1);
expect(markPrimaryCooldown).not.toHaveBeenCalled();
});
it('marks no-fallback cooldown when all Claude tokens are org-access-denied', async () => {
it('returns error when all Claude tokens are exhausted', async () => {
vi.mocked(getTokenCount).mockReturnValue(2);
const outcome = await runClaudeRotationLoop(
@@ -66,16 +59,12 @@ describe('runClaudeRotationLoop', () => {
);
expect(outcome).toEqual({
type: 'no-fallback',
type: 'error',
trigger: { reason: 'org-access-denied' },
});
expect(markPrimaryCooldown).toHaveBeenCalledWith(
'org-access-denied',
undefined,
);
});
it('returns success-null-result as a fallback trigger after rotation', async () => {
it('returns error with success-null-result trigger after rotation', async () => {
vi.mocked(getTokenCount).mockReturnValue(2);
vi.mocked(rotateToken).mockReturnValueOnce(true);
@@ -90,9 +79,8 @@ describe('runClaudeRotationLoop', () => {
);
expect(outcome).toEqual({
type: 'needs-fallback',
type: 'error',
trigger: { reason: 'success-null-result' },
});
expect(markPrimaryCooldown).not.toHaveBeenCalled();
});
});

View File

@@ -6,16 +6,12 @@
*/
import {
isNoFallbackCooldownReason,
classifyRotationTrigger,
shouldRotateClaudeToken,
type AgentTriggerReason,
} from './agent-error-detection.js';
import { logger } from './logger.js';
import { getErrorMessage } from './utils.js';
import {
detectFallbackTrigger,
markPrimaryCooldown,
} from './provider-fallback.js';
import {
rotateToken,
getTokenCount,
@@ -39,17 +35,15 @@ export interface RotationAttemptResult {
export type RotationOutcome =
| { type: 'success' }
| { type: 'error'; message?: string }
| { type: 'needs-fallback'; trigger: TriggerInfo }
| { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired/org-access-denied
| { type: 'error'; trigger?: TriggerInfo };
// ── Shared rotation loop ─────────────────────────────────────────
/**
* Retry a Claude request by rotating through available tokens.
*
* Returns a discriminated outcome — the caller decides what to do
* with 'needs-fallback' (e.g. run Kimi fallback) or 'no-fallback'.
* Returns 'success' if a rotated token worked, or 'error' if all
* tokens are exhausted or the error is non-retryable.
*/
export async function runClaudeRotationLoop(
initialTrigger: TriggerInfo,
@@ -63,7 +57,9 @@ export async function runClaudeRotationLoop(
while (
shouldRotateClaudeToken(trigger.reason) &&
getTokenCount() > 1 &&
rotateToken(lastRotationMessage, { ignoreRateLimits: true })
// Respect per-token cooldowns so exhausted auth/quota failures can
// terminate instead of cycling forever.
rotateToken(lastRotationMessage)
) {
logger.info(
{ ...logContext, reason: trigger.reason },
@@ -78,12 +74,12 @@ export async function runClaudeRotationLoop(
const errMsg = getErrorMessage(attempt.thrownError);
const retryTrigger = attempt.streamedTriggerReason
? {
shouldFallback: true,
shouldRetry: true,
reason: attempt.streamedTriggerReason.reason,
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
}
: detectFallbackTrigger(errMsg);
if (retryTrigger.shouldFallback) {
: classifyRotationTrigger(errMsg);
if (retryTrigger.shouldRetry) {
trigger = {
reason: retryTrigger.reason,
retryAfterMs: retryTrigger.retryAfterMs,
@@ -124,12 +120,13 @@ export async function runClaudeRotationLoop(
continue;
}
// ── Success with null result (MAE-specific, TaskScheduler ignores) ──
// ── Success with null result ──
if (!attempt.sawOutput && attempt.sawSuccessNullResult) {
return {
type: 'needs-fallback',
trigger: { reason: 'success-null-result' },
};
logger.warn(
{ ...logContext, provider: 'claude' },
'All rotated tokens returned success with null result',
);
return { type: 'error', trigger: { reason: 'success-null-result' } };
}
// ── Error status ──
@@ -137,12 +134,12 @@ export async function runClaudeRotationLoop(
if (!attempt.sawOutput) {
const retryTrigger = attempt.streamedTriggerReason
? {
shouldFallback: true,
shouldRetry: true,
reason: attempt.streamedTriggerReason.reason,
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
}
: detectFallbackTrigger(output.error);
if (retryTrigger.shouldFallback) {
: classifyRotationTrigger(output.error);
if (retryTrigger.shouldRetry) {
trigger = {
reason: retryTrigger.reason,
retryAfterMs: retryTrigger.retryAfterMs,
@@ -165,16 +162,9 @@ export async function runClaudeRotationLoop(
}
// ── All tokens exhausted ──
// Usage/auth/org access failures: don't fall back to Kimi
if (isNoFallbackCooldownReason(trigger.reason)) {
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
logger.info(
{ ...logContext, reason: trigger.reason },
`All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`,
);
return { type: 'no-fallback', trigger };
}
return { type: 'needs-fallback', trigger };
logger.warn(
{ ...logContext, reason: trigger.reason },
`All Claude tokens exhausted (${trigger.reason})`,
);
return { type: 'error', trigger };
}

View File

@@ -92,3 +92,15 @@ export function findChannel(
): Channel | undefined {
return channels.find((c) => c.ownsJid(jid));
}
/**
* Normalize message text for deduplication comparison.
* - Trim leading/trailing whitespace
* - Collapse consecutive whitespace/newlines into single space
*/
export function normalizeMessageForDedupe(text: string): string {
return text
.trim()
.replace(/\s+/g, ' ')
.toLowerCase();
}

View File

@@ -0,0 +1,77 @@
import { beforeEach, describe, expect, it } from 'vitest';
import { _initTestDatabase, setRegisteredGroup } from './db.js';
import {
activateCodexFailover,
getActiveCodexFailoverLeases,
getEffectiveChannelLease,
refreshChannelOwnerCache,
restoreDefaultChannelLease,
} from './service-routing.js';
beforeEach(() => {
_initTestDatabase();
refreshChannelOwnerCache(true);
});
describe('service-routing failover leases', () => {
it('uses codex-review as owner and codex-main as reviewer during failover', () => {
setRegisteredGroup('dc:paired', {
name: 'Paired Room Claude',
folder: 'paired-claude',
trigger: '@Andy',
added_at: '2024-01-01T00:00:00.000Z',
agentType: 'claude-code',
});
setRegisteredGroup('dc:paired', {
name: 'Paired Room Codex',
folder: 'paired-codex',
trigger: '@Codex',
added_at: '2024-01-01T00:00:00.000Z',
agentType: 'codex',
});
activateCodexFailover('dc:paired', 'claude-429');
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
chat_jid: 'dc:paired',
owner_service_id: 'codex-review',
reviewer_service_id: 'codex-main',
reason: 'claude-429',
explicit: true,
});
expect(getActiveCodexFailoverLeases()).toEqual([
{
chatJid: 'dc:paired',
activatedAt: expect.any(String),
},
]);
});
it('restores the default lease after failover is cleared', () => {
setRegisteredGroup('dc:paired', {
name: 'Paired Room Claude',
folder: 'paired-claude',
trigger: '@Andy',
added_at: '2024-01-01T00:00:00.000Z',
agentType: 'claude-code',
});
setRegisteredGroup('dc:paired', {
name: 'Paired Room Codex',
folder: 'paired-codex',
trigger: '@Codex',
added_at: '2024-01-01T00:00:00.000Z',
agentType: 'codex',
});
activateCodexFailover('dc:paired', 'claude-429');
restoreDefaultChannelLease('dc:paired');
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
chat_jid: 'dc:paired',
owner_service_id: 'claude',
reviewer_service_id: 'codex-main',
explicit: false,
});
});
});

189
src/service-routing.ts Normal file
View File

@@ -0,0 +1,189 @@
import {
CLAUDE_SERVICE_ID,
CODEX_MAIN_SERVICE_ID,
CODEX_REVIEW_SERVICE_ID,
SERVICE_AGENT_TYPE,
SERVICE_ID,
normalizeServiceId,
} from './config.js';
import {
clearChannelOwnerLease,
getAllChannelOwnerLeases,
getRegisteredAgentTypesForJid,
setChannelOwnerLease,
type ChannelOwnerLeaseRow,
} from './db.js';
export interface EffectiveChannelLease {
chat_jid: string;
owner_service_id: string;
reviewer_service_id: string | null;
activated_at: string | null;
reason: string | null;
explicit: boolean;
}
const LEASE_CACHE_REFRESH_MS = 2_000;
let lastLeaseRefreshAt = 0;
const leaseCache = new Map<string, ChannelOwnerLeaseRow>();
function normalizeLeaseRow(
row: ChannelOwnerLeaseRow,
explicit: boolean,
): EffectiveChannelLease {
return {
chat_jid: row.chat_jid,
owner_service_id: normalizeServiceId(row.owner_service_id),
reviewer_service_id: row.reviewer_service_id
? normalizeServiceId(row.reviewer_service_id)
: null,
activated_at: row.activated_at,
reason: row.reason,
explicit,
};
}
function getDefaultLease(chatJid: string): EffectiveChannelLease {
const types = getRegisteredAgentTypesForJid(chatJid);
const hasClaude = types.includes('claude-code');
const hasCodex = types.includes('codex');
if (hasClaude && hasCodex) {
return {
chat_jid: chatJid,
owner_service_id: CLAUDE_SERVICE_ID,
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
activated_at: null,
reason: null,
explicit: false,
};
}
if (hasCodex) {
return {
chat_jid: chatJid,
owner_service_id: CODEX_MAIN_SERVICE_ID,
reviewer_service_id: null,
activated_at: null,
reason: null,
explicit: false,
};
}
if (hasClaude) {
return {
chat_jid: chatJid,
owner_service_id: CLAUDE_SERVICE_ID,
reviewer_service_id: null,
activated_at: null,
reason: null,
explicit: false,
};
}
return {
chat_jid: chatJid,
owner_service_id:
SERVICE_AGENT_TYPE === 'codex' ? CODEX_MAIN_SERVICE_ID : CLAUDE_SERVICE_ID,
reviewer_service_id: null,
activated_at: null,
reason: null,
explicit: false,
};
}
export function refreshChannelOwnerCache(force = false): void {
const now = Date.now();
if (!force && now - lastLeaseRefreshAt < LEASE_CACHE_REFRESH_MS) {
return;
}
leaseCache.clear();
for (const row of getAllChannelOwnerLeases()) {
leaseCache.set(row.chat_jid, row);
}
lastLeaseRefreshAt = now;
}
export function getEffectiveChannelLease(chatJid: string): EffectiveChannelLease {
refreshChannelOwnerCache();
const row = leaseCache.get(chatJid);
if (row) {
return normalizeLeaseRow(row, true);
}
return getDefaultLease(chatJid);
}
export function isOwnerServiceForChat(
chatJid: string,
serviceId: string = SERVICE_ID,
): boolean {
const lease = getEffectiveChannelLease(chatJid);
return normalizeServiceId(serviceId) === lease.owner_service_id;
}
export function isReviewerServiceForChat(
chatJid: string,
serviceId: string = SERVICE_ID,
): boolean {
const lease = getEffectiveChannelLease(chatJid);
return (
lease.reviewer_service_id !== null &&
normalizeServiceId(serviceId) === lease.reviewer_service_id
);
}
export function shouldServiceProcessChat(
chatJid: string,
serviceId: string = SERVICE_ID,
): boolean {
const normalizedServiceId = normalizeServiceId(serviceId);
const lease = getEffectiveChannelLease(chatJid);
return (
normalizedServiceId === lease.owner_service_id ||
normalizedServiceId === lease.reviewer_service_id
);
}
export function activateCodexFailover(chatJid: string, reason: string): void {
const now = new Date().toISOString();
const row: ChannelOwnerLeaseRow = {
chat_jid: chatJid,
owner_service_id: CODEX_REVIEW_SERVICE_ID,
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
activated_at: now,
reason,
};
setChannelOwnerLease(row);
leaseCache.set(chatJid, row);
lastLeaseRefreshAt = Date.now();
}
export function restoreDefaultChannelLease(chatJid: string): void {
clearChannelOwnerLease(chatJid);
leaseCache.delete(chatJid);
lastLeaseRefreshAt = Date.now();
}
export interface ActiveFailoverLease {
chatJid: string;
activatedAt: string | null;
}
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
refreshChannelOwnerCache(true);
return [...leaseCache.values()]
.filter(
(row) =>
normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID &&
normalizeServiceId(row.reviewer_service_id || '') ===
CODEX_MAIN_SERVICE_ID,
)
.map((row) => ({ chatJid: row.chat_jid, activatedAt: row.activated_at ?? null }));
}
/** @deprecated Use getActiveCodexFailoverLeases() instead */
export function getActiveCodexFailoverChatJids(): string[] {
return getActiveCodexFailoverLeases().map((l) => l.chatJid);
}

View File

@@ -1,6 +1,9 @@
import { describe, expect, it } from 'vitest';
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
import {
shouldResetSessionOnAgentFailure,
shouldRetryFreshSessionOnAgentFailure,
} from './session-recovery.js';
describe('shouldResetSessionOnAgentFailure', () => {
it('matches many-image dimension limit errors', () => {
@@ -31,4 +34,45 @@ describe('shouldResetSessionOnAgentFailure', () => {
}),
).toBe(false);
});
it('matches thinking signature 400 errors', () => {
expect(
shouldResetSessionOnAgentFailure({
result:
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
error: undefined,
}),
).toBe(true);
});
});
describe('shouldRetryFreshSessionOnAgentFailure', () => {
it('matches stale session errors', () => {
expect(
shouldRetryFreshSessionOnAgentFailure({
result: null,
error: 'No conversation found with session ID: stale',
}),
).toBe(true);
});
it('matches thinking signature 400 errors', () => {
expect(
shouldRetryFreshSessionOnAgentFailure({
result:
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
error: undefined,
}),
).toBe(true);
});
it('does not retry image dimension limit errors', () => {
expect(
shouldRetryFreshSessionOnAgentFailure({
result:
'An image in the conversation exceeds the dimension limit for many-image requests (2000px). Start a new session with fewer images.',
error: undefined,
}),
).toBe(false);
});
});

View File

@@ -4,6 +4,16 @@ const SESSION_RESET_PATTERNS = [
/An image in the conversation exceeds the dimension limit for many-image requests \(2000px\)\./i,
/Start a new session with fewer images\./i,
/No conversation found with session ID/i,
/400[^\n]*thinking/i,
/thinking[^\n]*block[^\n]*(?:invalid|error|signature)/i,
/invalid[^\n]*signature[^\n]*thinking/i,
];
const SESSION_RETRY_PATTERNS = [
/No conversation found with session ID/i,
/400[^\n]*thinking/i,
/thinking[^\n]*block[^\n]*(?:invalid|error|signature)/i,
/invalid[^\n]*signature[^\n]*thinking/i,
];
function toText(value: string | object | null | undefined): string[] {
@@ -25,3 +35,12 @@ export function shouldResetSessionOnAgentFailure(
SESSION_RESET_PATTERNS.some((pattern) => pattern.test(text)),
);
}
export function shouldRetryFreshSessionOnAgentFailure(
output: Pick<AgentOutput, 'result' | 'error'>,
): boolean {
const texts = [...toText(output.result), ...toText(output.error)];
return texts.some((text) =>
SESSION_RETRY_PATTERNS.some((pattern) => pattern.test(text)),
);
}

View File

@@ -26,6 +26,7 @@ export interface UsageRowSnapshot {
}
export interface StatusSnapshot {
serviceId: string;
agentType: AgentType;
assistantName: string;
updatedAt: string;
@@ -41,7 +42,7 @@ export function writeStatusSnapshot(snapshot: StatusSnapshot): void {
fs.mkdirSync(STATUS_SNAPSHOT_DIR, { recursive: true });
const targetPath = path.join(
STATUS_SNAPSHOT_DIR,
`${snapshot.agentType}.json`,
`${snapshot.serviceId}.json`,
);
const tempPath = `${targetPath}.tmp`;
writeJsonFile(tempPath, snapshot, true);
@@ -63,6 +64,7 @@ export function readStatusSnapshots(maxAgeMs: number): StatusSnapshot[] {
const parsed = JSON.parse(raw) as StatusSnapshot;
if (
!parsed.updatedAt ||
!parsed.serviceId ||
!parsed.agentType ||
!Array.isArray(parsed.entries)
)

View File

@@ -4,6 +4,8 @@ import type { AgentOutput } from './agent-runner.js';
// ── Mocks ──────────────────────────────────────────────────────
vi.mock('./agent-error-detection.js', () => ({
classifyClaudeAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
classifyRotationTrigger: vi.fn(() => ({ shouldRetry: false, reason: '' })),
detectClaudeProviderFailureMessage: vi.fn(() => ''),
isClaudeUsageExhaustedMessage: vi.fn(() => false),
isClaudeOrgAccessDeniedMessage: vi.fn(() => false),
@@ -11,10 +13,6 @@ vi.mock('./agent-error-detection.js', () => ({
isClaudeAuthError: vi.fn(() => false),
}));
vi.mock('./provider-fallback.js', () => ({
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
}));
vi.mock('./codex-token-rotation.js', () => ({
detectCodexRotationTrigger: vi.fn(() => ({
shouldRotate: false,
@@ -22,15 +20,21 @@ vi.mock('./codex-token-rotation.js', () => ({
})),
}));
vi.mock('./session-recovery.js', () => ({
shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false),
}));
import {
classifyClaudeAuthError,
classifyRotationTrigger,
detectClaudeProviderFailureMessage,
isClaudeUsageExhaustedMessage,
isClaudeOrgAccessDeniedMessage,
isClaudeAuthExpiredMessage,
isClaudeAuthError,
} from './agent-error-detection.js';
import { detectFallbackTrigger } from './provider-fallback.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js';
import {
evaluateStreamedOutput,
@@ -40,7 +44,11 @@ import {
// ── Helpers ────────────────────────────────────────────────────
function freshState(): StreamedOutputState {
return { sawOutput: false, sawSuccessNullResultWithoutOutput: false };
return {
sawOutput: false,
sawVisibleOutput: false,
sawSuccessNullResultWithoutOutput: false,
};
}
const claudeOpts: EvaluateStreamedOutputOptions = {
@@ -53,8 +61,11 @@ const codexOpts: EvaluateStreamedOutputOptions = {
provider: 'codex',
};
function successOutput(result: string | null): AgentOutput {
return { status: 'success', result };
function successOutput(
result: string | null,
phase?: AgentOutput['phase'],
): AgentOutput {
return { status: 'success', result, ...(phase ? { phase } : {}) };
}
function errorOutput(error: string): AgentOutput {
@@ -63,19 +74,24 @@ function errorOutput(error: string): AgentOutput {
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(classifyClaudeAuthError).mockReturnValue({
category: 'none',
reason: '',
});
vi.mocked(classifyRotationTrigger).mockReturnValue({
shouldRetry: false,
reason: '',
});
vi.mocked(detectClaudeProviderFailureMessage).mockReturnValue('');
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false);
vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(false);
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false);
vi.mocked(isClaudeAuthError).mockReturnValue(false);
vi.mocked(detectFallbackTrigger).mockReturnValue({
shouldFallback: false,
reason: '',
});
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
shouldRotate: false,
reason: '',
});
vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(false);
});
// ── Tests ──────────────────────────────────────────────────────
@@ -112,6 +128,16 @@ describe('evaluateStreamedOutput', () => {
expect(result.shouldForwardOutput).toBe(true);
expect(result.state.sawOutput).toBe(false);
});
it('does not set sawOutput for progress output', () => {
const result = evaluateStreamedOutput(
successOutput('대화 요약 중...', 'progress'),
freshState(),
claudeOpts,
);
expect(result.shouldForwardOutput).toBe(true);
expect(result.state.sawOutput).toBe(false);
});
});
describe('Claude usage-exhausted banner', () => {
@@ -156,6 +182,39 @@ describe('evaluateStreamedOutput', () => {
});
});
describe('Claude retryable session failure suppression', () => {
it('suppresses retryable session failures before any visible output', () => {
vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(true);
const result = evaluateStreamedOutput(
successOutput(
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
'intermediate',
),
freshState(),
claudeOpts,
);
expect(result.shouldForwardOutput).toBe(false);
expect(result.suppressedRetryableSessionFailure).toBe(true);
expect(result.state.retryableSessionFailureDetected).toBe(true);
expect(result.state.sawOutput).toBe(false);
});
it('does not suppress retryable session failures after visible output', () => {
vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(true);
const result = evaluateStreamedOutput(
successOutput('API Error: 400 invalid thinking block', 'final'),
{ ...freshState(), sawOutput: true },
claudeOpts,
);
expect(result.shouldForwardOutput).toBe(true);
expect(result.suppressedRetryableSessionFailure).toBeUndefined();
});
});
describe('Claude auth-expired banner', () => {
it('suppresses output and returns newTrigger', () => {
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(true);
@@ -208,6 +267,24 @@ describe('evaluateStreamedOutput', () => {
reason: 'org-access-denied',
});
});
it('suppresses a 403 terminated auth banner surfaced as success text', () => {
vi.mocked(classifyClaudeAuthError).mockReturnValue({
category: 'org-access-denied',
reason: 'org-access-denied',
});
const result = evaluateStreamedOutput(
successOutput('Failed to authenticate. API Error: 403 terminated'),
freshState(),
claudeOpts,
);
expect(result.shouldForwardOutput).toBe(false);
expect(result.newTrigger).toEqual({ reason: 'org-access-denied' });
expect(result.state.streamedTriggerReason).toEqual({
reason: 'org-access-denied',
});
});
});
describe('duplicate trigger suppression', () => {
@@ -298,10 +375,10 @@ describe('evaluateStreamedOutput', () => {
});
});
describe('error → Claude fallback trigger', () => {
it('returns fallback trigger with retryAfterMs', () => {
vi.mocked(detectFallbackTrigger).mockReturnValue({
shouldFallback: true,
describe('error → Claude rotation trigger', () => {
it('returns rotation trigger with retryAfterMs', () => {
vi.mocked(classifyRotationTrigger).mockReturnValue({
shouldRetry: true,
reason: '429',
retryAfterMs: 30000,
});
@@ -324,8 +401,8 @@ describe('evaluateStreamedOutput', () => {
});
it('suppresses error forward when shortCircuitTriggeredErrors is set', () => {
vi.mocked(detectFallbackTrigger).mockReturnValue({
shouldFallback: true,
vi.mocked(classifyRotationTrigger).mockReturnValue({
shouldRetry: true,
reason: '429',
});
@@ -337,9 +414,9 @@ describe('evaluateStreamedOutput', () => {
expect(result.newTrigger).toEqual({ reason: '429' });
});
it('skips fallback check when output was already seen', () => {
vi.mocked(detectFallbackTrigger).mockReturnValue({
shouldFallback: true,
it('skips rotation check when output was already seen', () => {
vi.mocked(classifyRotationTrigger).mockReturnValue({
shouldRetry: true,
reason: '429',
});
@@ -352,9 +429,9 @@ describe('evaluateStreamedOutput', () => {
expect(result.shouldForwardOutput).toBe(true);
});
it('skips fallback check when already triggered', () => {
vi.mocked(detectFallbackTrigger).mockReturnValue({
shouldFallback: true,
it('skips rotation check when already triggered', () => {
vi.mocked(classifyRotationTrigger).mockReturnValue({
shouldRetry: true,
reason: '429',
});
@@ -397,7 +474,7 @@ describe('evaluateStreamedOutput', () => {
freshState(),
claudeOpts,
);
// Claude uses detectFallbackTrigger, not detectCodexRotationTrigger
// Claude uses classifyRotationTrigger, not detectCodexRotationTrigger
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
});
});

View File

@@ -1,4 +1,6 @@
import {
classifyClaudeAuthError,
classifyRotationTrigger,
detectClaudeProviderFailureMessage,
isClaudeAuthError,
isClaudeAuthExpiredMessage,
@@ -8,7 +10,7 @@ import {
} from './agent-error-detection.js';
import type { AgentOutput } from './agent-runner.js';
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
import { detectFallbackTrigger } from './provider-fallback.js';
import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js';
export interface StreamedTriggerReason {
reason: AgentTriggerReason;
@@ -17,8 +19,10 @@ export interface StreamedTriggerReason {
export interface StreamedOutputState {
sawOutput: boolean;
sawVisibleOutput: boolean;
sawSuccessNullResultWithoutOutput: boolean;
streamedTriggerReason?: StreamedTriggerReason;
retryableSessionFailureDetected?: boolean;
}
export interface EvaluateStreamedOutputOptions {
@@ -34,6 +38,7 @@ export interface EvaluateStreamedOutputResult {
shouldForwardOutput: boolean;
newTrigger?: StreamedTriggerReason;
suppressedAuthError?: boolean;
suppressedRetryableSessionFailure?: boolean;
}
export function evaluateStreamedOutput(
@@ -46,6 +51,21 @@ export function evaluateStreamedOutput(
options.agentType === 'claude-code' && options.provider === 'claude';
const isPrimaryCodex =
options.agentType === 'codex' && options.provider === 'codex';
const countsAsFinalOutput =
output.phase === undefined || output.phase === 'final';
if (
isPrimaryClaude &&
!state.sawOutput &&
shouldRetryFreshSessionOnAgentFailure(output)
) {
nextState.retryableSessionFailureDetected = true;
return {
state: nextState,
shouldForwardOutput: false,
suppressedRetryableSessionFailure: true,
};
}
if (
isPrimaryClaude &&
@@ -53,12 +73,15 @@ export function evaluateStreamedOutput(
!state.sawOutput &&
typeof output.result === 'string'
) {
const authClassification = classifyClaudeAuthError(output.result);
const triggerReason: AgentTriggerReason | undefined =
isClaudeUsageExhaustedMessage(output.result)
? 'usage-exhausted'
: isClaudeOrgAccessDeniedMessage(output.result)
: isClaudeOrgAccessDeniedMessage(output.result) ||
authClassification.category === 'org-access-denied'
? 'org-access-denied'
: isClaudeAuthExpiredMessage(output.result)
: isClaudeAuthExpiredMessage(output.result) ||
authClassification.category === 'auth-expired'
? 'auth-expired'
: detectClaudeProviderFailureMessage(output.result) || undefined;
@@ -87,7 +110,11 @@ export function evaluateStreamedOutput(
}
}
if (output.result !== null && output.result !== undefined) {
if (
countsAsFinalOutput &&
output.result !== null &&
output.result !== undefined
) {
nextState.sawOutput = true;
} else if (
options.trackSuccessNullResult &&
@@ -106,8 +133,8 @@ export function evaluateStreamedOutput(
let newTrigger: StreamedTriggerReason | undefined;
if (isPrimaryClaude) {
const trigger = detectFallbackTrigger(output.error);
if (trigger.shouldFallback) {
const trigger = classifyRotationTrigger(output.error);
if (trigger.shouldRetry) {
newTrigger = {
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,

View File

@@ -24,38 +24,31 @@ const {
),
}));
vi.mock('./provider-fallback.js', () => ({
detectFallbackTrigger: vi.fn((error?: string | null) => {
const lower = (error || '').toLowerCase();
if (
lower.includes('does not have access to claude') ||
(lower.includes('failed to authenticate') &&
lower.includes('403') &&
lower.includes('terminated'))
) {
return { shouldFallback: true, reason: 'org-access-denied' };
}
if (
lower.includes('429') ||
lower.includes('rate limit') ||
lower.includes('hit your limit')
) {
return { shouldFallback: true, reason: '429' };
}
return { shouldFallback: false, reason: '' };
}),
getActiveProvider: vi.fn(async () => 'claude'),
getFallbackEnvOverrides: vi.fn(() => ({
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
ANTHROPIC_MODEL: 'kimi-k2.5',
})),
getFallbackProviderName: vi.fn(() => 'kimi'),
hasGroupProviderOverride: vi.fn(() => false),
isFallbackEnabled: vi.fn(() => true),
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
markPrimaryCooldown: vi.fn(),
}));
vi.mock('./agent-error-detection.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('./agent-error-detection.js')>();
return {
...actual,
classifyRotationTrigger: vi.fn((error?: string | null) => {
const lower = (error || '').toLowerCase();
if (
lower.includes('does not have access to claude') ||
(lower.includes('failed to authenticate') &&
lower.includes('403') &&
lower.includes('terminated'))
) {
return { shouldRetry: true, reason: 'org-access-denied' };
}
if (
lower.includes('429') ||
lower.includes('rate limit') ||
lower.includes('hit your limit')
) {
return { shouldRetry: true, reason: '429' };
}
return { shouldRetry: false, reason: '' };
}),
};
});
vi.mock('./token-rotation.js', () => ({
rotateToken: vi.fn(() => false),
@@ -136,7 +129,6 @@ vi.mock('./github-ci.js', () => ({
}));
import { _initTestDatabase, createTask, getTaskById } from './db.js';
import * as providerFallback from './provider-fallback.js';
import * as codexTokenRotation from './codex-token-rotation.js';
import { createTaskStatusTracker } from './task-status-tracker.js';
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
@@ -163,10 +155,7 @@ describe('task scheduler', () => {
terminal: false,
resultSummary: 'GitHub Actions run 123 is in_progress',
});
vi.mocked(providerFallback.markPrimaryCooldown).mockClear();
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
// No fallback provider setup needed
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
vi.mocked(tokenRotation.markTokenHealthy).mockClear();
vi.mocked(tokenRotation.rotateToken).mockClear();
@@ -454,7 +443,7 @@ Check the run.
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
// No fallback cooldown to check
expect(sendMessage).toHaveBeenCalledTimes(1);
expect(sendMessage).toHaveBeenCalledWith(
'shared@g.us',
@@ -553,7 +542,7 @@ Check the run.
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
// No fallback cooldown to check
expect(sendMessage).toHaveBeenCalledTimes(1);
expect(sendMessage).toHaveBeenCalledWith(
'shared@g.us',
@@ -601,23 +590,6 @@ Check the run.
result: null,
};
},
)
.mockImplementationOnce(
async (
_group: unknown,
_input: unknown,
_onProcess: unknown,
onOutput?: (output: Record<string, unknown>) => Promise<void>,
) => {
await onOutput?.({
status: 'success',
result: 'scheduled fallback response',
});
return {
status: 'success',
result: null,
};
},
);
const enqueueTask = vi.fn(
@@ -646,17 +618,9 @@ Check the run.
await vi.advanceTimersByTimeAsync(10);
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
expect(runAgentProcessMock).toHaveBeenCalledTimes(1);
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
'overloaded',
undefined,
);
expect(sendMessage).toHaveBeenCalledTimes(1);
expect(sendMessage).toHaveBeenCalledWith(
'shared@g.us',
'scheduled fallback response',
);
// No fallback — 502 results in error without retrying on another provider
});
it('suppresses Claude org access denied banners for scheduled tasks and retries with a rotated account', async () => {
@@ -750,7 +714,7 @@ Check the run.
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
// No fallback cooldown to check
expect(sendMessage).toHaveBeenCalledTimes(1);
expect(sendMessage).toHaveBeenCalledWith(
'shared@g.us',

View File

@@ -1,12 +1,10 @@
import { ChildProcess } from 'child_process';
import { CronExpressionParser } from 'cron-parser';
import fs from 'fs';
import path from 'path';
import { getErrorMessage } from './utils.js';
import {
ASSISTANT_NAME,
DATA_DIR,
SCHEDULER_POLL_INTERVAL,
SERVICE_AGENT_TYPE,
TIMEZONE,
@@ -33,16 +31,6 @@ import {
} from './group-folder.js';
import { logger } from './logger.js';
import { createTaskStatusTracker } from './task-status-tracker.js';
import {
detectFallbackTrigger,
getActiveProvider,
getFallbackEnvOverrides,
getFallbackProviderName,
hasGroupProviderOverride,
isFallbackEnabled,
isPrimaryNoFallbackCooldownActive,
markPrimaryCooldown,
} from './provider-fallback.js';
import { runClaudeRotationLoop } from './provider-retry.js';
import {
detectCodexRotationTrigger,
@@ -50,9 +38,10 @@ import {
getCodexAccountCount,
markCodexTokenHealthy,
} from './codex-token-rotation.js';
import type {
AgentTriggerReason,
CodexRotationReason,
import {
classifyRotationTrigger,
type AgentTriggerReason,
type CodexRotationReason,
} from './agent-error-detection.js';
import {
getTokenCount,
@@ -306,19 +295,8 @@ async function runTask(
sendTrackedMessage: deps.sendTrackedMessage,
editTrackedMessage: deps.editTrackedMessage,
});
const settingsPath = path.join(
DATA_DIR,
'sessions',
task.group_folder,
'.claude',
'settings.json',
);
const isClaudeAgent = context.taskAgentType === 'claude-code';
const canRotateToken = isClaudeAgent && getTokenCount() > 1;
const canFallback =
isClaudeAgent &&
isFallbackEnabled() &&
!hasGroupProviderOverride(settingsPath);
try {
await statusTracker.update('checking');
@@ -337,6 +315,7 @@ async function runTask(
}> => {
let streamedState: StreamedOutputState = {
sawOutput: false,
sawVisibleOutput: false,
sawSuccessNullResultWithoutOutput: false,
};
let attemptResult: string | null = null;
@@ -391,7 +370,7 @@ async function runTask(
reason: evaluation.newTrigger.reason,
resultPreview: streamedOutput.result.slice(0, 120),
},
'Detected Claude fallback trigger during scheduled task output',
'Detected Claude rotation trigger during scheduled task output',
);
} else if (
evaluation.newTrigger &&
@@ -407,7 +386,7 @@ async function runTask(
errorPreview: streamedOutput.error.slice(0, 120),
},
provider === 'claude'
? 'Detected Claude fallback trigger during scheduled task error output'
? 'Detected Claude rotation trigger during scheduled task error output'
: 'Detected Codex rotation trigger during scheduled task error output',
);
}
@@ -428,9 +407,7 @@ async function runTask(
attemptError = streamedOutput.error || 'Unknown error';
}
},
isClaudeAgent && provider !== 'claude'
? getFallbackEnvOverrides()
: undefined,
undefined,
);
if (output.status === 'error' && !attemptError) {
@@ -448,38 +425,6 @@ async function runTask(
};
};
const runFallbackTaskAttempt = async (
reason: AgentTriggerReason,
retryAfterMs?: number,
): Promise<void> => {
if (!canFallback) {
error = reason;
return;
}
const fallbackName = getFallbackProviderName();
markPrimaryCooldown(reason, retryAfterMs);
logger.info(
{
taskId: task.id,
group: context.group.name,
groupFolder: task.group_folder,
reason,
retryAfterMs,
fallbackProvider: fallbackName,
},
`Falling back to provider: ${fallbackName} for scheduled task (reason: ${reason})`,
);
const fallbackAttempt = await runTaskAttempt(fallbackName);
result = fallbackAttempt.attemptResult;
error =
fallbackAttempt.output.status === 'error'
? fallbackAttempt.attemptError || 'Unknown error'
: null;
};
const retryClaudeTaskWithRotation = async (
initialTrigger: {
reason: AgentTriggerReason;
@@ -514,15 +459,9 @@ async function runTask(
error = null;
return;
case 'error':
return;
case 'no-fallback':
error = `Claude ${outcome.trigger.reason}`;
return;
case 'needs-fallback':
await runFallbackTaskAttempt(
outcome.trigger.reason,
outcome.trigger.retryAfterMs,
);
if (outcome.trigger) {
error = `Claude ${outcome.trigger.reason}`;
}
return;
}
};
@@ -597,25 +536,9 @@ async function runTask(
};
const provider =
context.taskAgentType === 'codex'
? 'codex'
: canFallback
? await getActiveProvider()
: 'claude';
context.taskAgentType === 'codex' ? 'codex' : 'claude';
// Already in no-fallback Claude cooldown — skip task instead of running on Kimi
if (
isClaudeAgent &&
provider !== 'claude' &&
isPrimaryNoFallbackCooldownActive()
) {
logger.info(
{ taskId: task.id, group: context.group.name, provider },
'Claude primary cooldown active, skipping scheduled task',
);
error = 'Claude primary cooldown active';
// Fall through to task completion handling below
} else {
{
const attempt = await runTaskAttempt(provider);
result = attempt.attemptResult;
error = attempt.attemptError;
@@ -642,12 +565,12 @@ async function runTask(
} else if (attempt.output.status === 'error' && provider === 'claude') {
const trigger = attempt.streamedTriggerReason
? {
shouldFallback: true,
shouldRetry: true,
reason: attempt.streamedTriggerReason.reason,
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
}
: detectFallbackTrigger(error);
if (trigger.shouldFallback) {
: classifyRotationTrigger(error);
if (trigger.shouldRetry) {
await retryClaudeTaskWithRotation({
reason: trigger.reason,
retryAfterMs: trigger.retryAfterMs,
@@ -725,8 +648,8 @@ async function runTask(
}
}
} else {
const trigger = detectFallbackTrigger(error);
if (trigger.shouldFallback) {
const trigger = classifyRotationTrigger(error);
if (trigger.shouldRetry) {
const rotated = getTokenCount() > 1 && rotateToken(error);
if (rotated) {
logger.info(

View File

@@ -7,7 +7,7 @@
* CLAUDE_CODE_OAUTH_TOKEN if multi-token is not configured.
*
* On rate-limit: rotate to next token
* All exhausted: fall through to provider fallback (Kimi etc.)
* All exhausted: surface error to caller
*/
import fs from 'fs';
@@ -153,7 +153,7 @@ export function rotateToken(
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
reason: errorMessage ?? null,
},
'All tokens are rate-limited, falling through to provider fallback',
'All tokens are rate-limited, no available tokens',
);
return false;
}
@@ -234,3 +234,13 @@ export function getTokenRotationInfo(): {
).length,
};
}
export function hasAvailableClaudeToken(): boolean {
if (tokens.length === 0) {
return Boolean(process.env.CLAUDE_CODE_OAUTH_TOKEN);
}
const now = Date.now();
return tokens.some(
(token) => !token.rateLimitedUntil || token.rateLimitedUntil <= now,
);
}

View File

@@ -1,7 +1,9 @@
import { describe, expect, it } from 'vitest';
import type { UsageRow } from './dashboard-usage-rows.js';
import {
formatStatusHeader,
renderUsageTable,
summarizeWatcherTasks,
} from './unified-dashboard.js';
@@ -57,3 +59,51 @@ describe('formatStatusHeader', () => {
).toBe('**📊 에이전트 상태** — 활성 3 / 8 | 감시 2 | 일시정지 1');
});
});
describe('renderUsageTable', () => {
const claudeRow: UsageRow = {
name: 'Claude pro',
h5pct: 50,
h5reset: '',
d7pct: 30,
d7reset: '',
};
const codexRow: UsageRow = {
name: 'Codex',
h5pct: 40,
h5reset: '',
d7pct: 25,
d7reset: '',
};
it('renders Claude before separator before Codex', () => {
const lines = renderUsageTable([claudeRow], [codexRow]);
const claudeIdx = lines.findIndex((l) => l.includes('Claude'));
const sepIdx = lines.findIndex((l) => /^─+$/.test(l));
const codexIdx = lines.findIndex((l) => l.includes('Codex'));
expect(claudeIdx).toBeGreaterThan(-1);
expect(sepIdx).toBeGreaterThan(claudeIdx);
expect(codexIdx).toBeGreaterThan(sepIdx);
});
it('omits separator when only Claude rows exist', () => {
const lines = renderUsageTable([claudeRow], []);
expect(lines.some((l) => /^─+$/.test(l))).toBe(false);
expect(lines.some((l) => l.includes('Claude'))).toBe(true);
});
it('omits separator when only Codex rows exist', () => {
const lines = renderUsageTable([], [codexRow]);
expect(lines.some((l) => /^─+$/.test(l))).toBe(false);
expect(lines.some((l) => l.includes('Codex'))).toBe(true);
});
it('returns fallback text when both groups are empty', () => {
const lines = renderUsageTable([], []);
expect(lines).toEqual(['_조회 불가_']);
});
});

View File

@@ -47,6 +47,7 @@ import type {
export interface UnifiedDashboardOptions {
assistantName: string;
serviceId: string;
serviceAgentType: AgentType;
statusChannelId: string;
statusUpdateInterval: number;
@@ -189,8 +190,12 @@ async function refreshChannelMeta(
}
}
function getAgentDisplayName(agentType: 'claude-code' | 'codex'): string {
return agentType === 'codex' ? '코덱스' : '클코';
function getAgentDisplayName(
agentType: 'claude-code' | 'codex',
serviceId: string,
): string {
if (agentType === 'claude-code') return '클코';
return serviceId === 'codex-review' ? '코리뷰' : '코덱스';
}
function formatRoomName(
@@ -216,6 +221,7 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
const statuses = opts.queue.getStatuses(Object.keys(groups));
writeStatusSnapshot({
serviceId: opts.serviceId,
agentType: opts.serviceAgentType,
assistantName: opts.assistantName,
updatedAt: new Date().toISOString(),
@@ -261,6 +267,7 @@ function buildStatusContent(): string {
);
interface RoomEntry {
serviceId: string;
agentType: 'claude-code' | 'codex';
status: 'processing' | 'waiting' | 'inactive';
elapsedMs: number | null;
@@ -277,6 +284,7 @@ function buildStatusContent(): string {
for (const entry of snapshot.entries) {
const existing = byJid.get(entry.jid) || [];
existing.push({
serviceId: snapshot.serviceId,
agentType,
status: entry.status,
elapsedMs: entry.elapsedMs,
@@ -345,7 +353,7 @@ function buildStatusContent(): string {
elapsedMs: agent.elapsedMs,
pendingTasks: agent.pendingTasks,
});
const tag = getAgentDisplayName(agent.agentType);
const tag = getAgentDisplayName(agent.agentType, agent.serviceId);
return `${tag} ${icon} ${label}`;
});
roomLines.push({
@@ -373,6 +381,76 @@ function buildStatusContent(): string {
return `${header}\n\n${sections}`;
}
/**
* Render usage table lines from two row groups (Claude and Codex).
* Returns rendered lines including code block markers.
* Ordering: Claude rows → separator → Codex rows.
* Exported for testing.
*/
export function renderUsageTable(
claudeBotRows: UsageRow[],
codexBotRows: UsageRow[],
): string[] {
const allRows = [...claudeBotRows, ...codexBotRows];
if (allRows.length === 0) return ['_조회 불가_'];
const bar = (pct: number) => {
const filled = Math.max(0, Math.min(5, Math.round(pct / 20)));
return '█'.repeat(filled) + '░'.repeat(5 - filled);
};
const visualWidth = (s: string) =>
[...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0);
const maxNameWidth =
Math.max(8, ...allRows.map((r) => visualWidth(r.name))) + 1;
const padName = (s: string) =>
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
const compactReset = (s: string) =>
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
const lines: string[] = [];
const renderRows = (rows: UsageRow[]) => {
for (const row of rows) {
const h5 =
row.h5pct >= 0
? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%`
: ' — ';
const d7 =
row.d7pct >= 0
? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%`
: ' — ';
lines.push(`${padName(row.name)}${h5} ${d7}`);
const r5 = compactReset(row.h5reset);
const r7 = compactReset(row.d7reset);
if (r5 || r7) {
const d7ColStart = maxNameWidth + 10;
let resetLine = ' '.repeat(maxNameWidth);
if (r5) resetLine += r5;
resetLine = resetLine.padEnd(d7ColStart);
if (r7) resetLine += r7;
lines.push(resetLine);
}
}
};
lines.push('```');
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
renderRows(claudeBotRows);
if (claudeBotRows.length > 0 && codexBotRows.length > 0) {
const separatorWidth = maxNameWidth + 20;
lines.push('─'.repeat(separatorWidth));
}
renderRows(codexBotRows);
lines.push('```');
return lines;
}
async function buildUsageContent(): Promise<string> {
const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED;
let liveClaudeAccounts: ClaudeAccountUsage[] | null = null;
@@ -391,64 +469,27 @@ async function buildUsageContent(): Promise<string> {
return '█'.repeat(filled) + '░'.repeat(5 - filled);
};
const rows: UsageRow[] = [];
// Group 1: Claude bot
const claudeBotRows: UsageRow[] = [];
if (shouldFetchClaudeUsage) {
cachedClaudeAccounts = mergeClaudeDashboardAccounts(
liveClaudeAccounts,
cachedClaudeAccounts,
);
rows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
claudeBotRows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
}
// Codex usage: read from Codex service's own status snapshot.
// Each service owns its usage data — no cross-service auth access needed.
// Group 2: Codex bot (separate service)
const codexBotRows: UsageRow[] = [];
const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS).find(
(s) => s.agentType === 'codex',
(s) => s.serviceId === 'codex-main' || s.serviceId === 'codex',
);
codexBotRows.push(
...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS),
);
rows.push(...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS));
if (rows.length > 0) {
// Emoji characters take 2 columns in monospace — count visual width
const visualWidth = (s: string) =>
[...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0);
const maxNameWidth =
Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1;
const padName = (s: string) =>
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
// Strip whitespace and trailing 'm' from reset strings for compact display
const compactReset = (s: string) =>
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
lines.push('```');
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
for (const row of rows) {
const h5 =
row.h5pct >= 0
? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%`
: ' — ';
const d7 =
row.d7pct >= 0
? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%`
: ' — ';
lines.push(`${padName(row.name)}${h5} ${d7}`);
const r5 = compactReset(row.h5reset);
const r7 = compactReset(row.d7reset);
if (r5 || r7) {
// Align reset values under 5h / 7d columns
// h5 column starts at maxNameWidth; d7 column starts at maxNameWidth + 9 + 1
const d7ColStart = maxNameWidth + 10;
let resetLine = ' '.repeat(maxNameWidth);
if (r5) resetLine += r5;
resetLine = resetLine.padEnd(d7ColStart);
if (r7) resetLine += r7;
lines.push(resetLine);
}
}
lines.push('```');
} else {
lines.push('_조회 불가_');
}
lines.push(...renderUsageTable(claudeBotRows, codexBotRows));
lines.push('');
lines.push('🖥️ *서버*');