feat: add codex review failover and suppress output hardening
This commit is contained in:
@@ -71,12 +71,6 @@ OPENAI_API_KEY=...
|
|||||||
### 선택 환경 변수
|
### 선택 환경 변수
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Provider fallback (Claude 429 시 대체)
|
|
||||||
FALLBACK_PROVIDER_NAME=kimi
|
|
||||||
FALLBACK_BASE_URL=https://api.kimi.com/coding
|
|
||||||
FALLBACK_AUTH_TOKEN=...
|
|
||||||
FALLBACK_MODEL=kimi-k2.5
|
|
||||||
|
|
||||||
# 사용량 대시보드
|
# 사용량 대시보드
|
||||||
STATUS_CHANNEL_ID=... # 상태 업데이트 디스코드 채널
|
STATUS_CHANNEL_ID=... # 상태 업데이트 디스코드 채널
|
||||||
USAGE_DASHBOARD=true
|
USAGE_DASHBOARD=true
|
||||||
|
|||||||
5
.env.codex-review.example
Normal file
5
.env.codex-review.example
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
# ===== EJClaw Codex Review Service Environment =====
|
||||||
|
# Copy this file to .env.codex-review for the ejclaw-review service.
|
||||||
|
# This service needs its own Discord bot token.
|
||||||
|
|
||||||
|
DISCORD_BOT_TOKEN= # Discord bot token (Codex review bot)
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
# ===== EJClaw Environment Configuration =====
|
# ===== EJClaw Environment Configuration =====
|
||||||
# Copy this file to .env and fill in the values.
|
# Copy this file to .env and fill in the values.
|
||||||
# For dual-service (Codex), also create .env.codex with DISCORD_BOT_TOKEN for the Codex bot.
|
# For dual-service (Codex), also create .env.codex with DISCORD_BOT_TOKEN for the Codex bot.
|
||||||
|
# For review failover, create .env.codex-review with a third Discord bot token.
|
||||||
|
|
||||||
# --- Required ---
|
# --- Required ---
|
||||||
DISCORD_BOT_TOKEN= # Discord bot token (Claude bot)
|
DISCORD_BOT_TOKEN= # Discord bot token (Claude bot)
|
||||||
@@ -25,13 +26,6 @@ ASSISTANT_NAME=claude # Trigger name (@claude)
|
|||||||
GROQ_API_KEY= # Groq Whisper (primary, free: console.groq.com)
|
GROQ_API_KEY= # Groq Whisper (primary, free: console.groq.com)
|
||||||
# OPENAI_API_KEY also used as Whisper fallback
|
# OPENAI_API_KEY also used as Whisper fallback
|
||||||
|
|
||||||
# --- Provider fallback (optional — Claude 429 → alternative) ---
|
|
||||||
# FALLBACK_PROVIDER_NAME=kimi
|
|
||||||
# FALLBACK_BASE_URL=https://api.kimi.com/coding
|
|
||||||
# FALLBACK_AUTH_TOKEN=
|
|
||||||
# FALLBACK_MODEL=kimi-k2.5
|
|
||||||
# FALLBACK_COOLDOWN_MS=600000
|
|
||||||
|
|
||||||
# --- Usage dashboard (optional) ---
|
# --- Usage dashboard (optional) ---
|
||||||
# STATUS_CHANNEL_ID= # Discord channel for live status updates
|
# STATUS_CHANNEL_ID= # Discord channel for live status updates
|
||||||
# USAGE_DASHBOARD=true # Enable usage dashboard
|
# USAGE_DASHBOARD=true # Enable usage dashboard
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ Both share the same codebase (`dist/index.js`), differentiated by environment va
|
|||||||
- **Browser automation** — [gstack browse](https://github.com/garrytan/gstack) skill, headless Chromium daemon, ~100ms/command
|
- **Browser automation** — [gstack browse](https://github.com/garrytan/gstack) skill, headless Chromium daemon, ~100ms/command
|
||||||
- **Voice transcription** — Groq Whisper (primary) / OpenAI Whisper (fallback), shared file cache with dedup
|
- **Voice transcription** — Groq Whisper (primary) / OpenAI Whisper (fallback), shared file cache with dedup
|
||||||
- **Bidirectional images** — receive Discord attachments as multimodal input, send screenshots back
|
- **Bidirectional images** — receive Discord attachments as multimodal input, send screenshots back
|
||||||
- **Provider fallback** — Claude 429 → configurable fallback provider (e.g. Kimi K2.5) with cooldown
|
- **Token rotation** — Claude 429 / usage exhaustion → automatic multi-account rotation
|
||||||
- **CI monitoring** — `watch_ci` MCP tool for GitHub Actions run polling (structured fast path, no LLM token cost)
|
- **CI monitoring** — `watch_ci` MCP tool for GitHub Actions run polling (structured fast path, no LLM token cost)
|
||||||
- **Usage dashboard** — real-time token usage and service status overview
|
- **Usage dashboard** — real-time token usage and service status overview
|
||||||
- **MCP integration** — Memento (persistent memory) + EJClaw host tools (send_message, schedule_task, watch_ci, etc.)
|
- **MCP integration** — Memento (persistent memory) + EJClaw host tools (send_message, schedule_task, watch_ci, etc.)
|
||||||
@@ -82,12 +82,6 @@ CLAUDE_CODE_OAUTH_TOKEN= # Claude Code OAuth token (primary)
|
|||||||
CLAUDE_CODE_OAUTH_TOKENS= # Comma-separated tokens for multi-account rotation
|
CLAUDE_CODE_OAUTH_TOKENS= # Comma-separated tokens for multi-account rotation
|
||||||
OPENAI_API_KEY= # For Codex
|
OPENAI_API_KEY= # For Codex
|
||||||
GROQ_API_KEY= # Voice transcription (Groq Whisper)
|
GROQ_API_KEY= # Voice transcription (Groq Whisper)
|
||||||
|
|
||||||
# Provider fallback (optional)
|
|
||||||
FALLBACK_PROVIDER_NAME= # e.g. kimi
|
|
||||||
FALLBACK_BASE_URL= # e.g. https://api.kimi.com/coding
|
|
||||||
FALLBACK_AUTH_TOKEN= # Fallback provider API key
|
|
||||||
FALLBACK_MODEL= # e.g. kimi-k2.5
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Codex Service (optional)
|
### Codex Service (optional)
|
||||||
|
|||||||
10
prompts/codex-review-failover-paired-room.md
Normal file
10
prompts/codex-review-failover-paired-room.md
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
# Codex Review Failover Paired Room Rules
|
||||||
|
|
||||||
|
This room has `codex-review` acting as the owner-side failover agent and `codex-main` acting as the separate Codex reviewer.
|
||||||
|
|
||||||
|
- You are the owner for this chat while failover is active
|
||||||
|
- `codex-main` is a separate reviewer and is not you
|
||||||
|
- Keep collaboration with `codex-main` public when useful
|
||||||
|
- Evaluate reviewer feedback on its merits, but answer the user as the owner
|
||||||
|
- The visible bot name in history may differ from room to room; do not infer role from the visible name
|
||||||
|
- Use `<internal>` only for repetitive non-user-facing coordination noise
|
||||||
44
prompts/codex-review-failover-platform.md
Normal file
44
prompts/codex-review-failover-platform.md
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
# Codex Review Failover Rules
|
||||||
|
|
||||||
|
You are `codex-review`, and in this chat you are currently acting as `클코`, the owner-side failover agent.
|
||||||
|
|
||||||
|
## Role
|
||||||
|
|
||||||
|
- Answer as the primary owner for the chat, not as a reviewer
|
||||||
|
- Do not describe yourself as the review-side Codex service while this failover prompt is active
|
||||||
|
- `codex-main` is a separate Codex participant and is not you
|
||||||
|
- The visible bot name or nickname in chat history may vary by room; do not infer your role from the visible name
|
||||||
|
|
||||||
|
## Communication
|
||||||
|
|
||||||
|
Your output is sent directly to the user or Discord group.
|
||||||
|
|
||||||
|
You may use `<internal>` to suppress repetitive agent-to-agent noise.
|
||||||
|
Keep status updates, conclusions, and handoffs visible.
|
||||||
|
|
||||||
|
```text
|
||||||
|
<internal>Collected the reviewer notes and folded them into the final answer.</internal>
|
||||||
|
|
||||||
|
Here is the answer for the user...
|
||||||
|
```
|
||||||
|
|
||||||
|
Text inside `<internal>` tags is logged but not sent to the user.
|
||||||
|
|
||||||
|
Keep replies concise and owner-oriented.
|
||||||
|
|
||||||
|
- Respond directly to the user
|
||||||
|
- Give conclusions and concrete next steps
|
||||||
|
- Do not expose internal routing details unless they matter to the answer
|
||||||
|
- Do not claim to be `codex-main`
|
||||||
|
|
||||||
|
## Memory
|
||||||
|
|
||||||
|
The group folder may contain a `conversations/` directory with searchable history from earlier sessions. Use it when you need prior context.
|
||||||
|
|
||||||
|
## Message formatting
|
||||||
|
|
||||||
|
Do not use markdown headings in chat replies. Keep messages clean and readable for Discord.
|
||||||
|
|
||||||
|
- Use concise paragraphs or simple lists
|
||||||
|
- Use fenced code blocks when showing code
|
||||||
|
- Prefer plain links over markdown link syntax
|
||||||
14
prompts/codex-review-paired-room.md
Normal file
14
prompts/codex-review-paired-room.md
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# Codex Review Paired Room Rules
|
||||||
|
|
||||||
|
This service is the reviewer side when paired-room failover is active.
|
||||||
|
|
||||||
|
- Default stance: review, challenge, and verify.
|
||||||
|
- Do not mirror the owner's answer unless you are adding a concrete correction, risk, or missing prerequisite.
|
||||||
|
- Prioritize:
|
||||||
|
1. wrong root cause
|
||||||
|
2. unsafe operational advice
|
||||||
|
3. missing verification
|
||||||
|
4. hidden assumptions
|
||||||
|
- If the owner answer is fine, keep your message short or skip it entirely.
|
||||||
|
- If the turn provides a suppress token and you are only agreeing or rephrasing without a concrete correction, risk, prerequisite, test gap, or code change, output only that token and nothing else.
|
||||||
|
- When code changes are proposed, focus on bugs, regressions, and test gaps.
|
||||||
10
prompts/codex-review-platform.md
Normal file
10
prompts/codex-review-platform.md
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
# Codex Review Role
|
||||||
|
|
||||||
|
You are `코리뷰`, the review-side Codex service.
|
||||||
|
|
||||||
|
- Your job is to improve answer quality by challenging weak assumptions, spotting bugs, and tightening plans.
|
||||||
|
- Prefer concise technical criticism over parallel repetition.
|
||||||
|
- If the main Codex answer is already correct and sufficient, stay brief or silent.
|
||||||
|
- If the turn provides a suppress token and you have no concrete correction, risk, prerequisite, test gap, or code change, output only that token and nothing else.
|
||||||
|
- When you disagree, say exactly what is wrong and what should change.
|
||||||
|
- Focus on correctness, regressions, missing tests, and operational risk.
|
||||||
@@ -5,7 +5,7 @@ import {
|
|||||||
classifyClaudeAuthError,
|
classifyClaudeAuthError,
|
||||||
detectClaudeProviderFailureMessage,
|
detectClaudeProviderFailureMessage,
|
||||||
isClaudeOrgAccessDeniedMessage,
|
isClaudeOrgAccessDeniedMessage,
|
||||||
isNoFallbackCooldownReason,
|
|
||||||
shouldRotateClaudeToken,
|
shouldRotateClaudeToken,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
|
|
||||||
@@ -65,11 +65,4 @@ describe('agent-error-detection', () => {
|
|||||||
expect(shouldRotateClaudeToken('success-null-result')).toBe(false);
|
expect(shouldRotateClaudeToken('success-null-result')).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('marks only no-fallback cooldown reasons as skip-worthy', () => {
|
|
||||||
expect(isNoFallbackCooldownReason('usage-exhausted')).toBe(true);
|
|
||||||
expect(isNoFallbackCooldownReason('auth-expired')).toBe(true);
|
|
||||||
expect(isNoFallbackCooldownReason('org-access-denied')).toBe(true);
|
|
||||||
expect(isNoFallbackCooldownReason('429')).toBe(false);
|
|
||||||
expect(isNoFallbackCooldownReason('success-null-result')).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -103,21 +103,14 @@ export type AgentTriggerReason =
|
|||||||
| 'network-error'
|
| 'network-error'
|
||||||
| 'success-null-result';
|
| 'success-null-result';
|
||||||
|
|
||||||
export type FallbackTriggerReason = Exclude<
|
|
||||||
AgentTriggerReason,
|
|
||||||
'usage-exhausted' | 'success-null-result'
|
|
||||||
>;
|
|
||||||
|
|
||||||
export type ClaudeRotationReason = Extract<
|
export type ClaudeRotationReason = Extract<
|
||||||
AgentTriggerReason,
|
AgentTriggerReason,
|
||||||
'429' | 'usage-exhausted' | 'auth-expired' | 'org-access-denied'
|
'429' | 'usage-exhausted' | 'auth-expired' | 'org-access-denied'
|
||||||
>;
|
>;
|
||||||
|
|
||||||
export type CodexRotationReason = FallbackTriggerReason;
|
export type CodexRotationReason = Extract<
|
||||||
|
|
||||||
export type NoFallbackCooldownReason = Extract<
|
|
||||||
AgentTriggerReason,
|
AgentTriggerReason,
|
||||||
'usage-exhausted' | 'auth-expired' | 'org-access-denied'
|
'429' | 'auth-expired' | 'org-access-denied' | 'overloaded' | 'network-error'
|
||||||
>;
|
>;
|
||||||
|
|
||||||
export function shouldRotateClaudeToken(
|
export function shouldRotateClaudeToken(
|
||||||
@@ -131,14 +124,50 @@ export function shouldRotateClaudeToken(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isNoFallbackCooldownReason(
|
// ── Rotation trigger classification ─────────────────────────────
|
||||||
reason: AgentTriggerReason,
|
|
||||||
): reason is NoFallbackCooldownReason {
|
export type RotationTriggerResult =
|
||||||
return (
|
| { shouldRetry: false; reason: '' }
|
||||||
reason === 'usage-exhausted' ||
|
| {
|
||||||
reason === 'auth-expired' ||
|
shouldRetry: true;
|
||||||
reason === 'org-access-denied'
|
reason: AgentTriggerReason;
|
||||||
);
|
retryAfterMs?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify an error string to determine if Claude token rotation
|
||||||
|
* should be attempted.
|
||||||
|
*
|
||||||
|
* Priority: 429 > auth/org errors > 503/network.
|
||||||
|
*/
|
||||||
|
export function classifyRotationTrigger(
|
||||||
|
error?: string | null,
|
||||||
|
): RotationTriggerResult {
|
||||||
|
if (!error) return { shouldRetry: false, reason: '' };
|
||||||
|
|
||||||
|
const common = classifyAgentError(error);
|
||||||
|
|
||||||
|
// 429 rate-limit (highest priority)
|
||||||
|
if (common.category === 'rate-limit') {
|
||||||
|
return {
|
||||||
|
shouldRetry: true,
|
||||||
|
reason: common.reason,
|
||||||
|
retryAfterMs: common.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Claude-specific auth checks (before 503/network)
|
||||||
|
const auth = classifyClaudeAuthError(error);
|
||||||
|
if (auth.category !== 'none') {
|
||||||
|
return { shouldRetry: true, reason: auth.reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 503 overloaded, network errors
|
||||||
|
if (common.category !== 'none') {
|
||||||
|
return { shouldRetry: true, reason: common.reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { shouldRetry: false, reason: '' };
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Unified error classification ────────────────────────────────
|
// ── Unified error classification ────────────────────────────────
|
||||||
|
|||||||
@@ -9,8 +9,12 @@ const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
|
CODEX_REVIEW_SERVICE_ID: 'codex-review',
|
||||||
GROUPS_DIR: '/tmp/ejclaw-test-groups',
|
GROUPS_DIR: '/tmp/ejclaw-test-groups',
|
||||||
|
SERVICE_ID: 'codex-main',
|
||||||
|
SERVICE_SESSION_SCOPE: 'codex-main',
|
||||||
TIMEZONE: 'Asia/Seoul',
|
TIMEZONE: 'Asia/Seoul',
|
||||||
|
isReviewService: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./db.js', () => ({
|
vi.mock('./db.js', () => ({
|
||||||
@@ -35,17 +39,31 @@ vi.mock('./platform-prompts.js', () => ({
|
|||||||
readPairedRoomPrompt: vi.fn(() => 'paired room prompt'),
|
readPairedRoomPrompt: vi.fn(() => 'paired room prompt'),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./service-routing.js', () => ({
|
||||||
|
getEffectiveChannelLease: vi.fn(() => ({
|
||||||
|
chat_jid: 'dc:test',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock('./group-folder.js', () => ({
|
vi.mock('./group-folder.js', () => ({
|
||||||
resolveGroupFolderPath: (folder: string) =>
|
resolveGroupFolderPath: (folder: string) =>
|
||||||
`${process.env.EJ_TEST_ROOT}/groups/${folder}`,
|
`${process.env.EJ_TEST_ROOT}/groups/${folder}`,
|
||||||
resolveGroupIpcPath: (folder: string) =>
|
resolveGroupIpcPath: (folder: string) =>
|
||||||
`${process.env.EJ_TEST_ROOT}/ipc/${folder}`,
|
`${process.env.EJ_TEST_ROOT}/ipc/${folder}`,
|
||||||
resolveGroupSessionsPath: (folder: string) =>
|
resolveServiceGroupSessionsPath: (folder: string, serviceId: string) =>
|
||||||
`${process.env.EJ_TEST_ROOT}/sessions/${folder}`,
|
`${process.env.EJ_TEST_ROOT}/sessions/${folder}/services/${serviceId}`,
|
||||||
resolveTaskRuntimeIpcPath: (folder: string, taskId: string) =>
|
resolveTaskRuntimeIpcPath: (folder: string, taskId: string) =>
|
||||||
`${process.env.EJ_TEST_ROOT}/task-ipc/${folder}/${taskId}`,
|
`${process.env.EJ_TEST_ROOT}/task-ipc/${folder}/${taskId}`,
|
||||||
resolveTaskSessionsPath: (folder: string, taskId: string) =>
|
resolveServiceTaskSessionsPath: (
|
||||||
`${process.env.EJ_TEST_ROOT}/task-sessions/${folder}/${taskId}`,
|
folder: string,
|
||||||
|
serviceId: string,
|
||||||
|
taskId: string,
|
||||||
|
) => `${process.env.EJ_TEST_ROOT}/task-sessions/${folder}/${serviceId}/${taskId}`,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('os', async () => {
|
vi.mock('os', async () => {
|
||||||
@@ -61,6 +79,9 @@ vi.mock('os', async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
import { prepareGroupEnvironment } from './agent-runner-environment.js';
|
import { prepareGroupEnvironment } from './agent-runner-environment.js';
|
||||||
|
import * as config from './config.js';
|
||||||
|
import * as db from './db.js';
|
||||||
|
import * as serviceRouting from './service-routing.js';
|
||||||
import type { RegisteredGroup } from './types.js';
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
const group: RegisteredGroup = {
|
const group: RegisteredGroup = {
|
||||||
@@ -131,6 +152,8 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
tempRoot,
|
tempRoot,
|
||||||
'sessions',
|
'sessions',
|
||||||
group.folder,
|
group.folder,
|
||||||
|
'services',
|
||||||
|
'codex-main',
|
||||||
'.codex',
|
'.codex',
|
||||||
'auth.json',
|
'auth.json',
|
||||||
);
|
);
|
||||||
@@ -165,6 +188,8 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
tempRoot,
|
tempRoot,
|
||||||
'sessions',
|
'sessions',
|
||||||
group.folder,
|
group.folder,
|
||||||
|
'services',
|
||||||
|
'codex-main',
|
||||||
'.codex',
|
'.codex',
|
||||||
'auth.json',
|
'auth.json',
|
||||||
);
|
);
|
||||||
@@ -172,4 +197,118 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
|
|
||||||
expect(auth).toEqual(rotatedAuth);
|
expect(auth).toEqual(rotatedAuth);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('uses the failover prompt pack for codex-review when it owns an explicit failover lease', () => {
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
|
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
|
||||||
|
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||||
|
chat_jid: 'dc:test',
|
||||||
|
owner_service_id: 'codex-review',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
activated_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
reason: 'claude-429',
|
||||||
|
explicit: true,
|
||||||
|
});
|
||||||
|
mockReadEnvFile.mockReturnValue({});
|
||||||
|
|
||||||
|
const promptsDir = path.join(tempRoot, 'prompts');
|
||||||
|
fs.mkdirSync(promptsDir, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-platform.md'),
|
||||||
|
'review platform prompt\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-paired-room.md'),
|
||||||
|
'review paired prompt\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-failover-platform.md'),
|
||||||
|
'failover platform prompt\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-failover-paired-room.md'),
|
||||||
|
'failover paired prompt\n',
|
||||||
|
);
|
||||||
|
|
||||||
|
prepareGroupEnvironment(
|
||||||
|
{ ...group, workDir: path.join(tempRoot, 'workdir') },
|
||||||
|
false,
|
||||||
|
'dc:test',
|
||||||
|
{
|
||||||
|
memoryBriefing: 'memory briefing',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const agentsPath = path.join(
|
||||||
|
tempRoot,
|
||||||
|
'sessions',
|
||||||
|
group.folder,
|
||||||
|
'services',
|
||||||
|
'codex-main',
|
||||||
|
'.codex',
|
||||||
|
'AGENTS.md',
|
||||||
|
);
|
||||||
|
const agents = fs.readFileSync(agentsPath, 'utf-8');
|
||||||
|
const segments = agents.trim().split('\n\n---\n\n');
|
||||||
|
|
||||||
|
expect(segments).toEqual([
|
||||||
|
'failover platform prompt',
|
||||||
|
'failover paired prompt',
|
||||||
|
'memory briefing',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns to the normal review prompt stack after failover is cleared', () => {
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
|
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
|
||||||
|
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||||
|
chat_jid: 'dc:test',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
});
|
||||||
|
mockReadEnvFile.mockReturnValue({});
|
||||||
|
|
||||||
|
const promptsDir = path.join(tempRoot, 'prompts');
|
||||||
|
fs.mkdirSync(promptsDir, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-platform.md'),
|
||||||
|
'review platform prompt\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-paired-room.md'),
|
||||||
|
'review paired prompt\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-failover-platform.md'),
|
||||||
|
'failover platform prompt\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(promptsDir, 'codex-review-failover-paired-room.md'),
|
||||||
|
'failover paired prompt\n',
|
||||||
|
);
|
||||||
|
|
||||||
|
prepareGroupEnvironment(group, false, 'dc:test');
|
||||||
|
|
||||||
|
const agentsPath = path.join(
|
||||||
|
tempRoot,
|
||||||
|
'sessions',
|
||||||
|
group.folder,
|
||||||
|
'services',
|
||||||
|
'codex-main',
|
||||||
|
'.codex',
|
||||||
|
'AGENTS.md',
|
||||||
|
);
|
||||||
|
const agents = fs.readFileSync(agentsPath, 'utf-8');
|
||||||
|
const segments = agents.trim().split('\n\n---\n\n');
|
||||||
|
|
||||||
|
expect(segments).toEqual([
|
||||||
|
'platform prompt',
|
||||||
|
'review platform prompt',
|
||||||
|
'paired room prompt',
|
||||||
|
'review paired prompt',
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,7 +2,14 @@ import fs from 'fs';
|
|||||||
import os from 'os';
|
import os from 'os';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { GROUPS_DIR, TIMEZONE } from './config.js';
|
import {
|
||||||
|
CODEX_REVIEW_SERVICE_ID,
|
||||||
|
GROUPS_DIR,
|
||||||
|
SERVICE_ID,
|
||||||
|
SERVICE_SESSION_SCOPE,
|
||||||
|
TIMEZONE,
|
||||||
|
isReviewService,
|
||||||
|
} from './config.js';
|
||||||
import { isPairedRoomJid } from './db.js';
|
import { isPairedRoomJid } from './db.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { readEnvFile } from './env.js';
|
||||||
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
||||||
@@ -10,14 +17,15 @@ import { getCurrentToken } from './token-rotation.js';
|
|||||||
import {
|
import {
|
||||||
resolveGroupFolderPath,
|
resolveGroupFolderPath,
|
||||||
resolveGroupIpcPath,
|
resolveGroupIpcPath,
|
||||||
resolveGroupSessionsPath,
|
resolveServiceGroupSessionsPath,
|
||||||
resolveTaskRuntimeIpcPath,
|
resolveTaskRuntimeIpcPath,
|
||||||
resolveTaskSessionsPath,
|
resolveServiceTaskSessionsPath,
|
||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import {
|
import {
|
||||||
readPairedRoomPrompt,
|
readPairedRoomPrompt,
|
||||||
readPlatformPrompt,
|
readPlatformPrompt,
|
||||||
} from './platform-prompts.js';
|
} from './platform-prompts.js';
|
||||||
|
import { getEffectiveChannelLease } from './service-routing.js';
|
||||||
import type { AgentType, RegisteredGroup } from './types.js';
|
import type { AgentType, RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
// writeCodexApiKeyAuth removed — Codex uses OAuth only.
|
// writeCodexApiKeyAuth removed — Codex uses OAuth only.
|
||||||
@@ -39,6 +47,16 @@ function syncDirectoryEntries(sources: string[], destination: string): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readOptionalPromptFile(
|
||||||
|
projectRoot: string,
|
||||||
|
filename: string,
|
||||||
|
): string | undefined {
|
||||||
|
const promptPath = path.join(projectRoot, 'prompts', filename);
|
||||||
|
if (!fs.existsSync(promptPath)) return undefined;
|
||||||
|
const prompt = fs.readFileSync(promptPath, 'utf-8').trim();
|
||||||
|
return prompt || undefined;
|
||||||
|
}
|
||||||
|
|
||||||
function ensureClaudeSessionSettings(groupSessionsDir: string): void {
|
function ensureClaudeSessionSettings(groupSessionsDir: string): void {
|
||||||
const settingsFile = path.join(groupSessionsDir, 'settings.json');
|
const settingsFile = path.join(groupSessionsDir, 'settings.json');
|
||||||
if (fs.existsSync(settingsFile)) return;
|
if (fs.existsSync(settingsFile)) return;
|
||||||
@@ -175,6 +193,7 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
chatJid: string;
|
chatJid: string;
|
||||||
isMain: boolean;
|
isMain: boolean;
|
||||||
isPairedRoom: boolean;
|
isPairedRoom: boolean;
|
||||||
|
useFailoverPromptPack: boolean;
|
||||||
memoryBriefing?: string;
|
memoryBriefing?: string;
|
||||||
}): void {
|
}): void {
|
||||||
// API key auth intentionally removed — Codex uses OAuth only.
|
// API key auth intentionally removed — Codex uses OAuth only.
|
||||||
@@ -236,13 +255,38 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const sessionAgentsPath = path.join(sessionCodexDir, 'AGENTS.md');
|
const sessionAgentsPath = path.join(sessionCodexDir, 'AGENTS.md');
|
||||||
const sessionAgents = [
|
const sessionAgents = (
|
||||||
readPlatformPrompt('codex', args.projectRoot),
|
args.useFailoverPromptPack
|
||||||
args.isPairedRoom
|
? [
|
||||||
? readPairedRoomPrompt('codex', args.projectRoot)
|
readOptionalPromptFile(
|
||||||
: undefined,
|
args.projectRoot,
|
||||||
args.memoryBriefing,
|
'codex-review-failover-platform.md',
|
||||||
]
|
),
|
||||||
|
args.isPairedRoom
|
||||||
|
? readOptionalPromptFile(
|
||||||
|
args.projectRoot,
|
||||||
|
'codex-review-failover-paired-room.md',
|
||||||
|
)
|
||||||
|
: undefined,
|
||||||
|
args.memoryBriefing,
|
||||||
|
]
|
||||||
|
: [
|
||||||
|
readPlatformPrompt('codex', args.projectRoot),
|
||||||
|
isReviewService(SERVICE_ID)
|
||||||
|
? readOptionalPromptFile(args.projectRoot, 'codex-review-platform.md')
|
||||||
|
: undefined,
|
||||||
|
args.isPairedRoom
|
||||||
|
? readPairedRoomPrompt('codex', args.projectRoot)
|
||||||
|
: undefined,
|
||||||
|
args.isPairedRoom && isReviewService(SERVICE_ID)
|
||||||
|
? readOptionalPromptFile(
|
||||||
|
args.projectRoot,
|
||||||
|
'codex-review-paired-room.md',
|
||||||
|
)
|
||||||
|
: undefined,
|
||||||
|
args.memoryBriefing,
|
||||||
|
]
|
||||||
|
)
|
||||||
.filter((value): value is string => Boolean(value))
|
.filter((value): value is string => Boolean(value))
|
||||||
.join('\n\n---\n\n')
|
.join('\n\n---\n\n')
|
||||||
.trim();
|
.trim();
|
||||||
@@ -342,8 +386,12 @@ export function prepareGroupEnvironment(
|
|||||||
options?.useTaskScopedSession === true && Boolean(runtimeTaskId);
|
options?.useTaskScopedSession === true && Boolean(runtimeTaskId);
|
||||||
const sessionRootDir =
|
const sessionRootDir =
|
||||||
runtimeTaskId && useTaskScopedSession
|
runtimeTaskId && useTaskScopedSession
|
||||||
? resolveTaskSessionsPath(group.folder, runtimeTaskId)
|
? resolveServiceTaskSessionsPath(
|
||||||
: resolveGroupSessionsPath(group.folder);
|
group.folder,
|
||||||
|
SERVICE_SESSION_SCOPE,
|
||||||
|
runtimeTaskId,
|
||||||
|
)
|
||||||
|
: resolveServiceGroupSessionsPath(group.folder, SERVICE_SESSION_SCOPE);
|
||||||
|
|
||||||
const groupSessionsDir = path.join(sessionRootDir, '.claude');
|
const groupSessionsDir = path.join(sessionRootDir, '.claude');
|
||||||
fs.mkdirSync(groupSessionsDir, { recursive: true });
|
fs.mkdirSync(groupSessionsDir, { recursive: true });
|
||||||
@@ -370,6 +418,11 @@ export function prepareGroupEnvironment(
|
|||||||
const globalDir = path.join(GROUPS_DIR, 'global');
|
const globalDir = path.join(GROUPS_DIR, 'global');
|
||||||
const globalClaudeMdPath = path.join(globalDir, 'CLAUDE.md');
|
const globalClaudeMdPath = path.join(globalDir, 'CLAUDE.md');
|
||||||
const isPairedRoom = isPairedRoomJid(chatJid);
|
const isPairedRoom = isPairedRoomJid(chatJid);
|
||||||
|
const effectiveLease = getEffectiveChannelLease(chatJid);
|
||||||
|
const useCodexReviewFailoverPromptPack =
|
||||||
|
isReviewService(SERVICE_ID) &&
|
||||||
|
effectiveLease.explicit &&
|
||||||
|
effectiveLease.owner_service_id === CODEX_REVIEW_SERVICE_ID;
|
||||||
|
|
||||||
const claudePlatformPrompt = readPlatformPrompt('claude-code', projectRoot);
|
const claudePlatformPrompt = readPlatformPrompt('claude-code', projectRoot);
|
||||||
const claudePairedRoomPrompt = isPairedRoom
|
const claudePairedRoomPrompt = isPairedRoom
|
||||||
@@ -440,6 +493,7 @@ export function prepareGroupEnvironment(
|
|||||||
chatJid,
|
chatJid,
|
||||||
isMain,
|
isMain,
|
||||||
isPairedRoom,
|
isPairedRoom,
|
||||||
|
useFailoverPromptPack: useCodexReviewFailoverPromptPack,
|
||||||
memoryBriefing: options?.memoryBriefing,
|
memoryBriefing: options?.memoryBriefing,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -13,7 +13,10 @@ vi.mock('./config.js', () => ({
|
|||||||
DATA_DIR: '/tmp/ejclaw-test-data',
|
DATA_DIR: '/tmp/ejclaw-test-data',
|
||||||
GROUPS_DIR: '/tmp/ejclaw-test-groups',
|
GROUPS_DIR: '/tmp/ejclaw-test-groups',
|
||||||
IDLE_TIMEOUT: 1800000, // 30min
|
IDLE_TIMEOUT: 1800000, // 30min
|
||||||
|
SERVICE_ID: 'claude',
|
||||||
|
SERVICE_SESSION_SCOPE: 'claude',
|
||||||
TIMEZONE: 'America/Los_Angeles',
|
TIMEZONE: 'America/Los_Angeles',
|
||||||
|
isReviewService: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Mock logger
|
// Mock logger
|
||||||
@@ -55,6 +58,17 @@ vi.mock('./env.js', () => ({
|
|||||||
getEnv: vi.fn(() => undefined),
|
getEnv: vi.fn(() => undefined),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./service-routing.js', () => ({
|
||||||
|
getEffectiveChannelLease: vi.fn(() => ({
|
||||||
|
chat_jid: 'test@g.us',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
// Create a controllable fake ChildProcess
|
// Create a controllable fake ChildProcess
|
||||||
function createFakeProcess() {
|
function createFakeProcess() {
|
||||||
const proc = new EventEmitter() as EventEmitter & {
|
const proc = new EventEmitter() as EventEmitter & {
|
||||||
@@ -328,7 +342,7 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
'/tmp/ejclaw-test-data/ipc/test-group',
|
'/tmp/ejclaw-test-data/ipc/test-group',
|
||||||
);
|
);
|
||||||
expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe(
|
expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe(
|
||||||
'/tmp/ejclaw-test-data/sessions/test-group/tasks/task-123/.claude',
|
'/tmp/ejclaw-test-data/sessions/test-group/services/claude/tasks/task-123/.claude',
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -407,7 +421,7 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
'/tmp/ejclaw-test-data/ipc/test-group',
|
'/tmp/ejclaw-test-data/ipc/test-group',
|
||||||
);
|
);
|
||||||
expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe(
|
expect(spawnEnv?.CLAUDE_CONFIG_DIR).toBe(
|
||||||
'/tmp/ejclaw-test-data/sessions/test-group/.claude',
|
'/tmp/ejclaw-test-data/sessions/test-group/services/claude/.claude',
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -416,7 +430,7 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
fakeProc = createFakeProcess();
|
fakeProc = createFakeProcess();
|
||||||
const overlayPath = '/tmp/ejclaw-test-groups/test-group/.codex/config.toml';
|
const overlayPath = '/tmp/ejclaw-test-groups/test-group/.codex/config.toml';
|
||||||
const sessionConfigPath =
|
const sessionConfigPath =
|
||||||
'/tmp/ejclaw-test-data/sessions/test-group/.codex/config.toml';
|
'/tmp/ejclaw-test-data/sessions/test-group/services/claude/.codex/config.toml';
|
||||||
let sessionToml = `model = "gpt-5.4"\n`;
|
let sessionToml = `model = "gpt-5.4"\n`;
|
||||||
|
|
||||||
vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => {
|
vi.mocked(fs.existsSync).mockImplementation((p: fs.PathLike) => {
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ export async function runAgentProcess(
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
// Apply provider fallback overrides (e.g. Kimi env vars when Claude is in cooldown)
|
// Apply env overrides (caller-provided)
|
||||||
if (envOverrides) {
|
if (envOverrides) {
|
||||||
for (const [key, value] of Object.entries(envOverrides)) {
|
for (const [key, value] of Object.entries(envOverrides)) {
|
||||||
if (value) env[key] = value;
|
if (value) env[key] = value;
|
||||||
|
|||||||
@@ -870,6 +870,31 @@ describe('DiscordChannel', () => {
|
|||||||
expect(currentClient().channels.fetch).not.toHaveBeenCalled();
|
expect(currentClient().channels.fetch).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not send stale typing after typing was disabled mid-flight', async () => {
|
||||||
|
const opts = createTestOpts();
|
||||||
|
const channel = new DiscordChannel('test-token', opts);
|
||||||
|
await channel.connect();
|
||||||
|
|
||||||
|
let resolveFetch!: (value: unknown) => void;
|
||||||
|
const fetchPromise = new Promise((resolve) => {
|
||||||
|
resolveFetch = resolve;
|
||||||
|
});
|
||||||
|
const mockChannel = {
|
||||||
|
send: vi.fn(),
|
||||||
|
sendTyping: vi.fn().mockResolvedValue(undefined),
|
||||||
|
};
|
||||||
|
|
||||||
|
currentClient().channels.fetch.mockImplementationOnce(() => fetchPromise);
|
||||||
|
|
||||||
|
const typingPromise = channel.setTyping('dc:1234567890123456', true);
|
||||||
|
await Promise.resolve();
|
||||||
|
await channel.setTyping('dc:1234567890123456', false);
|
||||||
|
resolveFetch(mockChannel);
|
||||||
|
await typingPromise;
|
||||||
|
|
||||||
|
expect(mockChannel.sendTyping).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it('does nothing when client is not initialized', async () => {
|
it('does nothing when client is not initialized', async () => {
|
||||||
const opts = createTestOpts();
|
const opts = createTestOpts();
|
||||||
const channel = new DiscordChannel('test-token', opts);
|
const channel = new DiscordChannel('test-token', opts);
|
||||||
|
|||||||
@@ -177,6 +177,7 @@ export class DiscordChannel implements Channel {
|
|||||||
private opts: DiscordChannelOpts;
|
private opts: DiscordChannelOpts;
|
||||||
private botToken: string;
|
private botToken: string;
|
||||||
private typingIntervals = new Map<string, NodeJS.Timeout>();
|
private typingIntervals = new Map<string, NodeJS.Timeout>();
|
||||||
|
private typingGenerations = new Map<string, number>();
|
||||||
private agentTypeFilter?: AgentType;
|
private agentTypeFilter?: AgentType;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@@ -539,6 +540,9 @@ export class DiscordChannel implements Channel {
|
|||||||
async setTyping(jid: string, isTyping: boolean): Promise<void> {
|
async setTyping(jid: string, isTyping: boolean): Promise<void> {
|
||||||
if (!this.client) return;
|
if (!this.client) return;
|
||||||
|
|
||||||
|
const generation = (this.typingGenerations.get(jid) ?? 0) + 1;
|
||||||
|
this.typingGenerations.set(jid, generation);
|
||||||
|
|
||||||
// Clear any existing interval for this channel
|
// Clear any existing interval for this channel
|
||||||
const existing = this.typingIntervals.get(jid);
|
const existing = this.typingIntervals.get(jid);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
@@ -548,10 +552,15 @@ export class DiscordChannel implements Channel {
|
|||||||
|
|
||||||
if (!isTyping) return;
|
if (!isTyping) return;
|
||||||
|
|
||||||
|
const isCurrentGeneration = () =>
|
||||||
|
this.typingGenerations.get(jid) === generation;
|
||||||
|
|
||||||
const sendOnce = async () => {
|
const sendOnce = async () => {
|
||||||
|
if (!isCurrentGeneration()) return;
|
||||||
try {
|
try {
|
||||||
const channelId = jid.replace(/^dc:/, '');
|
const channelId = jid.replace(/^dc:/, '');
|
||||||
const channel = await this.client!.channels.fetch(channelId);
|
const channel = await this.client!.channels.fetch(channelId);
|
||||||
|
if (!isCurrentGeneration()) return;
|
||||||
if (channel && 'sendTyping' in channel) {
|
if (channel && 'sendTyping' in channel) {
|
||||||
await (channel as TextChannel).sendTyping();
|
await (channel as TextChannel).sendTyping();
|
||||||
}
|
}
|
||||||
@@ -562,7 +571,13 @@ export class DiscordChannel implements Channel {
|
|||||||
|
|
||||||
// Send immediately, then refresh every 8 seconds (Discord expires at ~10s)
|
// Send immediately, then refresh every 8 seconds (Discord expires at ~10s)
|
||||||
await sendOnce();
|
await sendOnce();
|
||||||
this.typingIntervals.set(jid, setInterval(sendOnce, 8000));
|
if (!isCurrentGeneration()) return;
|
||||||
|
this.typingIntervals.set(
|
||||||
|
jid,
|
||||||
|
setInterval(() => {
|
||||||
|
void sendOnce();
|
||||||
|
}, 8000),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async sendAndTrack(jid: string, text: string): Promise<string | null> {
|
async sendAndTrack(jid: string, text: string): Promise<string | null> {
|
||||||
|
|||||||
@@ -10,15 +10,44 @@ export const ASSISTANT_HAS_OWN_NUMBER =
|
|||||||
const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase();
|
const ASSISTANT_SLUG = ASSISTANT_NAME.trim().toLowerCase();
|
||||||
const rawServiceAgentType = getEnv('SERVICE_AGENT_TYPE');
|
const rawServiceAgentType = getEnv('SERVICE_AGENT_TYPE');
|
||||||
export const SERVICE_ID = getEnv('SERVICE_ID') || ASSISTANT_SLUG;
|
export const SERVICE_ID = getEnv('SERVICE_ID') || ASSISTANT_SLUG;
|
||||||
|
export const CLAUDE_SERVICE_ID = getEnv('CLAUDE_SERVICE_ID') || 'claude';
|
||||||
|
export const CODEX_MAIN_SERVICE_ID =
|
||||||
|
getEnv('CODEX_MAIN_SERVICE_ID') || 'codex-main';
|
||||||
|
export const CODEX_REVIEW_SERVICE_ID =
|
||||||
|
getEnv('CODEX_REVIEW_SERVICE_ID') || 'codex-review';
|
||||||
export const SERVICE_AGENT_TYPE: AgentType =
|
export const SERVICE_AGENT_TYPE: AgentType =
|
||||||
rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code'
|
rawServiceAgentType === 'codex' || rawServiceAgentType === 'claude-code'
|
||||||
? rawServiceAgentType
|
? rawServiceAgentType
|
||||||
: ASSISTANT_SLUG === 'codex'
|
: ASSISTANT_SLUG === 'codex'
|
||||||
? 'codex'
|
? 'codex'
|
||||||
: 'claude-code';
|
: 'claude-code';
|
||||||
|
|
||||||
|
export function normalizeServiceId(serviceId: string): string {
|
||||||
|
if (serviceId === 'codex') {
|
||||||
|
return CODEX_MAIN_SERVICE_ID;
|
||||||
|
}
|
||||||
|
return serviceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const SERVICE_SESSION_SCOPE = normalizeServiceId(SERVICE_ID);
|
||||||
|
|
||||||
|
export function isClaudeService(serviceId: string = SERVICE_ID): boolean {
|
||||||
|
return normalizeServiceId(serviceId) === CLAUDE_SERVICE_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isCodexMainService(serviceId: string = SERVICE_ID): boolean {
|
||||||
|
return normalizeServiceId(serviceId) === CODEX_MAIN_SERVICE_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isReviewService(serviceId: string = SERVICE_ID): boolean {
|
||||||
|
return normalizeServiceId(serviceId) === CODEX_REVIEW_SERVICE_ID;
|
||||||
|
}
|
||||||
export const POLL_INTERVAL = 2000;
|
export const POLL_INTERVAL = 2000;
|
||||||
export const SCHEDULER_POLL_INTERVAL = 60000;
|
export const SCHEDULER_POLL_INTERVAL = 60000;
|
||||||
|
|
||||||
|
/** Minimum time (ms) a failover lease stays active before Claude can reclaim it. */
|
||||||
|
export const FAILOVER_MIN_DURATION_MS = 3 * 60 * 60 * 1000; // 3 hours
|
||||||
|
|
||||||
const PROJECT_ROOT = process.cwd();
|
const PROJECT_ROOT = process.cwd();
|
||||||
const HOME_DIR = process.env.HOME || os.homedir();
|
const HOME_DIR = process.env.HOME || os.homedir();
|
||||||
|
|
||||||
@@ -106,3 +135,26 @@ const SESSION_COMMAND_ALLOWED_SENDERS = new Set(
|
|||||||
export function isSessionCommandSenderAllowed(sender: string): boolean {
|
export function isSessionCommandSenderAllowed(sender: string): boolean {
|
||||||
return SESSION_COMMAND_ALLOWED_SENDERS.has(sender);
|
return SESSION_COMMAND_ALLOWED_SENDERS.has(sender);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delta handoff: cross-provider session continuity with probe + fallback
|
||||||
|
export const DELTA_HANDOFF_ENABLED =
|
||||||
|
getEnv('DELTA_HANDOFF_ENABLED') === 'true';
|
||||||
|
|
||||||
|
// Comma-separated list of group folders for canary testing
|
||||||
|
const rawDeltaHandoffCanaryGroups =
|
||||||
|
getEnv('DELTA_HANDOFF_CANARY_GROUPS') || '';
|
||||||
|
export const DELTA_HANDOFF_CANARY_GROUPS = new Set(
|
||||||
|
rawDeltaHandoffCanaryGroups
|
||||||
|
.split(',')
|
||||||
|
.map((v) => v.trim())
|
||||||
|
.filter(Boolean),
|
||||||
|
);
|
||||||
|
|
||||||
|
export function isDeltaHandoffEnabledForGroup(groupFolder: string): boolean {
|
||||||
|
if (!DELTA_HANDOFF_ENABLED) return false;
|
||||||
|
// If canary groups are specified, only enable for those groups
|
||||||
|
if (DELTA_HANDOFF_CANARY_GROUPS.size > 0) {
|
||||||
|
return DELTA_HANDOFF_CANARY_GROUPS.has(groupFolder);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ describe('extractCodexUsageRows staleness', () => {
|
|||||||
it('returns real rows when usageRowsFetchedAt is within maxAgeMs', () => {
|
it('returns real rows when usageRowsFetchedAt is within maxAgeMs', () => {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const snapshot: StatusSnapshot = {
|
const snapshot: StatusSnapshot = {
|
||||||
|
serviceId: 'codex-main',
|
||||||
agentType: 'codex',
|
agentType: 'codex',
|
||||||
assistantName: 'test',
|
assistantName: 'test',
|
||||||
updatedAt: new Date(now).toISOString(),
|
updatedAt: new Date(now).toISOString(),
|
||||||
@@ -40,6 +41,7 @@ describe('extractCodexUsageRows staleness', () => {
|
|||||||
it('returns degraded row when usageRowsFetchedAt exceeds maxAgeMs', () => {
|
it('returns degraded row when usageRowsFetchedAt exceeds maxAgeMs', () => {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const snapshot: StatusSnapshot = {
|
const snapshot: StatusSnapshot = {
|
||||||
|
serviceId: 'codex-main',
|
||||||
agentType: 'codex',
|
agentType: 'codex',
|
||||||
assistantName: 'test',
|
assistantName: 'test',
|
||||||
updatedAt: new Date(now).toISOString(), // heartbeat is fresh
|
updatedAt: new Date(now).toISOString(), // heartbeat is fresh
|
||||||
@@ -57,6 +59,7 @@ describe('extractCodexUsageRows staleness', () => {
|
|||||||
it('returns degraded row when usageRowsFetchedAt is missing', () => {
|
it('returns degraded row when usageRowsFetchedAt is missing', () => {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const snapshot: StatusSnapshot = {
|
const snapshot: StatusSnapshot = {
|
||||||
|
serviceId: 'codex-main',
|
||||||
agentType: 'codex',
|
agentType: 'codex',
|
||||||
assistantName: 'test',
|
assistantName: 'test',
|
||||||
updatedAt: new Date(now).toISOString(),
|
updatedAt: new Date(now).toISOString(),
|
||||||
@@ -77,6 +80,7 @@ describe('extractCodexUsageRows staleness', () => {
|
|||||||
|
|
||||||
it('returns empty array when usageRows is empty', () => {
|
it('returns empty array when usageRows is empty', () => {
|
||||||
const snapshot: StatusSnapshot = {
|
const snapshot: StatusSnapshot = {
|
||||||
|
serviceId: 'codex-main',
|
||||||
agentType: 'codex',
|
agentType: 'codex',
|
||||||
assistantName: 'test',
|
assistantName: 'test',
|
||||||
updatedAt: new Date().toISOString(),
|
updatedAt: new Date().toISOString(),
|
||||||
|
|||||||
@@ -45,9 +45,8 @@ describe('buildStatusContent', () => {
|
|||||||
): Promise<(opts: DashboardOptions) => string> {
|
): Promise<(opts: DashboardOptions) => string> {
|
||||||
vi.resetModules();
|
vi.resetModules();
|
||||||
process.env.STATUS_SHOW_ROOMS = options.statusShowRooms ?? 'true';
|
process.env.STATUS_SHOW_ROOMS = options.statusShowRooms ?? 'true';
|
||||||
if (options.statusShowRoomDetails !== undefined) {
|
process.env.STATUS_SHOW_ROOM_DETAILS =
|
||||||
process.env.STATUS_SHOW_ROOM_DETAILS = options.statusShowRoomDetails;
|
options.statusShowRoomDetails ?? 'true';
|
||||||
}
|
|
||||||
const mod = await import('./dashboard-status-content.js');
|
const mod = await import('./dashboard-status-content.js');
|
||||||
return mod.buildStatusContent;
|
return mod.buildStatusContent;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ export async function startStatusDashboard(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await startUnifiedDashboard({
|
await startUnifiedDashboard({
|
||||||
assistantName: opts.assistantName,
|
assistantName: opts.assistantName,
|
||||||
|
serviceId: opts.serviceAgentType === 'codex' ? 'codex-main' : 'claude',
|
||||||
serviceAgentType: opts.serviceAgentType || 'claude-code',
|
serviceAgentType: opts.serviceAgentType || 'claude-code',
|
||||||
statusChannelId: opts.statusChannelId,
|
statusChannelId: opts.statusChannelId,
|
||||||
statusUpdateInterval: opts.statusUpdateInterval,
|
statusUpdateInterval: opts.statusUpdateInterval,
|
||||||
|
|||||||
@@ -4,7 +4,10 @@ import { describe, it, expect, beforeEach } from 'vitest';
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
_initTestDatabase,
|
_initTestDatabase,
|
||||||
|
claimServiceHandoff,
|
||||||
|
completeServiceHandoffAndAdvanceTargetCursor,
|
||||||
createTask,
|
createTask,
|
||||||
|
createServiceHandoff,
|
||||||
createProducedWorkItem,
|
createProducedWorkItem,
|
||||||
deleteSession,
|
deleteSession,
|
||||||
deleteTask,
|
deleteTask,
|
||||||
@@ -15,9 +18,11 @@ import {
|
|||||||
getMessagesSinceSeq,
|
getMessagesSinceSeq,
|
||||||
getNewMessagesBySeq,
|
getNewMessagesBySeq,
|
||||||
getOpenWorkItem,
|
getOpenWorkItem,
|
||||||
|
getPendingServiceHandoffs,
|
||||||
getRegisteredAgentTypesForJid,
|
getRegisteredAgentTypesForJid,
|
||||||
getMessagesSince,
|
getMessagesSince,
|
||||||
getNewMessages,
|
getNewMessages,
|
||||||
|
getRouterStateForService,
|
||||||
isPairedRoomJid,
|
isPairedRoomJid,
|
||||||
getSession,
|
getSession,
|
||||||
getTaskById,
|
getTaskById,
|
||||||
@@ -25,6 +30,7 @@ import {
|
|||||||
markWorkItemDeliveryRetry,
|
markWorkItemDeliveryRetry,
|
||||||
setSession,
|
setSession,
|
||||||
setRegisteredGroup,
|
setRegisteredGroup,
|
||||||
|
setRouterStateForService,
|
||||||
storeChatMetadata,
|
storeChatMetadata,
|
||||||
storeMessage,
|
storeMessage,
|
||||||
updateTask,
|
updateTask,
|
||||||
@@ -678,6 +684,84 @@ describe('paired room registration', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('service handoff completion', () => {
|
||||||
|
it('atomically completes the handoff and advances the target cursor', () => {
|
||||||
|
storeChatMetadata('dc:handoff', '2024-01-01T00:00:00.000Z');
|
||||||
|
store({
|
||||||
|
id: 'handoff-msg-1',
|
||||||
|
chat_jid: 'dc:handoff',
|
||||||
|
sender: 'user',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2024-01-01T00:00:01.000Z',
|
||||||
|
});
|
||||||
|
const handoff = createServiceHandoff({
|
||||||
|
chat_jid: 'dc:handoff',
|
||||||
|
group_folder: 'test-group',
|
||||||
|
source_service_id: 'claude',
|
||||||
|
target_service_id: 'codex-review',
|
||||||
|
target_agent_type: 'codex',
|
||||||
|
prompt: 'hello',
|
||||||
|
end_seq: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(claimServiceHandoff(handoff.id)).toBe(true);
|
||||||
|
|
||||||
|
const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({
|
||||||
|
id: handoff.id,
|
||||||
|
target_service_id: 'codex-review',
|
||||||
|
chat_jid: 'dc:handoff',
|
||||||
|
end_seq: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(appliedCursor).toBe('1');
|
||||||
|
expect(getPendingServiceHandoffs('codex-review')).toEqual([]);
|
||||||
|
expect(
|
||||||
|
JSON.parse(
|
||||||
|
getRouterStateForService('last_agent_seq', 'codex-review') || '{}',
|
||||||
|
),
|
||||||
|
).toMatchObject({
|
||||||
|
'dc:handoff': '1',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not move the target cursor backwards when a newer cursor already exists', () => {
|
||||||
|
storeChatMetadata('dc:handoff', '2024-01-01T00:00:00.000Z');
|
||||||
|
setRouterStateForService(
|
||||||
|
'last_agent_seq',
|
||||||
|
JSON.stringify({ 'dc:handoff': '5' }),
|
||||||
|
'codex-review',
|
||||||
|
);
|
||||||
|
const handoff = createServiceHandoff({
|
||||||
|
chat_jid: 'dc:handoff',
|
||||||
|
group_folder: 'test-group',
|
||||||
|
source_service_id: 'claude',
|
||||||
|
target_service_id: 'codex-review',
|
||||||
|
target_agent_type: 'codex',
|
||||||
|
prompt: 'hello',
|
||||||
|
end_seq: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(claimServiceHandoff(handoff.id)).toBe(true);
|
||||||
|
|
||||||
|
const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({
|
||||||
|
id: handoff.id,
|
||||||
|
target_service_id: 'codex-review',
|
||||||
|
chat_jid: 'dc:handoff',
|
||||||
|
end_seq: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(appliedCursor).toBe('5');
|
||||||
|
expect(
|
||||||
|
JSON.parse(
|
||||||
|
getRouterStateForService('last_agent_seq', 'codex-review') || '{}',
|
||||||
|
),
|
||||||
|
).toMatchObject({
|
||||||
|
'dc:handoff': '5',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('message seq cursors', () => {
|
describe('message seq cursors', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
storeChatMetadata('group@g.us', '2024-01-01T00:00:00.000Z');
|
storeChatMetadata('group@g.us', '2024-01-01T00:00:00.000Z');
|
||||||
|
|||||||
519
src/db.ts
519
src/db.ts
@@ -4,14 +4,20 @@ import path from 'path';
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
ASSISTANT_NAME,
|
ASSISTANT_NAME,
|
||||||
|
CLAUDE_SERVICE_ID,
|
||||||
|
CODEX_MAIN_SERVICE_ID,
|
||||||
|
CODEX_REVIEW_SERVICE_ID,
|
||||||
DATA_DIR,
|
DATA_DIR,
|
||||||
|
normalizeServiceId,
|
||||||
SERVICE_AGENT_TYPE,
|
SERVICE_AGENT_TYPE,
|
||||||
SERVICE_ID,
|
SERVICE_ID,
|
||||||
|
SERVICE_SESSION_SCOPE,
|
||||||
STORE_DIR,
|
STORE_DIR,
|
||||||
} from './config.js';
|
} from './config.js';
|
||||||
import {
|
import {
|
||||||
isValidGroupFolder,
|
isValidGroupFolder,
|
||||||
resolveTaskRuntimeIpcPath as resolveTaskRuntimeIpcPathFromGroup,
|
resolveTaskRuntimeIpcPath as resolveTaskRuntimeIpcPathFromGroup,
|
||||||
|
resolveServiceTaskSessionsPath as resolveServiceTaskSessionsPathFromGroup,
|
||||||
resolveTaskSessionsPath as resolveTaskSessionsPathFromGroup,
|
resolveTaskSessionsPath as resolveTaskSessionsPathFromGroup,
|
||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
@@ -32,6 +38,7 @@ export interface WorkItem {
|
|||||||
group_folder: string;
|
group_folder: string;
|
||||||
chat_jid: string;
|
chat_jid: string;
|
||||||
agent_type: AgentType;
|
agent_type: AgentType;
|
||||||
|
service_id: string;
|
||||||
status: 'produced' | 'delivery_retry' | 'delivered';
|
status: 'produced' | 'delivery_retry' | 'delivered';
|
||||||
start_seq: number | null;
|
start_seq: number | null;
|
||||||
end_seq: number | null;
|
end_seq: number | null;
|
||||||
@@ -44,6 +51,32 @@ export interface WorkItem {
|
|||||||
delivered_at: string | null;
|
delivered_at: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ChannelOwnerLeaseRow {
|
||||||
|
chat_jid: string;
|
||||||
|
owner_service_id: string;
|
||||||
|
reviewer_service_id: string | null;
|
||||||
|
activated_at: string | null;
|
||||||
|
reason: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ServiceHandoff {
|
||||||
|
id: number;
|
||||||
|
chat_jid: string;
|
||||||
|
group_folder: string;
|
||||||
|
source_service_id: string;
|
||||||
|
target_service_id: string;
|
||||||
|
target_agent_type: AgentType;
|
||||||
|
prompt: string;
|
||||||
|
status: 'pending' | 'claimed' | 'completed' | 'failed';
|
||||||
|
start_seq: number | null;
|
||||||
|
end_seq: number | null;
|
||||||
|
reason: string | null;
|
||||||
|
created_at: string;
|
||||||
|
claimed_at: string | null;
|
||||||
|
completed_at: string | null;
|
||||||
|
last_error: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
function backfillMessageSeq(database: Database.Database): void {
|
function backfillMessageSeq(database: Database.Database): void {
|
||||||
const rows = database
|
const rows = database
|
||||||
.prepare(
|
.prepare(
|
||||||
@@ -114,6 +147,7 @@ function createSchema(database: Database.Database): void {
|
|||||||
group_folder TEXT NOT NULL,
|
group_folder TEXT NOT NULL,
|
||||||
chat_jid TEXT NOT NULL,
|
chat_jid TEXT NOT NULL,
|
||||||
agent_type TEXT NOT NULL,
|
agent_type TEXT NOT NULL,
|
||||||
|
service_id TEXT NOT NULL DEFAULT '',
|
||||||
status TEXT NOT NULL DEFAULT 'produced',
|
status TEXT NOT NULL DEFAULT 'produced',
|
||||||
start_seq INTEGER,
|
start_seq INTEGER,
|
||||||
end_seq INTEGER,
|
end_seq INTEGER,
|
||||||
@@ -127,9 +161,9 @@ function createSchema(database: Database.Database): void {
|
|||||||
CHECK (status IN ('produced', 'delivery_retry', 'delivered'))
|
CHECK (status IN ('produced', 'delivery_retry', 'delivered'))
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_work_items_status ON work_items(status, updated_at);
|
CREATE INDEX IF NOT EXISTS idx_work_items_status ON work_items(status, updated_at);
|
||||||
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, status);
|
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, service_id, status);
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open
|
||||||
ON work_items(chat_jid, agent_type)
|
ON work_items(chat_jid, agent_type, service_id)
|
||||||
WHERE status IN ('produced', 'delivery_retry');
|
WHERE status IN ('produced', 'delivery_retry');
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS scheduled_tasks (
|
CREATE TABLE IF NOT EXISTS scheduled_tasks (
|
||||||
@@ -176,6 +210,12 @@ function createSchema(database: Database.Database): void {
|
|||||||
session_id TEXT NOT NULL,
|
session_id TEXT NOT NULL,
|
||||||
PRIMARY KEY (group_folder, agent_type)
|
PRIMARY KEY (group_folder, agent_type)
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS service_sessions (
|
||||||
|
group_folder TEXT NOT NULL,
|
||||||
|
service_id TEXT NOT NULL,
|
||||||
|
session_id TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (group_folder, service_id)
|
||||||
|
);
|
||||||
CREATE TABLE IF NOT EXISTS registered_groups (
|
CREATE TABLE IF NOT EXISTS registered_groups (
|
||||||
jid TEXT NOT NULL,
|
jid TEXT NOT NULL,
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
@@ -190,6 +230,33 @@ function createSchema(database: Database.Database): void {
|
|||||||
PRIMARY KEY (jid, agent_type),
|
PRIMARY KEY (jid, agent_type),
|
||||||
UNIQUE (folder, agent_type)
|
UNIQUE (folder, agent_type)
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS channel_owner (
|
||||||
|
chat_jid TEXT PRIMARY KEY,
|
||||||
|
owner_service_id TEXT NOT NULL,
|
||||||
|
reviewer_service_id TEXT,
|
||||||
|
activated_at TEXT,
|
||||||
|
reason TEXT
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS service_handoffs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
chat_jid TEXT NOT NULL,
|
||||||
|
group_folder TEXT NOT NULL,
|
||||||
|
source_service_id TEXT NOT NULL,
|
||||||
|
target_service_id TEXT NOT NULL,
|
||||||
|
target_agent_type TEXT NOT NULL,
|
||||||
|
prompt TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
start_seq INTEGER,
|
||||||
|
end_seq INTEGER,
|
||||||
|
reason TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
claimed_at TEXT,
|
||||||
|
completed_at TEXT,
|
||||||
|
last_error TEXT,
|
||||||
|
CHECK (status IN ('pending', 'claimed', 'completed', 'failed'))
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_service_handoffs_target
|
||||||
|
ON service_handoffs(target_service_id, status, created_at);
|
||||||
`);
|
`);
|
||||||
|
|
||||||
// Add context_mode column if it doesn't exist (migration for existing DBs)
|
// Add context_mode column if it doesn't exist (migration for existing DBs)
|
||||||
@@ -293,12 +360,41 @@ function createSchema(database: Database.Database): void {
|
|||||||
/* column already exists */
|
/* column already exists */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
database.exec(`ALTER TABLE work_items ADD COLUMN service_id TEXT DEFAULT ''`);
|
||||||
|
} catch {
|
||||||
|
/* column already exists */
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
database
|
||||||
|
.prepare(
|
||||||
|
`UPDATE work_items
|
||||||
|
SET service_id = CASE
|
||||||
|
WHEN agent_type = 'codex' THEN ?
|
||||||
|
ELSE ?
|
||||||
|
END
|
||||||
|
WHERE COALESCE(service_id, '') = ''`,
|
||||||
|
)
|
||||||
|
.run(CODEX_MAIN_SERVICE_ID, CLAUDE_SERVICE_ID);
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
|
||||||
backfillMessageSeq(database);
|
backfillMessageSeq(database);
|
||||||
|
|
||||||
database.exec(`
|
database.exec(`
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_seq ON messages(seq);
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_seq ON messages(seq);
|
||||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_jid_seq ON messages(chat_jid, seq);
|
CREATE INDEX IF NOT EXISTS idx_messages_chat_jid_seq ON messages(chat_jid, seq);
|
||||||
`);
|
`);
|
||||||
|
database.exec(`DROP INDEX IF EXISTS idx_work_items_group_agent;`);
|
||||||
|
database.exec(`DROP INDEX IF EXISTS idx_work_items_open;`);
|
||||||
|
database.exec(`
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent
|
||||||
|
ON work_items(chat_jid, agent_type, service_id, status);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open
|
||||||
|
ON work_items(chat_jid, agent_type, service_id)
|
||||||
|
WHERE status IN ('produced', 'delivery_retry');
|
||||||
|
`);
|
||||||
|
|
||||||
// Migrate registered_groups to composite keys so Claude/Codex can share a jid/folder.
|
// Migrate registered_groups to composite keys so Claude/Codex can share a jid/folder.
|
||||||
const registeredGroupsSql = (
|
const registeredGroupsSql = (
|
||||||
@@ -769,6 +865,36 @@ export function getMessagesSinceSeq(
|
|||||||
return rows.map(normalizeMessageRow);
|
return rows.map(normalizeMessageRow);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the N most recent messages for a chat, ordered chronologically.
|
||||||
|
* Includes both human and bot messages for full conversation context.
|
||||||
|
* Used for conversation context retrieval.
|
||||||
|
*/
|
||||||
|
export function getRecentChatMessages(
|
||||||
|
chatJid: string,
|
||||||
|
limit: number = 20,
|
||||||
|
): NewMessage[] {
|
||||||
|
const sql = `
|
||||||
|
SELECT * FROM (
|
||||||
|
SELECT id, chat_jid, sender, sender_name, content, timestamp, is_from_me, is_bot_message
|
||||||
|
FROM messages
|
||||||
|
WHERE chat_jid = ?
|
||||||
|
AND content != '' AND content IS NOT NULL
|
||||||
|
ORDER BY timestamp DESC
|
||||||
|
LIMIT ?
|
||||||
|
) ORDER BY timestamp
|
||||||
|
`;
|
||||||
|
const rows = db
|
||||||
|
.prepare(sql)
|
||||||
|
.all(chatJid, limit) as Array<
|
||||||
|
NewMessage & {
|
||||||
|
is_from_me?: boolean | number;
|
||||||
|
is_bot_message?: boolean | number;
|
||||||
|
}
|
||||||
|
>;
|
||||||
|
return rows.map(normalizeMessageRow);
|
||||||
|
}
|
||||||
|
|
||||||
export function getLastHumanMessageTimestamp(chatJid: string): string | null {
|
export function getLastHumanMessageTimestamp(chatJid: string): string | null {
|
||||||
const row = db
|
const row = db
|
||||||
.prepare(
|
.prepare(
|
||||||
@@ -805,34 +931,39 @@ export function hasRecentRestartAnnouncement(
|
|||||||
export function getOpenWorkItem(
|
export function getOpenWorkItem(
|
||||||
chatJid: string,
|
chatJid: string,
|
||||||
agentType: AgentType = SERVICE_AGENT_TYPE,
|
agentType: AgentType = SERVICE_AGENT_TYPE,
|
||||||
|
serviceId: string = SERVICE_SESSION_SCOPE,
|
||||||
): WorkItem | undefined {
|
): WorkItem | undefined {
|
||||||
return db
|
return db
|
||||||
.prepare(
|
.prepare(
|
||||||
`SELECT *
|
`SELECT *
|
||||||
FROM work_items
|
FROM work_items
|
||||||
WHERE chat_jid = ? AND agent_type = ? AND status IN ('produced', 'delivery_retry')
|
WHERE chat_jid = ? AND agent_type = ? AND service_id = ?
|
||||||
|
AND status IN ('produced', 'delivery_retry')
|
||||||
ORDER BY id ASC
|
ORDER BY id ASC
|
||||||
LIMIT 1`,
|
LIMIT 1`,
|
||||||
)
|
)
|
||||||
.get(chatJid, agentType) as WorkItem | undefined;
|
.get(chatJid, agentType, serviceId) as WorkItem | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createProducedWorkItem(input: {
|
export function createProducedWorkItem(input: {
|
||||||
group_folder: string;
|
group_folder: string;
|
||||||
chat_jid: string;
|
chat_jid: string;
|
||||||
agent_type?: AgentType;
|
agent_type?: AgentType;
|
||||||
|
service_id?: string;
|
||||||
start_seq: number | null;
|
start_seq: number | null;
|
||||||
end_seq: number | null;
|
end_seq: number | null;
|
||||||
result_payload: string;
|
result_payload: string;
|
||||||
}): WorkItem {
|
}): WorkItem {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const agentType = input.agent_type || SERVICE_AGENT_TYPE;
|
const agentType = input.agent_type || SERVICE_AGENT_TYPE;
|
||||||
|
const serviceId = input.service_id || SERVICE_SESSION_SCOPE;
|
||||||
const result = db
|
const result = db
|
||||||
.prepare(
|
.prepare(
|
||||||
`INSERT INTO work_items (
|
`INSERT INTO work_items (
|
||||||
group_folder,
|
group_folder,
|
||||||
chat_jid,
|
chat_jid,
|
||||||
agent_type,
|
agent_type,
|
||||||
|
service_id,
|
||||||
status,
|
status,
|
||||||
start_seq,
|
start_seq,
|
||||||
end_seq,
|
end_seq,
|
||||||
@@ -840,12 +971,13 @@ export function createProducedWorkItem(input: {
|
|||||||
delivery_attempts,
|
delivery_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
) VALUES (?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`,
|
) VALUES (?, ?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`,
|
||||||
)
|
)
|
||||||
.run(
|
.run(
|
||||||
input.group_folder,
|
input.group_folder,
|
||||||
input.chat_jid,
|
input.chat_jid,
|
||||||
agentType,
|
agentType,
|
||||||
|
serviceId,
|
||||||
input.start_seq,
|
input.start_seq,
|
||||||
input.end_seq,
|
input.end_seq,
|
||||||
input.result_payload,
|
input.result_payload,
|
||||||
@@ -1085,6 +1217,21 @@ export function deleteTask(id: string): void {
|
|||||||
cleanupTargets.push(
|
cleanupTargets.push(
|
||||||
resolveTaskRuntimeIpcPathFromGroup(task.group_folder, runtimeTaskId),
|
resolveTaskRuntimeIpcPathFromGroup(task.group_folder, runtimeTaskId),
|
||||||
resolveTaskSessionsPathFromGroup(task.group_folder, runtimeTaskId),
|
resolveTaskSessionsPathFromGroup(task.group_folder, runtimeTaskId),
|
||||||
|
resolveServiceTaskSessionsPathFromGroup(
|
||||||
|
task.group_folder,
|
||||||
|
CLAUDE_SERVICE_ID,
|
||||||
|
runtimeTaskId,
|
||||||
|
),
|
||||||
|
resolveServiceTaskSessionsPathFromGroup(
|
||||||
|
task.group_folder,
|
||||||
|
CODEX_MAIN_SERVICE_ID,
|
||||||
|
runtimeTaskId,
|
||||||
|
),
|
||||||
|
resolveServiceTaskSessionsPathFromGroup(
|
||||||
|
task.group_folder,
|
||||||
|
CODEX_REVIEW_SERVICE_ID,
|
||||||
|
runtimeTaskId,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@@ -1175,7 +1322,14 @@ export function getRecentConsecutiveErrors(
|
|||||||
// --- Router state accessors ---
|
// --- Router state accessors ---
|
||||||
|
|
||||||
export function getRouterState(key: string): string | undefined {
|
export function getRouterState(key: string): string | undefined {
|
||||||
const prefixedKey = `${SERVICE_ID}:${key}`;
|
return getRouterStateForService(key, SERVICE_ID);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getRouterStateForService(
|
||||||
|
key: string,
|
||||||
|
serviceId: string,
|
||||||
|
): string | undefined {
|
||||||
|
const prefixedKey = `${normalizeServiceId(serviceId)}:${key}`;
|
||||||
const row = db
|
const row = db
|
||||||
.prepare('SELECT value FROM router_state WHERE key = ?')
|
.prepare('SELECT value FROM router_state WHERE key = ?')
|
||||||
.get(prefixedKey) as { value: string } | undefined;
|
.get(prefixedKey) as { value: string } | undefined;
|
||||||
@@ -1196,7 +1350,15 @@ export function getRouterState(key: string): string | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function setRouterState(key: string, value: string): void {
|
export function setRouterState(key: string, value: string): void {
|
||||||
const prefixedKey = `${SERVICE_ID}:${key}`;
|
setRouterStateForService(key, value, SERVICE_ID);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setRouterStateForService(
|
||||||
|
key: string,
|
||||||
|
value: string,
|
||||||
|
serviceId: string,
|
||||||
|
): void {
|
||||||
|
const prefixedKey = `${normalizeServiceId(serviceId)}:${key}`;
|
||||||
db.prepare(
|
db.prepare(
|
||||||
'INSERT OR REPLACE INTO router_state (key, value) VALUES (?, ?)',
|
'INSERT OR REPLACE INTO router_state (key, value) VALUES (?, ?)',
|
||||||
).run(prefixedKey, value);
|
).run(prefixedKey, value);
|
||||||
@@ -1205,6 +1367,17 @@ export function setRouterState(key: string, value: string): void {
|
|||||||
// --- Session accessors ---
|
// --- Session accessors ---
|
||||||
|
|
||||||
export function getSession(groupFolder: string): string | undefined {
|
export function getSession(groupFolder: string): string | undefined {
|
||||||
|
const serviceScopedRow = db
|
||||||
|
.prepare(
|
||||||
|
'SELECT session_id FROM service_sessions WHERE group_folder = ? AND service_id = ?',
|
||||||
|
)
|
||||||
|
.get(groupFolder, SERVICE_SESSION_SCOPE) as
|
||||||
|
| { session_id: string }
|
||||||
|
| undefined;
|
||||||
|
if (serviceScopedRow?.session_id) {
|
||||||
|
return serviceScopedRow.session_id;
|
||||||
|
}
|
||||||
|
|
||||||
const row = db
|
const row = db
|
||||||
.prepare(
|
.prepare(
|
||||||
'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?',
|
'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?',
|
||||||
@@ -1214,18 +1387,40 @@ export function getSession(groupFolder: string): string | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function setSession(groupFolder: string, sessionId: string): void {
|
export function setSession(groupFolder: string, sessionId: string): void {
|
||||||
|
db.prepare(
|
||||||
|
'INSERT OR REPLACE INTO service_sessions (group_folder, service_id, session_id) VALUES (?, ?, ?)',
|
||||||
|
).run(groupFolder, SERVICE_SESSION_SCOPE, sessionId);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)',
|
'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)',
|
||||||
).run(groupFolder, SERVICE_AGENT_TYPE, sessionId);
|
).run(groupFolder, SERVICE_AGENT_TYPE, sessionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function deleteSession(groupFolder: string): void {
|
export function deleteSession(groupFolder: string): void {
|
||||||
|
db.prepare(
|
||||||
|
'DELETE FROM service_sessions WHERE group_folder = ? AND service_id = ?',
|
||||||
|
).run(groupFolder, SERVICE_SESSION_SCOPE);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
'DELETE FROM sessions WHERE group_folder = ? AND agent_type = ?',
|
'DELETE FROM sessions WHERE group_folder = ? AND agent_type = ?',
|
||||||
).run(groupFolder, SERVICE_AGENT_TYPE);
|
).run(groupFolder, SERVICE_AGENT_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getAllSessions(): Record<string, string> {
|
export function getAllSessions(): Record<string, string> {
|
||||||
|
const serviceRows = db
|
||||||
|
.prepare(
|
||||||
|
'SELECT group_folder, session_id FROM service_sessions WHERE service_id = ?',
|
||||||
|
)
|
||||||
|
.all(SERVICE_SESSION_SCOPE) as Array<{
|
||||||
|
group_folder: string;
|
||||||
|
session_id: string;
|
||||||
|
}>;
|
||||||
|
const result: Record<string, string> = {};
|
||||||
|
for (const row of serviceRows) {
|
||||||
|
result[row.group_folder] = row.session_id;
|
||||||
|
}
|
||||||
|
if (serviceRows.length > 0) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
const rows = db
|
const rows = db
|
||||||
.prepare(
|
.prepare(
|
||||||
'SELECT group_folder, session_id FROM sessions WHERE agent_type = ?',
|
'SELECT group_folder, session_id FROM sessions WHERE agent_type = ?',
|
||||||
@@ -1234,11 +1429,66 @@ export function getAllSessions(): Record<string, string> {
|
|||||||
group_folder: string;
|
group_folder: string;
|
||||||
session_id: string;
|
session_id: string;
|
||||||
}>;
|
}>;
|
||||||
const result: Record<string, string> = {};
|
const legacyResult: Record<string, string> = {};
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
result[row.group_folder] = row.session_id;
|
legacyResult[row.group_folder] = row.session_id;
|
||||||
}
|
}
|
||||||
return result;
|
return legacyResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get session for a specific agent type (cross-provider access).
|
||||||
|
* Used for provider switch probe attempts.
|
||||||
|
*/
|
||||||
|
export function getSessionForAgentType(
|
||||||
|
groupFolder: string,
|
||||||
|
agentType: string,
|
||||||
|
): string | undefined {
|
||||||
|
const row = db
|
||||||
|
.prepare(
|
||||||
|
'SELECT session_id FROM sessions WHERE group_folder = ? AND agent_type = ?',
|
||||||
|
)
|
||||||
|
.get(groupFolder, agentType) as { session_id: string } | undefined;
|
||||||
|
return row?.session_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save session for a specific agent type without affecting current service's session.
|
||||||
|
* Used when probe succeeds and we want to save to target provider's slot only.
|
||||||
|
*/
|
||||||
|
export function setSessionForAgentType(
|
||||||
|
groupFolder: string,
|
||||||
|
agentType: string,
|
||||||
|
sessionId: string,
|
||||||
|
): void {
|
||||||
|
db.prepare(
|
||||||
|
'INSERT OR REPLACE INTO sessions (group_folder, agent_type, session_id) VALUES (?, ?, ?)',
|
||||||
|
).run(groupFolder, agentType, sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the agent type of the most recent bot response in a chat.
|
||||||
|
* Used to detect provider switches for delta handoff.
|
||||||
|
*/
|
||||||
|
export function getLastRespondingAgentType(
|
||||||
|
chatJid: string,
|
||||||
|
): AgentType | undefined {
|
||||||
|
const row = db
|
||||||
|
.prepare(
|
||||||
|
`SELECT sender FROM messages
|
||||||
|
WHERE chat_jid = ? AND is_bot_message = 1
|
||||||
|
ORDER BY timestamp DESC, seq DESC
|
||||||
|
LIMIT 1`,
|
||||||
|
)
|
||||||
|
.get(chatJid) as { sender: string } | undefined;
|
||||||
|
|
||||||
|
if (!row) return undefined;
|
||||||
|
|
||||||
|
// Map sender to agent type (sender contains the bot identifier)
|
||||||
|
const sender = row.sender.toLowerCase();
|
||||||
|
if (sender.includes('claude')) return 'claude-code';
|
||||||
|
if (sender.includes('codex')) return 'codex';
|
||||||
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Registered group accessors ---
|
// --- Registered group accessors ---
|
||||||
@@ -1388,6 +1638,255 @@ export function isPairedRoomJid(jid: string): boolean {
|
|||||||
return types.includes('claude-code') && types.includes('codex');
|
return types.includes('claude-code') && types.includes('codex');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the most recent bot message (is_bot_message=1) in a chat, regardless of which bot sent it.
|
||||||
|
* Used for duplicate detection in pair rooms.
|
||||||
|
*/
|
||||||
|
export function getLastBotFinalMessage(
|
||||||
|
chatJid: string,
|
||||||
|
_agentType: AgentType = SERVICE_AGENT_TYPE,
|
||||||
|
limit: number = 1,
|
||||||
|
): Array<{ content: string; timestamp: string }> {
|
||||||
|
const rows = db
|
||||||
|
.prepare(
|
||||||
|
`SELECT content, timestamp
|
||||||
|
FROM messages
|
||||||
|
WHERE chat_jid = ? AND is_bot_message = 1
|
||||||
|
ORDER BY timestamp DESC, seq DESC
|
||||||
|
LIMIT ?`,
|
||||||
|
)
|
||||||
|
.all(chatJid, limit) as Array<{ content: string; timestamp: string }>;
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Channel owner lease accessors ---
|
||||||
|
|
||||||
|
export function getChannelOwnerLease(
|
||||||
|
chatJid: string,
|
||||||
|
): ChannelOwnerLeaseRow | undefined {
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`SELECT chat_jid, owner_service_id, reviewer_service_id, activated_at, reason
|
||||||
|
FROM channel_owner
|
||||||
|
WHERE chat_jid = ?`,
|
||||||
|
)
|
||||||
|
.get(chatJid) as ChannelOwnerLeaseRow | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAllChannelOwnerLeases(): ChannelOwnerLeaseRow[] {
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`SELECT chat_jid, owner_service_id, reviewer_service_id, activated_at, reason
|
||||||
|
FROM channel_owner`,
|
||||||
|
)
|
||||||
|
.all() as ChannelOwnerLeaseRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setChannelOwnerLease(input: {
|
||||||
|
chat_jid: string;
|
||||||
|
owner_service_id: string;
|
||||||
|
reviewer_service_id?: string | null;
|
||||||
|
activated_at?: string | null;
|
||||||
|
reason?: string | null;
|
||||||
|
}): void {
|
||||||
|
db.prepare(
|
||||||
|
`INSERT OR REPLACE INTO channel_owner (
|
||||||
|
chat_jid,
|
||||||
|
owner_service_id,
|
||||||
|
reviewer_service_id,
|
||||||
|
activated_at,
|
||||||
|
reason
|
||||||
|
) VALUES (?, ?, ?, ?, ?)`,
|
||||||
|
).run(
|
||||||
|
input.chat_jid,
|
||||||
|
input.owner_service_id,
|
||||||
|
input.reviewer_service_id ?? null,
|
||||||
|
input.activated_at ?? new Date().toISOString(),
|
||||||
|
input.reason ?? null,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearChannelOwnerLease(chatJid: string): void {
|
||||||
|
db.prepare('DELETE FROM channel_owner WHERE chat_jid = ?').run(chatJid);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Cross-service handoff accessors ---
|
||||||
|
|
||||||
|
export function createServiceHandoff(input: {
|
||||||
|
chat_jid: string;
|
||||||
|
group_folder: string;
|
||||||
|
source_service_id: string;
|
||||||
|
target_service_id: string;
|
||||||
|
target_agent_type: AgentType;
|
||||||
|
prompt: string;
|
||||||
|
start_seq?: number | null;
|
||||||
|
end_seq?: number | null;
|
||||||
|
reason?: string | null;
|
||||||
|
}): ServiceHandoff {
|
||||||
|
const result = db
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO service_handoffs (
|
||||||
|
chat_jid,
|
||||||
|
group_folder,
|
||||||
|
source_service_id,
|
||||||
|
target_service_id,
|
||||||
|
target_agent_type,
|
||||||
|
prompt,
|
||||||
|
start_seq,
|
||||||
|
end_seq,
|
||||||
|
reason
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||||
|
)
|
||||||
|
.run(
|
||||||
|
input.chat_jid,
|
||||||
|
input.group_folder,
|
||||||
|
input.source_service_id,
|
||||||
|
input.target_service_id,
|
||||||
|
input.target_agent_type,
|
||||||
|
input.prompt,
|
||||||
|
input.start_seq ?? null,
|
||||||
|
input.end_seq ?? null,
|
||||||
|
input.reason ?? null,
|
||||||
|
);
|
||||||
|
|
||||||
|
return db
|
||||||
|
.prepare('SELECT * FROM service_handoffs WHERE id = ?')
|
||||||
|
.get(result.lastInsertRowid) as ServiceHandoff;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getPendingServiceHandoffs(
|
||||||
|
targetServiceId: string = SERVICE_SESSION_SCOPE,
|
||||||
|
): ServiceHandoff[] {
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`SELECT *
|
||||||
|
FROM service_handoffs
|
||||||
|
WHERE target_service_id = ?
|
||||||
|
AND status = 'pending'
|
||||||
|
ORDER BY created_at ASC, id ASC`,
|
||||||
|
)
|
||||||
|
.all(targetServiceId) as ServiceHandoff[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function claimServiceHandoff(id: number): boolean {
|
||||||
|
const result = db
|
||||||
|
.prepare(
|
||||||
|
`UPDATE service_handoffs
|
||||||
|
SET status = 'claimed',
|
||||||
|
claimed_at = datetime('now')
|
||||||
|
WHERE id = ?
|
||||||
|
AND status = 'pending'`,
|
||||||
|
)
|
||||||
|
.run(id);
|
||||||
|
return result.changes > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function completeServiceHandoff(id: number): void {
|
||||||
|
db.prepare(
|
||||||
|
`UPDATE service_handoffs
|
||||||
|
SET status = 'completed',
|
||||||
|
completed_at = datetime('now'),
|
||||||
|
last_error = NULL
|
||||||
|
WHERE id = ?`,
|
||||||
|
).run(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function failServiceHandoff(id: number, error: string): void {
|
||||||
|
db.prepare(
|
||||||
|
`UPDATE service_handoffs
|
||||||
|
SET status = 'failed',
|
||||||
|
completed_at = datetime('now'),
|
||||||
|
last_error = ?
|
||||||
|
WHERE id = ?`,
|
||||||
|
).run(error, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeStoredLastAgentSeqCursor(
|
||||||
|
cursor: string | number | null | undefined,
|
||||||
|
chatJid: string,
|
||||||
|
): number {
|
||||||
|
if (typeof cursor === 'number') {
|
||||||
|
return Number.isFinite(cursor) && cursor > 0 ? cursor : 0;
|
||||||
|
}
|
||||||
|
if (!cursor) return 0;
|
||||||
|
const trimmed = cursor.trim();
|
||||||
|
if (/^\d+$/.test(trimmed)) {
|
||||||
|
return normalizeSeqCursor(trimmed);
|
||||||
|
}
|
||||||
|
return getLatestMessageSeqAtOrBefore(trimmed, chatJid);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseLastAgentSeqState(
|
||||||
|
raw: string | undefined,
|
||||||
|
serviceId: string,
|
||||||
|
): Record<string, string> {
|
||||||
|
if (!raw) return {};
|
||||||
|
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(raw);
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(
|
||||||
|
`Invalid last_agent_seq JSON for ${serviceId}: ${
|
||||||
|
err instanceof Error ? err.message : String(err)
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||||
|
throw new Error(`Invalid last_agent_seq JSON for ${serviceId}: not an object`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cursors: Record<string, string> = {};
|
||||||
|
for (const [chatJid, cursor] of Object.entries(parsed)) {
|
||||||
|
if (typeof cursor === 'string' || typeof cursor === 'number') {
|
||||||
|
cursors[chatJid] = String(cursor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cursors;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function completeServiceHandoffAndAdvanceTargetCursor(input: {
|
||||||
|
id: number;
|
||||||
|
target_service_id: string;
|
||||||
|
chat_jid: string;
|
||||||
|
end_seq?: number | null;
|
||||||
|
}): string | null {
|
||||||
|
return db.transaction(() => {
|
||||||
|
let appliedCursor: string | null = null;
|
||||||
|
|
||||||
|
if (input.end_seq != null) {
|
||||||
|
const currentState = parseLastAgentSeqState(
|
||||||
|
getRouterStateForService('last_agent_seq', input.target_service_id),
|
||||||
|
input.target_service_id,
|
||||||
|
);
|
||||||
|
const existingSeq = normalizeStoredLastAgentSeqCursor(
|
||||||
|
currentState[input.chat_jid],
|
||||||
|
input.chat_jid,
|
||||||
|
);
|
||||||
|
currentState[input.chat_jid] = String(
|
||||||
|
Math.max(existingSeq, input.end_seq),
|
||||||
|
);
|
||||||
|
setRouterStateForService(
|
||||||
|
'last_agent_seq',
|
||||||
|
JSON.stringify(currentState),
|
||||||
|
input.target_service_id,
|
||||||
|
);
|
||||||
|
appliedCursor = currentState[input.chat_jid];
|
||||||
|
}
|
||||||
|
|
||||||
|
db.prepare(
|
||||||
|
`UPDATE service_handoffs
|
||||||
|
SET status = 'completed',
|
||||||
|
completed_at = datetime('now'),
|
||||||
|
last_error = NULL
|
||||||
|
WHERE id = ?`,
|
||||||
|
).run(input.id);
|
||||||
|
|
||||||
|
return appliedCursor;
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
// --- JSON migration ---
|
// --- JSON migration ---
|
||||||
|
|
||||||
function migrateJsonState(): void {
|
function migrateJsonState(): void {
|
||||||
|
|||||||
443
src/duplicate-suppression.test.ts
Normal file
443
src/duplicate-suppression.test.ts
Normal file
@@ -0,0 +1,443 @@
|
|||||||
|
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||||
|
|
||||||
|
import {
|
||||||
|
_initTestDatabase,
|
||||||
|
storeMessage,
|
||||||
|
storeChatMetadata,
|
||||||
|
createProducedWorkItem,
|
||||||
|
getOpenWorkItem,
|
||||||
|
setRegisteredGroup,
|
||||||
|
isPairedRoomJid,
|
||||||
|
getLastBotFinalMessage,
|
||||||
|
markWorkItemDelivered,
|
||||||
|
} from './db.js';
|
||||||
|
import { normalizeMessageForDedupe } from './router.js';
|
||||||
|
import { isDuplicateOfLastBotFinal } from './message-runtime.js';
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
_initTestDatabase();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Helper to create a message
|
||||||
|
const createMessage = (overrides: Partial<{
|
||||||
|
id: string;
|
||||||
|
chat_jid: string;
|
||||||
|
sender: string;
|
||||||
|
sender_name: string;
|
||||||
|
content: string;
|
||||||
|
timestamp: string;
|
||||||
|
is_from_me: boolean;
|
||||||
|
is_bot_message: boolean;
|
||||||
|
}> = {}) => ({
|
||||||
|
id: overrides.id ?? 'msg-1',
|
||||||
|
chat_jid: overrides.chat_jid ?? 'dc:test-room',
|
||||||
|
sender: overrides.sender ?? 'user1',
|
||||||
|
sender_name: overrides.sender_name ?? 'User',
|
||||||
|
content: overrides.content ?? 'Hello',
|
||||||
|
timestamp: overrides.timestamp ?? new Date().toISOString(),
|
||||||
|
is_from_me: overrides.is_from_me ?? false,
|
||||||
|
is_bot_message: overrides.is_bot_message ?? false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Helper to setup chat metadata
|
||||||
|
const setupChat = (jid: string) => {
|
||||||
|
storeChatMetadata(jid, new Date().toISOString(), 'Test Chat', 'discord', true);
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('isPairedRoomJid', () => {
|
||||||
|
it('returns true when both claude-code and codex are registered', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isPairedRoomJid(jid)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false when only claude-code is registered', () => {
|
||||||
|
const jid = 'dc:single-claude';
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Single Claude',
|
||||||
|
folder: 'single-claude',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isPairedRoomJid(jid)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false when only codex is registered', () => {
|
||||||
|
const jid = 'dc:single-codex';
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Single Codex',
|
||||||
|
folder: 'single-codex',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(isPairedRoomJid(jid)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getLastBotFinalMessage', () => {
|
||||||
|
it('returns the most recent bot message from any service (is_bot_message=1)', () => {
|
||||||
|
const jid = 'dc:test-room';
|
||||||
|
const now = new Date();
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Store older bot message (from any bot)
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'First bot message',
|
||||||
|
timestamp: new Date(now.getTime() - 1000).toISOString(),
|
||||||
|
is_from_me: true,
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Store newer bot message (from different bot - is_from_me=0 but is_bot_message=1)
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-2',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'Second bot message',
|
||||||
|
timestamp: now.toISOString(),
|
||||||
|
is_from_me: false, // Different bot
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Store human message (should not be returned)
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-3',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'Human message',
|
||||||
|
timestamp: new Date(now.getTime() + 1000).toISOString(),
|
||||||
|
is_from_me: false,
|
||||||
|
is_bot_message: false,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const lastMessages = getLastBotFinalMessage(jid, 'claude-code', 1);
|
||||||
|
expect(lastMessages).toHaveLength(1);
|
||||||
|
expect(lastMessages[0].content).toBe('Second bot message');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty array when no bot messages exist', () => {
|
||||||
|
const jid = 'dc:empty-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
const lastMessages = getLastBotFinalMessage(jid, 'claude-code', 1);
|
||||||
|
expect(lastMessages).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('normalizeMessageForDedupe', () => {
|
||||||
|
it('normalizes messages for comparison', () => {
|
||||||
|
expect(normalizeMessageForDedupe(' Hello World ')).toBe('hello world');
|
||||||
|
expect(normalizeMessageForDedupe('Hello\n\nWorld')).toBe('hello world');
|
||||||
|
expect(normalizeMessageForDedupe('Hello World')).toBe('hello world');
|
||||||
|
expect(normalizeMessageForDedupe('HELLO WORLD')).toBe('hello world');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles empty strings', () => {
|
||||||
|
expect(normalizeMessageForDedupe('')).toBe('');
|
||||||
|
expect(normalizeMessageForDedupe(' ')).toBe('');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('isDuplicateOfLastBotFinal (runtime function)', () => {
|
||||||
|
it('paired room: detects duplicate final message', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register as paired room
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message (from any bot)
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'DONE — Task completed successfully',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: false, // Different bot
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Verify it's a paired room
|
||||||
|
expect(isPairedRoomJid(jid)).toBe(true);
|
||||||
|
|
||||||
|
// Verify duplicate detection works via the actual runtime function
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'DONE — Task completed successfully')).toBe(true);
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'done — task completed successfully')).toBe(true); // Normalized match
|
||||||
|
});
|
||||||
|
|
||||||
|
it('non-paired room: duplicate check is bypassed', () => {
|
||||||
|
const jid = 'dc:single-bot';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register only one bot
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Single Bot',
|
||||||
|
folder: 'single-bot',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'DONE — Task completed',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: true,
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Verify it's NOT a paired room
|
||||||
|
expect(isPairedRoomJid(jid)).toBe(false);
|
||||||
|
|
||||||
|
// Duplicate check should be bypassed (return false)
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'DONE — Task completed')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('non-duplicate message in paired room is not suppressed', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register as paired room
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'First message content',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: true,
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Verify different content is not a duplicate
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'Different message content')).toBe(false);
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'First message content')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('cross-bot duplicate detection: claude detects codex message as duplicate', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register as paired room
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message from "codex" (is_from_me=0)
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'DONE — Analysis complete',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: false, // Other bot
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Verify claude service detects this as duplicate (cross-bot detection)
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'DONE — Analysis complete')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('normalization handles whitespace and case differences', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register as paired room
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message with specific formatting
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'DONE — Task completed\n\nSuccessfully',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: true,
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Same content with different whitespace should be detected as duplicate
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'done — task completed successfully')).toBe(true);
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, ' DONE — Task completed Successfully ')).toBe(true);
|
||||||
|
|
||||||
|
// Different content should not be duplicate
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, 'FAILED — Task failed')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('work item lifecycle: produced -> delivered (duplicate)', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register as paired room
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message (simulating previous delivery)
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'DONE — Task completed successfully',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: true,
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Create a work item with the same content (duplicate)
|
||||||
|
const workItem = createProducedWorkItem({
|
||||||
|
group_folder: 'paired-room',
|
||||||
|
chat_jid: jid,
|
||||||
|
agent_type: 'claude-code',
|
||||||
|
start_seq: 1,
|
||||||
|
end_seq: 2,
|
||||||
|
result_payload: 'DONE — Task completed successfully',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(workItem.status).toBe('produced');
|
||||||
|
|
||||||
|
// Verify duplicate detection via actual runtime function
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, workItem.result_payload)).toBe(true);
|
||||||
|
|
||||||
|
// Simulate suppression: mark as delivered without sending
|
||||||
|
markWorkItemDelivered(workItem.id, null);
|
||||||
|
|
||||||
|
// Verify work item is marked delivered
|
||||||
|
const openItem = getOpenWorkItem(jid, 'claude-code');
|
||||||
|
expect(openItem).toBeUndefined(); // No open items because it was marked delivered
|
||||||
|
});
|
||||||
|
|
||||||
|
it('work item lifecycle: produced -> pending -> delivered (non-duplicate)', () => {
|
||||||
|
const jid = 'dc:paired-room';
|
||||||
|
setupChat(jid);
|
||||||
|
|
||||||
|
// Register as paired room
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@claude',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
|
||||||
|
setRegisteredGroup(jid, {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store a bot message
|
||||||
|
storeMessage(createMessage({
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: jid,
|
||||||
|
content: 'First message',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
is_from_me: true,
|
||||||
|
is_bot_message: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Create a work item with DIFFERENT content (non-duplicate)
|
||||||
|
const workItem = createProducedWorkItem({
|
||||||
|
group_folder: 'paired-room',
|
||||||
|
chat_jid: jid,
|
||||||
|
agent_type: 'claude-code',
|
||||||
|
start_seq: 1,
|
||||||
|
end_seq: 2,
|
||||||
|
result_payload: 'Second message (different)',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(workItem.status).toBe('produced');
|
||||||
|
|
||||||
|
// Verify NOT a duplicate
|
||||||
|
expect(isDuplicateOfLastBotFinal(jid, workItem.result_payload)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -68,6 +68,18 @@ export function resolveGroupSessionsPath(folder: string): string {
|
|||||||
return sessionsPath;
|
return sessionsPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveServiceGroupSessionsPath(
|
||||||
|
folder: string,
|
||||||
|
serviceId: string,
|
||||||
|
): string {
|
||||||
|
assertValidGroupFolder(folder);
|
||||||
|
assertValidRuntimeSegment(serviceId, 'service ID');
|
||||||
|
const sessionsBaseDir = path.resolve(DATA_DIR, 'sessions');
|
||||||
|
const sessionsPath = path.resolve(sessionsBaseDir, folder, 'services', serviceId);
|
||||||
|
ensureWithinBase(sessionsBaseDir, sessionsPath);
|
||||||
|
return sessionsPath;
|
||||||
|
}
|
||||||
|
|
||||||
export function resolveTaskRuntimeIpcPath(
|
export function resolveTaskRuntimeIpcPath(
|
||||||
folder: string,
|
folder: string,
|
||||||
taskId: string,
|
taskId: string,
|
||||||
@@ -91,3 +103,24 @@ export function resolveTaskSessionsPath(
|
|||||||
ensureWithinBase(sessionsBaseDir, sessionsPath);
|
ensureWithinBase(sessionsBaseDir, sessionsPath);
|
||||||
return sessionsPath;
|
return sessionsPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveServiceTaskSessionsPath(
|
||||||
|
folder: string,
|
||||||
|
serviceId: string,
|
||||||
|
taskId: string,
|
||||||
|
): string {
|
||||||
|
assertValidGroupFolder(folder);
|
||||||
|
assertValidRuntimeSegment(serviceId, 'service ID');
|
||||||
|
assertValidRuntimeSegment(taskId, 'task ID');
|
||||||
|
const sessionsBaseDir = path.resolve(DATA_DIR, 'sessions');
|
||||||
|
const sessionsPath = path.resolve(
|
||||||
|
sessionsBaseDir,
|
||||||
|
folder,
|
||||||
|
'services',
|
||||||
|
serviceId,
|
||||||
|
'tasks',
|
||||||
|
taskId,
|
||||||
|
);
|
||||||
|
ensureWithinBase(sessionsBaseDir, sessionsPath);
|
||||||
|
return sessionsPath;
|
||||||
|
}
|
||||||
|
|||||||
90
src/index.ts
90
src/index.ts
@@ -6,7 +6,10 @@ import {
|
|||||||
DATA_DIR,
|
DATA_DIR,
|
||||||
IDLE_TIMEOUT,
|
IDLE_TIMEOUT,
|
||||||
POLL_INTERVAL,
|
POLL_INTERVAL,
|
||||||
|
SERVICE_ID,
|
||||||
SERVICE_AGENT_TYPE,
|
SERVICE_AGENT_TYPE,
|
||||||
|
isClaudeService,
|
||||||
|
isReviewService,
|
||||||
isSessionCommandSenderAllowed,
|
isSessionCommandSenderAllowed,
|
||||||
STATUS_CHANNEL_ID,
|
STATUS_CHANNEL_ID,
|
||||||
STATUS_UPDATE_INTERVAL,
|
STATUS_UPDATE_INTERVAL,
|
||||||
@@ -41,7 +44,11 @@ import { composeDashboardContent } from './dashboard-render.js';
|
|||||||
import { GroupQueue } from './group-queue.js';
|
import { GroupQueue } from './group-queue.js';
|
||||||
import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js';
|
import { resolveGroupFolderPath, resolveGroupIpcPath } from './group-folder.js';
|
||||||
import { startIpcWatcher } from './ipc.js';
|
import { startIpcWatcher } from './ipc.js';
|
||||||
import { findChannel, formatOutbound } from './router.js';
|
import {
|
||||||
|
findChannel,
|
||||||
|
formatOutbound,
|
||||||
|
normalizeMessageForDedupe,
|
||||||
|
} from './router.js';
|
||||||
import {
|
import {
|
||||||
buildRestartAnnouncement,
|
buildRestartAnnouncement,
|
||||||
buildInterruptedRestartAnnouncement,
|
buildInterruptedRestartAnnouncement,
|
||||||
@@ -64,12 +71,17 @@ import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
|||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { normalizeStoredSeqCursor } from './message-cursor.js';
|
import { normalizeStoredSeqCursor } from './message-cursor.js';
|
||||||
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
||||||
import { initTokenRotation } from './token-rotation.js';
|
import { hasAvailableClaudeToken, initTokenRotation } from './token-rotation.js';
|
||||||
import {
|
import {
|
||||||
shouldStartTokenRefreshLoop,
|
shouldStartTokenRefreshLoop,
|
||||||
startTokenRefreshLoop,
|
startTokenRefreshLoop,
|
||||||
stopTokenRefreshLoop,
|
stopTokenRefreshLoop,
|
||||||
} from './token-refresh.js';
|
} from './token-refresh.js';
|
||||||
|
import {
|
||||||
|
getActiveCodexFailoverLeases,
|
||||||
|
restoreDefaultChannelLease,
|
||||||
|
} from './service-routing.js';
|
||||||
|
import { FAILOVER_MIN_DURATION_MS } from './config.js';
|
||||||
|
|
||||||
// Token rotation is initialized lazily on first use or at startup below
|
// Token rotation is initialized lazily on first use or at startup below
|
||||||
|
|
||||||
@@ -315,9 +327,14 @@ async function main(): Promise<void> {
|
|||||||
loadState();
|
loadState();
|
||||||
|
|
||||||
// Graceful shutdown handlers
|
// Graceful shutdown handlers
|
||||||
|
let leaseRecoveryTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
const shutdown = async (signal: string) => {
|
const shutdown = async (signal: string) => {
|
||||||
logger.info({ signal }, 'Shutdown signal received');
|
logger.info({ signal }, 'Shutdown signal received');
|
||||||
stopTokenRefreshLoop();
|
stopTokenRefreshLoop();
|
||||||
|
if (leaseRecoveryTimer) {
|
||||||
|
clearInterval(leaseRecoveryTimer);
|
||||||
|
leaseRecoveryTimer = null;
|
||||||
|
}
|
||||||
const interruptedGroups = queue
|
const interruptedGroups = queue
|
||||||
.getStatuses(Object.keys(registeredGroups))
|
.getStatuses(Object.keys(registeredGroups))
|
||||||
.filter(
|
.filter(
|
||||||
@@ -410,19 +427,23 @@ async function main(): Promise<void> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Start subsystems (independently of connection handler)
|
// Start subsystems (independently of connection handler)
|
||||||
startSchedulerLoop({
|
if (!isReviewService()) {
|
||||||
registeredGroups: () => registeredGroups,
|
startSchedulerLoop({
|
||||||
getSessions: () => sessions,
|
registeredGroups: () => registeredGroups,
|
||||||
queue,
|
getSessions: () => sessions,
|
||||||
onProcess: (groupJid, proc, processName, ipcDir) =>
|
queue,
|
||||||
queue.registerProcess(groupJid, proc, processName, ipcDir),
|
onProcess: (groupJid, proc, processName, ipcDir) =>
|
||||||
sendMessage: (jid, rawText) =>
|
queue.registerProcess(groupJid, proc, processName, ipcDir),
|
||||||
sendFormattedChannelMessage(channels, jid, rawText),
|
sendMessage: (jid, rawText) =>
|
||||||
sendTrackedMessage: (jid, rawText) =>
|
sendFormattedChannelMessage(channels, jid, rawText),
|
||||||
sendFormattedTrackedChannelMessage(channels, jid, rawText),
|
sendTrackedMessage: (jid, rawText) =>
|
||||||
editTrackedMessage: (jid, messageId, rawText) =>
|
sendFormattedTrackedChannelMessage(channels, jid, rawText),
|
||||||
editFormattedTrackedChannelMessage(channels, jid, messageId, rawText),
|
editTrackedMessage: (jid, messageId, rawText) =>
|
||||||
});
|
editFormattedTrackedChannelMessage(channels, jid, messageId, rawText),
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
logger.info({ serviceId: SERVICE_ID }, 'Skipping scheduler for review service');
|
||||||
|
}
|
||||||
startIpcWatcher({
|
startIpcWatcher({
|
||||||
sendMessage: (jid, text) => {
|
sendMessage: (jid, text) => {
|
||||||
const channel = findChannel(channels, jid);
|
const channel = findChannel(channels, jid);
|
||||||
@@ -467,6 +488,7 @@ async function main(): Promise<void> {
|
|||||||
}
|
}
|
||||||
await startUnifiedDashboard({
|
await startUnifiedDashboard({
|
||||||
assistantName: ASSISTANT_NAME,
|
assistantName: ASSISTANT_NAME,
|
||||||
|
serviceId: SERVICE_ID,
|
||||||
serviceAgentType: SERVICE_AGENT_TYPE,
|
serviceAgentType: SERVICE_AGENT_TYPE,
|
||||||
statusChannelId: STATUS_CHANNEL_ID,
|
statusChannelId: STATUS_CHANNEL_ID,
|
||||||
statusUpdateInterval: STATUS_UPDATE_INTERVAL,
|
statusUpdateInterval: STATUS_UPDATE_INTERVAL,
|
||||||
@@ -482,6 +504,44 @@ async function main(): Promise<void> {
|
|||||||
},
|
},
|
||||||
purgeOnStart: true,
|
purgeOnStart: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (isClaudeService()) {
|
||||||
|
leaseRecoveryTimer = setInterval(() => {
|
||||||
|
if (!hasAvailableClaudeToken()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const now = Date.now();
|
||||||
|
for (const lease of getActiveCodexFailoverLeases()) {
|
||||||
|
const activatedMs = lease.activatedAt
|
||||||
|
? new Date(lease.activatedAt).getTime()
|
||||||
|
: NaN;
|
||||||
|
if (Number.isNaN(activatedMs)) {
|
||||||
|
logger.warn(
|
||||||
|
{ chatJid: lease.chatJid, activatedAt: lease.activatedAt },
|
||||||
|
'Failover lease has unparseable activated_at, skipping auto-restore',
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const elapsed = now - activatedMs;
|
||||||
|
if (elapsed < FAILOVER_MIN_DURATION_MS) {
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
chatJid: lease.chatJid,
|
||||||
|
elapsedMin: Math.round(elapsed / 60_000),
|
||||||
|
minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000),
|
||||||
|
},
|
||||||
|
'Failover lease still within minimum hold period, skipping restore',
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
restoreDefaultChannelLease(lease.chatJid);
|
||||||
|
logger.info(
|
||||||
|
{ chatJid: lease.chatJid, serviceId: SERVICE_ID, elapsedMin: Math.round(elapsed / 60_000) },
|
||||||
|
'Claude token available and failover hold period elapsed, restored default channel lease',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, 5_000);
|
||||||
|
}
|
||||||
runtime.startMessageLoop().catch((err) => {
|
runtime.startMessageLoop().catch((err) => {
|
||||||
logger.fatal({ err }, 'Message loop crashed unexpectedly');
|
logger.fatal({ err }, 'Message loop crashed unexpectedly');
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|||||||
@@ -11,13 +11,29 @@ vi.mock('./available-groups.js', () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
|
CODEX_MAIN_SERVICE_ID: 'codex-main',
|
||||||
|
CODEX_REVIEW_SERVICE_ID: 'codex-review',
|
||||||
DATA_DIR: '/tmp/ejclaw-test-data',
|
DATA_DIR: '/tmp/ejclaw-test-data',
|
||||||
|
SERVICE_SESSION_SCOPE: 'claude',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./db.js', () => ({
|
vi.mock('./db.js', () => ({
|
||||||
|
createServiceHandoff: vi.fn(),
|
||||||
getAllTasks: vi.fn(() => []),
|
getAllTasks: vi.fn(() => []),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./service-routing.js', () => ({
|
||||||
|
activateCodexFailover: vi.fn(),
|
||||||
|
getEffectiveChannelLease: vi.fn(() => ({
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock('./logger.js', () => ({
|
vi.mock('./logger.js', () => ({
|
||||||
logger: {
|
logger: {
|
||||||
debug: vi.fn(),
|
debug: vi.fn(),
|
||||||
@@ -27,41 +43,35 @@ vi.mock('./logger.js', () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./provider-fallback.js', () => ({
|
vi.mock('./agent-error-detection.js', async (importOriginal) => {
|
||||||
detectFallbackTrigger: vi.fn((error?: string | null) => {
|
const actual = await importOriginal<typeof import('./agent-error-detection.js')>();
|
||||||
const lower = (error || '').toLowerCase();
|
return {
|
||||||
if (
|
...actual,
|
||||||
lower.includes('does not have access to claude') ||
|
classifyRotationTrigger: vi.fn((error?: string | null) => {
|
||||||
(lower.includes('failed to authenticate') &&
|
const lower = (error || '').toLowerCase();
|
||||||
lower.includes('403') &&
|
if (
|
||||||
lower.includes('terminated'))
|
lower.includes('does not have access to claude') ||
|
||||||
) {
|
(lower.includes('failed to authenticate') &&
|
||||||
return { shouldFallback: true, reason: 'org-access-denied' };
|
lower.includes('403') &&
|
||||||
}
|
lower.includes('terminated'))
|
||||||
if (
|
) {
|
||||||
lower.includes('429') ||
|
return { shouldRetry: true, reason: 'org-access-denied' };
|
||||||
lower.includes('rate limit') ||
|
}
|
||||||
lower.includes('hit your limit')
|
if (
|
||||||
) {
|
lower.includes('429') ||
|
||||||
return { shouldFallback: true, reason: '429' };
|
lower.includes('rate limit') ||
|
||||||
}
|
lower.includes('hit your limit')
|
||||||
return { shouldFallback: false, reason: '' };
|
) {
|
||||||
}),
|
return { shouldRetry: true, reason: '429' };
|
||||||
getActiveProvider: vi.fn(async () => 'claude'),
|
}
|
||||||
getFallbackEnvOverrides: vi.fn(() => ({
|
return { shouldRetry: false, reason: '' };
|
||||||
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
}),
|
||||||
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
|
};
|
||||||
ANTHROPIC_MODEL: 'kimi-k2.5',
|
});
|
||||||
})),
|
|
||||||
getFallbackProviderName: vi.fn(() => 'kimi'),
|
|
||||||
hasGroupProviderOverride: vi.fn(() => false),
|
|
||||||
isFallbackEnabled: vi.fn(() => true),
|
|
||||||
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
|
|
||||||
markPrimaryCooldown: vi.fn(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('./session-recovery.js', () => ({
|
vi.mock('./session-recovery.js', () => ({
|
||||||
shouldResetSessionOnAgentFailure: vi.fn(() => false),
|
shouldResetSessionOnAgentFailure: vi.fn(() => false),
|
||||||
|
shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./token-rotation.js', () => ({
|
vi.mock('./token-rotation.js', () => ({
|
||||||
@@ -96,9 +106,11 @@ vi.mock('./memento-client.js', () => ({
|
|||||||
|
|
||||||
import * as agentRunner from './agent-runner.js';
|
import * as agentRunner from './agent-runner.js';
|
||||||
import * as codexTokenRotation from './codex-token-rotation.js';
|
import * as codexTokenRotation from './codex-token-rotation.js';
|
||||||
|
import * as db from './db.js';
|
||||||
import { buildRoomMemoryBriefing } from './memento-client.js';
|
import { buildRoomMemoryBriefing } from './memento-client.js';
|
||||||
import { runAgentForGroup } from './message-agent-executor.js';
|
import { runAgentForGroup } from './message-agent-executor.js';
|
||||||
import * as providerFallback from './provider-fallback.js';
|
import * as sessionRecovery from './session-recovery.js';
|
||||||
|
import * as serviceRouting from './service-routing.js';
|
||||||
import * as tokenRotation from './token-rotation.js';
|
import * as tokenRotation from './token-rotation.js';
|
||||||
import type { RegisteredGroup } from './types.js';
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
@@ -129,9 +141,6 @@ function makeDeps() {
|
|||||||
describe('runAgentForGroup room memory', () => {
|
describe('runAgentForGroup room memory', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
|
|
||||||
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(false);
|
|
||||||
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
|
|
||||||
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||||
status: 'success',
|
status: 'success',
|
||||||
result: 'ok',
|
result: 'ok',
|
||||||
@@ -167,7 +176,6 @@ describe('runAgentForGroup room memory', () => {
|
|||||||
}),
|
}),
|
||||||
expect.any(Function),
|
expect.any(Function),
|
||||||
undefined,
|
undefined,
|
||||||
undefined,
|
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -196,6 +204,59 @@ describe('runAgentForGroup room memory', () => {
|
|||||||
}),
|
}),
|
||||||
expect.any(Function),
|
expect.any(Function),
|
||||||
undefined,
|
undefined,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('injects suppress token instructions into the agent prompt', async () => {
|
||||||
|
const group = { ...makeGroup(), folder: 'test-group' };
|
||||||
|
|
||||||
|
await runAgentForGroup(makeDeps(), {
|
||||||
|
group,
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-suppress',
|
||||||
|
suppressToken: '__TEST_SUPPRESS__',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledWith(
|
||||||
|
group,
|
||||||
|
expect.objectContaining({
|
||||||
|
prompt: expect.stringContaining(
|
||||||
|
'If you have no user-visible content to send for this turn, output exactly this token and nothing else: __TEST_SUPPRESS__',
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
expect.any(Function),
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('adds reviewer silence guidance when the current service is the reviewer for the chat', async () => {
|
||||||
|
const group = { ...makeGroup(), folder: 'test-group' };
|
||||||
|
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'claude',
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
await runAgentForGroup(makeDeps(), {
|
||||||
|
group,
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-review-suppress',
|
||||||
|
suppressToken: '__TEST_SUPPRESS__',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledWith(
|
||||||
|
group,
|
||||||
|
expect.objectContaining({
|
||||||
|
prompt: expect.stringContaining(
|
||||||
|
'If you are only agreeing, mirroring, or restating without adding a concrete correction, risk, missing prerequisite, test gap, or code change, output only the token.',
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
expect.any(Function),
|
||||||
undefined,
|
undefined,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -205,14 +266,11 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
|
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
|
||||||
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
|
|
||||||
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
|
|
||||||
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
|
|
||||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
||||||
vi.mocked(tokenRotation.rotateToken).mockReturnValue(false);
|
vi.mocked(tokenRotation.rotateToken).mockReturnValue(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rotates to another Claude account before falling back to Kimi', async () => {
|
it('rotates to another Claude account on usage exhaustion', async () => {
|
||||||
const outputs: string[] = [];
|
const outputs: string[] = [];
|
||||||
|
|
||||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
@@ -223,7 +281,7 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
await onOutput?.({
|
await onOutput?.({
|
||||||
status: 'success',
|
status: 'success',
|
||||||
phase: 'final',
|
phase: 'final',
|
||||||
result: 'You’re out of extra usage · resets 4am (Asia/Seoul)',
|
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
@@ -256,7 +314,7 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
// No fallback provider — rotation is the only recovery mechanism
|
||||||
expect(outputs).toEqual(['회전된 Claude 응답입니다.']);
|
expect(outputs).toEqual(['회전된 Claude 응답입니다.']);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -311,15 +369,15 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
// No fallback provider — rotation is the only recovery mechanism
|
||||||
expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']);
|
expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('suppresses Claude 502 HTML and falls back without forwarding it', async () => {
|
it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => {
|
||||||
const outputs: string[] = [];
|
const outputs: string[] = [];
|
||||||
|
|
||||||
vi.mocked(agentRunner.runAgentProcess)
|
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
await onOutput?.({
|
await onOutput?.({
|
||||||
status: 'success',
|
status: 'success',
|
||||||
phase: 'intermediate',
|
phase: 'intermediate',
|
||||||
@@ -336,24 +394,64 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
status: 'success',
|
status: 'success',
|
||||||
result: null,
|
result: null,
|
||||||
};
|
};
|
||||||
})
|
},
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
);
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'Kimi 폴백 응답입니다.',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await runAgentForGroup(makeDeps(), {
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
group: makeGroup(),
|
group: makeGroup(),
|
||||||
prompt: 'hello',
|
prompt: 'hello',
|
||||||
chatJid: 'group@test',
|
chatJid: 'group@test',
|
||||||
runId: 'run-claude-502-fallback',
|
runId: 'run-claude-502-error',
|
||||||
|
onOutput: async (output) => {
|
||||||
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('error');
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
||||||
|
expect(outputs).toEqual([]);
|
||||||
|
expect(db.createServiceHandoff).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears the Claude session and retries fresh when a retryable thinking 400 is surfaced as text', async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
const deps = makeDeps();
|
||||||
|
|
||||||
|
vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure)
|
||||||
|
.mockReturnValueOnce(true)
|
||||||
|
.mockReturnValueOnce(false);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'intermediate',
|
||||||
|
result:
|
||||||
|
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'fresh Claude retry success',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
newSessionId: 'claude-session-fresh',
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(deps, {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-retryable-thinking-400-retry',
|
||||||
onOutput: async (output) => {
|
onOutput: async (output) => {
|
||||||
if (typeof output.result === 'string') outputs.push(output.result);
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
},
|
},
|
||||||
@@ -361,15 +459,63 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
|
|
||||||
expect(result).toBe('success');
|
expect(result).toBe('success');
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
|
expect(deps.clearSession).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
expect(deps.clearSession).toHaveBeenCalledWith('test-claude');
|
||||||
'overloaded',
|
// No fallback provider — rotation is the only recovery mechanism
|
||||||
undefined,
|
expect(outputs).toEqual(['fresh Claude retry success']);
|
||||||
);
|
|
||||||
expect(outputs).toEqual(['Kimi 폴백 응답입니다.']);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('stops after all Claude accounts are usage-exhausted without falling back to Kimi', async () => {
|
it('returns error when the fresh Claude retry also hits the same retryable thinking 400', async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
const deps = makeDeps();
|
||||||
|
|
||||||
|
vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure)
|
||||||
|
.mockReturnValueOnce(true)
|
||||||
|
.mockReturnValueOnce(true);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'intermediate',
|
||||||
|
result:
|
||||||
|
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'intermediate',
|
||||||
|
result:
|
||||||
|
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(deps, {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-retryable-thinking-400-error',
|
||||||
|
onOutput: async (output) => {
|
||||||
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('error');
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(deps.clearSession).toHaveBeenCalledTimes(2);
|
||||||
|
expect(outputs).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns error after all Claude accounts are usage-exhausted', async () => {
|
||||||
const outputs: string[] = [];
|
const outputs: string[] = [];
|
||||||
|
|
||||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
@@ -382,7 +528,7 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
await onOutput?.({
|
await onOutput?.({
|
||||||
status: 'success',
|
status: 'success',
|
||||||
phase: 'final',
|
phase: 'final',
|
||||||
result: 'You’re out of extra usage · resets 4am (Asia/Seoul)',
|
result: 'You\u2019re out of extra usage \u00b7 resets 4am (Asia/Seoul)',
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
@@ -393,18 +539,7 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
await onOutput?.({
|
await onOutput?.({
|
||||||
status: 'success',
|
status: 'success',
|
||||||
phase: 'final',
|
phase: 'final',
|
||||||
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
|
result: "You're out of extra usage \u00b7 resets 4am (Asia/Seoul)",
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'Kimi 폴백 응답입니다.',
|
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
@@ -416,20 +551,71 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
group: makeGroup(),
|
group: makeGroup(),
|
||||||
prompt: 'hello',
|
prompt: 'hello',
|
||||||
chatJid: 'group@test',
|
chatJid: 'group@test',
|
||||||
runId: 'run-fallback-after-rotation',
|
runId: 'run-all-exhausted-error',
|
||||||
|
startSeq: 10,
|
||||||
|
endSeq: 12,
|
||||||
|
onOutput: async (output) => {
|
||||||
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('success');
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
|
||||||
|
expect(outputs).toEqual([]);
|
||||||
|
expect(serviceRouting.activateCodexFailover).toHaveBeenCalledWith(
|
||||||
|
'group@test',
|
||||||
|
'claude-usage-exhausted',
|
||||||
|
);
|
||||||
|
expect(db.createServiceHandoff).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
target_service_id: 'codex-review',
|
||||||
|
target_agent_type: 'codex',
|
||||||
|
start_seq: 10,
|
||||||
|
end_seq: 12,
|
||||||
|
reason: 'claude-usage-exhausted',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
|
||||||
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||||
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'progress',
|
||||||
|
result: '대화 요약 중...',
|
||||||
|
});
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: "You've hit your limit · resets 2am (Asia/Seoul)",
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-progress-before-usage-banner',
|
||||||
onOutput: async (output) => {
|
onOutput: async (output) => {
|
||||||
if (typeof output.result === 'string') outputs.push(output.result);
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result).toBe('error');
|
expect(result).toBe('error');
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
expect(outputs).toEqual(['대화 요약 중...']);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
|
expect(db.createServiceHandoff).not.toHaveBeenCalled();
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'usage-exhausted',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
expect(outputs).toEqual([]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rotates to another Claude account when Claude streams an org access denied banner', async () => {
|
it('rotates to another Claude account when Claude streams an org access denied banner', async () => {
|
||||||
@@ -483,11 +669,57 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
// No fallback provider — rotation is the only recovery mechanism
|
||||||
expect(outputs).toEqual(['org access denied 회전 성공 응답']);
|
expect(outputs).toEqual(['org access denied 회전 성공 응답']);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('stops after all Claude accounts are org-access-denied without falling back to Kimi', async () => {
|
it('rotates when Claude surfaces 403 terminated as a success result', async () => {
|
||||||
|
const outputs: string[] = [];
|
||||||
|
|
||||||
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
|
vi.mocked(tokenRotation.rotateToken).mockReturnValueOnce(true);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'Failed to authenticate. API Error: 403 terminated',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: '403 회전 성공 응답',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
|
group: makeGroup(),
|
||||||
|
prompt: 'hello',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-403-success-rotation',
|
||||||
|
onOutput: async (output) => {
|
||||||
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe('success');
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
|
expect(outputs).toEqual(['403 회전 성공 응답']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns error after all Claude accounts are org-access-denied', async () => {
|
||||||
const outputs: string[] = [];
|
const outputs: string[] = [];
|
||||||
|
|
||||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(2);
|
||||||
@@ -519,54 +751,34 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
result: null,
|
result: null,
|
||||||
error: 'Failed to authenticate. API Error: 403 terminated',
|
error: 'Failed to authenticate. API Error: 403 terminated',
|
||||||
};
|
};
|
||||||
})
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'Kimi 폴백 응답입니다.',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = await runAgentForGroup(makeDeps(), {
|
const result = await runAgentForGroup(makeDeps(), {
|
||||||
group: makeGroup(),
|
group: makeGroup(),
|
||||||
prompt: 'hello',
|
prompt: 'hello',
|
||||||
chatJid: 'group@test',
|
chatJid: 'group@test',
|
||||||
runId: 'run-org-access-denied-no-fallback',
|
runId: 'run-org-access-denied-error',
|
||||||
onOutput: async (output) => {
|
onOutput: async (output) => {
|
||||||
if (typeof output.result === 'string') outputs.push(output.result);
|
if (typeof output.result === 'string') outputs.push(output.result);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(result).toBe('error');
|
expect(result).toBe('success');
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(2);
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'org-access-denied',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
expect(outputs).toEqual([]);
|
expect(outputs).toEqual([]);
|
||||||
});
|
expect(serviceRouting.activateCodexFailover).toHaveBeenCalledWith(
|
||||||
|
'group@test',
|
||||||
it('skips execution entirely when Claude no-fallback cooldown is already active', async () => {
|
'claude-org-access-denied',
|
||||||
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('kimi');
|
);
|
||||||
vi.mocked(
|
expect(db.createServiceHandoff).toHaveBeenCalledWith(
|
||||||
providerFallback.isPrimaryNoFallbackCooldownActive,
|
expect.objectContaining({
|
||||||
).mockReturnValue(true);
|
chat_jid: 'group@test',
|
||||||
|
target_service_id: 'codex-review',
|
||||||
const result = await runAgentForGroup(makeDeps(), {
|
target_agent_type: 'codex',
|
||||||
group: makeGroup(),
|
reason: 'claude-org-access-denied',
|
||||||
prompt: 'hello',
|
}),
|
||||||
chatJid: 'group@test',
|
);
|
||||||
runId: 'run-skip-primary-cooldown',
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toBe('error');
|
|
||||||
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not mistake a normal response quoting the banner text for a usage error', async () => {
|
it('does not mistake a normal response quoting the banner text for a usage error', async () => {
|
||||||
@@ -601,7 +813,6 @@ describe('runAgentForGroup Claude rotation', () => {
|
|||||||
|
|
||||||
expect(result).toBe('success');
|
expect(result).toBe('success');
|
||||||
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
|
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
|
||||||
expect(outputs).toEqual([
|
expect(outputs).toEqual([
|
||||||
"상태 문구 예시: You're out of extra usage · resets 4am (Asia/Seoul) 라는 배너가 뜰 수 있습니다.",
|
"상태 문구 예시: You're out of extra usage · resets 4am (Asia/Seoul) 라는 배너가 뜰 수 있습니다.",
|
||||||
]);
|
]);
|
||||||
@@ -612,9 +823,6 @@ describe('runAgentForGroup Codex rotation', () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
|
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
|
||||||
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
|
|
||||||
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(false);
|
|
||||||
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
|
|
||||||
vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(2);
|
vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(2);
|
||||||
vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValueOnce(true);
|
vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValueOnce(true);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import path from 'path';
|
|
||||||
import { getErrorMessage } from './utils.js';
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@@ -8,23 +7,25 @@ import {
|
|||||||
writeTasksSnapshot,
|
writeTasksSnapshot,
|
||||||
} from './agent-runner.js';
|
} from './agent-runner.js';
|
||||||
import { listAvailableGroups } from './available-groups.js';
|
import { listAvailableGroups } from './available-groups.js';
|
||||||
import { DATA_DIR } from './config.js';
|
import { createServiceHandoff, getAllTasks } from './db.js';
|
||||||
import { getAllTasks } from './db.js';
|
|
||||||
import { GroupQueue } from './group-queue.js';
|
import { GroupQueue } from './group-queue.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { buildRoomMemoryBriefing } from './memento-client.js';
|
import { buildRoomMemoryBriefing } from './memento-client.js';
|
||||||
import {
|
import {
|
||||||
detectFallbackTrigger,
|
classifyRotationTrigger,
|
||||||
getActiveProvider,
|
type AgentTriggerReason,
|
||||||
getFallbackEnvOverrides,
|
} from './agent-error-detection.js';
|
||||||
getFallbackProviderName,
|
|
||||||
hasGroupProviderOverride,
|
|
||||||
isFallbackEnabled,
|
|
||||||
isPrimaryNoFallbackCooldownActive,
|
|
||||||
markPrimaryCooldown,
|
|
||||||
} from './provider-fallback.js';
|
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
import {
|
||||||
|
shouldResetSessionOnAgentFailure,
|
||||||
|
shouldRetryFreshSessionOnAgentFailure,
|
||||||
|
} from './session-recovery.js';
|
||||||
|
import { CODEX_MAIN_SERVICE_ID, CODEX_REVIEW_SERVICE_ID, SERVICE_SESSION_SCOPE } from './config.js';
|
||||||
|
import {
|
||||||
|
buildSuppressTokenPrompt,
|
||||||
|
classifySuppressTokenOutput,
|
||||||
|
} from './output-suppression.js';
|
||||||
|
import { activateCodexFailover, getEffectiveChannelLease } from './service-routing.js';
|
||||||
import {
|
import {
|
||||||
evaluateStreamedOutput,
|
evaluateStreamedOutput,
|
||||||
type StreamedOutputState,
|
type StreamedOutputState,
|
||||||
@@ -35,13 +36,12 @@ import {
|
|||||||
getCodexAccountCount,
|
getCodexAccountCount,
|
||||||
markCodexTokenHealthy,
|
markCodexTokenHealthy,
|
||||||
} from './codex-token-rotation.js';
|
} from './codex-token-rotation.js';
|
||||||
import type {
|
import type { CodexRotationReason } from './agent-error-detection.js';
|
||||||
AgentTriggerReason,
|
|
||||||
CodexRotationReason,
|
|
||||||
} from './agent-error-detection.js';
|
|
||||||
import { getTokenCount } from './token-rotation.js';
|
import { getTokenCount } from './token-rotation.js';
|
||||||
import type { RegisteredGroup } from './types.js';
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
|
// ── Main executor ─────────────────────────────────────────────────
|
||||||
|
|
||||||
export interface MessageAgentExecutorDeps {
|
export interface MessageAgentExecutorDeps {
|
||||||
assistantName: string;
|
assistantName: string;
|
||||||
queue: Pick<GroupQueue, 'registerProcess'>;
|
queue: Pick<GroupQueue, 'registerProcess'>;
|
||||||
@@ -58,10 +58,22 @@ export async function runAgentForGroup(
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
|
suppressToken?: string;
|
||||||
|
startSeq?: number | null;
|
||||||
|
endSeq?: number | null;
|
||||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||||
},
|
},
|
||||||
): Promise<'success' | 'error'> {
|
): Promise<'success' | 'error'> {
|
||||||
const { group, prompt, chatJid, runId, onOutput } = args;
|
const {
|
||||||
|
group,
|
||||||
|
prompt,
|
||||||
|
chatJid,
|
||||||
|
runId,
|
||||||
|
suppressToken,
|
||||||
|
startSeq,
|
||||||
|
endSeq,
|
||||||
|
onOutput,
|
||||||
|
} = args;
|
||||||
const isMain = group.isMain === true;
|
const isMain = group.isMain === true;
|
||||||
const isClaudeCodeAgent =
|
const isClaudeCodeAgent =
|
||||||
(group.agentType || 'claude-code') === 'claude-code';
|
(group.agentType || 'claude-code') === 'claude-code';
|
||||||
@@ -97,20 +109,61 @@ export async function runAgentForGroup(
|
|||||||
|
|
||||||
let resetSessionRequested = false;
|
let resetSessionRequested = false;
|
||||||
|
|
||||||
const settingsPath = path.join(
|
|
||||||
DATA_DIR,
|
|
||||||
'sessions',
|
|
||||||
group.folder,
|
|
||||||
'.claude',
|
|
||||||
'settings.json',
|
|
||||||
);
|
|
||||||
const groupHasOverride = hasGroupProviderOverride(settingsPath);
|
|
||||||
const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1;
|
const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1;
|
||||||
const canFallback =
|
const currentLease = getEffectiveChannelLease(chatJid);
|
||||||
isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride;
|
const reviewerMode = currentLease.reviewer_service_id === SERVICE_SESSION_SCOPE;
|
||||||
|
const effectivePrompt = buildSuppressTokenPrompt(prompt, suppressToken, {
|
||||||
|
reviewerMode,
|
||||||
|
});
|
||||||
|
|
||||||
|
const shouldHandoffToCodex = (
|
||||||
|
reason: AgentTriggerReason,
|
||||||
|
sawVisibleOutput: boolean,
|
||||||
|
): boolean => {
|
||||||
|
if (sawVisibleOutput) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
reason === '429' ||
|
||||||
|
reason === 'usage-exhausted' ||
|
||||||
|
reason === 'auth-expired' ||
|
||||||
|
reason === 'org-access-denied'
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const maybeHandoffToCodex = (
|
||||||
|
reason: AgentTriggerReason,
|
||||||
|
sawVisibleOutput: boolean,
|
||||||
|
): boolean => {
|
||||||
|
if (!isClaudeCodeAgent) return false;
|
||||||
|
if (!shouldHandoffToCodex(reason, sawVisibleOutput)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (currentLease.reviewer_service_id === null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
activateCodexFailover(chatJid, `claude-${reason}`);
|
||||||
|
createServiceHandoff({
|
||||||
|
chat_jid: chatJid,
|
||||||
|
group_folder: group.folder,
|
||||||
|
source_service_id: SERVICE_SESSION_SCOPE,
|
||||||
|
target_service_id: CODEX_REVIEW_SERVICE_ID,
|
||||||
|
target_agent_type: 'codex',
|
||||||
|
prompt,
|
||||||
|
start_seq: startSeq ?? null,
|
||||||
|
end_seq: endSeq ?? null,
|
||||||
|
reason: `claude-${reason}`,
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{ chatJid, group: group.name, runId, reason },
|
||||||
|
'Claude unavailable, handed off current turn to codex-review',
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
const agentInput = {
|
const agentInput = {
|
||||||
prompt,
|
prompt: effectivePrompt,
|
||||||
sessionId,
|
sessionId,
|
||||||
memoryBriefing,
|
memoryBriefing,
|
||||||
groupFolder: group.folder,
|
groupFolder: group.folder,
|
||||||
@@ -126,29 +179,31 @@ export async function runAgentForGroup(
|
|||||||
output?: AgentOutput;
|
output?: AgentOutput;
|
||||||
error?: unknown;
|
error?: unknown;
|
||||||
sawOutput: boolean;
|
sawOutput: boolean;
|
||||||
|
sawVisibleOutput: boolean;
|
||||||
sawSuccessNullResultWithoutOutput: boolean;
|
sawSuccessNullResultWithoutOutput: boolean;
|
||||||
|
retryableSessionFailureDetected: boolean;
|
||||||
streamedTriggerReason?: {
|
streamedTriggerReason?: {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
};
|
};
|
||||||
}> => {
|
}> => {
|
||||||
const persistSessionIds = provider === 'claude';
|
|
||||||
let streamedState: StreamedOutputState = {
|
let streamedState: StreamedOutputState = {
|
||||||
sawOutput: false,
|
sawOutput: false,
|
||||||
|
sawVisibleOutput: false,
|
||||||
sawSuccessNullResultWithoutOutput: false,
|
sawSuccessNullResultWithoutOutput: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
const wrappedOnOutput = onOutput
|
const wrappedOnOutput = onOutput
|
||||||
? async (output: AgentOutput) => {
|
? async (output: AgentOutput) => {
|
||||||
if (
|
if (
|
||||||
persistSessionIds &&
|
|
||||||
isClaudeCodeAgent &&
|
isClaudeCodeAgent &&
|
||||||
|
provider === 'claude' &&
|
||||||
shouldResetSessionOnAgentFailure(output)
|
shouldResetSessionOnAgentFailure(output)
|
||||||
) {
|
) {
|
||||||
resetSessionRequested = true;
|
resetSessionRequested = true;
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
persistSessionIds &&
|
provider === 'claude' &&
|
||||||
output.newSessionId &&
|
output.newSessionId &&
|
||||||
!resetSessionRequested
|
!resetSessionRequested
|
||||||
) {
|
) {
|
||||||
@@ -161,7 +216,7 @@ export async function runAgentForGroup(
|
|||||||
trackSuccessNullResult: true,
|
trackSuccessNullResult: true,
|
||||||
shortCircuitTriggeredErrors:
|
shortCircuitTriggeredErrors:
|
||||||
provider === 'claude'
|
provider === 'claude'
|
||||||
? canFallback || canRotateToken
|
? canRotateToken
|
||||||
: getCodexAccountCount() > 1,
|
: getCodexAccountCount() > 1,
|
||||||
});
|
});
|
||||||
streamedState = evaluation.state;
|
streamedState = evaluation.state;
|
||||||
@@ -179,7 +234,7 @@ export async function runAgentForGroup(
|
|||||||
reason: evaluation.newTrigger.reason,
|
reason: evaluation.newTrigger.reason,
|
||||||
resultPreview: output.result.slice(0, 120),
|
resultPreview: output.result.slice(0, 120),
|
||||||
},
|
},
|
||||||
'Detected Claude fallback trigger in successful output',
|
'Detected Claude rotation trigger in successful output',
|
||||||
);
|
);
|
||||||
} else if (
|
} else if (
|
||||||
evaluation.newTrigger &&
|
evaluation.newTrigger &&
|
||||||
@@ -194,7 +249,7 @@ export async function runAgentForGroup(
|
|||||||
errorPreview: output.error.slice(0, 120),
|
errorPreview: output.error.slice(0, 120),
|
||||||
},
|
},
|
||||||
provider === 'claude'
|
provider === 'claude'
|
||||||
? 'Detected Claude fallback trigger in streamed error output'
|
? 'Detected Claude rotation trigger in streamed error output'
|
||||||
: 'Detected Codex rotation trigger in streamed error output',
|
: 'Detected Codex rotation trigger in streamed error output',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -215,39 +270,53 @@ export async function runAgentForGroup(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (evaluation.suppressedRetryableSessionFailure) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
runId,
|
||||||
|
resultPreview:
|
||||||
|
typeof output.result === 'string'
|
||||||
|
? output.result.slice(0, 160)
|
||||||
|
: output.error?.slice(0, 160),
|
||||||
|
},
|
||||||
|
'Suppressed retryable Claude session failure from chat output',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!evaluation.shouldForwardOutput) {
|
if (!evaluation.shouldForwardOutput) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const suppressState =
|
||||||
|
typeof output.result === 'string'
|
||||||
|
? classifySuppressTokenOutput(output.result, suppressToken)
|
||||||
|
: 'none';
|
||||||
|
if (
|
||||||
|
typeof output.result === 'string' &&
|
||||||
|
output.result.length > 0 &&
|
||||||
|
suppressState === 'none'
|
||||||
|
) {
|
||||||
|
streamedState = {
|
||||||
|
...evaluation.state,
|
||||||
|
sawVisibleOutput: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
await onOutput(output);
|
await onOutput(output);
|
||||||
}
|
}
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
if (provider !== 'claude') {
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
provider,
|
|
||||||
},
|
|
||||||
`Claude provider in cooldown, routing request to ${provider}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const agentType = group.agentType || 'claude-code';
|
const agentType = group.agentType || 'claude-code';
|
||||||
const providerLabel = canFallback ? provider : agentType;
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
chatJid,
|
chatJid,
|
||||||
group: group.name,
|
group: group.name,
|
||||||
groupFolder: group.folder,
|
groupFolder: group.folder,
|
||||||
runId,
|
runId,
|
||||||
provider: providerLabel,
|
provider: agentType,
|
||||||
canFallback,
|
|
||||||
groupHasOverride,
|
|
||||||
},
|
},
|
||||||
`Using provider: ${providerLabel}`,
|
`Using provider: ${agentType}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -255,17 +324,14 @@ export async function runAgentForGroup(
|
|||||||
group,
|
group,
|
||||||
{
|
{
|
||||||
...agentInput,
|
...agentInput,
|
||||||
sessionId: persistSessionIds ? sessionId : undefined,
|
sessionId: provider === 'claude' ? sessionId : undefined,
|
||||||
},
|
},
|
||||||
(proc, processName, ipcDir) =>
|
(proc, processName, ipcDir) =>
|
||||||
deps.queue.registerProcess(chatJid, proc, processName, ipcDir),
|
deps.queue.registerProcess(chatJid, proc, processName, ipcDir),
|
||||||
wrappedOnOutput,
|
wrappedOnOutput,
|
||||||
isClaudeCodeAgent && provider !== 'claude'
|
|
||||||
? getFallbackEnvOverrides()
|
|
||||||
: undefined,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
if (persistSessionIds && output.newSessionId) {
|
if (provider === 'claude' && output.newSessionId) {
|
||||||
deps.persistSession(group.folder, output.newSessionId);
|
deps.persistSession(group.folder, output.newSessionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -285,75 +351,27 @@ export async function runAgentForGroup(
|
|||||||
return {
|
return {
|
||||||
output,
|
output,
|
||||||
sawOutput: streamedState.sawOutput,
|
sawOutput: streamedState.sawOutput,
|
||||||
|
sawVisibleOutput: streamedState.sawVisibleOutput,
|
||||||
sawSuccessNullResultWithoutOutput:
|
sawSuccessNullResultWithoutOutput:
|
||||||
streamedState.sawSuccessNullResultWithoutOutput,
|
streamedState.sawSuccessNullResultWithoutOutput,
|
||||||
|
retryableSessionFailureDetected:
|
||||||
|
streamedState.retryableSessionFailureDetected === true,
|
||||||
streamedTriggerReason: streamedState.streamedTriggerReason,
|
streamedTriggerReason: streamedState.streamedTriggerReason,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return {
|
return {
|
||||||
error,
|
error,
|
||||||
sawOutput: streamedState.sawOutput,
|
sawOutput: streamedState.sawOutput,
|
||||||
|
sawVisibleOutput: streamedState.sawVisibleOutput,
|
||||||
sawSuccessNullResultWithoutOutput:
|
sawSuccessNullResultWithoutOutput:
|
||||||
streamedState.sawSuccessNullResultWithoutOutput,
|
streamedState.sawSuccessNullResultWithoutOutput,
|
||||||
|
retryableSessionFailureDetected:
|
||||||
|
streamedState.retryableSessionFailureDetected === true,
|
||||||
streamedTriggerReason: streamedState.streamedTriggerReason,
|
streamedTriggerReason: streamedState.streamedTriggerReason,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const runFallbackAttempt = async (
|
|
||||||
reason: AgentTriggerReason,
|
|
||||||
retryAfterMs?: number,
|
|
||||||
): Promise<'success' | 'error'> => {
|
|
||||||
const fallbackName = getFallbackProviderName();
|
|
||||||
markPrimaryCooldown(reason, retryAfterMs);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
reason,
|
|
||||||
retryAfterMs,
|
|
||||||
fallbackProvider: fallbackName,
|
|
||||||
},
|
|
||||||
`Falling back to provider: ${fallbackName} (reason: ${reason})`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const fallbackAttempt = await runAttempt(fallbackName);
|
|
||||||
if (fallbackAttempt.error) {
|
|
||||||
logger.error(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
provider: fallbackName,
|
|
||||||
err: fallbackAttempt.error,
|
|
||||||
},
|
|
||||||
'Fallback provider also threw',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fallbackAttempt.output?.status === 'error') {
|
|
||||||
logger.error(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
provider: fallbackName,
|
|
||||||
error: fallbackAttempt.output.error,
|
|
||||||
},
|
|
||||||
`Fallback provider (${fallbackName}) also failed`,
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
return 'success';
|
|
||||||
};
|
|
||||||
|
|
||||||
const retryCodexWithRotation = async (
|
const retryCodexWithRotation = async (
|
||||||
initialTrigger: { reason: CodexRotationReason },
|
initialTrigger: { reason: CodexRotationReason },
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
@@ -496,63 +514,86 @@ export async function runAgentForGroup(
|
|||||||
return 'success';
|
return 'success';
|
||||||
case 'error':
|
case 'error':
|
||||||
return 'error';
|
return 'error';
|
||||||
case 'no-fallback':
|
|
||||||
return 'error';
|
|
||||||
case 'needs-fallback':
|
|
||||||
if (outcome.trigger.reason === 'success-null-result') {
|
|
||||||
return canFallback
|
|
||||||
? runFallbackAttempt('success-null-result')
|
|
||||||
: 'error';
|
|
||||||
}
|
|
||||||
if (!canFallback) {
|
|
||||||
logger.warn(
|
|
||||||
{ ...logCtx, reason: outcome.trigger.reason },
|
|
||||||
'All Claude tokens exhausted and fallback disabled',
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
return runFallbackAttempt(
|
|
||||||
outcome.trigger.reason,
|
|
||||||
outcome.trigger.retryAfterMs,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const provider = canFallback ? await getActiveProvider() : 'claude';
|
const maybeHandoffAfterError = (
|
||||||
|
reason: AgentTriggerReason,
|
||||||
// Already in no-fallback Claude cooldown — log only, no response
|
attempt: Awaited<ReturnType<typeof runAttempt>>,
|
||||||
if (provider !== 'claude' && isPrimaryNoFallbackCooldownActive()) {
|
): 'success' | 'error' => {
|
||||||
logger.info(
|
if (maybeHandoffToCodex(reason, attempt.sawVisibleOutput)) {
|
||||||
{ chatJid, group: group.name, runId, provider },
|
return 'success';
|
||||||
'Claude primary cooldown active, silently skipping',
|
}
|
||||||
);
|
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
};
|
||||||
|
|
||||||
const primaryAttempt = await runAttempt(provider);
|
const provider = 'claude';
|
||||||
|
|
||||||
|
let primaryAttempt = await runAttempt(provider);
|
||||||
|
|
||||||
|
const isRetryableClaudeSessionFailure = (
|
||||||
|
attempt: Awaited<ReturnType<typeof runAttempt>>,
|
||||||
|
): boolean =>
|
||||||
|
isClaudeCodeAgent &&
|
||||||
|
provider === 'claude' &&
|
||||||
|
!attempt.sawOutput &&
|
||||||
|
(attempt.retryableSessionFailureDetected === true ||
|
||||||
|
(attempt.error != null &&
|
||||||
|
shouldRetryFreshSessionOnAgentFailure({
|
||||||
|
result: null,
|
||||||
|
error: getErrorMessage(attempt.error),
|
||||||
|
})));
|
||||||
|
|
||||||
|
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
|
||||||
|
deps.clearSession(group.folder);
|
||||||
|
logger.warn(
|
||||||
|
{ group: group.name, chatJid, runId },
|
||||||
|
'Cleared poisoned Claude session before visible output, retrying fresh session',
|
||||||
|
);
|
||||||
|
|
||||||
|
primaryAttempt = await runAttempt('claude');
|
||||||
|
|
||||||
|
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
|
||||||
|
deps.clearSession(group.folder);
|
||||||
|
logger.warn(
|
||||||
|
{ group: group.name, chatJid, runId },
|
||||||
|
'Fresh Claude retry also hit a retryable session failure',
|
||||||
|
);
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{ group: group.name, chatJid, runId },
|
||||||
|
'Retryable Claude session failure persisted after fresh retry',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (primaryAttempt.error) {
|
if (primaryAttempt.error) {
|
||||||
if (
|
if (
|
||||||
(canFallback || canRotateToken) &&
|
canRotateToken &&
|
||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
!primaryAttempt.sawOutput
|
!primaryAttempt.sawOutput
|
||||||
) {
|
) {
|
||||||
const errMsg = getErrorMessage(primaryAttempt.error);
|
const errMsg = getErrorMessage(primaryAttempt.error);
|
||||||
const trigger = primaryAttempt.streamedTriggerReason
|
const trigger = primaryAttempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
||||||
}
|
}
|
||||||
: detectFallbackTrigger(errMsg);
|
: classifyRotationTrigger(errMsg);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldRetry) {
|
||||||
return retryClaudeWithRotation(
|
const result = await retryClaudeWithRotation(
|
||||||
{
|
{
|
||||||
reason: trigger.reason,
|
reason: trigger.reason,
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
},
|
},
|
||||||
errMsg,
|
errMsg,
|
||||||
);
|
);
|
||||||
|
if (result === 'error') {
|
||||||
|
return maybeHandoffAfterError(trigger.reason, primaryAttempt);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -594,25 +635,23 @@ export async function runAgentForGroup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
(canFallback || canRotateToken) &&
|
canRotateToken &&
|
||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
!primaryAttempt.sawOutput &&
|
!primaryAttempt.sawOutput &&
|
||||||
primaryAttempt.streamedTriggerReason &&
|
primaryAttempt.streamedTriggerReason &&
|
||||||
output.status !== 'error'
|
output.status !== 'error'
|
||||||
) {
|
) {
|
||||||
return retryClaudeWithRotation({
|
const result = await retryClaudeWithRotation({
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
||||||
});
|
});
|
||||||
}
|
if (result === 'error') {
|
||||||
|
return maybeHandoffAfterError(
|
||||||
if (
|
primaryAttempt.streamedTriggerReason.reason,
|
||||||
canFallback &&
|
primaryAttempt,
|
||||||
provider === 'claude' &&
|
);
|
||||||
!primaryAttempt.sawOutput &&
|
}
|
||||||
primaryAttempt.sawSuccessNullResultWithoutOutput
|
return result;
|
||||||
) {
|
|
||||||
return runFallbackAttempt('success-null-result');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -628,25 +667,29 @@ export async function runAgentForGroup(
|
|||||||
|
|
||||||
if (output.status === 'error') {
|
if (output.status === 'error') {
|
||||||
if (
|
if (
|
||||||
(canFallback || canRotateToken) &&
|
canRotateToken &&
|
||||||
provider === 'claude' &&
|
provider === 'claude' &&
|
||||||
!primaryAttempt.sawOutput
|
!primaryAttempt.sawOutput
|
||||||
) {
|
) {
|
||||||
const trigger = primaryAttempt.streamedTriggerReason
|
const trigger = primaryAttempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: primaryAttempt.streamedTriggerReason.reason,
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
||||||
}
|
}
|
||||||
: detectFallbackTrigger(output.error);
|
: classifyRotationTrigger(output.error);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldRetry) {
|
||||||
return retryClaudeWithRotation(
|
const result = await retryClaudeWithRotation(
|
||||||
{
|
{
|
||||||
reason: trigger.reason,
|
reason: trigger.reason,
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
},
|
},
|
||||||
output.error ?? undefined,
|
output.error ?? undefined,
|
||||||
);
|
);
|
||||||
|
if (result === 'error') {
|
||||||
|
return maybeHandoffAfterError(trigger.reason, primaryAttempt);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -687,5 +730,38 @@ export async function runAgentForGroup(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Unresolved streamed trigger — rotation was unavailable or output was
|
||||||
|
// already forwarded. Surfaces as an error since there is no alternative provider.
|
||||||
|
if (primaryAttempt.streamedTriggerReason) {
|
||||||
|
if (
|
||||||
|
isClaudeCodeAgent &&
|
||||||
|
maybeHandoffToCodex(
|
||||||
|
primaryAttempt.streamedTriggerReason.reason,
|
||||||
|
primaryAttempt.sawVisibleOutput,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return 'success';
|
||||||
|
}
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
group: group.name,
|
||||||
|
chatJid,
|
||||||
|
runId,
|
||||||
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
|
},
|
||||||
|
'Agent trigger detected but could not be resolved',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
// success-null-result with no visible output — agent returned nothing useful
|
||||||
|
if (primaryAttempt.sawSuccessNullResultWithoutOutput) {
|
||||||
|
logger.error(
|
||||||
|
{ group: group.name, chatJid, runId },
|
||||||
|
'Agent returned success with null result and no visible output',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
return 'success';
|
return 'success';
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,8 +11,21 @@ vi.mock('./agent-runner.js', () => ({
|
|||||||
writeTasksSnapshot: vi.fn(),
|
writeTasksSnapshot: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./output-suppression.js', async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import('./output-suppression.js')>();
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
createSuppressToken: vi.fn(() => '__TEST_SUPPRESS__'),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
DATA_DIR: '/tmp/ejclaw-test-data',
|
DATA_DIR: '/tmp/ejclaw-test-data',
|
||||||
|
SERVICE_ID: 'claude',
|
||||||
|
SERVICE_AGENT_TYPE: 'claude-code',
|
||||||
|
SERVICE_SESSION_SCOPE: 'claude',
|
||||||
|
isClaudeService: vi.fn(() => true),
|
||||||
|
isReviewService: vi.fn(() => false),
|
||||||
isSessionCommandSenderAllowed: vi.fn(() => false),
|
isSessionCommandSenderAllowed: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -40,6 +53,10 @@ vi.mock('./db.js', () => {
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
claimServiceHandoff: vi.fn(() => true),
|
||||||
|
completeServiceHandoff: vi.fn(),
|
||||||
|
completeServiceHandoffAndAdvanceTargetCursor: vi.fn(),
|
||||||
|
failServiceHandoff: vi.fn(),
|
||||||
getAllChats: vi.fn(() => []),
|
getAllChats: vi.fn(() => []),
|
||||||
getAllTasks: vi.fn(() => []),
|
getAllTasks: vi.fn(() => []),
|
||||||
getLastHumanMessageTimestamp: vi.fn(() => null),
|
getLastHumanMessageTimestamp: vi.fn(() => null),
|
||||||
@@ -89,11 +106,13 @@ vi.mock('./db.js', () => {
|
|||||||
},
|
},
|
||||||
),
|
),
|
||||||
getOpenWorkItem: vi.fn(() => undefined),
|
getOpenWorkItem: vi.fn(() => undefined),
|
||||||
|
getPendingServiceHandoffs: vi.fn(() => []),
|
||||||
createProducedWorkItem: vi.fn((input) => ({
|
createProducedWorkItem: vi.fn((input) => ({
|
||||||
id: 1,
|
id: 1,
|
||||||
group_folder: input.group_folder,
|
group_folder: input.group_folder,
|
||||||
chat_jid: input.chat_jid,
|
chat_jid: input.chat_jid,
|
||||||
agent_type: input.agent_type || 'claude-code',
|
agent_type: input.agent_type || 'claude-code',
|
||||||
|
service_id: 'claude',
|
||||||
status: 'produced',
|
status: 'produced',
|
||||||
start_seq: input.start_seq,
|
start_seq: input.start_seq,
|
||||||
end_seq: input.end_seq,
|
end_seq: input.end_seq,
|
||||||
@@ -108,9 +127,22 @@ vi.mock('./db.js', () => {
|
|||||||
markWorkItemDelivered: vi.fn(),
|
markWorkItemDelivered: vi.fn(),
|
||||||
markWorkItemDeliveryRetry: vi.fn(),
|
markWorkItemDeliveryRetry: vi.fn(),
|
||||||
isPairedRoomJid: vi.fn(() => false),
|
isPairedRoomJid: vi.fn(() => false),
|
||||||
|
getLastBotFinalMessage: vi.fn(() => []),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
vi.mock('./service-routing.js', () => ({
|
||||||
|
getEffectiveChannelLease: vi.fn((chatJid: string) => ({
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
})),
|
||||||
|
shouldServiceProcessChat: vi.fn(() => true),
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock('./logger.js', () => ({
|
vi.mock('./logger.js', () => ({
|
||||||
logger: {
|
logger: {
|
||||||
debug: vi.fn(),
|
debug: vi.fn(),
|
||||||
@@ -120,21 +152,6 @@ vi.mock('./logger.js', () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./provider-fallback.js', () => ({
|
|
||||||
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
|
|
||||||
getActiveProvider: vi.fn(async () => 'claude'),
|
|
||||||
getFallbackEnvOverrides: vi.fn(() => ({
|
|
||||||
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
|
||||||
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
|
|
||||||
ANTHROPIC_MODEL: 'kimi-k2.5',
|
|
||||||
})),
|
|
||||||
getFallbackProviderName: vi.fn(() => 'kimi'),
|
|
||||||
hasGroupProviderOverride: vi.fn(() => false),
|
|
||||||
isFallbackEnabled: vi.fn(() => true),
|
|
||||||
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
|
|
||||||
markPrimaryCooldown: vi.fn(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('./sender-allowlist.js', () => ({
|
vi.mock('./sender-allowlist.js', () => ({
|
||||||
isTriggerAllowed: vi.fn(() => true),
|
isTriggerAllowed: vi.fn(() => true),
|
||||||
loadSenderAllowlist: vi.fn(() => ({})),
|
loadSenderAllowlist: vi.fn(() => ({})),
|
||||||
@@ -151,7 +168,8 @@ import * as agentRunner from './agent-runner.js';
|
|||||||
import * as db from './db.js';
|
import * as db from './db.js';
|
||||||
import { resolveGroupIpcPath } from './group-folder.js';
|
import { resolveGroupIpcPath } from './group-folder.js';
|
||||||
import { createMessageRuntime } from './message-runtime.js';
|
import { createMessageRuntime } from './message-runtime.js';
|
||||||
import * as providerFallback from './provider-fallback.js';
|
import * as outputSuppression from './output-suppression.js';
|
||||||
|
import * as config from './config.js';
|
||||||
import type { Channel, RegisteredGroup } from './types.js';
|
import type { Channel, RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
function makeGroup(agentType: 'claude-code' | 'codex'): RegisteredGroup {
|
function makeGroup(agentType: 'claude-code' | 'codex'): RegisteredGroup {
|
||||||
@@ -182,19 +200,10 @@ function makeChannel(chatJid: string): Channel {
|
|||||||
describe('createMessageRuntime', () => {
|
describe('createMessageRuntime', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
|
vi.mocked(db.getLastBotFinalMessage).mockReturnValue([]);
|
||||||
vi.mocked(providerFallback.getFallbackProviderName).mockReturnValue('kimi');
|
vi.mocked(db.isPairedRoomJid).mockReturnValue(false);
|
||||||
vi.mocked(providerFallback.getFallbackEnvOverrides).mockReturnValue({
|
vi.mocked(config.isClaudeService).mockReturnValue(true);
|
||||||
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
vi.mocked(config.isReviewService).mockReturnValue(false);
|
||||||
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
|
|
||||||
ANTHROPIC_MODEL: 'kimi-k2.5',
|
|
||||||
});
|
|
||||||
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
|
|
||||||
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
|
|
||||||
vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({
|
|
||||||
shouldFallback: false,
|
|
||||||
reason: '',
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('ignores generic failure bot messages in paired rooms', async () => {
|
it('ignores generic failure bot messages in paired rooms', async () => {
|
||||||
@@ -257,6 +266,8 @@ describe('createMessageRuntime', () => {
|
|||||||
const saveState = vi.fn();
|
const saveState = vi.fn();
|
||||||
const lastAgentTimestamps: Record<string, string> = {};
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
|
||||||
|
vi.mocked(config.isClaudeService).mockReturnValue(false);
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(false);
|
||||||
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
|
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
|
||||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
{
|
{
|
||||||
@@ -317,10 +328,82 @@ describe('createMessageRuntime', () => {
|
|||||||
chatJid,
|
chatJid,
|
||||||
'그 방향이 맞습니다.',
|
'그 방향이 맞습니다.',
|
||||||
);
|
);
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false);
|
||||||
expect(lastAgentTimestamps[chatJid]).toBe('1');
|
expect(lastAgentTimestamps[chatJid]).toBe('1');
|
||||||
expect(saveState).toHaveBeenCalled();
|
expect(saveState).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not defer typing-on for suppress-capable review turns', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const saveState = vi.fn();
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
|
||||||
|
vi.mocked(config.isClaudeService).mockReturnValue(false);
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
|
vi.mocked(db.isPairedRoomJid).mockReturnValue(true);
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'other-bot@test',
|
||||||
|
sender_name: 'Other Bot',
|
||||||
|
content: '이어서 확인해줘.',
|
||||||
|
timestamp: '2026-03-18T09:00:00.000Z',
|
||||||
|
is_bot_message: true,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||||
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: '리뷰 확인 완료입니다.',
|
||||||
|
newSessionId: 'session-review-follow-up-immediate',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: '리뷰 확인 완료입니다.',
|
||||||
|
newSessionId: 'session-review-follow-up-immediate',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-review-follow-up-immediate-typing',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false);
|
||||||
|
});
|
||||||
|
|
||||||
it('ignores watcher status control messages in paired rooms', async () => {
|
it('ignores watcher status control messages in paired rooms', async () => {
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
const group = makeGroup('codex');
|
const group = makeGroup('codex');
|
||||||
@@ -1280,6 +1363,374 @@ describe('createMessageRuntime', () => {
|
|||||||
expect(channel.sendAndTrack).not.toHaveBeenCalled();
|
expect(channel.sendAndTrack).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not emit a visible message when the final output is the suppress token only', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
const saveState = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
seq: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||||
|
status: 'success',
|
||||||
|
result: '__TEST_SUPPRESS__',
|
||||||
|
newSessionId: 'session-suppress-run',
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-suppress-only',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(saveState).toHaveBeenCalled();
|
||||||
|
expect(lastAgentTimestamps[chatJid]).toBe('1');
|
||||||
|
expect(channel.sendMessage).not.toHaveBeenCalled();
|
||||||
|
expect(channel.sendAndTrack).not.toHaveBeenCalled();
|
||||||
|
expect(channel.setTyping).not.toHaveBeenCalledWith(chatJid, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('defers typing-on until the first visible output for suppress-capable turns', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('claude-code');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
const saveState = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
seq: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||||
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'visible suppress-capable reply',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
newSessionId: 'session-visible-suppress-capable',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-visible-suppress-capable',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||||
|
chatJid,
|
||||||
|
'visible suppress-capable reply',
|
||||||
|
);
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, true);
|
||||||
|
expect(channel.setTyping).toHaveBeenCalledWith(chatJid, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not grant suppress-token silence authority to codex-main', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
const saveState = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(config.isClaudeService).mockReturnValue(false);
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(false);
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
seq: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||||
|
status: 'success',
|
||||||
|
result: 'visible codex reply',
|
||||||
|
newSessionId: 'session-codex-main',
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-codex-main-no-suppress',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(outputSuppression.createSuppressToken).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('blocks malformed output when the suppress token is mixed with visible text', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
const saveState = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
seq: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||||
|
status: 'success',
|
||||||
|
result: '동의합니다 __TEST_SUPPRESS__',
|
||||||
|
newSessionId: 'session-suppress-mixed',
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-suppress-mixed',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(saveState).toHaveBeenCalled();
|
||||||
|
expect(lastAgentTimestamps[chatJid]).toBe('1');
|
||||||
|
expect(channel.sendMessage).not.toHaveBeenCalled();
|
||||||
|
expect(channel.sendAndTrack).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('suppresses a leaked foreign suppress token even when it differs from the current turn token', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
const saveState = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(config.isClaudeService).mockReturnValue(false);
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
seq: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||||
|
status: 'success',
|
||||||
|
result: '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
|
||||||
|
newSessionId: 'session-foreign-suppress-token',
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-foreign-suppress-token',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(saveState).toHaveBeenCalled();
|
||||||
|
expect(lastAgentTimestamps[chatJid]).toBe('1');
|
||||||
|
expect(channel.sendMessage).not.toHaveBeenCalled();
|
||||||
|
expect(channel.sendAndTrack).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('suppresses a malformed leaked foreign suppress token without the closing suffix', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const lastAgentTimestamps: Record<string, string> = {};
|
||||||
|
const saveState = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(config.isClaudeService).mockReturnValue(false);
|
||||||
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
seq: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||||
|
status: 'success',
|
||||||
|
result: '__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef',
|
||||||
|
newSessionId: 'session-malformed-foreign-suppress-token',
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||||
|
saveState,
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-malformed-foreign-suppress-token',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(saveState).toHaveBeenCalled();
|
||||||
|
expect(lastAgentTimestamps[chatJid]).toBe('1');
|
||||||
|
expect(channel.sendMessage).not.toHaveBeenCalled();
|
||||||
|
expect(channel.sendAndTrack).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it('resets tracked progress after a final output that becomes empty after formatting', async () => {
|
it('resets tracked progress after a final output that becomes empty after formatting', async () => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
@@ -1417,7 +1868,7 @@ describe('createMessageRuntime', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it('promotes the last progress output to a final message when the agent completes without a final phase', async () => {
|
it('promotes the last flushed progress output to a final message when the agent completes without a final phase', async () => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
const group = makeGroup('codex');
|
const group = makeGroup('codex');
|
||||||
@@ -1508,24 +1959,24 @@ describe('createMessageRuntime', () => {
|
|||||||
chatJid,
|
chatJid,
|
||||||
P('검증 중입니다.\n\n0초'),
|
P('검증 중입니다.\n\n0초'),
|
||||||
);
|
);
|
||||||
// Ticker fires after advanceTimersByTime — edits tracked message with latest heading
|
// Ticker fires after advanceTimersByTime — edits tracked message with the last flushed heading
|
||||||
expect(channel.editMessage).toHaveBeenCalledWith(
|
expect(channel.editMessage).toHaveBeenCalledWith(
|
||||||
chatJid,
|
chatJid,
|
||||||
'progress-1',
|
'progress-1',
|
||||||
P('테스트도 통과했습니다.\n\n5초'),
|
P('커밋은 정상 들어갔고 pre-commit도 통과했습니다.\n\n5초'),
|
||||||
);
|
);
|
||||||
// finish() promotes the last flushed progress text to a final message
|
// finish() promotes the last flushed progress text to a final message
|
||||||
expect(channel.sendMessage).toHaveBeenCalledTimes(1);
|
expect(channel.sendMessage).toHaveBeenCalledTimes(1);
|
||||||
expect(channel.sendMessage).toHaveBeenCalledWith(
|
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||||
chatJid,
|
chatJid,
|
||||||
'검증 중입니다.',
|
'커밋은 정상 들어갔고 pre-commit도 통과했습니다.',
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it('retries editing progress message instead of creating a duplicate when edit fails', async () => {
|
it('keeps going after a tracked progress edit fails and still emits the last flushed final message', async () => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
const group = makeGroup('codex');
|
const group = makeGroup('codex');
|
||||||
@@ -1614,8 +2065,7 @@ describe('createMessageRuntime', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(result).toBe(true);
|
expect(result).toBe(true);
|
||||||
// Only one progress message created via sendAndTrack — no duplicate
|
// The first flushed progress is still tracked
|
||||||
expect(channel.sendAndTrack).toHaveBeenCalledTimes(1);
|
|
||||||
expect(channel.sendAndTrack).toHaveBeenCalledWith(
|
expect(channel.sendAndTrack).toHaveBeenCalledWith(
|
||||||
chatJid,
|
chatJid,
|
||||||
P('진행 중입니다.\n\n0초'),
|
P('진행 중입니다.\n\n0초'),
|
||||||
@@ -1630,7 +2080,7 @@ describe('createMessageRuntime', () => {
|
|||||||
expect(channel.sendMessage).toHaveBeenCalledTimes(1);
|
expect(channel.sendMessage).toHaveBeenCalledTimes(1);
|
||||||
expect(channel.sendMessage).toHaveBeenCalledWith(
|
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||||
chatJid,
|
chatJid,
|
||||||
'진행 중입니다.',
|
'아직 진행 중.',
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
@@ -1868,331 +2318,6 @@ describe('createMessageRuntime', () => {
|
|||||||
expect(saveState).toHaveBeenCalled();
|
expect(saveState).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('retries with the fallback provider when Claude returns a 429 error before any output', async () => {
|
|
||||||
const chatJid = 'group@test';
|
|
||||||
const group = makeGroup('claude-code');
|
|
||||||
const channel = makeChannel(chatJid);
|
|
||||||
|
|
||||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
|
||||||
{
|
|
||||||
id: 'msg-1',
|
|
||||||
chat_jid: chatJid,
|
|
||||||
sender: 'user@test',
|
|
||||||
sender_name: 'User',
|
|
||||||
content: 'hello',
|
|
||||||
timestamp: '2026-03-19T00:00:00.000Z',
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
|
|
||||||
vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: '429',
|
|
||||||
retryAfterMs: 60_000,
|
|
||||||
});
|
|
||||||
|
|
||||||
vi.mocked(agentRunner.runAgentProcess)
|
|
||||||
.mockResolvedValueOnce({
|
|
||||||
status: 'error',
|
|
||||||
result: null,
|
|
||||||
error: '429 rate limited retry after 60',
|
|
||||||
})
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'fallback 응답입니다.',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const runtime = createMessageRuntime({
|
|
||||||
assistantName: 'Andy',
|
|
||||||
idleTimeout: 1_000,
|
|
||||||
pollInterval: 1_000,
|
|
||||||
timezone: 'UTC',
|
|
||||||
triggerPattern: /^@Andy\b/i,
|
|
||||||
channels: [channel],
|
|
||||||
queue: {
|
|
||||||
registerProcess: vi.fn(),
|
|
||||||
closeStdin: vi.fn(),
|
|
||||||
notifyIdle: vi.fn(),
|
|
||||||
} as any,
|
|
||||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
|
||||||
getSessions: () => ({}),
|
|
||||||
getLastTimestamp: () => '',
|
|
||||||
setLastTimestamp: vi.fn(),
|
|
||||||
getLastAgentTimestamps: () => ({}),
|
|
||||||
saveState: vi.fn(),
|
|
||||||
persistSession: vi.fn(),
|
|
||||||
clearSession: vi.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await runtime.processGroupMessages(chatJid, {
|
|
||||||
runId: 'run-fallback-429',
|
|
||||||
reason: 'messages',
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toBe(true);
|
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenNthCalledWith(
|
|
||||||
2,
|
|
||||||
expect.anything(),
|
|
||||||
expect.objectContaining({ sessionId: undefined }),
|
|
||||||
expect.any(Function),
|
|
||||||
expect.any(Function),
|
|
||||||
expect.objectContaining({
|
|
||||||
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
|
||||||
ANTHROPIC_MODEL: 'kimi-k2.5',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'429',
|
|
||||||
60_000,
|
|
||||||
);
|
|
||||||
expect(channel.sendMessage).toHaveBeenCalledWith(
|
|
||||||
chatJid,
|
|
||||||
'fallback 응답입니다.',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('silently suppresses a usage exhaustion banner without falling back', async () => {
|
|
||||||
const chatJid = 'group@test';
|
|
||||||
const group = makeGroup('claude-code');
|
|
||||||
const channel = makeChannel(chatJid);
|
|
||||||
|
|
||||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
|
||||||
{
|
|
||||||
id: 'msg-1',
|
|
||||||
chat_jid: chatJid,
|
|
||||||
sender: 'user@test',
|
|
||||||
sender_name: 'User',
|
|
||||||
content: 'hello',
|
|
||||||
timestamp: '2026-03-24T00:00:00.000Z',
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
|
|
||||||
vi.mocked(agentRunner.runAgentProcess)
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'usage fallback 응답입니다.',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const runtime = createMessageRuntime({
|
|
||||||
assistantName: 'Andy',
|
|
||||||
idleTimeout: 1_000,
|
|
||||||
pollInterval: 1_000,
|
|
||||||
timezone: 'UTC',
|
|
||||||
triggerPattern: /^@Andy\b/i,
|
|
||||||
channels: [channel],
|
|
||||||
queue: {
|
|
||||||
registerProcess: vi.fn(),
|
|
||||||
closeStdin: vi.fn(),
|
|
||||||
notifyIdle: vi.fn(),
|
|
||||||
} as any,
|
|
||||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
|
||||||
getSessions: () => ({}),
|
|
||||||
getLastTimestamp: () => '',
|
|
||||||
setLastTimestamp: vi.fn(),
|
|
||||||
getLastAgentTimestamps: () => ({}),
|
|
||||||
saveState: vi.fn(),
|
|
||||||
persistSession: vi.fn(),
|
|
||||||
clearSession: vi.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await runtime.processGroupMessages(chatJid, {
|
|
||||||
runId: 'run-fallback-usage-exhausted',
|
|
||||||
reason: 'messages',
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toBe(true);
|
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'usage-exhausted',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
expect(channel.sendMessage).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('suppresses duplicate streamed usage banners without emitting a visible reply', async () => {
|
|
||||||
const chatJid = 'group@test';
|
|
||||||
const group = makeGroup('claude-code');
|
|
||||||
const channel = makeChannel(chatJid);
|
|
||||||
|
|
||||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
|
||||||
{
|
|
||||||
id: 'msg-1',
|
|
||||||
chat_jid: chatJid,
|
|
||||||
sender: 'user@test',
|
|
||||||
sender_name: 'User',
|
|
||||||
content: 'hello',
|
|
||||||
timestamp: '2026-03-24T00:00:00.000Z',
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
|
|
||||||
vi.mocked(agentRunner.runAgentProcess)
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'intermediate',
|
|
||||||
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
|
|
||||||
});
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: "You're out of extra usage · resets 4am (Asia/Seoul)",
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'duplicate banner fallback 응답입니다.',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const runtime = createMessageRuntime({
|
|
||||||
assistantName: 'Andy',
|
|
||||||
idleTimeout: 1_000,
|
|
||||||
pollInterval: 1_000,
|
|
||||||
timezone: 'UTC',
|
|
||||||
triggerPattern: /^@Andy\b/i,
|
|
||||||
channels: [channel],
|
|
||||||
queue: {
|
|
||||||
registerProcess: vi.fn(),
|
|
||||||
closeStdin: vi.fn(),
|
|
||||||
notifyIdle: vi.fn(),
|
|
||||||
} as any,
|
|
||||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
|
||||||
getSessions: () => ({}),
|
|
||||||
getLastTimestamp: () => '',
|
|
||||||
setLastTimestamp: vi.fn(),
|
|
||||||
getLastAgentTimestamps: () => ({}),
|
|
||||||
saveState: vi.fn(),
|
|
||||||
persistSession: vi.fn(),
|
|
||||||
clearSession: vi.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await runtime.processGroupMessages(chatJid, {
|
|
||||||
runId: 'run-fallback-usage-exhausted-duplicate',
|
|
||||||
reason: 'messages',
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toBe(true);
|
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'usage-exhausted',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
expect(channel.sendMessage).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('retries with the fallback provider when Claude ends with success-null-result before any output', async () => {
|
|
||||||
const chatJid = 'group@test';
|
|
||||||
const group = makeGroup('claude-code');
|
|
||||||
const channel = makeChannel(chatJid);
|
|
||||||
|
|
||||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
|
||||||
{
|
|
||||||
id: 'msg-1',
|
|
||||||
chat_jid: chatJid,
|
|
||||||
sender: 'user@test',
|
|
||||||
sender_name: 'User',
|
|
||||||
content: 'hello',
|
|
||||||
timestamp: '2026-03-19T00:00:00.000Z',
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
|
|
||||||
vi.mocked(agentRunner.runAgentProcess)
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
phase: 'final',
|
|
||||||
result: 'success-null-result 폴백 응답입니다.',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
const runtime = createMessageRuntime({
|
|
||||||
assistantName: 'Andy',
|
|
||||||
idleTimeout: 1_000,
|
|
||||||
pollInterval: 1_000,
|
|
||||||
timezone: 'UTC',
|
|
||||||
triggerPattern: /^@Andy\b/i,
|
|
||||||
channels: [channel],
|
|
||||||
queue: {
|
|
||||||
registerProcess: vi.fn(),
|
|
||||||
closeStdin: vi.fn(),
|
|
||||||
notifyIdle: vi.fn(),
|
|
||||||
} as any,
|
|
||||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
|
||||||
getSessions: () => ({}),
|
|
||||||
getLastTimestamp: () => '',
|
|
||||||
setLastTimestamp: vi.fn(),
|
|
||||||
getLastAgentTimestamps: () => ({}),
|
|
||||||
saveState: vi.fn(),
|
|
||||||
persistSession: vi.fn(),
|
|
||||||
clearSession: vi.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = await runtime.processGroupMessages(chatJid, {
|
|
||||||
runId: 'run-fallback-success-null',
|
|
||||||
reason: 'messages',
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toBe(true);
|
|
||||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'success-null-result',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
expect(channel.sendMessage).toHaveBeenCalledWith(
|
|
||||||
chatJid,
|
|
||||||
'success-null-result 폴백 응답입니다.',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats missing streamed phase as final output', async () => {
|
it('treats missing streamed phase as final output', async () => {
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
const group = makeGroup('claude-code');
|
const group = makeGroup('claude-code');
|
||||||
@@ -2266,6 +2391,7 @@ describe('createMessageRuntime', () => {
|
|||||||
group_folder: group.folder,
|
group_folder: group.folder,
|
||||||
chat_jid: chatJid,
|
chat_jid: chatJid,
|
||||||
agent_type: 'claude-code',
|
agent_type: 'claude-code',
|
||||||
|
service_id: 'claude',
|
||||||
status: 'produced',
|
status: 'produced',
|
||||||
start_seq: 1,
|
start_seq: 1,
|
||||||
end_seq: 1,
|
end_seq: 1,
|
||||||
|
|||||||
@@ -1,17 +1,30 @@
|
|||||||
import { AgentOutput } from './agent-runner.js';
|
import { AgentOutput } from './agent-runner.js';
|
||||||
import { getErrorMessage } from './utils.js';
|
import { getErrorMessage } from './utils.js';
|
||||||
import {
|
import {
|
||||||
|
claimServiceHandoff,
|
||||||
|
completeServiceHandoffAndAdvanceTargetCursor,
|
||||||
|
createProducedWorkItem,
|
||||||
|
failServiceHandoff,
|
||||||
|
getOpenWorkItem,
|
||||||
|
getPendingServiceHandoffs,
|
||||||
getMessagesSinceSeq,
|
getMessagesSinceSeq,
|
||||||
getNewMessagesBySeq,
|
getNewMessagesBySeq,
|
||||||
getOpenWorkItem,
|
|
||||||
createProducedWorkItem,
|
|
||||||
markWorkItemDelivered,
|
markWorkItemDelivered,
|
||||||
markWorkItemDeliveryRetry,
|
markWorkItemDeliveryRetry,
|
||||||
|
getLastBotFinalMessage,
|
||||||
|
isPairedRoomJid,
|
||||||
|
type ServiceHandoff,
|
||||||
type WorkItem,
|
type WorkItem,
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
import { isSessionCommandSenderAllowed } from './config.js';
|
import {
|
||||||
|
isClaudeService,
|
||||||
|
isReviewService,
|
||||||
|
isSessionCommandSenderAllowed,
|
||||||
|
SERVICE_AGENT_TYPE,
|
||||||
|
SERVICE_ID,
|
||||||
|
} from './config.js';
|
||||||
import { GroupQueue, GroupRunContext } from './group-queue.js';
|
import { GroupQueue, GroupRunContext } from './group-queue.js';
|
||||||
import { findChannel, formatMessages } from './router.js';
|
import { findChannel, formatMessages, normalizeMessageForDedupe } from './router.js';
|
||||||
import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js';
|
import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js';
|
||||||
import {
|
import {
|
||||||
advanceLastAgentCursor,
|
advanceLastAgentCursor,
|
||||||
@@ -23,6 +36,7 @@ import {
|
|||||||
} from './message-runtime-rules.js';
|
} from './message-runtime-rules.js';
|
||||||
import { runAgentForGroup } from './message-agent-executor.js';
|
import { runAgentForGroup } from './message-agent-executor.js';
|
||||||
import { MessageTurnController } from './message-turn-controller.js';
|
import { MessageTurnController } from './message-turn-controller.js';
|
||||||
|
import { createSuppressToken } from './output-suppression.js';
|
||||||
import {
|
import {
|
||||||
extractSessionCommand,
|
extractSessionCommand,
|
||||||
handleSessionCommand,
|
handleSessionCommand,
|
||||||
@@ -32,6 +46,30 @@ import {
|
|||||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { resolveGroupIpcPath } from './group-folder.js';
|
import { resolveGroupIpcPath } from './group-folder.js';
|
||||||
|
import { shouldServiceProcessChat } from './service-routing.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a message is a duplicate of the last bot final message in a paired room.
|
||||||
|
* Exported for testing purposes.
|
||||||
|
*/
|
||||||
|
export function isDuplicateOfLastBotFinal(chatJid: string, text: string): boolean {
|
||||||
|
// Only check in paired rooms (both claude and codex registered)
|
||||||
|
if (!isPairedRoomJid(chatJid)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the last bot final message from DB (any bot, not just this service)
|
||||||
|
const lastMessages = getLastBotFinalMessage(chatJid, SERVICE_AGENT_TYPE, 1);
|
||||||
|
if (lastMessages.length === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastMessage = lastMessages[0];
|
||||||
|
const normalizedLast = normalizeMessageForDedupe(lastMessage.content);
|
||||||
|
const normalizedCurrent = normalizeMessageForDedupe(text);
|
||||||
|
|
||||||
|
return normalizedLast === normalizedCurrent && normalizedLast.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
export interface MessageRuntimeDeps {
|
export interface MessageRuntimeDeps {
|
||||||
assistantName: string;
|
assistantName: string;
|
||||||
@@ -64,6 +102,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
const continuationTracker = createImplicitContinuationTracker(
|
const continuationTracker = createImplicitContinuationTracker(
|
||||||
deps.idleTimeout,
|
deps.idleTimeout,
|
||||||
);
|
);
|
||||||
|
const isBotOnlyPairedRoomTurn = (
|
||||||
|
chatJid: string,
|
||||||
|
messages: NewMessage[],
|
||||||
|
): boolean =>
|
||||||
|
isPairedRoomJid(chatJid) &&
|
||||||
|
messages.every(
|
||||||
|
(message) => message.is_from_me === true || !!message.is_bot_message,
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a message is a duplicate of the last bot final message in a paired room.
|
||||||
|
* Returns true if duplicate (should be suppressed).
|
||||||
|
*/
|
||||||
|
const checkDuplicateOfLastBotFinal = (chatJid: string, text: string): boolean => {
|
||||||
|
return isDuplicateOfLastBotFinal(chatJid, text);
|
||||||
|
};
|
||||||
|
|
||||||
const deliverOpenWorkItem = async (
|
const deliverOpenWorkItem = async (
|
||||||
channel: Channel,
|
channel: Channel,
|
||||||
@@ -73,6 +127,24 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
},
|
},
|
||||||
): Promise<boolean> => {
|
): Promise<boolean> => {
|
||||||
const replaceMessageId = options?.replaceMessageId ?? null;
|
const replaceMessageId = options?.replaceMessageId ?? null;
|
||||||
|
|
||||||
|
// Check for duplicate in paired rooms before attempting delivery
|
||||||
|
const isDuplicate = checkDuplicateOfLastBotFinal(item.chat_jid, item.result_payload);
|
||||||
|
|
||||||
|
if (isDuplicate) {
|
||||||
|
// Mark as delivered without sending, and don't open continuation
|
||||||
|
markWorkItemDelivered(item.id, null);
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid: item.chat_jid,
|
||||||
|
workItemId: item.id,
|
||||||
|
preview: item.result_payload.slice(0, 100),
|
||||||
|
},
|
||||||
|
'Suppressed duplicate final message in paired room (marked as delivered)',
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (replaceMessageId && channel.editMessage) {
|
if (replaceMessageId && channel.editMessage) {
|
||||||
await channel.editMessage(
|
await channel.editMessage(
|
||||||
@@ -141,6 +213,11 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
chatJid: string,
|
chatJid: string,
|
||||||
runId: string,
|
runId: string,
|
||||||
onOutput?: (output: AgentOutput) => Promise<void>,
|
onOutput?: (output: AgentOutput) => Promise<void>,
|
||||||
|
options?: {
|
||||||
|
suppressToken?: string;
|
||||||
|
startSeq?: number | null;
|
||||||
|
endSeq?: number | null;
|
||||||
|
},
|
||||||
): Promise<'success' | 'error'> =>
|
): Promise<'success' | 'error'> =>
|
||||||
runAgentForGroup(
|
runAgentForGroup(
|
||||||
{
|
{
|
||||||
@@ -156,10 +233,185 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
prompt,
|
prompt,
|
||||||
chatJid,
|
chatJid,
|
||||||
runId,
|
runId,
|
||||||
|
suppressToken: options?.suppressToken,
|
||||||
|
startSeq: options?.startSeq,
|
||||||
|
endSeq: options?.endSeq,
|
||||||
onOutput,
|
onOutput,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const executeTurn = async (args: {
|
||||||
|
group: RegisteredGroup;
|
||||||
|
prompt: string;
|
||||||
|
chatJid: string;
|
||||||
|
runId: string;
|
||||||
|
channel: Channel;
|
||||||
|
startSeq: number | null;
|
||||||
|
endSeq: number | null;
|
||||||
|
}): Promise<{
|
||||||
|
outputStatus: 'success' | 'error';
|
||||||
|
deliverySucceeded: boolean;
|
||||||
|
visiblePhase: ReturnType<MessageTurnController['finish']> extends Promise<
|
||||||
|
infer T
|
||||||
|
>
|
||||||
|
? T extends { visiblePhase: infer V }
|
||||||
|
? V
|
||||||
|
: never
|
||||||
|
: never;
|
||||||
|
}> => {
|
||||||
|
const { group, prompt, chatJid, runId, channel, startSeq, endSeq } = args;
|
||||||
|
const isClaudeCodeAgent =
|
||||||
|
(group.agentType || 'claude-code') === 'claude-code';
|
||||||
|
const suppressToken =
|
||||||
|
isClaudeService() || isReviewService() ? createSuppressToken() : undefined;
|
||||||
|
const deferTypingUntilVisible = Boolean(suppressToken) && isClaudeService();
|
||||||
|
|
||||||
|
const turnController = new MessageTurnController({
|
||||||
|
chatJid,
|
||||||
|
group,
|
||||||
|
runId,
|
||||||
|
channel,
|
||||||
|
idleTimeout: deps.idleTimeout,
|
||||||
|
failureFinalText: FAILURE_FINAL_TEXT,
|
||||||
|
isClaudeCodeAgent,
|
||||||
|
deferTypingUntilVisible,
|
||||||
|
suppressToken,
|
||||||
|
clearSession: () => deps.clearSession(group.folder),
|
||||||
|
requestClose: (reason) => deps.queue.closeStdin(chatJid, { runId, reason }),
|
||||||
|
deliverFinalText: async (text) => {
|
||||||
|
try {
|
||||||
|
const workItem = createProducedWorkItem({
|
||||||
|
group_folder: group.folder,
|
||||||
|
chat_jid: chatJid,
|
||||||
|
agent_type: group.agentType || 'claude-code',
|
||||||
|
start_seq: startSeq,
|
||||||
|
end_seq: endSeq,
|
||||||
|
result_payload: text,
|
||||||
|
});
|
||||||
|
return deliverOpenWorkItem(channel, workItem);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ group: group.name, chatJid, runId, err },
|
||||||
|
'Failed to persist produced output for delivery',
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await turnController.start();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const outputStatus = await runAgent(
|
||||||
|
group,
|
||||||
|
prompt,
|
||||||
|
chatJid,
|
||||||
|
runId,
|
||||||
|
(result) => turnController.handleOutput(result),
|
||||||
|
{ suppressToken, startSeq, endSeq },
|
||||||
|
);
|
||||||
|
|
||||||
|
const { deliverySucceeded, visiblePhase } =
|
||||||
|
await turnController.finish(outputStatus);
|
||||||
|
|
||||||
|
return {
|
||||||
|
outputStatus,
|
||||||
|
deliverySucceeded,
|
||||||
|
visiblePhase,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
turnController.cancelPendingTypingDelay();
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
transition: 'typing:off',
|
||||||
|
source: 'message-runtime:safety-net',
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
},
|
||||||
|
'Typing indicator transition',
|
||||||
|
);
|
||||||
|
await channel.setTyping?.(chatJid, false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const enqueuePendingHandoffs = (): void => {
|
||||||
|
for (const handoff of getPendingServiceHandoffs(SERVICE_ID)) {
|
||||||
|
if (!claimServiceHandoff(handoff.id)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
deps.queue.enqueueTask(
|
||||||
|
handoff.chat_jid,
|
||||||
|
`handoff:${handoff.id}`,
|
||||||
|
async () => {
|
||||||
|
await processClaimedHandoff(handoff);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const processClaimedHandoff = async (handoff: ServiceHandoff): Promise<void> => {
|
||||||
|
const group = deps.getRegisteredGroups()[handoff.chat_jid];
|
||||||
|
if (!group) {
|
||||||
|
failServiceHandoff(handoff.id, 'Group not registered on target service');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const channel = findChannel(deps.channels, handoff.chat_jid);
|
||||||
|
if (!channel) {
|
||||||
|
failServiceHandoff(handoff.id, 'No channel owns handoff jid');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const runId = `handoff-${handoff.id}`;
|
||||||
|
try {
|
||||||
|
const result = await executeTurn({
|
||||||
|
group,
|
||||||
|
prompt: handoff.prompt,
|
||||||
|
chatJid: handoff.chat_jid,
|
||||||
|
runId,
|
||||||
|
channel,
|
||||||
|
startSeq: handoff.start_seq,
|
||||||
|
endSeq: handoff.end_seq,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!result.deliverySucceeded) {
|
||||||
|
failServiceHandoff(handoff.id, 'Handoff delivery failed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const appliedCursor = completeServiceHandoffAndAdvanceTargetCursor({
|
||||||
|
id: handoff.id,
|
||||||
|
target_service_id: handoff.target_service_id,
|
||||||
|
chat_jid: handoff.chat_jid,
|
||||||
|
end_seq: handoff.end_seq,
|
||||||
|
});
|
||||||
|
if (appliedCursor) {
|
||||||
|
deps.getLastAgentTimestamps()[handoff.chat_jid] = appliedCursor;
|
||||||
|
}
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid: handoff.chat_jid,
|
||||||
|
handoffId: handoff.id,
|
||||||
|
runId,
|
||||||
|
outputStatus: result.outputStatus,
|
||||||
|
visiblePhase: result.visiblePhase,
|
||||||
|
appliedCursor,
|
||||||
|
},
|
||||||
|
'Completed claimed service handoff',
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
const errorMessage = getErrorMessage(err);
|
||||||
|
failServiceHandoff(handoff.id, errorMessage);
|
||||||
|
logger.error(
|
||||||
|
{ chatJid: handoff.chat_jid, handoffId: handoff.id, err },
|
||||||
|
'Claimed service handoff failed',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const processGroupMessages = async (
|
const processGroupMessages = async (
|
||||||
chatJid: string,
|
chatJid: string,
|
||||||
context: GroupRunContext,
|
context: GroupRunContext,
|
||||||
@@ -183,9 +435,29 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
if (!delivered) return false;
|
if (!delivered) return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) {
|
||||||
|
const rawMissedMessages = getMessagesSinceSeq(
|
||||||
|
chatJid,
|
||||||
|
deps.getLastAgentTimestamps()[chatJid] || '0',
|
||||||
|
deps.assistantName,
|
||||||
|
);
|
||||||
|
const lastIgnored = rawMissedMessages[rawMissedMessages.length - 1];
|
||||||
|
if (lastIgnored?.seq != null) {
|
||||||
|
advanceLastAgentCursor(
|
||||||
|
deps.getLastAgentTimestamps(),
|
||||||
|
deps.saveState,
|
||||||
|
chatJid,
|
||||||
|
lastIgnored.seq,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
logger.debug(
|
||||||
|
{ chatJid, serviceId: SERVICE_ID },
|
||||||
|
'Skipping message processing for unassigned service',
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
const isMainGroup = group.isMain === true;
|
const isMainGroup = group.isMain === true;
|
||||||
const isClaudeCodeAgent =
|
|
||||||
(group.agentType || 'claude-code') === 'claude-code';
|
|
||||||
while (true) {
|
while (true) {
|
||||||
const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '0';
|
const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '0';
|
||||||
const rawMissedMessages = getMessagesSinceSeq(
|
const rawMissedMessages = getMessagesSinceSeq(
|
||||||
@@ -311,85 +583,36 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
},
|
},
|
||||||
'Dispatching queued messages to agent',
|
'Dispatching queued messages to agent',
|
||||||
);
|
);
|
||||||
const turnController = new MessageTurnController({
|
const { deliverySucceeded, visiblePhase } = await executeTurn({
|
||||||
chatJid,
|
|
||||||
group,
|
group,
|
||||||
|
prompt,
|
||||||
|
chatJid,
|
||||||
runId,
|
runId,
|
||||||
channel,
|
channel,
|
||||||
idleTimeout: deps.idleTimeout,
|
startSeq,
|
||||||
failureFinalText: FAILURE_FINAL_TEXT,
|
endSeq,
|
||||||
isClaudeCodeAgent,
|
|
||||||
clearSession: () => deps.clearSession(group.folder),
|
|
||||||
requestClose: (reason) =>
|
|
||||||
deps.queue.closeStdin(chatJid, { runId, reason }),
|
|
||||||
deliverFinalText: async (text) => {
|
|
||||||
try {
|
|
||||||
const workItem = createProducedWorkItem({
|
|
||||||
group_folder: group.folder,
|
|
||||||
chat_jid: chatJid,
|
|
||||||
agent_type: group.agentType || 'claude-code',
|
|
||||||
start_seq: startSeq,
|
|
||||||
end_seq: endSeq,
|
|
||||||
result_payload: text,
|
|
||||||
});
|
|
||||||
return deliverOpenWorkItem(channel, workItem);
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn(
|
|
||||||
{ group: group.name, chatJid, runId, err },
|
|
||||||
'Failed to persist produced output for delivery',
|
|
||||||
);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await turnController.start();
|
if (!deliverySucceeded) {
|
||||||
|
logger.warn(
|
||||||
try {
|
{ chatJid, group: group.name, groupFolder: group.folder, runId },
|
||||||
const output = await runAgent(group, prompt, chatJid, runId, (result) =>
|
'Persisted produced output for delivery retry without rerunning agent',
|
||||||
turnController.handleOutput(result),
|
|
||||||
);
|
);
|
||||||
|
return false;
|
||||||
const { deliverySucceeded, visiblePhase } =
|
|
||||||
await turnController.finish(output);
|
|
||||||
|
|
||||||
if (!deliverySucceeded) {
|
|
||||||
logger.warn(
|
|
||||||
{ chatJid, group: group.name, groupFolder: group.folder, runId },
|
|
||||||
'Persisted produced output for delivery retry without rerunning agent',
|
|
||||||
);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
visiblePhase,
|
|
||||||
},
|
|
||||||
'Queued run completed successfully',
|
|
||||||
);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
} finally {
|
|
||||||
// Safety net: always clear typing even if runAgent() or finish() throws.
|
|
||||||
// Prevents stuck typing indicators when exceptions bypass the normal
|
|
||||||
// turnController.finish() -> setTyping(false) path.
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
transition: 'typing:off',
|
|
||||||
source: 'message-runtime:safety-net',
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
runId,
|
|
||||||
},
|
|
||||||
'Typing indicator transition',
|
|
||||||
);
|
|
||||||
await channel.setTyping?.(chatJid, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
visiblePhase,
|
||||||
|
},
|
||||||
|
'Queued run completed successfully',
|
||||||
|
);
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -404,6 +627,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
|
enqueuePendingHandoffs();
|
||||||
const registeredGroups = deps.getRegisteredGroups();
|
const registeredGroups = deps.getRegisteredGroups();
|
||||||
const jids = Object.keys(registeredGroups);
|
const jids = Object.keys(registeredGroups);
|
||||||
const { messages, newSeqCursor } = getNewMessagesBySeq(
|
const { messages, newSeqCursor } = getNewMessagesBySeq(
|
||||||
@@ -461,6 +685,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) {
|
||||||
|
const lastIgnored =
|
||||||
|
processableGroupMessages[processableGroupMessages.length - 1];
|
||||||
|
if (lastIgnored?.seq != null) {
|
||||||
|
advanceLastAgentCursor(
|
||||||
|
deps.getLastAgentTimestamps(),
|
||||||
|
deps.saveState,
|
||||||
|
chatJid,
|
||||||
|
lastIgnored.seq,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)
|
shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)
|
||||||
) {
|
) {
|
||||||
@@ -530,6 +768,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
? pendingMessages
|
? pendingMessages
|
||||||
: processableGroupMessages;
|
: processableGroupMessages;
|
||||||
const formatted = formatMessages(messagesToSend, deps.timezone);
|
const formatted = formatMessages(messagesToSend, deps.timezone);
|
||||||
|
const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn(
|
||||||
|
chatJid,
|
||||||
|
messagesToSend,
|
||||||
|
);
|
||||||
|
|
||||||
if (deps.queue.sendMessage(chatJid, formatted)) {
|
if (deps.queue.sendMessage(chatJid, formatted)) {
|
||||||
const endSeq = messagesToSend[messagesToSend.length - 1]?.seq;
|
const endSeq = messagesToSend[messagesToSend.length - 1]?.seq;
|
||||||
@@ -549,17 +791,20 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
group: group.name,
|
group: group.name,
|
||||||
groupFolder: group.folder,
|
groupFolder: group.folder,
|
||||||
endSeq: endSeq ?? null,
|
endSeq: endSeq ?? null,
|
||||||
|
suppressed: isBotOnlyPairedFollowUp,
|
||||||
},
|
},
|
||||||
'Typing indicator transition',
|
'Typing indicator transition',
|
||||||
);
|
);
|
||||||
await channel
|
if (!isBotOnlyPairedFollowUp) {
|
||||||
.setTyping?.(chatJid, true)
|
await channel
|
||||||
?.catch((err) =>
|
.setTyping?.(chatJid, true)
|
||||||
logger.warn(
|
?.catch((err) =>
|
||||||
{ chatJid, err },
|
logger.warn(
|
||||||
'Failed to set typing indicator',
|
{ chatJid, err },
|
||||||
),
|
'Failed to set typing indicator',
|
||||||
);
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -595,6 +840,10 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!shouldServiceProcessChat(chatJid, SERVICE_ID)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '';
|
const sinceSeqCursor = deps.getLastAgentTimestamps()[chatJid] || '';
|
||||||
const rawPending = getMessagesSinceSeq(
|
const rawPending = getMessagesSinceSeq(
|
||||||
chatJid,
|
chatJid,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { type AgentOutput } from './agent-runner.js';
|
import { type AgentOutput } from './agent-runner.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import { classifySuppressTokenOutput } from './output-suppression.js';
|
||||||
import { formatOutbound } from './router.js';
|
import { formatOutbound } from './router.js';
|
||||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
||||||
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
||||||
@@ -28,6 +29,8 @@ interface MessageTurnControllerOptions {
|
|||||||
idleTimeout: number;
|
idleTimeout: number;
|
||||||
failureFinalText: string;
|
failureFinalText: string;
|
||||||
isClaudeCodeAgent: boolean;
|
isClaudeCodeAgent: boolean;
|
||||||
|
deferTypingUntilVisible?: boolean;
|
||||||
|
suppressToken?: string;
|
||||||
clearSession: () => void;
|
clearSession: () => void;
|
||||||
requestClose: (reason: string) => void;
|
requestClose: (reason: string) => void;
|
||||||
deliverFinalText: (text: string) => Promise<boolean>;
|
deliverFinalText: (text: string) => Promise<boolean>;
|
||||||
@@ -53,6 +56,7 @@ export class MessageTurnController {
|
|||||||
private lastIntermediateText: string | null = null;
|
private lastIntermediateText: string | null = null;
|
||||||
private poisonedSessionDetected = false;
|
private poisonedSessionDetected = false;
|
||||||
private closeRequested = false;
|
private closeRequested = false;
|
||||||
|
private typingActive = false;
|
||||||
|
|
||||||
constructor(private readonly options: MessageTurnControllerOptions) {}
|
constructor(private readonly options: MessageTurnControllerOptions) {}
|
||||||
|
|
||||||
@@ -78,7 +82,10 @@ export class MessageTurnController {
|
|||||||
|
|
||||||
async start(): Promise<void> {
|
async start(): Promise<void> {
|
||||||
this.resetIdleTimer();
|
this.resetIdleTimer();
|
||||||
await this.setTyping(true, 'turn:start');
|
if (this.options.deferTypingUntilVisible) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.activateTyping('turn:start');
|
||||||
}
|
}
|
||||||
|
|
||||||
async handleOutput(result: AgentOutput): Promise<void> {
|
async handleOutput(result: AgentOutput): Promise<void> {
|
||||||
@@ -123,7 +130,11 @@ export class MessageTurnController {
|
|||||||
: typeof result.result === 'string'
|
: typeof result.result === 'string'
|
||||||
? result.result
|
? result.result
|
||||||
: JSON.stringify(result.result);
|
: JSON.stringify(result.result);
|
||||||
const text = raw ? formatOutbound(raw) : null;
|
const suppressState =
|
||||||
|
raw && this.options.suppressToken
|
||||||
|
? classifySuppressTokenOutput(raw, this.options.suppressToken)
|
||||||
|
: 'none';
|
||||||
|
const text = raw && suppressState === 'none' ? formatOutbound(raw) : null;
|
||||||
|
|
||||||
if (raw) {
|
if (raw) {
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -138,6 +149,31 @@ export class MessageTurnController {
|
|||||||
},
|
},
|
||||||
`Agent output: ${raw.slice(0, 200)}`,
|
`Agent output: ${raw.slice(0, 200)}`,
|
||||||
);
|
);
|
||||||
|
if (suppressState === 'exact') {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid: this.options.chatJid,
|
||||||
|
group: this.options.group.name,
|
||||||
|
groupFolder: this.options.group.folder,
|
||||||
|
runId: this.options.runId,
|
||||||
|
resultStatus: result.status,
|
||||||
|
resultPhase: result.phase,
|
||||||
|
},
|
||||||
|
'Suppressed exact-match silent output token',
|
||||||
|
);
|
||||||
|
} else if (suppressState === 'mixed') {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
chatJid: this.options.chatJid,
|
||||||
|
group: this.options.group.name,
|
||||||
|
groupFolder: this.options.group.folder,
|
||||||
|
runId: this.options.runId,
|
||||||
|
resultStatus: result.status,
|
||||||
|
resultPhase: result.phase,
|
||||||
|
},
|
||||||
|
'Blocked malformed output that mixed the silent output token with visible text',
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const phase: AgentOutputPhase = normalizeAgentOutputPhase(result.phase);
|
const phase: AgentOutputPhase = normalizeAgentOutputPhase(result.phase);
|
||||||
@@ -275,6 +311,9 @@ export class MessageTurnController {
|
|||||||
await this.finalizeProgressMessage();
|
await this.finalizeProgressMessage();
|
||||||
await this.deliverFinalText(text);
|
await this.deliverFinalText(text);
|
||||||
}
|
}
|
||||||
|
} else if (suppressState !== 'none') {
|
||||||
|
await this.finalizeProgressMessage();
|
||||||
|
this.latestProgressTextForFinal = null;
|
||||||
} else if (raw) {
|
} else if (raw) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
@@ -294,7 +333,7 @@ export class MessageTurnController {
|
|||||||
await this.finalizeProgressMessage();
|
await this.finalizeProgressMessage();
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.setTyping(false, 'turn:handle-output', {
|
await this.deactivateTyping('turn:handle-output', {
|
||||||
outputStatus: result.status,
|
outputStatus: result.status,
|
||||||
phase,
|
phase,
|
||||||
});
|
});
|
||||||
@@ -311,7 +350,7 @@ export class MessageTurnController {
|
|||||||
deliverySucceeded: boolean;
|
deliverySucceeded: boolean;
|
||||||
visiblePhase: VisiblePhase;
|
visiblePhase: VisiblePhase;
|
||||||
}> {
|
}> {
|
||||||
await this.setTyping(false, 'turn:finish', { outputStatus });
|
await this.deactivateTyping('turn:finish', { outputStatus });
|
||||||
|
|
||||||
if (outputStatus === 'error') {
|
if (outputStatus === 'error') {
|
||||||
this.hadError = true;
|
this.hadError = true;
|
||||||
@@ -566,6 +605,7 @@ export class MessageTurnController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async deliverFinalText(text: string): Promise<void> {
|
private async deliverFinalText(text: string): Promise<void> {
|
||||||
|
await this.activateTyping('turn:deliver-final');
|
||||||
this.visiblePhase = toVisiblePhase('final');
|
this.visiblePhase = toVisiblePhase('final');
|
||||||
const delivered = await this.options.deliverFinalText(text);
|
const delivered = await this.options.deliverFinalText(text);
|
||||||
if (!delivered) {
|
if (!delivered) {
|
||||||
@@ -593,6 +633,8 @@ export class MessageTurnController {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await this.activateTyping('turn:send-progress');
|
||||||
|
|
||||||
if (this.progressStartedAt === null) {
|
if (this.progressStartedAt === null) {
|
||||||
this.progressStartedAt = Date.now();
|
this.progressStartedAt = Date.now();
|
||||||
}
|
}
|
||||||
@@ -670,6 +712,28 @@ export class MessageTurnController {
|
|||||||
this.visiblePhase = toVisiblePhase('progress');
|
this.visiblePhase = toVisiblePhase('progress');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async activateTyping(
|
||||||
|
source: string,
|
||||||
|
extra?: Record<string, unknown>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (this.typingActive) return;
|
||||||
|
await this.setTyping(true, source, extra);
|
||||||
|
this.typingActive = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deactivateTyping(
|
||||||
|
source: string,
|
||||||
|
extra?: Record<string, unknown>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!this.typingActive) return;
|
||||||
|
await this.setTyping(false, source, extra);
|
||||||
|
this.typingActive = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
cancelPendingTypingDelay(): void {
|
||||||
|
// No-op: typing delay removed. Kept for call-site compatibility.
|
||||||
|
}
|
||||||
|
|
||||||
private resetIdleTimer(): void {
|
private resetIdleTimer(): void {
|
||||||
if (this.idleTimer) clearTimeout(this.idleTimer);
|
if (this.idleTimer) clearTimeout(this.idleTimer);
|
||||||
if (this.hasVisibleOutput()) {
|
if (this.hasVisibleOutput()) {
|
||||||
|
|||||||
41
src/output-suppression.test.ts
Normal file
41
src/output-suppression.test.ts
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { classifySuppressTokenOutput } from './output-suppression.js';
|
||||||
|
|
||||||
|
describe('classifySuppressTokenOutput', () => {
|
||||||
|
it('treats the current turn suppress token as exact', () => {
|
||||||
|
expect(
|
||||||
|
classifySuppressTokenOutput(
|
||||||
|
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
|
||||||
|
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
|
||||||
|
),
|
||||||
|
).toBe('exact');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('treats a leaked foreign suppress token as exact silent output', () => {
|
||||||
|
expect(
|
||||||
|
classifySuppressTokenOutput(
|
||||||
|
'__EJ_SUPPRESS_feedfacefeedfacefeedface__',
|
||||||
|
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
|
||||||
|
),
|
||||||
|
).toBe('exact');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('treats a malformed foreign suppress token without the closing suffix as exact silent output', () => {
|
||||||
|
expect(
|
||||||
|
classifySuppressTokenOutput(
|
||||||
|
'__EJ_SUPPRESS_feedfacefeedfacefeedface',
|
||||||
|
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
|
||||||
|
),
|
||||||
|
).toBe('exact');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('treats a suppress token embedded in visible text as mixed', () => {
|
||||||
|
expect(
|
||||||
|
classifySuppressTokenOutput(
|
||||||
|
'prefix __EJ_SUPPRESS_feedfacefeedfacefeedface__ suffix',
|
||||||
|
'__EJ_SUPPRESS_deadbeefdeadbeefdeadbeef__',
|
||||||
|
),
|
||||||
|
).toBe('mixed');
|
||||||
|
});
|
||||||
|
});
|
||||||
65
src/output-suppression.ts
Normal file
65
src/output-suppression.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
import { randomBytes } from 'crypto';
|
||||||
|
|
||||||
|
import {
|
||||||
|
CLAUDE_SERVICE_ID,
|
||||||
|
CODEX_REVIEW_SERVICE_ID,
|
||||||
|
normalizeServiceId,
|
||||||
|
} from './config.js';
|
||||||
|
|
||||||
|
const ANY_SUPPRESS_TOKEN_PATTERN = /__EJ_SUPPRESS_[a-f0-9]{24,}(?:__)?/g;
|
||||||
|
const EXACT_ANY_SUPPRESS_TOKEN_PATTERN = /^__EJ_SUPPRESS_[a-f0-9]{24,}(?:__)?$/;
|
||||||
|
|
||||||
|
export function createSuppressToken(): string {
|
||||||
|
return `__EJ_SUPPRESS_${randomBytes(12).toString('hex')}__`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldEnableSuppressOutputForService(
|
||||||
|
serviceId: string | undefined,
|
||||||
|
): boolean {
|
||||||
|
if (!serviceId) return false;
|
||||||
|
const normalized = normalizeServiceId(serviceId);
|
||||||
|
return (
|
||||||
|
normalized === CLAUDE_SERVICE_ID ||
|
||||||
|
normalized === CODEX_REVIEW_SERVICE_ID
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function classifySuppressTokenOutput(
|
||||||
|
rawText: string,
|
||||||
|
suppressToken: string | undefined,
|
||||||
|
): 'exact' | 'mixed' | 'none' {
|
||||||
|
const trimmed = rawText.trim();
|
||||||
|
if ((suppressToken && trimmed === suppressToken) || EXACT_ANY_SUPPRESS_TOKEN_PATTERN.test(trimmed)) {
|
||||||
|
return 'exact';
|
||||||
|
}
|
||||||
|
if (suppressToken && rawText.includes(suppressToken)) {
|
||||||
|
return 'mixed';
|
||||||
|
}
|
||||||
|
ANY_SUPPRESS_TOKEN_PATTERN.lastIndex = 0;
|
||||||
|
return ANY_SUPPRESS_TOKEN_PATTERN.test(rawText) ? 'mixed' : 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSuppressTokenPrompt(
|
||||||
|
prompt: string,
|
||||||
|
suppressToken: string | undefined,
|
||||||
|
options?: {
|
||||||
|
reviewerMode?: boolean;
|
||||||
|
},
|
||||||
|
): string {
|
||||||
|
if (!suppressToken) return prompt;
|
||||||
|
|
||||||
|
const lines = [
|
||||||
|
'[OUTPUT CONTROL]',
|
||||||
|
`If you have no user-visible content to send for this turn, output exactly this token and nothing else: ${suppressToken}`,
|
||||||
|
'Do not wrap the token in backticks or code fences.',
|
||||||
|
'Do not combine the token with any other text.',
|
||||||
|
];
|
||||||
|
|
||||||
|
if (options?.reviewerMode) {
|
||||||
|
lines.push(
|
||||||
|
'If you are only agreeing, mirroring, or restating without adding a concrete correction, risk, missing prerequisite, test gap, or code change, output only the token.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${lines.join('\n')}\n\n${prompt}`;
|
||||||
|
}
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
||||||
|
|
||||||
vi.mock('./claude-usage.js', () => ({
|
|
||||||
fetchClaudeUsage: vi.fn(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('./env.js', () => {
|
|
||||||
const store: Record<string, string> = {
|
|
||||||
FALLBACK_PROVIDER_NAME: 'kimi',
|
|
||||||
FALLBACK_BASE_URL: 'https://api.kimi.com/coding/',
|
|
||||||
FALLBACK_AUTH_TOKEN: 'test-kimi-key',
|
|
||||||
FALLBACK_MODEL: 'kimi-k2.5',
|
|
||||||
FALLBACK_SMALL_MODEL: 'kimi-k2.5',
|
|
||||||
FALLBACK_COOLDOWN_MS: '600000',
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
readEnvFile: vi.fn((keys: string[]) => {
|
|
||||||
const result: Record<string, string> = {};
|
|
||||||
for (const k of keys) {
|
|
||||||
if (store[k]) result[k] = store[k];
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}),
|
|
||||||
getEnv: vi.fn((key: string) => store[key]),
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
vi.mock('./logger.js', () => ({
|
|
||||||
logger: {
|
|
||||||
debug: vi.fn(),
|
|
||||||
info: vi.fn(),
|
|
||||||
warn: vi.fn(),
|
|
||||||
error: vi.fn(),
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
import { fetchClaudeUsage } from './claude-usage.js';
|
|
||||||
import {
|
|
||||||
clearPrimaryCooldown,
|
|
||||||
detectFallbackTrigger,
|
|
||||||
getActiveProvider,
|
|
||||||
getCooldownInfo,
|
|
||||||
isPrimaryNoFallbackCooldownActive,
|
|
||||||
markPrimaryCooldown,
|
|
||||||
resetFallbackConfig,
|
|
||||||
} from './provider-fallback.js';
|
|
||||||
|
|
||||||
describe('provider fallback usage recovery', () => {
|
|
||||||
beforeEach(() => {
|
|
||||||
vi.useFakeTimers();
|
|
||||||
vi.setSystemTime(new Date('2026-03-24T00:00:00.000Z'));
|
|
||||||
vi.clearAllMocks();
|
|
||||||
clearPrimaryCooldown();
|
|
||||||
resetFallbackConfig();
|
|
||||||
delete process.env.FALLBACK_PROVIDER_NAME;
|
|
||||||
delete process.env.FALLBACK_BASE_URL;
|
|
||||||
delete process.env.FALLBACK_AUTH_TOKEN;
|
|
||||||
delete process.env.FALLBACK_MODEL;
|
|
||||||
delete process.env.FALLBACK_SMALL_MODEL;
|
|
||||||
delete process.env.FALLBACK_COOLDOWN_MS;
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
clearPrimaryCooldown();
|
|
||||||
resetFallbackConfig();
|
|
||||||
vi.useRealTimers();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('keeps the fallback provider active while Claude usage is still exhausted', async () => {
|
|
||||||
vi.mocked(fetchClaudeUsage).mockResolvedValue({
|
|
||||||
five_hour: {
|
|
||||||
utilization: 100,
|
|
||||||
resets_at: '2026-03-24T04:00:00.000+09:00',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
markPrimaryCooldown('usage-exhausted', 1_000);
|
|
||||||
vi.advanceTimersByTime(5_000);
|
|
||||||
|
|
||||||
await expect(getActiveProvider()).resolves.toBe('kimi');
|
|
||||||
expect(getCooldownInfo()).toMatchObject({
|
|
||||||
active: true,
|
|
||||||
reason: 'usage-exhausted',
|
|
||||||
remainingMs: 0,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns to Claude immediately when usage is no longer exhausted', async () => {
|
|
||||||
vi.mocked(fetchClaudeUsage).mockResolvedValue({
|
|
||||||
five_hour: {
|
|
||||||
utilization: 72,
|
|
||||||
resets_at: '2026-03-24T04:00:00.000+09:00',
|
|
||||||
},
|
|
||||||
seven_day: {
|
|
||||||
utilization: 55,
|
|
||||||
resets_at: '2026-03-31T04:00:00.000+09:00',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
markPrimaryCooldown('usage-exhausted', 600_000);
|
|
||||||
|
|
||||||
await expect(getActiveProvider()).resolves.toBe('claude');
|
|
||||||
expect(getCooldownInfo()).toEqual({ active: false });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('falls back to time-based retry when usage status cannot be fetched', async () => {
|
|
||||||
vi.mocked(fetchClaudeUsage).mockResolvedValue(null);
|
|
||||||
|
|
||||||
markPrimaryCooldown('usage-exhausted', 1_000);
|
|
||||||
vi.advanceTimersByTime(5_000);
|
|
||||||
|
|
||||||
await expect(getActiveProvider()).resolves.toBe('claude');
|
|
||||||
expect(getCooldownInfo()).toEqual({ active: false });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats terminated 401 auth failures as an auth-expired fallback trigger', () => {
|
|
||||||
expect(
|
|
||||||
detectFallbackTrigger(
|
|
||||||
'Failed to authenticate. API Error: 401 terminated',
|
|
||||||
),
|
|
||||||
).toEqual({
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: 'auth-expired',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats invalid authentication credentials as an auth-expired fallback trigger', () => {
|
|
||||||
expect(
|
|
||||||
detectFallbackTrigger(
|
|
||||||
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
|
|
||||||
),
|
|
||||||
).toEqual({
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: 'auth-expired',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats Cloudflare 502 HTML as an overloaded fallback trigger', () => {
|
|
||||||
expect(
|
|
||||||
detectFallbackTrigger(
|
|
||||||
'API Error: 502 <html><head><title>502 Bad Gateway</title></head><body><center><h1>502 Bad Gateway</h1></center><hr><center>cloudflare</center></body></html>',
|
|
||||||
),
|
|
||||||
).toEqual({
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: 'overloaded',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats Claude org access denied banners as an org-access-denied fallback trigger', () => {
|
|
||||||
expect(
|
|
||||||
detectFallbackTrigger(
|
|
||||||
'Your organization does not have access to Claude. Please login again or contact your administrator.',
|
|
||||||
),
|
|
||||||
).toEqual({
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: 'org-access-denied',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats terminated 403 auth failures as an org-access-denied fallback trigger', () => {
|
|
||||||
expect(
|
|
||||||
detectFallbackTrigger(
|
|
||||||
'Failed to authenticate. API Error: 403 terminated',
|
|
||||||
),
|
|
||||||
).toEqual({
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: 'org-access-denied',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('marks org-access-denied as a no-fallback cooldown reason', () => {
|
|
||||||
markPrimaryCooldown('org-access-denied', 60_000);
|
|
||||||
|
|
||||||
expect(isPrimaryNoFallbackCooldownActive()).toBe(true);
|
|
||||||
expect(getCooldownInfo()).toMatchObject({
|
|
||||||
active: true,
|
|
||||||
reason: 'org-access-denied',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,450 +0,0 @@
|
|||||||
/**
|
|
||||||
* Provider Fallback Module
|
|
||||||
*
|
|
||||||
* Manages automatic fallback from the primary provider (Claude) to a
|
|
||||||
* fallback provider (e.g. Kimi K2.5) when 429/rate-limit or network
|
|
||||||
* errors are detected.
|
|
||||||
*
|
|
||||||
* Cooldown-based recovery:
|
|
||||||
* Claude 429 → immediate Kimi retry for that turn
|
|
||||||
* Claude enters cooldown (retry-after header or default 10 min)
|
|
||||||
* During cooldown → skip Claude, route directly to fallback
|
|
||||||
* After cooldown → try Claude first again
|
|
||||||
*/
|
|
||||||
|
|
||||||
import fs from 'fs';
|
|
||||||
|
|
||||||
import {
|
|
||||||
classifyAgentError,
|
|
||||||
classifyClaudeAuthError,
|
|
||||||
isNoFallbackCooldownReason,
|
|
||||||
type AgentTriggerReason,
|
|
||||||
type FallbackTriggerReason,
|
|
||||||
} from './agent-error-detection.js';
|
|
||||||
import { fetchClaudeUsage, type ClaudeUsageData } from './claude-usage.js';
|
|
||||||
import { getEnv } from './env.js';
|
|
||||||
import { logger } from './logger.js';
|
|
||||||
import { rotateToken, getTokenCount } from './token-rotation.js';
|
|
||||||
|
|
||||||
// ── Types ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
export type ProviderName = 'claude' | string; // fallback name is configurable
|
|
||||||
|
|
||||||
export type FallbackTriggerResult =
|
|
||||||
| {
|
|
||||||
shouldFallback: false;
|
|
||||||
reason: '';
|
|
||||||
retryAfterMs?: undefined;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
shouldFallback: true;
|
|
||||||
reason: FallbackTriggerReason;
|
|
||||||
retryAfterMs?: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
interface CooldownState {
|
|
||||||
startedAt: number;
|
|
||||||
expiresAt: number;
|
|
||||||
reason: AgentTriggerReason;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FallbackConfig {
|
|
||||||
enabled: boolean;
|
|
||||||
providerName: string; // e.g. "kimi"
|
|
||||||
baseUrl: string;
|
|
||||||
authToken: string;
|
|
||||||
model: string;
|
|
||||||
smallModel: string;
|
|
||||||
defaultCooldownMs: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── State ────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
let cooldown: CooldownState | null = null;
|
|
||||||
let lastUsageAvailabilityCheck: {
|
|
||||||
checkedAt: number;
|
|
||||||
result: 'available' | 'exhausted' | 'unknown';
|
|
||||||
} | null = null;
|
|
||||||
let usageAvailabilityCheckPromise: Promise<
|
|
||||||
'available' | 'exhausted' | 'unknown'
|
|
||||||
> | null = null;
|
|
||||||
|
|
||||||
const USAGE_RECOVERY_RECHECK_MS = 30_000;
|
|
||||||
|
|
||||||
// ── Config ───────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
let _config: FallbackConfig | null = null;
|
|
||||||
|
|
||||||
function loadConfig(): FallbackConfig {
|
|
||||||
if (_config) return _config;
|
|
||||||
|
|
||||||
const baseUrl = getEnv('FALLBACK_BASE_URL') || '';
|
|
||||||
const authToken = getEnv('FALLBACK_AUTH_TOKEN') || '';
|
|
||||||
const model = getEnv('FALLBACK_MODEL') || '';
|
|
||||||
const explicitlyDisabled =
|
|
||||||
(getEnv('FALLBACK_ENABLED') || '').toLowerCase() === 'false';
|
|
||||||
|
|
||||||
_config = {
|
|
||||||
enabled: !explicitlyDisabled && Boolean(baseUrl && authToken && model),
|
|
||||||
providerName: getEnv('FALLBACK_PROVIDER_NAME') || 'kimi',
|
|
||||||
baseUrl,
|
|
||||||
authToken,
|
|
||||||
model,
|
|
||||||
smallModel: getEnv('FALLBACK_SMALL_MODEL') || model,
|
|
||||||
defaultCooldownMs: parseInt(getEnv('FALLBACK_COOLDOWN_MS') || '600000', 10),
|
|
||||||
};
|
|
||||||
|
|
||||||
if (_config.enabled) {
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
provider: _config.providerName,
|
|
||||||
model: _config.model,
|
|
||||||
cooldownMs: _config.defaultCooldownMs,
|
|
||||||
},
|
|
||||||
'Provider fallback configured',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return _config;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Force re-read of config (useful after .env changes). */
|
|
||||||
export function resetFallbackConfig(): void {
|
|
||||||
_config = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Public API ───────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/** Check whether the fallback system is configured and available. */
|
|
||||||
export function isFallbackEnabled(): boolean {
|
|
||||||
return loadConfig().enabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Get the display name of the fallback provider (e.g. "kimi"). */
|
|
||||||
export function getFallbackProviderName(): string {
|
|
||||||
return loadConfig().providerName;
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeUtilization(utilization: number): number {
|
|
||||||
return utilization > 1 ? utilization : utilization * 100;
|
|
||||||
}
|
|
||||||
|
|
||||||
type ClaudeUsageWindow = NonNullable<ClaudeUsageData[keyof ClaudeUsageData]>;
|
|
||||||
|
|
||||||
function hasExhaustedClaudeUsageWindow(
|
|
||||||
usage: ClaudeUsageData | null,
|
|
||||||
): boolean | null {
|
|
||||||
if (!usage) return null;
|
|
||||||
const windows: ClaudeUsageWindow[] = [];
|
|
||||||
if (usage.five_hour) windows.push(usage.five_hour);
|
|
||||||
if (usage.seven_day) windows.push(usage.seven_day);
|
|
||||||
if (usage.seven_day_sonnet) windows.push(usage.seven_day_sonnet);
|
|
||||||
if (usage.seven_day_opus) windows.push(usage.seven_day_opus);
|
|
||||||
if (windows.length === 0) return null;
|
|
||||||
return windows.some(
|
|
||||||
(window) => normalizeUtilization(window.utilization) >= 100,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function clearUsageAvailabilityCache(): void {
|
|
||||||
lastUsageAvailabilityCheck = null;
|
|
||||||
usageAvailabilityCheckPromise = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function logCooldownTransition(
|
|
||||||
level: 'debug' | 'info' | 'warn',
|
|
||||||
transition: string,
|
|
||||||
fields: Record<string, unknown>,
|
|
||||||
message: string,
|
|
||||||
): void {
|
|
||||||
logger[level](
|
|
||||||
{
|
|
||||||
transition,
|
|
||||||
provider: 'claude',
|
|
||||||
...fields,
|
|
||||||
},
|
|
||||||
message,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getClaudeUsageAvailability(): Promise<
|
|
||||||
'available' | 'exhausted' | 'unknown'
|
|
||||||
> {
|
|
||||||
const now = Date.now();
|
|
||||||
if (
|
|
||||||
lastUsageAvailabilityCheck &&
|
|
||||||
now - lastUsageAvailabilityCheck.checkedAt < USAGE_RECOVERY_RECHECK_MS
|
|
||||||
) {
|
|
||||||
return lastUsageAvailabilityCheck.result;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!usageAvailabilityCheckPromise) {
|
|
||||||
usageAvailabilityCheckPromise = (async () => {
|
|
||||||
const usage = await fetchClaudeUsage();
|
|
||||||
const exhausted = hasExhaustedClaudeUsageWindow(usage);
|
|
||||||
const result =
|
|
||||||
exhausted === null ? 'unknown' : exhausted ? 'exhausted' : 'available';
|
|
||||||
lastUsageAvailabilityCheck = {
|
|
||||||
checkedAt: Date.now(),
|
|
||||||
result,
|
|
||||||
};
|
|
||||||
return result;
|
|
||||||
})();
|
|
||||||
|
|
||||||
void usageAvailabilityCheckPromise.finally(() => {
|
|
||||||
usageAvailabilityCheckPromise = null;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return usageAvailabilityCheckPromise;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine which provider should be used for the next request.
|
|
||||||
* Returns 'claude' when Claude is healthy or cooldown has expired,
|
|
||||||
* or the fallback provider name during an active cooldown.
|
|
||||||
*/
|
|
||||||
export async function getActiveProvider(): Promise<string> {
|
|
||||||
const config = loadConfig();
|
|
||||||
if (!config.enabled) return 'claude';
|
|
||||||
|
|
||||||
if (cooldown) {
|
|
||||||
if (cooldown.reason === 'usage-exhausted') {
|
|
||||||
const usageAvailability = await getClaudeUsageAvailability();
|
|
||||||
if (usageAvailability === 'available') {
|
|
||||||
logCooldownTransition(
|
|
||||||
'info',
|
|
||||||
'cooldown:recover',
|
|
||||||
{
|
|
||||||
reason: cooldown.reason,
|
|
||||||
fallbackProvider: config.providerName,
|
|
||||||
},
|
|
||||||
'Claude usage recovered, retrying primary provider',
|
|
||||||
);
|
|
||||||
cooldown = null;
|
|
||||||
clearUsageAvailabilityCache();
|
|
||||||
return 'claude';
|
|
||||||
}
|
|
||||||
if (usageAvailability === 'exhausted') {
|
|
||||||
// Current token exhausted — try rotating to another token (ignore cooldowns)
|
|
||||||
if (
|
|
||||||
getTokenCount() > 1 &&
|
|
||||||
rotateToken(undefined, { ignoreRateLimits: true })
|
|
||||||
) {
|
|
||||||
logCooldownTransition(
|
|
||||||
'info',
|
|
||||||
'cooldown:recover',
|
|
||||||
{
|
|
||||||
reason: cooldown.reason,
|
|
||||||
fallbackProvider: config.providerName,
|
|
||||||
recovery: 'token-rotation',
|
|
||||||
},
|
|
||||||
'Claude current token exhausted, rotated to next token — retrying',
|
|
||||||
);
|
|
||||||
cooldown = null;
|
|
||||||
clearUsageAvailabilityCache();
|
|
||||||
return 'claude';
|
|
||||||
}
|
|
||||||
logCooldownTransition(
|
|
||||||
'debug',
|
|
||||||
'cooldown:stay',
|
|
||||||
{
|
|
||||||
reason: cooldown.reason,
|
|
||||||
fallbackProvider: config.providerName,
|
|
||||||
usageAvailability,
|
|
||||||
},
|
|
||||||
'All Claude tokens exhausted, keeping cooldown active',
|
|
||||||
);
|
|
||||||
return config.providerName;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Date.now() < cooldown.expiresAt) {
|
|
||||||
logCooldownTransition(
|
|
||||||
'debug',
|
|
||||||
'cooldown:stay',
|
|
||||||
{
|
|
||||||
reason: cooldown.reason,
|
|
||||||
fallbackProvider: config.providerName,
|
|
||||||
remainingMs: cooldown.expiresAt - Date.now(),
|
|
||||||
},
|
|
||||||
'Claude cooldown still active, routing to fallback provider',
|
|
||||||
);
|
|
||||||
return config.providerName;
|
|
||||||
}
|
|
||||||
// Cooldown expired — try Claude again
|
|
||||||
logCooldownTransition(
|
|
||||||
'info',
|
|
||||||
'cooldown:expire',
|
|
||||||
{
|
|
||||||
cooldownDurationMs: cooldown.expiresAt - cooldown.startedAt,
|
|
||||||
reason: cooldown.reason,
|
|
||||||
fallbackProvider: config.providerName,
|
|
||||||
},
|
|
||||||
'Claude cooldown expired, retrying primary provider',
|
|
||||||
);
|
|
||||||
cooldown = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return 'claude';
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Mark Claude as rate-limited. All subsequent requests will route to
|
|
||||||
* the fallback provider until the cooldown expires.
|
|
||||||
*/
|
|
||||||
export function markPrimaryCooldown(
|
|
||||||
reason: AgentTriggerReason,
|
|
||||||
retryAfterMs?: number,
|
|
||||||
): void {
|
|
||||||
const config = loadConfig();
|
|
||||||
const durationMs = retryAfterMs || config.defaultCooldownMs;
|
|
||||||
const now = Date.now();
|
|
||||||
|
|
||||||
cooldown = {
|
|
||||||
startedAt: now,
|
|
||||||
expiresAt: now + durationMs,
|
|
||||||
reason,
|
|
||||||
};
|
|
||||||
clearUsageAvailabilityCache();
|
|
||||||
|
|
||||||
logCooldownTransition(
|
|
||||||
'info',
|
|
||||||
'cooldown:enter',
|
|
||||||
{
|
|
||||||
reason,
|
|
||||||
cooldownMs: durationMs,
|
|
||||||
expiresAt: new Date(cooldown.expiresAt).toISOString(),
|
|
||||||
fallbackProvider: config.providerName,
|
|
||||||
},
|
|
||||||
`Falling back to provider: ${config.providerName} (reason: ${reason}, cooldownMs: ${durationMs})`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Manually clear cooldown (e.g. after a successful Claude response). */
|
|
||||||
export function clearPrimaryCooldown(): void {
|
|
||||||
clearUsageAvailabilityCache();
|
|
||||||
if (cooldown) {
|
|
||||||
logCooldownTransition(
|
|
||||||
'info',
|
|
||||||
'cooldown:clear',
|
|
||||||
{ reason: cooldown.reason },
|
|
||||||
'Claude cooldown cleared manually',
|
|
||||||
);
|
|
||||||
cooldown = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Check whether the active primary cooldown should suppress fallback entirely. */
|
|
||||||
export function isPrimaryNoFallbackCooldownActive(): boolean {
|
|
||||||
return cooldown ? isNoFallbackCooldownReason(cooldown.reason) : false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Get current cooldown info (for diagnostics / status dashboard). */
|
|
||||||
export function getCooldownInfo(): {
|
|
||||||
active: boolean;
|
|
||||||
reason?: string;
|
|
||||||
expiresAt?: string;
|
|
||||||
remainingMs?: number;
|
|
||||||
} {
|
|
||||||
if (!cooldown) {
|
|
||||||
return { active: false };
|
|
||||||
}
|
|
||||||
const remainingMs = Math.max(cooldown.expiresAt - Date.now(), 0);
|
|
||||||
if (cooldown.reason !== 'usage-exhausted' && remainingMs === 0) {
|
|
||||||
return { active: false };
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
active: true,
|
|
||||||
reason: cooldown.reason,
|
|
||||||
expiresAt: new Date(cooldown.expiresAt).toISOString(),
|
|
||||||
remainingMs,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build the env-var overrides that make Claude Code SDK talk to
|
|
||||||
* the fallback provider instead of Claude.
|
|
||||||
*/
|
|
||||||
export function getFallbackEnvOverrides(): Record<string, string> {
|
|
||||||
const config = loadConfig();
|
|
||||||
return {
|
|
||||||
ANTHROPIC_BASE_URL: config.baseUrl,
|
|
||||||
ANTHROPIC_AUTH_TOKEN: config.authToken,
|
|
||||||
ANTHROPIC_MODEL: config.model,
|
|
||||||
ANTHROPIC_SMALL_FAST_MODEL: config.smallModel,
|
|
||||||
// Disable non-essential traffic (usage telemetry etc.) on fallback
|
|
||||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
|
|
||||||
// Generous timeout for third-party APIs
|
|
||||||
API_TIMEOUT_MS: '3000000',
|
|
||||||
// Disable tool search (not supported by most fallback providers)
|
|
||||||
ENABLE_TOOL_SEARCH: 'false',
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Inspect an agent error string and decide whether it warrants
|
|
||||||
* a provider fallback.
|
|
||||||
*
|
|
||||||
* Triggers:
|
|
||||||
* - 429 / rate limit / too many requests
|
|
||||||
* - Claude auth/org access failures
|
|
||||||
* - 503 / overloaded (transient provider issue)
|
|
||||||
* - Network / connection errors
|
|
||||||
*
|
|
||||||
* Does NOT trigger for:
|
|
||||||
* - Poisoned sessions
|
|
||||||
* - Prompt / tool failures
|
|
||||||
* - Timeouts (agent took too long, not a provider issue)
|
|
||||||
*/
|
|
||||||
export function detectFallbackTrigger(
|
|
||||||
error?: string | null,
|
|
||||||
): FallbackTriggerResult {
|
|
||||||
if (!error) return { shouldFallback: false, reason: '' };
|
|
||||||
|
|
||||||
// Delegated to shared SSOT — original priority preserved:
|
|
||||||
// 429 first, then Claude auth/org errors, then 503/network
|
|
||||||
const common = classifyAgentError(error);
|
|
||||||
|
|
||||||
// 429 rate-limit (highest priority)
|
|
||||||
if (common.category === 'rate-limit') {
|
|
||||||
return {
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: common.reason,
|
|
||||||
retryAfterMs: common.retryAfterMs,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Claude-specific strict auth check (before 503/network)
|
|
||||||
const auth = classifyClaudeAuthError(error);
|
|
||||||
if (auth.category !== 'none') {
|
|
||||||
return { shouldFallback: true, reason: auth.reason };
|
|
||||||
}
|
|
||||||
|
|
||||||
// 503 overloaded, network errors
|
|
||||||
if (common.category !== 'none') {
|
|
||||||
return {
|
|
||||||
shouldFallback: true,
|
|
||||||
reason: common.reason,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { shouldFallback: false, reason: '' };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check whether a per-group settings.json already overrides the
|
|
||||||
* provider (e.g. the Kimi test channel). If so, we should NOT
|
|
||||||
* apply fallback env overrides on top — the channel already has
|
|
||||||
* its own provider configuration.
|
|
||||||
*/
|
|
||||||
export function hasGroupProviderOverride(settingsJsonPath: string): boolean {
|
|
||||||
try {
|
|
||||||
const raw = fs.readFileSync(settingsJsonPath, 'utf-8');
|
|
||||||
const settings = JSON.parse(raw);
|
|
||||||
const env = settings?.env || {};
|
|
||||||
return Boolean(env.ANTHROPIC_BASE_URL || env.ANTHROPIC_MODEL);
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,18 +8,12 @@ vi.mock('./logger.js', () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./provider-fallback.js', () => ({
|
|
||||||
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
|
|
||||||
markPrimaryCooldown: vi.fn(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('./token-rotation.js', () => ({
|
vi.mock('./token-rotation.js', () => ({
|
||||||
rotateToken: vi.fn(() => false),
|
rotateToken: vi.fn(() => false),
|
||||||
getTokenCount: vi.fn(() => 1),
|
getTokenCount: vi.fn(() => 1),
|
||||||
markTokenHealthy: vi.fn(),
|
markTokenHealthy: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
import { markPrimaryCooldown } from './provider-fallback.js';
|
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
import {
|
import {
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
@@ -50,10 +44,9 @@ describe('runClaudeRotationLoop', () => {
|
|||||||
expect(outcome).toEqual({ type: 'success' });
|
expect(outcome).toEqual({ type: 'success' });
|
||||||
expect(rotateToken).toHaveBeenCalledTimes(1);
|
expect(rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(markPrimaryCooldown).not.toHaveBeenCalled();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('marks no-fallback cooldown when all Claude tokens are org-access-denied', async () => {
|
it('returns error when all Claude tokens are exhausted', async () => {
|
||||||
vi.mocked(getTokenCount).mockReturnValue(2);
|
vi.mocked(getTokenCount).mockReturnValue(2);
|
||||||
|
|
||||||
const outcome = await runClaudeRotationLoop(
|
const outcome = await runClaudeRotationLoop(
|
||||||
@@ -66,16 +59,12 @@ describe('runClaudeRotationLoop', () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(outcome).toEqual({
|
expect(outcome).toEqual({
|
||||||
type: 'no-fallback',
|
type: 'error',
|
||||||
trigger: { reason: 'org-access-denied' },
|
trigger: { reason: 'org-access-denied' },
|
||||||
});
|
});
|
||||||
expect(markPrimaryCooldown).toHaveBeenCalledWith(
|
|
||||||
'org-access-denied',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns success-null-result as a fallback trigger after rotation', async () => {
|
it('returns error with success-null-result trigger after rotation', async () => {
|
||||||
vi.mocked(getTokenCount).mockReturnValue(2);
|
vi.mocked(getTokenCount).mockReturnValue(2);
|
||||||
vi.mocked(rotateToken).mockReturnValueOnce(true);
|
vi.mocked(rotateToken).mockReturnValueOnce(true);
|
||||||
|
|
||||||
@@ -90,9 +79,8 @@ describe('runClaudeRotationLoop', () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(outcome).toEqual({
|
expect(outcome).toEqual({
|
||||||
type: 'needs-fallback',
|
type: 'error',
|
||||||
trigger: { reason: 'success-null-result' },
|
trigger: { reason: 'success-null-result' },
|
||||||
});
|
});
|
||||||
expect(markPrimaryCooldown).not.toHaveBeenCalled();
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,16 +6,12 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import {
|
||||||
isNoFallbackCooldownReason,
|
classifyRotationTrigger,
|
||||||
shouldRotateClaudeToken,
|
shouldRotateClaudeToken,
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { getErrorMessage } from './utils.js';
|
import { getErrorMessage } from './utils.js';
|
||||||
import {
|
|
||||||
detectFallbackTrigger,
|
|
||||||
markPrimaryCooldown,
|
|
||||||
} from './provider-fallback.js';
|
|
||||||
import {
|
import {
|
||||||
rotateToken,
|
rotateToken,
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
@@ -39,17 +35,15 @@ export interface RotationAttemptResult {
|
|||||||
|
|
||||||
export type RotationOutcome =
|
export type RotationOutcome =
|
||||||
| { type: 'success' }
|
| { type: 'success' }
|
||||||
| { type: 'error'; message?: string }
|
| { type: 'error'; trigger?: TriggerInfo };
|
||||||
| { type: 'needs-fallback'; trigger: TriggerInfo }
|
|
||||||
| { type: 'no-fallback'; trigger: TriggerInfo }; // usage-exhausted/auth-expired/org-access-denied
|
|
||||||
|
|
||||||
// ── Shared rotation loop ─────────────────────────────────────────
|
// ── Shared rotation loop ─────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Retry a Claude request by rotating through available tokens.
|
* Retry a Claude request by rotating through available tokens.
|
||||||
*
|
*
|
||||||
* Returns a discriminated outcome — the caller decides what to do
|
* Returns 'success' if a rotated token worked, or 'error' if all
|
||||||
* with 'needs-fallback' (e.g. run Kimi fallback) or 'no-fallback'.
|
* tokens are exhausted or the error is non-retryable.
|
||||||
*/
|
*/
|
||||||
export async function runClaudeRotationLoop(
|
export async function runClaudeRotationLoop(
|
||||||
initialTrigger: TriggerInfo,
|
initialTrigger: TriggerInfo,
|
||||||
@@ -63,7 +57,9 @@ export async function runClaudeRotationLoop(
|
|||||||
while (
|
while (
|
||||||
shouldRotateClaudeToken(trigger.reason) &&
|
shouldRotateClaudeToken(trigger.reason) &&
|
||||||
getTokenCount() > 1 &&
|
getTokenCount() > 1 &&
|
||||||
rotateToken(lastRotationMessage, { ignoreRateLimits: true })
|
// Respect per-token cooldowns so exhausted auth/quota failures can
|
||||||
|
// terminate instead of cycling forever.
|
||||||
|
rotateToken(lastRotationMessage)
|
||||||
) {
|
) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{ ...logContext, reason: trigger.reason },
|
{ ...logContext, reason: trigger.reason },
|
||||||
@@ -78,12 +74,12 @@ export async function runClaudeRotationLoop(
|
|||||||
const errMsg = getErrorMessage(attempt.thrownError);
|
const errMsg = getErrorMessage(attempt.thrownError);
|
||||||
const retryTrigger = attempt.streamedTriggerReason
|
const retryTrigger = attempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
}
|
}
|
||||||
: detectFallbackTrigger(errMsg);
|
: classifyRotationTrigger(errMsg);
|
||||||
if (retryTrigger.shouldFallback) {
|
if (retryTrigger.shouldRetry) {
|
||||||
trigger = {
|
trigger = {
|
||||||
reason: retryTrigger.reason,
|
reason: retryTrigger.reason,
|
||||||
retryAfterMs: retryTrigger.retryAfterMs,
|
retryAfterMs: retryTrigger.retryAfterMs,
|
||||||
@@ -124,12 +120,13 @@ export async function runClaudeRotationLoop(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Success with null result (MAE-specific, TaskScheduler ignores) ──
|
// ── Success with null result ──
|
||||||
if (!attempt.sawOutput && attempt.sawSuccessNullResult) {
|
if (!attempt.sawOutput && attempt.sawSuccessNullResult) {
|
||||||
return {
|
logger.warn(
|
||||||
type: 'needs-fallback',
|
{ ...logContext, provider: 'claude' },
|
||||||
trigger: { reason: 'success-null-result' },
|
'All rotated tokens returned success with null result',
|
||||||
};
|
);
|
||||||
|
return { type: 'error', trigger: { reason: 'success-null-result' } };
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Error status ──
|
// ── Error status ──
|
||||||
@@ -137,12 +134,12 @@ export async function runClaudeRotationLoop(
|
|||||||
if (!attempt.sawOutput) {
|
if (!attempt.sawOutput) {
|
||||||
const retryTrigger = attempt.streamedTriggerReason
|
const retryTrigger = attempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
}
|
}
|
||||||
: detectFallbackTrigger(output.error);
|
: classifyRotationTrigger(output.error);
|
||||||
if (retryTrigger.shouldFallback) {
|
if (retryTrigger.shouldRetry) {
|
||||||
trigger = {
|
trigger = {
|
||||||
reason: retryTrigger.reason,
|
reason: retryTrigger.reason,
|
||||||
retryAfterMs: retryTrigger.retryAfterMs,
|
retryAfterMs: retryTrigger.retryAfterMs,
|
||||||
@@ -165,16 +162,9 @@ export async function runClaudeRotationLoop(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── All tokens exhausted ──
|
// ── All tokens exhausted ──
|
||||||
|
logger.warn(
|
||||||
// Usage/auth/org access failures: don't fall back to Kimi
|
{ ...logContext, reason: trigger.reason },
|
||||||
if (isNoFallbackCooldownReason(trigger.reason)) {
|
`All Claude tokens exhausted (${trigger.reason})`,
|
||||||
markPrimaryCooldown(trigger.reason, trigger.retryAfterMs);
|
);
|
||||||
logger.info(
|
return { type: 'error', trigger };
|
||||||
{ ...logContext, reason: trigger.reason },
|
|
||||||
`All Claude tokens ${trigger.reason}, silently skipping (no Kimi fallback)`,
|
|
||||||
);
|
|
||||||
return { type: 'no-fallback', trigger };
|
|
||||||
}
|
|
||||||
|
|
||||||
return { type: 'needs-fallback', trigger };
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,3 +92,15 @@ export function findChannel(
|
|||||||
): Channel | undefined {
|
): Channel | undefined {
|
||||||
return channels.find((c) => c.ownsJid(jid));
|
return channels.find((c) => c.ownsJid(jid));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize message text for deduplication comparison.
|
||||||
|
* - Trim leading/trailing whitespace
|
||||||
|
* - Collapse consecutive whitespace/newlines into single space
|
||||||
|
*/
|
||||||
|
export function normalizeMessageForDedupe(text: string): string {
|
||||||
|
return text
|
||||||
|
.trim()
|
||||||
|
.replace(/\s+/g, ' ')
|
||||||
|
.toLowerCase();
|
||||||
|
}
|
||||||
|
|||||||
77
src/service-routing.test.ts
Normal file
77
src/service-routing.test.ts
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
import { beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { _initTestDatabase, setRegisteredGroup } from './db.js';
|
||||||
|
import {
|
||||||
|
activateCodexFailover,
|
||||||
|
getActiveCodexFailoverLeases,
|
||||||
|
getEffectiveChannelLease,
|
||||||
|
refreshChannelOwnerCache,
|
||||||
|
restoreDefaultChannelLease,
|
||||||
|
} from './service-routing.js';
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
_initTestDatabase();
|
||||||
|
refreshChannelOwnerCache(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('service-routing failover leases', () => {
|
||||||
|
it('uses codex-review as owner and codex-main as reviewer during failover', () => {
|
||||||
|
setRegisteredGroup('dc:paired', {
|
||||||
|
name: 'Paired Room Claude',
|
||||||
|
folder: 'paired-claude',
|
||||||
|
trigger: '@Andy',
|
||||||
|
added_at: '2024-01-01T00:00:00.000Z',
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
setRegisteredGroup('dc:paired', {
|
||||||
|
name: 'Paired Room Codex',
|
||||||
|
folder: 'paired-codex',
|
||||||
|
trigger: '@Codex',
|
||||||
|
added_at: '2024-01-01T00:00:00.000Z',
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
activateCodexFailover('dc:paired', 'claude-429');
|
||||||
|
|
||||||
|
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
||||||
|
chat_jid: 'dc:paired',
|
||||||
|
owner_service_id: 'codex-review',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
reason: 'claude-429',
|
||||||
|
explicit: true,
|
||||||
|
});
|
||||||
|
expect(getActiveCodexFailoverLeases()).toEqual([
|
||||||
|
{
|
||||||
|
chatJid: 'dc:paired',
|
||||||
|
activatedAt: expect.any(String),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('restores the default lease after failover is cleared', () => {
|
||||||
|
setRegisteredGroup('dc:paired', {
|
||||||
|
name: 'Paired Room Claude',
|
||||||
|
folder: 'paired-claude',
|
||||||
|
trigger: '@Andy',
|
||||||
|
added_at: '2024-01-01T00:00:00.000Z',
|
||||||
|
agentType: 'claude-code',
|
||||||
|
});
|
||||||
|
setRegisteredGroup('dc:paired', {
|
||||||
|
name: 'Paired Room Codex',
|
||||||
|
folder: 'paired-codex',
|
||||||
|
trigger: '@Codex',
|
||||||
|
added_at: '2024-01-01T00:00:00.000Z',
|
||||||
|
agentType: 'codex',
|
||||||
|
});
|
||||||
|
|
||||||
|
activateCodexFailover('dc:paired', 'claude-429');
|
||||||
|
restoreDefaultChannelLease('dc:paired');
|
||||||
|
|
||||||
|
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
||||||
|
chat_jid: 'dc:paired',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
explicit: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
189
src/service-routing.ts
Normal file
189
src/service-routing.ts
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
import {
|
||||||
|
CLAUDE_SERVICE_ID,
|
||||||
|
CODEX_MAIN_SERVICE_ID,
|
||||||
|
CODEX_REVIEW_SERVICE_ID,
|
||||||
|
SERVICE_AGENT_TYPE,
|
||||||
|
SERVICE_ID,
|
||||||
|
normalizeServiceId,
|
||||||
|
} from './config.js';
|
||||||
|
import {
|
||||||
|
clearChannelOwnerLease,
|
||||||
|
getAllChannelOwnerLeases,
|
||||||
|
getRegisteredAgentTypesForJid,
|
||||||
|
setChannelOwnerLease,
|
||||||
|
type ChannelOwnerLeaseRow,
|
||||||
|
} from './db.js';
|
||||||
|
|
||||||
|
export interface EffectiveChannelLease {
|
||||||
|
chat_jid: string;
|
||||||
|
owner_service_id: string;
|
||||||
|
reviewer_service_id: string | null;
|
||||||
|
activated_at: string | null;
|
||||||
|
reason: string | null;
|
||||||
|
explicit: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const LEASE_CACHE_REFRESH_MS = 2_000;
|
||||||
|
|
||||||
|
let lastLeaseRefreshAt = 0;
|
||||||
|
const leaseCache = new Map<string, ChannelOwnerLeaseRow>();
|
||||||
|
|
||||||
|
function normalizeLeaseRow(
|
||||||
|
row: ChannelOwnerLeaseRow,
|
||||||
|
explicit: boolean,
|
||||||
|
): EffectiveChannelLease {
|
||||||
|
return {
|
||||||
|
chat_jid: row.chat_jid,
|
||||||
|
owner_service_id: normalizeServiceId(row.owner_service_id),
|
||||||
|
reviewer_service_id: row.reviewer_service_id
|
||||||
|
? normalizeServiceId(row.reviewer_service_id)
|
||||||
|
: null,
|
||||||
|
activated_at: row.activated_at,
|
||||||
|
reason: row.reason,
|
||||||
|
explicit,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDefaultLease(chatJid: string): EffectiveChannelLease {
|
||||||
|
const types = getRegisteredAgentTypesForJid(chatJid);
|
||||||
|
const hasClaude = types.includes('claude-code');
|
||||||
|
const hasCodex = types.includes('codex');
|
||||||
|
|
||||||
|
if (hasClaude && hasCodex) {
|
||||||
|
return {
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id: CLAUDE_SERVICE_ID,
|
||||||
|
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasCodex) {
|
||||||
|
return {
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id: CODEX_MAIN_SERVICE_ID,
|
||||||
|
reviewer_service_id: null,
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasClaude) {
|
||||||
|
return {
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id: CLAUDE_SERVICE_ID,
|
||||||
|
reviewer_service_id: null,
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id:
|
||||||
|
SERVICE_AGENT_TYPE === 'codex' ? CODEX_MAIN_SERVICE_ID : CLAUDE_SERVICE_ID,
|
||||||
|
reviewer_service_id: null,
|
||||||
|
activated_at: null,
|
||||||
|
reason: null,
|
||||||
|
explicit: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function refreshChannelOwnerCache(force = false): void {
|
||||||
|
const now = Date.now();
|
||||||
|
if (!force && now - lastLeaseRefreshAt < LEASE_CACHE_REFRESH_MS) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
leaseCache.clear();
|
||||||
|
for (const row of getAllChannelOwnerLeases()) {
|
||||||
|
leaseCache.set(row.chat_jid, row);
|
||||||
|
}
|
||||||
|
lastLeaseRefreshAt = now;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getEffectiveChannelLease(chatJid: string): EffectiveChannelLease {
|
||||||
|
refreshChannelOwnerCache();
|
||||||
|
const row = leaseCache.get(chatJid);
|
||||||
|
if (row) {
|
||||||
|
return normalizeLeaseRow(row, true);
|
||||||
|
}
|
||||||
|
return getDefaultLease(chatJid);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOwnerServiceForChat(
|
||||||
|
chatJid: string,
|
||||||
|
serviceId: string = SERVICE_ID,
|
||||||
|
): boolean {
|
||||||
|
const lease = getEffectiveChannelLease(chatJid);
|
||||||
|
return normalizeServiceId(serviceId) === lease.owner_service_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isReviewerServiceForChat(
|
||||||
|
chatJid: string,
|
||||||
|
serviceId: string = SERVICE_ID,
|
||||||
|
): boolean {
|
||||||
|
const lease = getEffectiveChannelLease(chatJid);
|
||||||
|
return (
|
||||||
|
lease.reviewer_service_id !== null &&
|
||||||
|
normalizeServiceId(serviceId) === lease.reviewer_service_id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shouldServiceProcessChat(
|
||||||
|
chatJid: string,
|
||||||
|
serviceId: string = SERVICE_ID,
|
||||||
|
): boolean {
|
||||||
|
const normalizedServiceId = normalizeServiceId(serviceId);
|
||||||
|
const lease = getEffectiveChannelLease(chatJid);
|
||||||
|
return (
|
||||||
|
normalizedServiceId === lease.owner_service_id ||
|
||||||
|
normalizedServiceId === lease.reviewer_service_id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function activateCodexFailover(chatJid: string, reason: string): void {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const row: ChannelOwnerLeaseRow = {
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id: CODEX_REVIEW_SERVICE_ID,
|
||||||
|
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
|
||||||
|
activated_at: now,
|
||||||
|
reason,
|
||||||
|
};
|
||||||
|
setChannelOwnerLease(row);
|
||||||
|
leaseCache.set(chatJid, row);
|
||||||
|
lastLeaseRefreshAt = Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function restoreDefaultChannelLease(chatJid: string): void {
|
||||||
|
clearChannelOwnerLease(chatJid);
|
||||||
|
leaseCache.delete(chatJid);
|
||||||
|
lastLeaseRefreshAt = Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ActiveFailoverLease {
|
||||||
|
chatJid: string;
|
||||||
|
activatedAt: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
|
||||||
|
refreshChannelOwnerCache(true);
|
||||||
|
return [...leaseCache.values()]
|
||||||
|
.filter(
|
||||||
|
(row) =>
|
||||||
|
normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID &&
|
||||||
|
normalizeServiceId(row.reviewer_service_id || '') ===
|
||||||
|
CODEX_MAIN_SERVICE_ID,
|
||||||
|
)
|
||||||
|
.map((row) => ({ chatJid: row.chat_jid, activatedAt: row.activated_at ?? null }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @deprecated Use getActiveCodexFailoverLeases() instead */
|
||||||
|
export function getActiveCodexFailoverChatJids(): string[] {
|
||||||
|
return getActiveCodexFailoverLeases().map((l) => l.chatJid);
|
||||||
|
}
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
import {
|
||||||
|
shouldResetSessionOnAgentFailure,
|
||||||
|
shouldRetryFreshSessionOnAgentFailure,
|
||||||
|
} from './session-recovery.js';
|
||||||
|
|
||||||
describe('shouldResetSessionOnAgentFailure', () => {
|
describe('shouldResetSessionOnAgentFailure', () => {
|
||||||
it('matches many-image dimension limit errors', () => {
|
it('matches many-image dimension limit errors', () => {
|
||||||
@@ -31,4 +34,45 @@ describe('shouldResetSessionOnAgentFailure', () => {
|
|||||||
}),
|
}),
|
||||||
).toBe(false);
|
).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('matches thinking signature 400 errors', () => {
|
||||||
|
expect(
|
||||||
|
shouldResetSessionOnAgentFailure({
|
||||||
|
result:
|
||||||
|
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
|
||||||
|
error: undefined,
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('shouldRetryFreshSessionOnAgentFailure', () => {
|
||||||
|
it('matches stale session errors', () => {
|
||||||
|
expect(
|
||||||
|
shouldRetryFreshSessionOnAgentFailure({
|
||||||
|
result: null,
|
||||||
|
error: 'No conversation found with session ID: stale',
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('matches thinking signature 400 errors', () => {
|
||||||
|
expect(
|
||||||
|
shouldRetryFreshSessionOnAgentFailure({
|
||||||
|
result:
|
||||||
|
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
|
||||||
|
error: undefined,
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not retry image dimension limit errors', () => {
|
||||||
|
expect(
|
||||||
|
shouldRetryFreshSessionOnAgentFailure({
|
||||||
|
result:
|
||||||
|
'An image in the conversation exceeds the dimension limit for many-image requests (2000px). Start a new session with fewer images.',
|
||||||
|
error: undefined,
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -4,6 +4,16 @@ const SESSION_RESET_PATTERNS = [
|
|||||||
/An image in the conversation exceeds the dimension limit for many-image requests \(2000px\)\./i,
|
/An image in the conversation exceeds the dimension limit for many-image requests \(2000px\)\./i,
|
||||||
/Start a new session with fewer images\./i,
|
/Start a new session with fewer images\./i,
|
||||||
/No conversation found with session ID/i,
|
/No conversation found with session ID/i,
|
||||||
|
/400[^\n]*thinking/i,
|
||||||
|
/thinking[^\n]*block[^\n]*(?:invalid|error|signature)/i,
|
||||||
|
/invalid[^\n]*signature[^\n]*thinking/i,
|
||||||
|
];
|
||||||
|
|
||||||
|
const SESSION_RETRY_PATTERNS = [
|
||||||
|
/No conversation found with session ID/i,
|
||||||
|
/400[^\n]*thinking/i,
|
||||||
|
/thinking[^\n]*block[^\n]*(?:invalid|error|signature)/i,
|
||||||
|
/invalid[^\n]*signature[^\n]*thinking/i,
|
||||||
];
|
];
|
||||||
|
|
||||||
function toText(value: string | object | null | undefined): string[] {
|
function toText(value: string | object | null | undefined): string[] {
|
||||||
@@ -25,3 +35,12 @@ export function shouldResetSessionOnAgentFailure(
|
|||||||
SESSION_RESET_PATTERNS.some((pattern) => pattern.test(text)),
|
SESSION_RESET_PATTERNS.some((pattern) => pattern.test(text)),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function shouldRetryFreshSessionOnAgentFailure(
|
||||||
|
output: Pick<AgentOutput, 'result' | 'error'>,
|
||||||
|
): boolean {
|
||||||
|
const texts = [...toText(output.result), ...toText(output.error)];
|
||||||
|
return texts.some((text) =>
|
||||||
|
SESSION_RETRY_PATTERNS.some((pattern) => pattern.test(text)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ export interface UsageRowSnapshot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface StatusSnapshot {
|
export interface StatusSnapshot {
|
||||||
|
serviceId: string;
|
||||||
agentType: AgentType;
|
agentType: AgentType;
|
||||||
assistantName: string;
|
assistantName: string;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
@@ -41,7 +42,7 @@ export function writeStatusSnapshot(snapshot: StatusSnapshot): void {
|
|||||||
fs.mkdirSync(STATUS_SNAPSHOT_DIR, { recursive: true });
|
fs.mkdirSync(STATUS_SNAPSHOT_DIR, { recursive: true });
|
||||||
const targetPath = path.join(
|
const targetPath = path.join(
|
||||||
STATUS_SNAPSHOT_DIR,
|
STATUS_SNAPSHOT_DIR,
|
||||||
`${snapshot.agentType}.json`,
|
`${snapshot.serviceId}.json`,
|
||||||
);
|
);
|
||||||
const tempPath = `${targetPath}.tmp`;
|
const tempPath = `${targetPath}.tmp`;
|
||||||
writeJsonFile(tempPath, snapshot, true);
|
writeJsonFile(tempPath, snapshot, true);
|
||||||
@@ -63,6 +64,7 @@ export function readStatusSnapshots(maxAgeMs: number): StatusSnapshot[] {
|
|||||||
const parsed = JSON.parse(raw) as StatusSnapshot;
|
const parsed = JSON.parse(raw) as StatusSnapshot;
|
||||||
if (
|
if (
|
||||||
!parsed.updatedAt ||
|
!parsed.updatedAt ||
|
||||||
|
!parsed.serviceId ||
|
||||||
!parsed.agentType ||
|
!parsed.agentType ||
|
||||||
!Array.isArray(parsed.entries)
|
!Array.isArray(parsed.entries)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import type { AgentOutput } from './agent-runner.js';
|
|||||||
|
|
||||||
// ── Mocks ──────────────────────────────────────────────────────
|
// ── Mocks ──────────────────────────────────────────────────────
|
||||||
vi.mock('./agent-error-detection.js', () => ({
|
vi.mock('./agent-error-detection.js', () => ({
|
||||||
|
classifyClaudeAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||||
|
classifyRotationTrigger: vi.fn(() => ({ shouldRetry: false, reason: '' })),
|
||||||
detectClaudeProviderFailureMessage: vi.fn(() => ''),
|
detectClaudeProviderFailureMessage: vi.fn(() => ''),
|
||||||
isClaudeUsageExhaustedMessage: vi.fn(() => false),
|
isClaudeUsageExhaustedMessage: vi.fn(() => false),
|
||||||
isClaudeOrgAccessDeniedMessage: vi.fn(() => false),
|
isClaudeOrgAccessDeniedMessage: vi.fn(() => false),
|
||||||
@@ -11,10 +13,6 @@ vi.mock('./agent-error-detection.js', () => ({
|
|||||||
isClaudeAuthError: vi.fn(() => false),
|
isClaudeAuthError: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./provider-fallback.js', () => ({
|
|
||||||
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('./codex-token-rotation.js', () => ({
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
detectCodexRotationTrigger: vi.fn(() => ({
|
detectCodexRotationTrigger: vi.fn(() => ({
|
||||||
shouldRotate: false,
|
shouldRotate: false,
|
||||||
@@ -22,15 +20,21 @@ vi.mock('./codex-token-rotation.js', () => ({
|
|||||||
})),
|
})),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./session-recovery.js', () => ({
|
||||||
|
shouldRetryFreshSessionOnAgentFailure: vi.fn(() => false),
|
||||||
|
}));
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
classifyClaudeAuthError,
|
||||||
|
classifyRotationTrigger,
|
||||||
detectClaudeProviderFailureMessage,
|
detectClaudeProviderFailureMessage,
|
||||||
isClaudeUsageExhaustedMessage,
|
isClaudeUsageExhaustedMessage,
|
||||||
isClaudeOrgAccessDeniedMessage,
|
isClaudeOrgAccessDeniedMessage,
|
||||||
isClaudeAuthExpiredMessage,
|
isClaudeAuthExpiredMessage,
|
||||||
isClaudeAuthError,
|
isClaudeAuthError,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import { detectFallbackTrigger } from './provider-fallback.js';
|
|
||||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
|
import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
evaluateStreamedOutput,
|
evaluateStreamedOutput,
|
||||||
@@ -40,7 +44,11 @@ import {
|
|||||||
|
|
||||||
// ── Helpers ────────────────────────────────────────────────────
|
// ── Helpers ────────────────────────────────────────────────────
|
||||||
function freshState(): StreamedOutputState {
|
function freshState(): StreamedOutputState {
|
||||||
return { sawOutput: false, sawSuccessNullResultWithoutOutput: false };
|
return {
|
||||||
|
sawOutput: false,
|
||||||
|
sawVisibleOutput: false,
|
||||||
|
sawSuccessNullResultWithoutOutput: false,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const claudeOpts: EvaluateStreamedOutputOptions = {
|
const claudeOpts: EvaluateStreamedOutputOptions = {
|
||||||
@@ -53,8 +61,11 @@ const codexOpts: EvaluateStreamedOutputOptions = {
|
|||||||
provider: 'codex',
|
provider: 'codex',
|
||||||
};
|
};
|
||||||
|
|
||||||
function successOutput(result: string | null): AgentOutput {
|
function successOutput(
|
||||||
return { status: 'success', result };
|
result: string | null,
|
||||||
|
phase?: AgentOutput['phase'],
|
||||||
|
): AgentOutput {
|
||||||
|
return { status: 'success', result, ...(phase ? { phase } : {}) };
|
||||||
}
|
}
|
||||||
|
|
||||||
function errorOutput(error: string): AgentOutput {
|
function errorOutput(error: string): AgentOutput {
|
||||||
@@ -63,19 +74,24 @@ function errorOutput(error: string): AgentOutput {
|
|||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(classifyClaudeAuthError).mockReturnValue({
|
||||||
|
category: 'none',
|
||||||
|
reason: '',
|
||||||
|
});
|
||||||
|
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||||
|
shouldRetry: false,
|
||||||
|
reason: '',
|
||||||
|
});
|
||||||
vi.mocked(detectClaudeProviderFailureMessage).mockReturnValue('');
|
vi.mocked(detectClaudeProviderFailureMessage).mockReturnValue('');
|
||||||
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false);
|
vi.mocked(isClaudeUsageExhaustedMessage).mockReturnValue(false);
|
||||||
vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(false);
|
vi.mocked(isClaudeOrgAccessDeniedMessage).mockReturnValue(false);
|
||||||
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false);
|
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(false);
|
||||||
vi.mocked(isClaudeAuthError).mockReturnValue(false);
|
vi.mocked(isClaudeAuthError).mockReturnValue(false);
|
||||||
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
|
||||||
shouldFallback: false,
|
|
||||||
reason: '',
|
|
||||||
});
|
|
||||||
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
||||||
shouldRotate: false,
|
shouldRotate: false,
|
||||||
reason: '',
|
reason: '',
|
||||||
});
|
});
|
||||||
|
vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
// ── Tests ──────────────────────────────────────────────────────
|
// ── Tests ──────────────────────────────────────────────────────
|
||||||
@@ -112,6 +128,16 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
expect(result.shouldForwardOutput).toBe(true);
|
expect(result.shouldForwardOutput).toBe(true);
|
||||||
expect(result.state.sawOutput).toBe(false);
|
expect(result.state.sawOutput).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not set sawOutput for progress output', () => {
|
||||||
|
const result = evaluateStreamedOutput(
|
||||||
|
successOutput('대화 요약 중...', 'progress'),
|
||||||
|
freshState(),
|
||||||
|
claudeOpts,
|
||||||
|
);
|
||||||
|
expect(result.shouldForwardOutput).toBe(true);
|
||||||
|
expect(result.state.sawOutput).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('Claude usage-exhausted banner', () => {
|
describe('Claude usage-exhausted banner', () => {
|
||||||
@@ -156,6 +182,39 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Claude retryable session failure suppression', () => {
|
||||||
|
it('suppresses retryable session failures before any visible output', () => {
|
||||||
|
vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(true);
|
||||||
|
|
||||||
|
const result = evaluateStreamedOutput(
|
||||||
|
successOutput(
|
||||||
|
'API Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.11.content.0: Invalid `signature` in `thinking` block"}}',
|
||||||
|
'intermediate',
|
||||||
|
),
|
||||||
|
freshState(),
|
||||||
|
claudeOpts,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.shouldForwardOutput).toBe(false);
|
||||||
|
expect(result.suppressedRetryableSessionFailure).toBe(true);
|
||||||
|
expect(result.state.retryableSessionFailureDetected).toBe(true);
|
||||||
|
expect(result.state.sawOutput).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not suppress retryable session failures after visible output', () => {
|
||||||
|
vi.mocked(shouldRetryFreshSessionOnAgentFailure).mockReturnValue(true);
|
||||||
|
|
||||||
|
const result = evaluateStreamedOutput(
|
||||||
|
successOutput('API Error: 400 invalid thinking block', 'final'),
|
||||||
|
{ ...freshState(), sawOutput: true },
|
||||||
|
claudeOpts,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.shouldForwardOutput).toBe(true);
|
||||||
|
expect(result.suppressedRetryableSessionFailure).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('Claude auth-expired banner', () => {
|
describe('Claude auth-expired banner', () => {
|
||||||
it('suppresses output and returns newTrigger', () => {
|
it('suppresses output and returns newTrigger', () => {
|
||||||
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(true);
|
vi.mocked(isClaudeAuthExpiredMessage).mockReturnValue(true);
|
||||||
@@ -208,6 +267,24 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
reason: 'org-access-denied',
|
reason: 'org-access-denied',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('suppresses a 403 terminated auth banner surfaced as success text', () => {
|
||||||
|
vi.mocked(classifyClaudeAuthError).mockReturnValue({
|
||||||
|
category: 'org-access-denied',
|
||||||
|
reason: 'org-access-denied',
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = evaluateStreamedOutput(
|
||||||
|
successOutput('Failed to authenticate. API Error: 403 terminated'),
|
||||||
|
freshState(),
|
||||||
|
claudeOpts,
|
||||||
|
);
|
||||||
|
expect(result.shouldForwardOutput).toBe(false);
|
||||||
|
expect(result.newTrigger).toEqual({ reason: 'org-access-denied' });
|
||||||
|
expect(result.state.streamedTriggerReason).toEqual({
|
||||||
|
reason: 'org-access-denied',
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('duplicate trigger suppression', () => {
|
describe('duplicate trigger suppression', () => {
|
||||||
@@ -298,10 +375,10 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('error → Claude fallback trigger', () => {
|
describe('error → Claude rotation trigger', () => {
|
||||||
it('returns fallback trigger with retryAfterMs', () => {
|
it('returns rotation trigger with retryAfterMs', () => {
|
||||||
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: '429',
|
reason: '429',
|
||||||
retryAfterMs: 30000,
|
retryAfterMs: 30000,
|
||||||
});
|
});
|
||||||
@@ -324,8 +401,8 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('suppresses error forward when shortCircuitTriggeredErrors is set', () => {
|
it('suppresses error forward when shortCircuitTriggeredErrors is set', () => {
|
||||||
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: '429',
|
reason: '429',
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -337,9 +414,9 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
expect(result.newTrigger).toEqual({ reason: '429' });
|
expect(result.newTrigger).toEqual({ reason: '429' });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('skips fallback check when output was already seen', () => {
|
it('skips rotation check when output was already seen', () => {
|
||||||
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: '429',
|
reason: '429',
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -352,9 +429,9 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
expect(result.shouldForwardOutput).toBe(true);
|
expect(result.shouldForwardOutput).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('skips fallback check when already triggered', () => {
|
it('skips rotation check when already triggered', () => {
|
||||||
vi.mocked(detectFallbackTrigger).mockReturnValue({
|
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: '429',
|
reason: '429',
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -397,7 +474,7 @@ describe('evaluateStreamedOutput', () => {
|
|||||||
freshState(),
|
freshState(),
|
||||||
claudeOpts,
|
claudeOpts,
|
||||||
);
|
);
|
||||||
// Claude uses detectFallbackTrigger, not detectCodexRotationTrigger
|
// Claude uses classifyRotationTrigger, not detectCodexRotationTrigger
|
||||||
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
|
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
import {
|
import {
|
||||||
|
classifyClaudeAuthError,
|
||||||
|
classifyRotationTrigger,
|
||||||
detectClaudeProviderFailureMessage,
|
detectClaudeProviderFailureMessage,
|
||||||
isClaudeAuthError,
|
isClaudeAuthError,
|
||||||
isClaudeAuthExpiredMessage,
|
isClaudeAuthExpiredMessage,
|
||||||
@@ -8,7 +10,7 @@ import {
|
|||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import type { AgentOutput } from './agent-runner.js';
|
import type { AgentOutput } from './agent-runner.js';
|
||||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
import { detectFallbackTrigger } from './provider-fallback.js';
|
import { shouldRetryFreshSessionOnAgentFailure } from './session-recovery.js';
|
||||||
|
|
||||||
export interface StreamedTriggerReason {
|
export interface StreamedTriggerReason {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
@@ -17,8 +19,10 @@ export interface StreamedTriggerReason {
|
|||||||
|
|
||||||
export interface StreamedOutputState {
|
export interface StreamedOutputState {
|
||||||
sawOutput: boolean;
|
sawOutput: boolean;
|
||||||
|
sawVisibleOutput: boolean;
|
||||||
sawSuccessNullResultWithoutOutput: boolean;
|
sawSuccessNullResultWithoutOutput: boolean;
|
||||||
streamedTriggerReason?: StreamedTriggerReason;
|
streamedTriggerReason?: StreamedTriggerReason;
|
||||||
|
retryableSessionFailureDetected?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface EvaluateStreamedOutputOptions {
|
export interface EvaluateStreamedOutputOptions {
|
||||||
@@ -34,6 +38,7 @@ export interface EvaluateStreamedOutputResult {
|
|||||||
shouldForwardOutput: boolean;
|
shouldForwardOutput: boolean;
|
||||||
newTrigger?: StreamedTriggerReason;
|
newTrigger?: StreamedTriggerReason;
|
||||||
suppressedAuthError?: boolean;
|
suppressedAuthError?: boolean;
|
||||||
|
suppressedRetryableSessionFailure?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function evaluateStreamedOutput(
|
export function evaluateStreamedOutput(
|
||||||
@@ -46,6 +51,21 @@ export function evaluateStreamedOutput(
|
|||||||
options.agentType === 'claude-code' && options.provider === 'claude';
|
options.agentType === 'claude-code' && options.provider === 'claude';
|
||||||
const isPrimaryCodex =
|
const isPrimaryCodex =
|
||||||
options.agentType === 'codex' && options.provider === 'codex';
|
options.agentType === 'codex' && options.provider === 'codex';
|
||||||
|
const countsAsFinalOutput =
|
||||||
|
output.phase === undefined || output.phase === 'final';
|
||||||
|
|
||||||
|
if (
|
||||||
|
isPrimaryClaude &&
|
||||||
|
!state.sawOutput &&
|
||||||
|
shouldRetryFreshSessionOnAgentFailure(output)
|
||||||
|
) {
|
||||||
|
nextState.retryableSessionFailureDetected = true;
|
||||||
|
return {
|
||||||
|
state: nextState,
|
||||||
|
shouldForwardOutput: false,
|
||||||
|
suppressedRetryableSessionFailure: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
isPrimaryClaude &&
|
isPrimaryClaude &&
|
||||||
@@ -53,12 +73,15 @@ export function evaluateStreamedOutput(
|
|||||||
!state.sawOutput &&
|
!state.sawOutput &&
|
||||||
typeof output.result === 'string'
|
typeof output.result === 'string'
|
||||||
) {
|
) {
|
||||||
|
const authClassification = classifyClaudeAuthError(output.result);
|
||||||
const triggerReason: AgentTriggerReason | undefined =
|
const triggerReason: AgentTriggerReason | undefined =
|
||||||
isClaudeUsageExhaustedMessage(output.result)
|
isClaudeUsageExhaustedMessage(output.result)
|
||||||
? 'usage-exhausted'
|
? 'usage-exhausted'
|
||||||
: isClaudeOrgAccessDeniedMessage(output.result)
|
: isClaudeOrgAccessDeniedMessage(output.result) ||
|
||||||
|
authClassification.category === 'org-access-denied'
|
||||||
? 'org-access-denied'
|
? 'org-access-denied'
|
||||||
: isClaudeAuthExpiredMessage(output.result)
|
: isClaudeAuthExpiredMessage(output.result) ||
|
||||||
|
authClassification.category === 'auth-expired'
|
||||||
? 'auth-expired'
|
? 'auth-expired'
|
||||||
: detectClaudeProviderFailureMessage(output.result) || undefined;
|
: detectClaudeProviderFailureMessage(output.result) || undefined;
|
||||||
|
|
||||||
@@ -87,7 +110,11 @@ export function evaluateStreamedOutput(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (output.result !== null && output.result !== undefined) {
|
if (
|
||||||
|
countsAsFinalOutput &&
|
||||||
|
output.result !== null &&
|
||||||
|
output.result !== undefined
|
||||||
|
) {
|
||||||
nextState.sawOutput = true;
|
nextState.sawOutput = true;
|
||||||
} else if (
|
} else if (
|
||||||
options.trackSuccessNullResult &&
|
options.trackSuccessNullResult &&
|
||||||
@@ -106,8 +133,8 @@ export function evaluateStreamedOutput(
|
|||||||
let newTrigger: StreamedTriggerReason | undefined;
|
let newTrigger: StreamedTriggerReason | undefined;
|
||||||
|
|
||||||
if (isPrimaryClaude) {
|
if (isPrimaryClaude) {
|
||||||
const trigger = detectFallbackTrigger(output.error);
|
const trigger = classifyRotationTrigger(output.error);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldRetry) {
|
||||||
newTrigger = {
|
newTrigger = {
|
||||||
reason: trigger.reason,
|
reason: trigger.reason,
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
|
|||||||
@@ -24,38 +24,31 @@ const {
|
|||||||
),
|
),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./provider-fallback.js', () => ({
|
vi.mock('./agent-error-detection.js', async (importOriginal) => {
|
||||||
detectFallbackTrigger: vi.fn((error?: string | null) => {
|
const actual = await importOriginal<typeof import('./agent-error-detection.js')>();
|
||||||
const lower = (error || '').toLowerCase();
|
return {
|
||||||
if (
|
...actual,
|
||||||
lower.includes('does not have access to claude') ||
|
classifyRotationTrigger: vi.fn((error?: string | null) => {
|
||||||
(lower.includes('failed to authenticate') &&
|
const lower = (error || '').toLowerCase();
|
||||||
lower.includes('403') &&
|
if (
|
||||||
lower.includes('terminated'))
|
lower.includes('does not have access to claude') ||
|
||||||
) {
|
(lower.includes('failed to authenticate') &&
|
||||||
return { shouldFallback: true, reason: 'org-access-denied' };
|
lower.includes('403') &&
|
||||||
}
|
lower.includes('terminated'))
|
||||||
if (
|
) {
|
||||||
lower.includes('429') ||
|
return { shouldRetry: true, reason: 'org-access-denied' };
|
||||||
lower.includes('rate limit') ||
|
}
|
||||||
lower.includes('hit your limit')
|
if (
|
||||||
) {
|
lower.includes('429') ||
|
||||||
return { shouldFallback: true, reason: '429' };
|
lower.includes('rate limit') ||
|
||||||
}
|
lower.includes('hit your limit')
|
||||||
return { shouldFallback: false, reason: '' };
|
) {
|
||||||
}),
|
return { shouldRetry: true, reason: '429' };
|
||||||
getActiveProvider: vi.fn(async () => 'claude'),
|
}
|
||||||
getFallbackEnvOverrides: vi.fn(() => ({
|
return { shouldRetry: false, reason: '' };
|
||||||
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
}),
|
||||||
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
|
};
|
||||||
ANTHROPIC_MODEL: 'kimi-k2.5',
|
});
|
||||||
})),
|
|
||||||
getFallbackProviderName: vi.fn(() => 'kimi'),
|
|
||||||
hasGroupProviderOverride: vi.fn(() => false),
|
|
||||||
isFallbackEnabled: vi.fn(() => true),
|
|
||||||
isPrimaryNoFallbackCooldownActive: vi.fn(() => false),
|
|
||||||
markPrimaryCooldown: vi.fn(),
|
|
||||||
}));
|
|
||||||
|
|
||||||
vi.mock('./token-rotation.js', () => ({
|
vi.mock('./token-rotation.js', () => ({
|
||||||
rotateToken: vi.fn(() => false),
|
rotateToken: vi.fn(() => false),
|
||||||
@@ -136,7 +129,6 @@ vi.mock('./github-ci.js', () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
import { _initTestDatabase, createTask, getTaskById } from './db.js';
|
import { _initTestDatabase, createTask, getTaskById } from './db.js';
|
||||||
import * as providerFallback from './provider-fallback.js';
|
|
||||||
import * as codexTokenRotation from './codex-token-rotation.js';
|
import * as codexTokenRotation from './codex-token-rotation.js';
|
||||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||||
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
||||||
@@ -163,10 +155,7 @@ describe('task scheduler', () => {
|
|||||||
terminal: false,
|
terminal: false,
|
||||||
resultSummary: 'GitHub Actions run 123 is in_progress',
|
resultSummary: 'GitHub Actions run 123 is in_progress',
|
||||||
});
|
});
|
||||||
vi.mocked(providerFallback.markPrimaryCooldown).mockClear();
|
// No fallback provider setup needed
|
||||||
vi.mocked(providerFallback.getActiveProvider).mockResolvedValue('claude');
|
|
||||||
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
|
|
||||||
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(false);
|
|
||||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
||||||
vi.mocked(tokenRotation.markTokenHealthy).mockClear();
|
vi.mocked(tokenRotation.markTokenHealthy).mockClear();
|
||||||
vi.mocked(tokenRotation.rotateToken).mockClear();
|
vi.mocked(tokenRotation.rotateToken).mockClear();
|
||||||
@@ -454,7 +443,7 @@ Check the run.
|
|||||||
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
// No fallback cooldown to check
|
||||||
expect(sendMessage).toHaveBeenCalledTimes(1);
|
expect(sendMessage).toHaveBeenCalledTimes(1);
|
||||||
expect(sendMessage).toHaveBeenCalledWith(
|
expect(sendMessage).toHaveBeenCalledWith(
|
||||||
'shared@g.us',
|
'shared@g.us',
|
||||||
@@ -553,7 +542,7 @@ Check the run.
|
|||||||
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
// No fallback cooldown to check
|
||||||
expect(sendMessage).toHaveBeenCalledTimes(1);
|
expect(sendMessage).toHaveBeenCalledTimes(1);
|
||||||
expect(sendMessage).toHaveBeenCalledWith(
|
expect(sendMessage).toHaveBeenCalledWith(
|
||||||
'shared@g.us',
|
'shared@g.us',
|
||||||
@@ -601,23 +590,6 @@ Check the run.
|
|||||||
result: null,
|
result: null,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
)
|
|
||||||
.mockImplementationOnce(
|
|
||||||
async (
|
|
||||||
_group: unknown,
|
|
||||||
_input: unknown,
|
|
||||||
_onProcess: unknown,
|
|
||||||
onOutput?: (output: Record<string, unknown>) => Promise<void>,
|
|
||||||
) => {
|
|
||||||
await onOutput?.({
|
|
||||||
status: 'success',
|
|
||||||
result: 'scheduled fallback response',
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
status: 'success',
|
|
||||||
result: null,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const enqueueTask = vi.fn(
|
const enqueueTask = vi.fn(
|
||||||
@@ -646,17 +618,9 @@ Check the run.
|
|||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
expect(runAgentProcessMock).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
|
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
|
||||||
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
// No fallback — 502 results in error without retrying on another provider
|
||||||
'overloaded',
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
expect(sendMessage).toHaveBeenCalledTimes(1);
|
|
||||||
expect(sendMessage).toHaveBeenCalledWith(
|
|
||||||
'shared@g.us',
|
|
||||||
'scheduled fallback response',
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('suppresses Claude org access denied banners for scheduled tasks and retries with a rotated account', async () => {
|
it('suppresses Claude org access denied banners for scheduled tasks and retries with a rotated account', async () => {
|
||||||
@@ -750,7 +714,7 @@ Check the run.
|
|||||||
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
expect(runAgentProcessMock).toHaveBeenCalledTimes(2);
|
||||||
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.rotateToken).toHaveBeenCalledTimes(1);
|
||||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||||
expect(providerFallback.markPrimaryCooldown).not.toHaveBeenCalled();
|
// No fallback cooldown to check
|
||||||
expect(sendMessage).toHaveBeenCalledTimes(1);
|
expect(sendMessage).toHaveBeenCalledTimes(1);
|
||||||
expect(sendMessage).toHaveBeenCalledWith(
|
expect(sendMessage).toHaveBeenCalledWith(
|
||||||
'shared@g.us',
|
'shared@g.us',
|
||||||
|
|||||||
@@ -1,12 +1,10 @@
|
|||||||
import { ChildProcess } from 'child_process';
|
import { ChildProcess } from 'child_process';
|
||||||
import { CronExpressionParser } from 'cron-parser';
|
import { CronExpressionParser } from 'cron-parser';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
|
||||||
import { getErrorMessage } from './utils.js';
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
ASSISTANT_NAME,
|
ASSISTANT_NAME,
|
||||||
DATA_DIR,
|
|
||||||
SCHEDULER_POLL_INTERVAL,
|
SCHEDULER_POLL_INTERVAL,
|
||||||
SERVICE_AGENT_TYPE,
|
SERVICE_AGENT_TYPE,
|
||||||
TIMEZONE,
|
TIMEZONE,
|
||||||
@@ -33,16 +31,6 @@ import {
|
|||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||||
import {
|
|
||||||
detectFallbackTrigger,
|
|
||||||
getActiveProvider,
|
|
||||||
getFallbackEnvOverrides,
|
|
||||||
getFallbackProviderName,
|
|
||||||
hasGroupProviderOverride,
|
|
||||||
isFallbackEnabled,
|
|
||||||
isPrimaryNoFallbackCooldownActive,
|
|
||||||
markPrimaryCooldown,
|
|
||||||
} from './provider-fallback.js';
|
|
||||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||||
import {
|
import {
|
||||||
detectCodexRotationTrigger,
|
detectCodexRotationTrigger,
|
||||||
@@ -50,9 +38,10 @@ import {
|
|||||||
getCodexAccountCount,
|
getCodexAccountCount,
|
||||||
markCodexTokenHealthy,
|
markCodexTokenHealthy,
|
||||||
} from './codex-token-rotation.js';
|
} from './codex-token-rotation.js';
|
||||||
import type {
|
import {
|
||||||
AgentTriggerReason,
|
classifyRotationTrigger,
|
||||||
CodexRotationReason,
|
type AgentTriggerReason,
|
||||||
|
type CodexRotationReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
@@ -306,19 +295,8 @@ async function runTask(
|
|||||||
sendTrackedMessage: deps.sendTrackedMessage,
|
sendTrackedMessage: deps.sendTrackedMessage,
|
||||||
editTrackedMessage: deps.editTrackedMessage,
|
editTrackedMessage: deps.editTrackedMessage,
|
||||||
});
|
});
|
||||||
const settingsPath = path.join(
|
|
||||||
DATA_DIR,
|
|
||||||
'sessions',
|
|
||||||
task.group_folder,
|
|
||||||
'.claude',
|
|
||||||
'settings.json',
|
|
||||||
);
|
|
||||||
const isClaudeAgent = context.taskAgentType === 'claude-code';
|
const isClaudeAgent = context.taskAgentType === 'claude-code';
|
||||||
const canRotateToken = isClaudeAgent && getTokenCount() > 1;
|
const canRotateToken = isClaudeAgent && getTokenCount() > 1;
|
||||||
const canFallback =
|
|
||||||
isClaudeAgent &&
|
|
||||||
isFallbackEnabled() &&
|
|
||||||
!hasGroupProviderOverride(settingsPath);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await statusTracker.update('checking');
|
await statusTracker.update('checking');
|
||||||
@@ -337,6 +315,7 @@ async function runTask(
|
|||||||
}> => {
|
}> => {
|
||||||
let streamedState: StreamedOutputState = {
|
let streamedState: StreamedOutputState = {
|
||||||
sawOutput: false,
|
sawOutput: false,
|
||||||
|
sawVisibleOutput: false,
|
||||||
sawSuccessNullResultWithoutOutput: false,
|
sawSuccessNullResultWithoutOutput: false,
|
||||||
};
|
};
|
||||||
let attemptResult: string | null = null;
|
let attemptResult: string | null = null;
|
||||||
@@ -391,7 +370,7 @@ async function runTask(
|
|||||||
reason: evaluation.newTrigger.reason,
|
reason: evaluation.newTrigger.reason,
|
||||||
resultPreview: streamedOutput.result.slice(0, 120),
|
resultPreview: streamedOutput.result.slice(0, 120),
|
||||||
},
|
},
|
||||||
'Detected Claude fallback trigger during scheduled task output',
|
'Detected Claude rotation trigger during scheduled task output',
|
||||||
);
|
);
|
||||||
} else if (
|
} else if (
|
||||||
evaluation.newTrigger &&
|
evaluation.newTrigger &&
|
||||||
@@ -407,7 +386,7 @@ async function runTask(
|
|||||||
errorPreview: streamedOutput.error.slice(0, 120),
|
errorPreview: streamedOutput.error.slice(0, 120),
|
||||||
},
|
},
|
||||||
provider === 'claude'
|
provider === 'claude'
|
||||||
? 'Detected Claude fallback trigger during scheduled task error output'
|
? 'Detected Claude rotation trigger during scheduled task error output'
|
||||||
: 'Detected Codex rotation trigger during scheduled task error output',
|
: 'Detected Codex rotation trigger during scheduled task error output',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -428,9 +407,7 @@ async function runTask(
|
|||||||
attemptError = streamedOutput.error || 'Unknown error';
|
attemptError = streamedOutput.error || 'Unknown error';
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
isClaudeAgent && provider !== 'claude'
|
undefined,
|
||||||
? getFallbackEnvOverrides()
|
|
||||||
: undefined,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
if (output.status === 'error' && !attemptError) {
|
if (output.status === 'error' && !attemptError) {
|
||||||
@@ -448,38 +425,6 @@ async function runTask(
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
const runFallbackTaskAttempt = async (
|
|
||||||
reason: AgentTriggerReason,
|
|
||||||
retryAfterMs?: number,
|
|
||||||
): Promise<void> => {
|
|
||||||
if (!canFallback) {
|
|
||||||
error = reason;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const fallbackName = getFallbackProviderName();
|
|
||||||
markPrimaryCooldown(reason, retryAfterMs);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
taskId: task.id,
|
|
||||||
group: context.group.name,
|
|
||||||
groupFolder: task.group_folder,
|
|
||||||
reason,
|
|
||||||
retryAfterMs,
|
|
||||||
fallbackProvider: fallbackName,
|
|
||||||
},
|
|
||||||
`Falling back to provider: ${fallbackName} for scheduled task (reason: ${reason})`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const fallbackAttempt = await runTaskAttempt(fallbackName);
|
|
||||||
result = fallbackAttempt.attemptResult;
|
|
||||||
error =
|
|
||||||
fallbackAttempt.output.status === 'error'
|
|
||||||
? fallbackAttempt.attemptError || 'Unknown error'
|
|
||||||
: null;
|
|
||||||
};
|
|
||||||
|
|
||||||
const retryClaudeTaskWithRotation = async (
|
const retryClaudeTaskWithRotation = async (
|
||||||
initialTrigger: {
|
initialTrigger: {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
@@ -514,15 +459,9 @@ async function runTask(
|
|||||||
error = null;
|
error = null;
|
||||||
return;
|
return;
|
||||||
case 'error':
|
case 'error':
|
||||||
return;
|
if (outcome.trigger) {
|
||||||
case 'no-fallback':
|
error = `Claude ${outcome.trigger.reason}`;
|
||||||
error = `Claude ${outcome.trigger.reason}`;
|
}
|
||||||
return;
|
|
||||||
case 'needs-fallback':
|
|
||||||
await runFallbackTaskAttempt(
|
|
||||||
outcome.trigger.reason,
|
|
||||||
outcome.trigger.retryAfterMs,
|
|
||||||
);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -597,25 +536,9 @@ async function runTask(
|
|||||||
};
|
};
|
||||||
|
|
||||||
const provider =
|
const provider =
|
||||||
context.taskAgentType === 'codex'
|
context.taskAgentType === 'codex' ? 'codex' : 'claude';
|
||||||
? 'codex'
|
|
||||||
: canFallback
|
|
||||||
? await getActiveProvider()
|
|
||||||
: 'claude';
|
|
||||||
|
|
||||||
// Already in no-fallback Claude cooldown — skip task instead of running on Kimi
|
{
|
||||||
if (
|
|
||||||
isClaudeAgent &&
|
|
||||||
provider !== 'claude' &&
|
|
||||||
isPrimaryNoFallbackCooldownActive()
|
|
||||||
) {
|
|
||||||
logger.info(
|
|
||||||
{ taskId: task.id, group: context.group.name, provider },
|
|
||||||
'Claude primary cooldown active, skipping scheduled task',
|
|
||||||
);
|
|
||||||
error = 'Claude primary cooldown active';
|
|
||||||
// Fall through to task completion handling below
|
|
||||||
} else {
|
|
||||||
const attempt = await runTaskAttempt(provider);
|
const attempt = await runTaskAttempt(provider);
|
||||||
result = attempt.attemptResult;
|
result = attempt.attemptResult;
|
||||||
error = attempt.attemptError;
|
error = attempt.attemptError;
|
||||||
@@ -642,12 +565,12 @@ async function runTask(
|
|||||||
} else if (attempt.output.status === 'error' && provider === 'claude') {
|
} else if (attempt.output.status === 'error' && provider === 'claude') {
|
||||||
const trigger = attempt.streamedTriggerReason
|
const trigger = attempt.streamedTriggerReason
|
||||||
? {
|
? {
|
||||||
shouldFallback: true,
|
shouldRetry: true,
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
reason: attempt.streamedTriggerReason.reason,
|
||||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||||
}
|
}
|
||||||
: detectFallbackTrigger(error);
|
: classifyRotationTrigger(error);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldRetry) {
|
||||||
await retryClaudeTaskWithRotation({
|
await retryClaudeTaskWithRotation({
|
||||||
reason: trigger.reason,
|
reason: trigger.reason,
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
@@ -725,8 +648,8 @@ async function runTask(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const trigger = detectFallbackTrigger(error);
|
const trigger = classifyRotationTrigger(error);
|
||||||
if (trigger.shouldFallback) {
|
if (trigger.shouldRetry) {
|
||||||
const rotated = getTokenCount() > 1 && rotateToken(error);
|
const rotated = getTokenCount() > 1 && rotateToken(error);
|
||||||
if (rotated) {
|
if (rotated) {
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
* CLAUDE_CODE_OAUTH_TOKEN if multi-token is not configured.
|
* CLAUDE_CODE_OAUTH_TOKEN if multi-token is not configured.
|
||||||
*
|
*
|
||||||
* On rate-limit: rotate to next token
|
* On rate-limit: rotate to next token
|
||||||
* All exhausted: fall through to provider fallback (Kimi etc.)
|
* All exhausted: surface error to caller
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
@@ -153,7 +153,7 @@ export function rotateToken(
|
|||||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||||
reason: errorMessage ?? null,
|
reason: errorMessage ?? null,
|
||||||
},
|
},
|
||||||
'All tokens are rate-limited, falling through to provider fallback',
|
'All tokens are rate-limited, no available tokens',
|
||||||
);
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -234,3 +234,13 @@ export function getTokenRotationInfo(): {
|
|||||||
).length,
|
).length,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function hasAvailableClaudeToken(): boolean {
|
||||||
|
if (tokens.length === 0) {
|
||||||
|
return Boolean(process.env.CLAUDE_CODE_OAUTH_TOKEN);
|
||||||
|
}
|
||||||
|
const now = Date.now();
|
||||||
|
return tokens.some(
|
||||||
|
(token) => !token.rateLimitedUntil || token.rateLimitedUntil <= now,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import type { UsageRow } from './dashboard-usage-rows.js';
|
||||||
import {
|
import {
|
||||||
formatStatusHeader,
|
formatStatusHeader,
|
||||||
|
renderUsageTable,
|
||||||
summarizeWatcherTasks,
|
summarizeWatcherTasks,
|
||||||
} from './unified-dashboard.js';
|
} from './unified-dashboard.js';
|
||||||
|
|
||||||
@@ -57,3 +59,51 @@ describe('formatStatusHeader', () => {
|
|||||||
).toBe('**📊 에이전트 상태** — 활성 3 / 8 | 감시 2 | 일시정지 1');
|
).toBe('**📊 에이전트 상태** — 활성 3 / 8 | 감시 2 | 일시정지 1');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('renderUsageTable', () => {
|
||||||
|
const claudeRow: UsageRow = {
|
||||||
|
name: 'Claude pro',
|
||||||
|
h5pct: 50,
|
||||||
|
h5reset: '',
|
||||||
|
d7pct: 30,
|
||||||
|
d7reset: '',
|
||||||
|
};
|
||||||
|
const codexRow: UsageRow = {
|
||||||
|
name: 'Codex',
|
||||||
|
h5pct: 40,
|
||||||
|
h5reset: '',
|
||||||
|
d7pct: 25,
|
||||||
|
d7reset: '',
|
||||||
|
};
|
||||||
|
|
||||||
|
it('renders Claude before separator before Codex', () => {
|
||||||
|
const lines = renderUsageTable([claudeRow], [codexRow]);
|
||||||
|
|
||||||
|
const claudeIdx = lines.findIndex((l) => l.includes('Claude'));
|
||||||
|
const sepIdx = lines.findIndex((l) => /^─+$/.test(l));
|
||||||
|
const codexIdx = lines.findIndex((l) => l.includes('Codex'));
|
||||||
|
|
||||||
|
expect(claudeIdx).toBeGreaterThan(-1);
|
||||||
|
expect(sepIdx).toBeGreaterThan(claudeIdx);
|
||||||
|
expect(codexIdx).toBeGreaterThan(sepIdx);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('omits separator when only Claude rows exist', () => {
|
||||||
|
const lines = renderUsageTable([claudeRow], []);
|
||||||
|
|
||||||
|
expect(lines.some((l) => /^─+$/.test(l))).toBe(false);
|
||||||
|
expect(lines.some((l) => l.includes('Claude'))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('omits separator when only Codex rows exist', () => {
|
||||||
|
const lines = renderUsageTable([], [codexRow]);
|
||||||
|
|
||||||
|
expect(lines.some((l) => /^─+$/.test(l))).toBe(false);
|
||||||
|
expect(lines.some((l) => l.includes('Codex'))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns fallback text when both groups are empty', () => {
|
||||||
|
const lines = renderUsageTable([], []);
|
||||||
|
expect(lines).toEqual(['_조회 불가_']);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ import type {
|
|||||||
|
|
||||||
export interface UnifiedDashboardOptions {
|
export interface UnifiedDashboardOptions {
|
||||||
assistantName: string;
|
assistantName: string;
|
||||||
|
serviceId: string;
|
||||||
serviceAgentType: AgentType;
|
serviceAgentType: AgentType;
|
||||||
statusChannelId: string;
|
statusChannelId: string;
|
||||||
statusUpdateInterval: number;
|
statusUpdateInterval: number;
|
||||||
@@ -189,8 +190,12 @@ async function refreshChannelMeta(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function getAgentDisplayName(agentType: 'claude-code' | 'codex'): string {
|
function getAgentDisplayName(
|
||||||
return agentType === 'codex' ? '코덱스' : '클코';
|
agentType: 'claude-code' | 'codex',
|
||||||
|
serviceId: string,
|
||||||
|
): string {
|
||||||
|
if (agentType === 'claude-code') return '클코';
|
||||||
|
return serviceId === 'codex-review' ? '코리뷰' : '코덱스';
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatRoomName(
|
function formatRoomName(
|
||||||
@@ -216,6 +221,7 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
|
|||||||
const statuses = opts.queue.getStatuses(Object.keys(groups));
|
const statuses = opts.queue.getStatuses(Object.keys(groups));
|
||||||
|
|
||||||
writeStatusSnapshot({
|
writeStatusSnapshot({
|
||||||
|
serviceId: opts.serviceId,
|
||||||
agentType: opts.serviceAgentType,
|
agentType: opts.serviceAgentType,
|
||||||
assistantName: opts.assistantName,
|
assistantName: opts.assistantName,
|
||||||
updatedAt: new Date().toISOString(),
|
updatedAt: new Date().toISOString(),
|
||||||
@@ -261,6 +267,7 @@ function buildStatusContent(): string {
|
|||||||
);
|
);
|
||||||
|
|
||||||
interface RoomEntry {
|
interface RoomEntry {
|
||||||
|
serviceId: string;
|
||||||
agentType: 'claude-code' | 'codex';
|
agentType: 'claude-code' | 'codex';
|
||||||
status: 'processing' | 'waiting' | 'inactive';
|
status: 'processing' | 'waiting' | 'inactive';
|
||||||
elapsedMs: number | null;
|
elapsedMs: number | null;
|
||||||
@@ -277,6 +284,7 @@ function buildStatusContent(): string {
|
|||||||
for (const entry of snapshot.entries) {
|
for (const entry of snapshot.entries) {
|
||||||
const existing = byJid.get(entry.jid) || [];
|
const existing = byJid.get(entry.jid) || [];
|
||||||
existing.push({
|
existing.push({
|
||||||
|
serviceId: snapshot.serviceId,
|
||||||
agentType,
|
agentType,
|
||||||
status: entry.status,
|
status: entry.status,
|
||||||
elapsedMs: entry.elapsedMs,
|
elapsedMs: entry.elapsedMs,
|
||||||
@@ -345,7 +353,7 @@ function buildStatusContent(): string {
|
|||||||
elapsedMs: agent.elapsedMs,
|
elapsedMs: agent.elapsedMs,
|
||||||
pendingTasks: agent.pendingTasks,
|
pendingTasks: agent.pendingTasks,
|
||||||
});
|
});
|
||||||
const tag = getAgentDisplayName(agent.agentType);
|
const tag = getAgentDisplayName(agent.agentType, agent.serviceId);
|
||||||
return `${tag} ${icon} ${label}`;
|
return `${tag} ${icon} ${label}`;
|
||||||
});
|
});
|
||||||
roomLines.push({
|
roomLines.push({
|
||||||
@@ -373,6 +381,76 @@ function buildStatusContent(): string {
|
|||||||
return `${header}\n\n${sections}`;
|
return `${header}\n\n${sections}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render usage table lines from two row groups (Claude and Codex).
|
||||||
|
* Returns rendered lines including code block markers.
|
||||||
|
* Ordering: Claude rows → separator → Codex rows.
|
||||||
|
* Exported for testing.
|
||||||
|
*/
|
||||||
|
export function renderUsageTable(
|
||||||
|
claudeBotRows: UsageRow[],
|
||||||
|
codexBotRows: UsageRow[],
|
||||||
|
): string[] {
|
||||||
|
const allRows = [...claudeBotRows, ...codexBotRows];
|
||||||
|
if (allRows.length === 0) return ['_조회 불가_'];
|
||||||
|
|
||||||
|
const bar = (pct: number) => {
|
||||||
|
const filled = Math.max(0, Math.min(5, Math.round(pct / 20)));
|
||||||
|
return '█'.repeat(filled) + '░'.repeat(5 - filled);
|
||||||
|
};
|
||||||
|
|
||||||
|
const visualWidth = (s: string) =>
|
||||||
|
[...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0);
|
||||||
|
const maxNameWidth =
|
||||||
|
Math.max(8, ...allRows.map((r) => visualWidth(r.name))) + 1;
|
||||||
|
const padName = (s: string) =>
|
||||||
|
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
|
||||||
|
const compactReset = (s: string) =>
|
||||||
|
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
|
||||||
|
|
||||||
|
const lines: string[] = [];
|
||||||
|
|
||||||
|
const renderRows = (rows: UsageRow[]) => {
|
||||||
|
for (const row of rows) {
|
||||||
|
const h5 =
|
||||||
|
row.h5pct >= 0
|
||||||
|
? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%`
|
||||||
|
: ' — ';
|
||||||
|
const d7 =
|
||||||
|
row.d7pct >= 0
|
||||||
|
? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%`
|
||||||
|
: ' — ';
|
||||||
|
lines.push(`${padName(row.name)}${h5} ${d7}`);
|
||||||
|
const r5 = compactReset(row.h5reset);
|
||||||
|
const r7 = compactReset(row.d7reset);
|
||||||
|
if (r5 || r7) {
|
||||||
|
const d7ColStart = maxNameWidth + 10;
|
||||||
|
let resetLine = ' '.repeat(maxNameWidth);
|
||||||
|
if (r5) resetLine += r5;
|
||||||
|
resetLine = resetLine.padEnd(d7ColStart);
|
||||||
|
if (r7) resetLine += r7;
|
||||||
|
lines.push(resetLine);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
lines.push('```');
|
||||||
|
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
|
||||||
|
|
||||||
|
renderRows(claudeBotRows);
|
||||||
|
|
||||||
|
if (claudeBotRows.length > 0 && codexBotRows.length > 0) {
|
||||||
|
const separatorWidth = maxNameWidth + 20;
|
||||||
|
lines.push('─'.repeat(separatorWidth));
|
||||||
|
}
|
||||||
|
|
||||||
|
renderRows(codexBotRows);
|
||||||
|
|
||||||
|
lines.push('```');
|
||||||
|
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
|
||||||
async function buildUsageContent(): Promise<string> {
|
async function buildUsageContent(): Promise<string> {
|
||||||
const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED;
|
const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED;
|
||||||
let liveClaudeAccounts: ClaudeAccountUsage[] | null = null;
|
let liveClaudeAccounts: ClaudeAccountUsage[] | null = null;
|
||||||
@@ -391,64 +469,27 @@ async function buildUsageContent(): Promise<string> {
|
|||||||
return '█'.repeat(filled) + '░'.repeat(5 - filled);
|
return '█'.repeat(filled) + '░'.repeat(5 - filled);
|
||||||
};
|
};
|
||||||
|
|
||||||
const rows: UsageRow[] = [];
|
// Group 1: Claude bot
|
||||||
|
const claudeBotRows: UsageRow[] = [];
|
||||||
|
|
||||||
if (shouldFetchClaudeUsage) {
|
if (shouldFetchClaudeUsage) {
|
||||||
cachedClaudeAccounts = mergeClaudeDashboardAccounts(
|
cachedClaudeAccounts = mergeClaudeDashboardAccounts(
|
||||||
liveClaudeAccounts,
|
liveClaudeAccounts,
|
||||||
cachedClaudeAccounts,
|
cachedClaudeAccounts,
|
||||||
);
|
);
|
||||||
rows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
|
claudeBotRows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Codex usage: read from Codex service's own status snapshot.
|
// Group 2: Codex bot (separate service)
|
||||||
// Each service owns its usage data — no cross-service auth access needed.
|
const codexBotRows: UsageRow[] = [];
|
||||||
const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS).find(
|
const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS).find(
|
||||||
(s) => s.agentType === 'codex',
|
(s) => s.serviceId === 'codex-main' || s.serviceId === 'codex',
|
||||||
|
);
|
||||||
|
codexBotRows.push(
|
||||||
|
...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS),
|
||||||
);
|
);
|
||||||
rows.push(...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS));
|
|
||||||
|
|
||||||
if (rows.length > 0) {
|
lines.push(...renderUsageTable(claudeBotRows, codexBotRows));
|
||||||
// Emoji characters take 2 columns in monospace — count visual width
|
|
||||||
const visualWidth = (s: string) =>
|
|
||||||
[...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0);
|
|
||||||
const maxNameWidth =
|
|
||||||
Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1;
|
|
||||||
const padName = (s: string) =>
|
|
||||||
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
|
|
||||||
// Strip whitespace and trailing 'm' from reset strings for compact display
|
|
||||||
const compactReset = (s: string) =>
|
|
||||||
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
|
|
||||||
|
|
||||||
lines.push('```');
|
|
||||||
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
|
|
||||||
for (const row of rows) {
|
|
||||||
const h5 =
|
|
||||||
row.h5pct >= 0
|
|
||||||
? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%`
|
|
||||||
: ' — ';
|
|
||||||
const d7 =
|
|
||||||
row.d7pct >= 0
|
|
||||||
? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%`
|
|
||||||
: ' — ';
|
|
||||||
lines.push(`${padName(row.name)}${h5} ${d7}`);
|
|
||||||
const r5 = compactReset(row.h5reset);
|
|
||||||
const r7 = compactReset(row.d7reset);
|
|
||||||
if (r5 || r7) {
|
|
||||||
// Align reset values under 5h / 7d columns
|
|
||||||
// h5 column starts at maxNameWidth; d7 column starts at maxNameWidth + 9 + 1
|
|
||||||
const d7ColStart = maxNameWidth + 10;
|
|
||||||
let resetLine = ' '.repeat(maxNameWidth);
|
|
||||||
if (r5) resetLine += r5;
|
|
||||||
resetLine = resetLine.padEnd(d7ColStart);
|
|
||||||
if (r7) resetLine += r7;
|
|
||||||
lines.push(resetLine);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
lines.push('```');
|
|
||||||
} else {
|
|
||||||
lines.push('_조회 불가_');
|
|
||||||
}
|
|
||||||
|
|
||||||
lines.push('');
|
lines.push('');
|
||||||
lines.push('🖥️ *서버*');
|
lines.push('🖥️ *서버*');
|
||||||
|
|||||||
Reference in New Issue
Block a user