feat: add seq cursor, work_items, provider fallback, and delivery reliability
Major reliability improvements to the message processing pipeline: - Add messages.seq monotonic cursor replacing timestamp-based cursors, preventing message loss from timestamp collisions with LIMIT queries - Add work_items table separating agent production from delivery, enabling delivery retry without re-running the agent - Propagate Discord send failures instead of silently swallowing them - Add Claude 429 → Kimi K2.5 automatic provider fallback with cooldown - Fix follow-up turn state reset in live index.ts path (not just message-runtime.ts) to prevent final output from being lost - Add restart context tracking for graceful restart announcements - Lazy migration from timestamp cursors to seq cursors for existing data Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -26,6 +26,7 @@ groups/global/*
|
|||||||
*.keys.json
|
*.keys.json
|
||||||
.env
|
.env
|
||||||
.env.codex
|
.env.codex
|
||||||
|
.env.minimax
|
||||||
|
|
||||||
# Temp files
|
# Temp files
|
||||||
.tmp-*
|
.tmp-*
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"build:runners": "npm --prefix runners/agent-runner install && npm --prefix runners/agent-runner run build && npm --prefix runners/codex-runner install && npm --prefix runners/codex-runner run build",
|
"build:runners": "npm --prefix runners/agent-runner install && npm --prefix runners/agent-runner run build && npm --prefix runners/codex-runner install && npm --prefix runners/codex-runner run build",
|
||||||
|
"restart:hint": "tsx src/restart-context-cli.ts write",
|
||||||
"start": "node dist/index.js",
|
"start": "node dist/index.js",
|
||||||
"dev": "tsx src/index.ts",
|
"dev": "tsx src/index.ts",
|
||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
|
|||||||
@@ -156,6 +156,8 @@ function prepareGroupEnvironment(
|
|||||||
// Build environment variables for the runner process
|
// Build environment variables for the runner process
|
||||||
const envVars = readEnvFile([
|
const envVars = readEnvFile([
|
||||||
'ANTHROPIC_API_KEY',
|
'ANTHROPIC_API_KEY',
|
||||||
|
'ANTHROPIC_AUTH_TOKEN',
|
||||||
|
'ANTHROPIC_BASE_URL',
|
||||||
'CLAUDE_CODE_OAUTH_TOKEN',
|
'CLAUDE_CODE_OAUTH_TOKEN',
|
||||||
'CLAUDE_MODEL',
|
'CLAUDE_MODEL',
|
||||||
'CLAUDE_THINKING',
|
'CLAUDE_THINKING',
|
||||||
@@ -340,6 +342,8 @@ args = [${JSON.stringify(mementoSseUrl)}, "--header", ${JSON.stringify(`Authoriz
|
|||||||
|
|
||||||
// Sanitize secrets: prevent API keys from leaking to codex subprocesses
|
// Sanitize secrets: prevent API keys from leaking to codex subprocesses
|
||||||
delete env.ANTHROPIC_API_KEY;
|
delete env.ANTHROPIC_API_KEY;
|
||||||
|
delete env.ANTHROPIC_AUTH_TOKEN;
|
||||||
|
delete env.ANTHROPIC_BASE_URL;
|
||||||
delete env.CLAUDE_CODE_OAUTH_TOKEN;
|
delete env.CLAUDE_CODE_OAUTH_TOKEN;
|
||||||
|
|
||||||
env.CODEX_HOME = sessionCodexDir;
|
env.CODEX_HOME = sessionCodexDir;
|
||||||
@@ -349,6 +353,19 @@ args = [${JSON.stringify(mementoSseUrl)}, "--header", ${JSON.stringify(`Authoriz
|
|||||||
env.ANTHROPIC_API_KEY =
|
env.ANTHROPIC_API_KEY =
|
||||||
envVars.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY || '';
|
envVars.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY || '';
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
envVars.ANTHROPIC_AUTH_TOKEN ||
|
||||||
|
process.env.ANTHROPIC_AUTH_TOKEN
|
||||||
|
) {
|
||||||
|
env.ANTHROPIC_AUTH_TOKEN =
|
||||||
|
envVars.ANTHROPIC_AUTH_TOKEN ||
|
||||||
|
process.env.ANTHROPIC_AUTH_TOKEN ||
|
||||||
|
'';
|
||||||
|
}
|
||||||
|
if (envVars.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL) {
|
||||||
|
env.ANTHROPIC_BASE_URL =
|
||||||
|
envVars.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL || '';
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
envVars.CLAUDE_CODE_OAUTH_TOKEN ||
|
envVars.CLAUDE_CODE_OAUTH_TOKEN ||
|
||||||
process.env.CLAUDE_CODE_OAUTH_TOKEN
|
process.env.CLAUDE_CODE_OAUTH_TOKEN
|
||||||
@@ -384,6 +401,7 @@ export async function runAgentProcess(
|
|||||||
input: AgentInput,
|
input: AgentInput,
|
||||||
onProcess: (proc: ChildProcess, processName: string) => void,
|
onProcess: (proc: ChildProcess, processName: string) => void,
|
||||||
onOutput?: (output: AgentOutput) => Promise<void>,
|
onOutput?: (output: AgentOutput) => Promise<void>,
|
||||||
|
envOverrides?: Record<string, string>,
|
||||||
): Promise<AgentOutput> {
|
): Promise<AgentOutput> {
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const { env, groupDir, runnerDir } = prepareGroupEnvironment(
|
const { env, groupDir, runnerDir } = prepareGroupEnvironment(
|
||||||
@@ -391,6 +409,13 @@ export async function runAgentProcess(
|
|||||||
input.isMain,
|
input.isMain,
|
||||||
input.chatJid,
|
input.chatJid,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Apply provider fallback overrides (e.g. Kimi env vars when Claude is in cooldown)
|
||||||
|
if (envOverrides) {
|
||||||
|
for (const [key, value] of Object.entries(envOverrides)) {
|
||||||
|
if (value) env[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (input.runId) {
|
if (input.runId) {
|
||||||
env.NANOCLAW_RUN_ID = input.runId;
|
env.NANOCLAW_RUN_ID = input.runId;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -761,7 +761,7 @@ describe('DiscordChannel', () => {
|
|||||||
expect(currentClient().channels.fetch).toHaveBeenCalledWith('9876543210');
|
expect(currentClient().channels.fetch).toHaveBeenCalledWith('9876543210');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('handles send failure gracefully', async () => {
|
it('propagates send failure to the caller', async () => {
|
||||||
const opts = createTestOpts();
|
const opts = createTestOpts();
|
||||||
const channel = new DiscordChannel('test-token', opts);
|
const channel = new DiscordChannel('test-token', opts);
|
||||||
await channel.connect();
|
await channel.connect();
|
||||||
@@ -770,10 +770,9 @@ describe('DiscordChannel', () => {
|
|||||||
new Error('Channel not found'),
|
new Error('Channel not found'),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Should not throw
|
|
||||||
await expect(
|
await expect(
|
||||||
channel.sendMessage('dc:1234567890123456', 'Will fail'),
|
channel.sendMessage('dc:1234567890123456', 'Will fail'),
|
||||||
).resolves.toBeUndefined();
|
).rejects.toThrow('Channel not found');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does nothing when client is not initialized', async () => {
|
it('does nothing when client is not initialized', async () => {
|
||||||
|
|||||||
@@ -484,6 +484,7 @@ export class DiscordChannel implements Channel {
|
|||||||
logger.info({ jid, length: text.length }, 'Discord message sent');
|
logger.info({ jid, length: text.length }, 'Discord message sent');
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error({ jid, err }, 'Failed to send Discord message');
|
logger.error({ jid, err }, 'Failed to send Discord message');
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -551,7 +552,7 @@ export class DiscordChannel implements Channel {
|
|||||||
return msg.id;
|
return msg.id;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error({ jid, err }, 'Failed to send tracked Discord message');
|
logger.error({ jid, err }, 'Failed to send tracked Discord message');
|
||||||
return null;
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,17 +3,24 @@ import { describe, it, expect, beforeEach } from 'vitest';
|
|||||||
import {
|
import {
|
||||||
_initTestDatabase,
|
_initTestDatabase,
|
||||||
createTask,
|
createTask,
|
||||||
|
createProducedWorkItem,
|
||||||
deleteSession,
|
deleteSession,
|
||||||
deleteTask,
|
deleteTask,
|
||||||
getAllChats,
|
getAllChats,
|
||||||
getAllRegisteredGroups,
|
getAllRegisteredGroups,
|
||||||
getDueTasks,
|
getDueTasks,
|
||||||
|
getLatestMessageSeqAtOrBefore,
|
||||||
|
getMessagesSinceSeq,
|
||||||
|
getNewMessagesBySeq,
|
||||||
|
getOpenWorkItem,
|
||||||
getRegisteredAgentTypesForJid,
|
getRegisteredAgentTypesForJid,
|
||||||
getMessagesSince,
|
getMessagesSince,
|
||||||
getNewMessages,
|
getNewMessages,
|
||||||
isPairedRoomJid,
|
isPairedRoomJid,
|
||||||
getSession,
|
getSession,
|
||||||
getTaskById,
|
getTaskById,
|
||||||
|
markWorkItemDelivered,
|
||||||
|
markWorkItemDeliveryRetry,
|
||||||
setSession,
|
setSession,
|
||||||
setRegisteredGroup,
|
setRegisteredGroup,
|
||||||
storeChatMetadata,
|
storeChatMetadata,
|
||||||
@@ -412,6 +419,7 @@ describe('task CRUD', () => {
|
|||||||
id: 'task-claude',
|
id: 'task-claude',
|
||||||
group_folder: 'main',
|
group_folder: 'main',
|
||||||
chat_jid: 'group@g.us',
|
chat_jid: 'group@g.us',
|
||||||
|
agent_type: 'claude-code',
|
||||||
prompt: 'claude task',
|
prompt: 'claude task',
|
||||||
schedule_type: 'once',
|
schedule_type: 'once',
|
||||||
schedule_value: dueAt,
|
schedule_value: dueAt,
|
||||||
@@ -595,3 +603,81 @@ describe('paired room registration', () => {
|
|||||||
expect(isPairedRoomJid('dc:solo')).toBe(false);
|
expect(isPairedRoomJid('dc:solo')).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('message seq cursors', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
storeChatMetadata('group@g.us', '2024-01-01T00:00:00.000Z');
|
||||||
|
store({
|
||||||
|
id: 'seq-1',
|
||||||
|
chat_jid: 'group@g.us',
|
||||||
|
sender: 'alice',
|
||||||
|
sender_name: 'Alice',
|
||||||
|
content: 'first',
|
||||||
|
timestamp: '2024-01-01T00:00:01.000Z',
|
||||||
|
});
|
||||||
|
store({
|
||||||
|
id: 'seq-2',
|
||||||
|
chat_jid: 'group@g.us',
|
||||||
|
sender: 'bob',
|
||||||
|
sender_name: 'Bob',
|
||||||
|
content: 'second',
|
||||||
|
timestamp: '2024-01-01T00:00:02.000Z',
|
||||||
|
});
|
||||||
|
store({
|
||||||
|
id: 'seq-3',
|
||||||
|
chat_jid: 'group@g.us',
|
||||||
|
sender: 'carol',
|
||||||
|
sender_name: 'Carol',
|
||||||
|
content: 'third',
|
||||||
|
timestamp: '2024-01-01T00:00:03.000Z',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('assigns monotonic seq values and preserves them on upsert', () => {
|
||||||
|
const { messages } = getNewMessagesBySeq(['group@g.us'], 0, 'Andy');
|
||||||
|
expect(messages.map((m) => m.seq)).toEqual([1, 2, 3]);
|
||||||
|
|
||||||
|
store({
|
||||||
|
id: 'seq-2',
|
||||||
|
chat_jid: 'group@g.us',
|
||||||
|
sender: 'bob',
|
||||||
|
sender_name: 'Bob',
|
||||||
|
content: 'second updated',
|
||||||
|
timestamp: '2024-01-01T00:00:02.500Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
const afterUpdate = getMessagesSinceSeq('group@g.us', 0, 'Andy');
|
||||||
|
expect(afterUpdate.map((m) => m.seq)).toEqual([1, 2, 3]);
|
||||||
|
expect(afterUpdate[1].content).toBe('second updated');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps legacy timestamp cursors to the latest seq at or before that time', () => {
|
||||||
|
expect(
|
||||||
|
getLatestMessageSeqAtOrBefore('2024-01-01T00:00:02.000Z', 'group@g.us'),
|
||||||
|
).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('work items', () => {
|
||||||
|
it('tracks produced, retry, and delivered states', () => {
|
||||||
|
const item = createProducedWorkItem({
|
||||||
|
group_folder: 'discord_test',
|
||||||
|
chat_jid: 'dc:123',
|
||||||
|
agent_type: 'claude-code',
|
||||||
|
start_seq: 10,
|
||||||
|
end_seq: 12,
|
||||||
|
result_payload: 'hello',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getOpenWorkItem('dc:123', 'claude-code')?.id).toBe(item.id);
|
||||||
|
|
||||||
|
markWorkItemDeliveryRetry(item.id, 'send failed');
|
||||||
|
const retried = getOpenWorkItem('dc:123', 'claude-code');
|
||||||
|
expect(retried?.status).toBe('delivery_retry');
|
||||||
|
expect(retried?.delivery_attempts).toBe(1);
|
||||||
|
expect(retried?.last_error).toBe('send failed');
|
||||||
|
|
||||||
|
markWorkItemDelivered(item.id, 'msg-1');
|
||||||
|
expect(getOpenWorkItem('dc:123', 'claude-code')).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
364
src/db.ts
364
src/db.ts
@@ -21,6 +21,61 @@ import {
|
|||||||
|
|
||||||
let db: Database.Database;
|
let db: Database.Database;
|
||||||
|
|
||||||
|
export interface WorkItem {
|
||||||
|
id: number;
|
||||||
|
group_folder: string;
|
||||||
|
chat_jid: string;
|
||||||
|
agent_type: AgentType;
|
||||||
|
status: 'produced' | 'delivery_retry' | 'delivered';
|
||||||
|
start_seq: number | null;
|
||||||
|
end_seq: number | null;
|
||||||
|
result_payload: string;
|
||||||
|
delivery_attempts: number;
|
||||||
|
delivery_message_id: string | null;
|
||||||
|
last_error: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
delivered_at: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function backfillMessageSeq(database: Database.Database): void {
|
||||||
|
const rows = database
|
||||||
|
.prepare(
|
||||||
|
`SELECT rowid, seq
|
||||||
|
FROM messages
|
||||||
|
ORDER BY CASE WHEN seq IS NULL THEN 1 ELSE 0 END, seq, timestamp, rowid`,
|
||||||
|
)
|
||||||
|
.all() as Array<{ rowid: number; seq: number | null }>;
|
||||||
|
|
||||||
|
if (rows.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let nextSeq = 1;
|
||||||
|
const assignSeq = database.prepare(
|
||||||
|
'UPDATE messages SET seq = ? WHERE rowid = ? AND seq IS NULL',
|
||||||
|
);
|
||||||
|
const tx = database.transaction(() => {
|
||||||
|
for (const row of rows) {
|
||||||
|
if (row.seq === null) {
|
||||||
|
assignSeq.run(nextSeq, row.rowid);
|
||||||
|
}
|
||||||
|
nextSeq = Math.max(nextSeq, (row.seq ?? nextSeq) + 1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
tx();
|
||||||
|
|
||||||
|
const maxSeqRow = database
|
||||||
|
.prepare('SELECT MAX(seq) AS maxSeq FROM messages')
|
||||||
|
.get() as { maxSeq: number | null };
|
||||||
|
const maxSeq = maxSeqRow.maxSeq ?? 0;
|
||||||
|
if (maxSeq > 0) {
|
||||||
|
database
|
||||||
|
.prepare('INSERT OR IGNORE INTO message_sequence (id) VALUES (?)')
|
||||||
|
.run(maxSeq);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function createSchema(database: Database.Database): void {
|
function createSchema(database: Database.Database): void {
|
||||||
database.exec(`
|
database.exec(`
|
||||||
CREATE TABLE IF NOT EXISTS chats (
|
CREATE TABLE IF NOT EXISTS chats (
|
||||||
@@ -37,12 +92,39 @@ function createSchema(database: Database.Database): void {
|
|||||||
sender_name TEXT,
|
sender_name TEXT,
|
||||||
content TEXT,
|
content TEXT,
|
||||||
timestamp TEXT,
|
timestamp TEXT,
|
||||||
|
seq INTEGER,
|
||||||
is_from_me INTEGER,
|
is_from_me INTEGER,
|
||||||
is_bot_message INTEGER DEFAULT 0,
|
is_bot_message INTEGER DEFAULT 0,
|
||||||
PRIMARY KEY (id, chat_jid),
|
PRIMARY KEY (id, chat_jid),
|
||||||
FOREIGN KEY (chat_jid) REFERENCES chats(jid)
|
FOREIGN KEY (chat_jid) REFERENCES chats(jid)
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_timestamp ON messages(timestamp);
|
CREATE INDEX IF NOT EXISTS idx_timestamp ON messages(timestamp);
|
||||||
|
CREATE TABLE IF NOT EXISTS message_sequence (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS work_items (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
group_folder TEXT NOT NULL,
|
||||||
|
chat_jid TEXT NOT NULL,
|
||||||
|
agent_type TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'produced',
|
||||||
|
start_seq INTEGER,
|
||||||
|
end_seq INTEGER,
|
||||||
|
result_payload TEXT NOT NULL,
|
||||||
|
delivery_attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
delivery_message_id TEXT,
|
||||||
|
last_error TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
delivered_at TEXT,
|
||||||
|
CHECK (status IN ('produced', 'delivery_retry', 'delivered'))
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_work_items_status ON work_items(status, updated_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_work_items_group_agent ON work_items(chat_jid, agent_type, status);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_work_items_open
|
||||||
|
ON work_items(chat_jid, agent_type)
|
||||||
|
WHERE status IN ('produced', 'delivery_retry');
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS scheduled_tasks (
|
CREATE TABLE IF NOT EXISTS scheduled_tasks (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
@@ -168,6 +250,19 @@ function createSchema(database: Database.Database): void {
|
|||||||
/* column already exists */
|
/* column already exists */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
database.exec(`ALTER TABLE messages ADD COLUMN seq INTEGER`);
|
||||||
|
} catch {
|
||||||
|
/* column already exists */
|
||||||
|
}
|
||||||
|
|
||||||
|
backfillMessageSeq(database);
|
||||||
|
|
||||||
|
database.exec(`
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_seq ON messages(seq);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_chat_jid_seq ON messages(chat_jid, seq);
|
||||||
|
`);
|
||||||
|
|
||||||
// Migrate registered_groups to composite keys so Claude/Codex can share a jid/folder.
|
// Migrate registered_groups to composite keys so Claude/Codex can share a jid/folder.
|
||||||
const registeredGroupsSql = (
|
const registeredGroupsSql = (
|
||||||
database
|
database
|
||||||
@@ -412,18 +507,60 @@ export function getAllChats(): ChatInfo[] {
|
|||||||
* Only call this for registered groups where message history is needed.
|
* Only call this for registered groups where message history is needed.
|
||||||
*/
|
*/
|
||||||
export function storeMessage(msg: NewMessage): void {
|
export function storeMessage(msg: NewMessage): void {
|
||||||
db.prepare(
|
const existing = db
|
||||||
`INSERT OR REPLACE INTO messages (id, chat_jid, sender, sender_name, content, timestamp, is_from_me, is_bot_message) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
.prepare('SELECT seq FROM messages WHERE id = ? AND chat_jid = ?')
|
||||||
).run(
|
.get(msg.id, msg.chat_jid) as { seq: number | null } | undefined;
|
||||||
msg.id,
|
|
||||||
msg.chat_jid,
|
const nextSeq = () => {
|
||||||
msg.sender,
|
const result = db
|
||||||
msg.sender_name,
|
.prepare('INSERT INTO message_sequence DEFAULT VALUES')
|
||||||
msg.content,
|
.run() as Database.RunResult;
|
||||||
msg.timestamp,
|
return Number(result.lastInsertRowid);
|
||||||
msg.is_from_me ? 1 : 0,
|
};
|
||||||
msg.is_bot_message ? 1 : 0,
|
|
||||||
);
|
db.transaction(() => {
|
||||||
|
if (existing?.seq != null) {
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO messages (
|
||||||
|
id, chat_jid, sender, sender_name, content, timestamp, seq, is_from_me, is_bot_message
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id, chat_jid) DO UPDATE SET
|
||||||
|
sender = excluded.sender,
|
||||||
|
sender_name = excluded.sender_name,
|
||||||
|
content = excluded.content,
|
||||||
|
timestamp = excluded.timestamp,
|
||||||
|
is_from_me = excluded.is_from_me,
|
||||||
|
is_bot_message = excluded.is_bot_message`,
|
||||||
|
).run(
|
||||||
|
msg.id,
|
||||||
|
msg.chat_jid,
|
||||||
|
msg.sender,
|
||||||
|
msg.sender_name,
|
||||||
|
msg.content,
|
||||||
|
msg.timestamp,
|
||||||
|
existing.seq,
|
||||||
|
msg.is_from_me ? 1 : 0,
|
||||||
|
msg.is_bot_message ? 1 : 0,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
db.prepare(
|
||||||
|
`INSERT INTO messages (
|
||||||
|
id, chat_jid, sender, sender_name, content, timestamp, seq, is_from_me, is_bot_message
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||||
|
).run(
|
||||||
|
msg.id,
|
||||||
|
msg.chat_jid,
|
||||||
|
msg.sender,
|
||||||
|
msg.sender_name,
|
||||||
|
msg.content,
|
||||||
|
msg.timestamp,
|
||||||
|
nextSeq(),
|
||||||
|
msg.is_from_me ? 1 : 0,
|
||||||
|
msg.is_bot_message ? 1 : 0,
|
||||||
|
);
|
||||||
|
})();
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeMessageRow(
|
function normalizeMessageRow(
|
||||||
@@ -511,6 +648,105 @@ export function getMessagesSince(
|
|||||||
return rows.map(normalizeMessageRow);
|
return rows.map(normalizeMessageRow);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeSeqCursor(cursor: string | number | null | undefined): number {
|
||||||
|
if (typeof cursor === 'number') {
|
||||||
|
return Number.isFinite(cursor) && cursor > 0 ? cursor : 0;
|
||||||
|
}
|
||||||
|
if (!cursor) return 0;
|
||||||
|
const parsed = Number.parseInt(cursor, 10);
|
||||||
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getLatestMessageSeqAtOrBefore(
|
||||||
|
timestamp: string,
|
||||||
|
chatJid?: string,
|
||||||
|
): number {
|
||||||
|
if (!timestamp) return 0;
|
||||||
|
const row = (chatJid
|
||||||
|
? db
|
||||||
|
.prepare(
|
||||||
|
`SELECT COALESCE(MAX(seq), 0) AS maxSeq
|
||||||
|
FROM messages
|
||||||
|
WHERE chat_jid = ? AND timestamp <= ?`,
|
||||||
|
)
|
||||||
|
.get(chatJid, timestamp)
|
||||||
|
: db
|
||||||
|
.prepare(
|
||||||
|
`SELECT COALESCE(MAX(seq), 0) AS maxSeq
|
||||||
|
FROM messages
|
||||||
|
WHERE timestamp <= ?`,
|
||||||
|
)
|
||||||
|
.get(timestamp)) as { maxSeq: number | null };
|
||||||
|
return row.maxSeq ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getNewMessagesBySeq(
|
||||||
|
jids: string[],
|
||||||
|
lastSeqCursor: string | number,
|
||||||
|
botPrefix: string,
|
||||||
|
limit: number = 200,
|
||||||
|
): { messages: NewMessage[]; newSeqCursor: string } {
|
||||||
|
const sinceSeq = normalizeSeqCursor(lastSeqCursor);
|
||||||
|
if (jids.length === 0) {
|
||||||
|
return { messages: [], newSeqCursor: String(sinceSeq) };
|
||||||
|
}
|
||||||
|
|
||||||
|
const placeholders = jids.map(() => '?').join(',');
|
||||||
|
const sql = `
|
||||||
|
SELECT id, chat_jid, sender, sender_name, content, timestamp, seq, is_from_me, is_bot_message
|
||||||
|
FROM messages
|
||||||
|
WHERE seq > ? AND chat_jid IN (${placeholders})
|
||||||
|
AND content NOT LIKE ?
|
||||||
|
AND content != '' AND content IS NOT NULL
|
||||||
|
ORDER BY seq
|
||||||
|
LIMIT ?
|
||||||
|
`;
|
||||||
|
|
||||||
|
const rows = db
|
||||||
|
.prepare(sql)
|
||||||
|
.all(sinceSeq, ...jids, `${botPrefix}:%`, limit) as Array<
|
||||||
|
NewMessage & {
|
||||||
|
seq: number;
|
||||||
|
is_from_me?: boolean | number;
|
||||||
|
is_bot_message?: boolean | number;
|
||||||
|
}
|
||||||
|
>;
|
||||||
|
|
||||||
|
const lastSeq = rows.length > 0 ? rows[rows.length - 1].seq : sinceSeq;
|
||||||
|
return {
|
||||||
|
messages: rows.map(normalizeMessageRow),
|
||||||
|
newSeqCursor: String(lastSeq),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getMessagesSinceSeq(
|
||||||
|
chatJid: string,
|
||||||
|
sinceSeqCursor: string | number,
|
||||||
|
botPrefix: string,
|
||||||
|
limit: number = 200,
|
||||||
|
): NewMessage[] {
|
||||||
|
const sinceSeq = normalizeSeqCursor(sinceSeqCursor);
|
||||||
|
const sql = `
|
||||||
|
SELECT id, chat_jid, sender, sender_name, content, timestamp, seq, is_from_me, is_bot_message
|
||||||
|
FROM messages
|
||||||
|
WHERE chat_jid = ? AND seq > ?
|
||||||
|
AND content NOT LIKE ?
|
||||||
|
AND content != '' AND content IS NOT NULL
|
||||||
|
ORDER BY seq
|
||||||
|
LIMIT ?
|
||||||
|
`;
|
||||||
|
const rows = db
|
||||||
|
.prepare(sql)
|
||||||
|
.all(chatJid, sinceSeq, `${botPrefix}:%`, limit) as Array<
|
||||||
|
NewMessage & {
|
||||||
|
seq: number;
|
||||||
|
is_from_me?: boolean | number;
|
||||||
|
is_bot_message?: boolean | number;
|
||||||
|
}
|
||||||
|
>;
|
||||||
|
return rows.map(normalizeMessageRow);
|
||||||
|
}
|
||||||
|
|
||||||
export function getLastHumanMessageTimestamp(chatJid: string): string | null {
|
export function getLastHumanMessageTimestamp(chatJid: string): string | null {
|
||||||
const row = db
|
const row = db
|
||||||
.prepare(
|
.prepare(
|
||||||
@@ -523,6 +759,110 @@ export function getLastHumanMessageTimestamp(chatJid: string): string | null {
|
|||||||
return row?.timestamp ?? null;
|
return row?.timestamp ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function hasRecentRestartAnnouncement(
|
||||||
|
chatJid: string,
|
||||||
|
sinceTimestamp: string,
|
||||||
|
): boolean {
|
||||||
|
const row = db
|
||||||
|
.prepare(
|
||||||
|
`SELECT 1 FROM messages
|
||||||
|
WHERE chat_jid = ?
|
||||||
|
AND timestamp >= ?
|
||||||
|
AND is_bot_message = 1
|
||||||
|
AND (
|
||||||
|
content LIKE '재시작 완료.%'
|
||||||
|
OR content LIKE '재시작 감지.%'
|
||||||
|
OR content LIKE '서비스 재시작으로 이전 작업이 중단됐습니다.%'
|
||||||
|
)
|
||||||
|
LIMIT 1`,
|
||||||
|
)
|
||||||
|
.get(chatJid, sinceTimestamp) as { 1: number } | undefined;
|
||||||
|
return !!row;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getOpenWorkItem(
|
||||||
|
chatJid: string,
|
||||||
|
agentType: AgentType = SERVICE_AGENT_TYPE,
|
||||||
|
): WorkItem | undefined {
|
||||||
|
return db
|
||||||
|
.prepare(
|
||||||
|
`SELECT *
|
||||||
|
FROM work_items
|
||||||
|
WHERE chat_jid = ? AND agent_type = ? AND status IN ('produced', 'delivery_retry')
|
||||||
|
ORDER BY id ASC
|
||||||
|
LIMIT 1`,
|
||||||
|
)
|
||||||
|
.get(chatJid, agentType) as WorkItem | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createProducedWorkItem(input: {
|
||||||
|
group_folder: string;
|
||||||
|
chat_jid: string;
|
||||||
|
agent_type?: AgentType;
|
||||||
|
start_seq: number | null;
|
||||||
|
end_seq: number | null;
|
||||||
|
result_payload: string;
|
||||||
|
}): WorkItem {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const agentType = input.agent_type || SERVICE_AGENT_TYPE;
|
||||||
|
const result = db
|
||||||
|
.prepare(
|
||||||
|
`INSERT INTO work_items (
|
||||||
|
group_folder,
|
||||||
|
chat_jid,
|
||||||
|
agent_type,
|
||||||
|
status,
|
||||||
|
start_seq,
|
||||||
|
end_seq,
|
||||||
|
result_payload,
|
||||||
|
delivery_attempts,
|
||||||
|
created_at,
|
||||||
|
updated_at
|
||||||
|
) VALUES (?, ?, ?, 'produced', ?, ?, ?, 0, ?, ?)`,
|
||||||
|
)
|
||||||
|
.run(
|
||||||
|
input.group_folder,
|
||||||
|
input.chat_jid,
|
||||||
|
agentType,
|
||||||
|
input.start_seq,
|
||||||
|
input.end_seq,
|
||||||
|
input.result_payload,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
) as Database.RunResult;
|
||||||
|
|
||||||
|
return db
|
||||||
|
.prepare('SELECT * FROM work_items WHERE id = ?')
|
||||||
|
.get(Number(result.lastInsertRowid)) as WorkItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markWorkItemDelivered(
|
||||||
|
id: number,
|
||||||
|
deliveryMessageId?: string | null,
|
||||||
|
): void {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
db.prepare(
|
||||||
|
`UPDATE work_items
|
||||||
|
SET status = 'delivered',
|
||||||
|
delivered_at = ?,
|
||||||
|
delivery_message_id = ?,
|
||||||
|
updated_at = ?
|
||||||
|
WHERE id = ?`,
|
||||||
|
).run(now, deliveryMessageId || null, now, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function markWorkItemDeliveryRetry(id: number, error: string): void {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
db.prepare(
|
||||||
|
`UPDATE work_items
|
||||||
|
SET status = 'delivery_retry',
|
||||||
|
delivery_attempts = delivery_attempts + 1,
|
||||||
|
last_error = ?,
|
||||||
|
updated_at = ?
|
||||||
|
WHERE id = ?`,
|
||||||
|
).run(error, now, id);
|
||||||
|
}
|
||||||
|
|
||||||
export function createTask(
|
export function createTask(
|
||||||
task: Omit<
|
task: Omit<
|
||||||
ScheduledTask,
|
ScheduledTask,
|
||||||
|
|||||||
@@ -414,6 +414,30 @@ describe('GroupQueue', () => {
|
|||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('sendMessage touches active run activity after piping follow-up', async () => {
|
||||||
|
let resolveProcess: () => void;
|
||||||
|
const touch = vi.fn();
|
||||||
|
|
||||||
|
const processMessages = vi.fn(async () => {
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
resolveProcess = resolve;
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
queue.setProcessMessagesFn(processMessages);
|
||||||
|
queue.enqueueMessageCheck('group1@g.us');
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
queue.registerProcess('group1@g.us', {} as any, 'agent-1', 'test-group');
|
||||||
|
queue.setActivityTouch('group1@g.us', touch);
|
||||||
|
|
||||||
|
expect(queue.sendMessage('group1@g.us', 'hello')).toBe(true);
|
||||||
|
expect(touch).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
resolveProcess!();
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
});
|
||||||
|
|
||||||
it('sendMessage returns false for task agents so user messages queue up', async () => {
|
it('sendMessage returns false for task agents so user messages queue up', async () => {
|
||||||
let resolveTask: () => void;
|
let resolveTask: () => void;
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,12 @@ interface QueuedTask {
|
|||||||
fn: () => Promise<void>;
|
fn: () => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface GroupActivityTouchMeta {
|
||||||
|
source: 'follow-up';
|
||||||
|
textLength: number;
|
||||||
|
filename: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface GroupRunContext {
|
export interface GroupRunContext {
|
||||||
runId: string;
|
runId: string;
|
||||||
reason: 'messages' | 'drain';
|
reason: 'messages' | 'drain';
|
||||||
@@ -35,6 +41,7 @@ interface GroupState {
|
|||||||
retryTimer: ReturnType<typeof setTimeout> | null;
|
retryTimer: ReturnType<typeof setTimeout> | null;
|
||||||
retryScheduledAt: number | null;
|
retryScheduledAt: number | null;
|
||||||
startedAt: number | null;
|
startedAt: number | null;
|
||||||
|
activityTouch: ((meta?: GroupActivityTouchMeta) => void) | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface GroupStatus {
|
export interface GroupStatus {
|
||||||
@@ -73,6 +80,7 @@ export class GroupQueue {
|
|||||||
retryTimer: null,
|
retryTimer: null,
|
||||||
retryScheduledAt: null,
|
retryScheduledAt: null,
|
||||||
startedAt: null,
|
startedAt: null,
|
||||||
|
activityTouch: null,
|
||||||
};
|
};
|
||||||
this.groups.set(groupJid, state);
|
this.groups.set(groupJid, state);
|
||||||
}
|
}
|
||||||
@@ -85,6 +93,14 @@ export class GroupQueue {
|
|||||||
this.processMessagesFn = fn;
|
this.processMessagesFn = fn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setActivityTouch(
|
||||||
|
groupJid: string,
|
||||||
|
touch: ((meta?: GroupActivityTouchMeta) => void) | null,
|
||||||
|
): void {
|
||||||
|
const state = this.getGroup(groupJid);
|
||||||
|
state.activityTouch = touch;
|
||||||
|
}
|
||||||
|
|
||||||
private createRunId(): string {
|
private createRunId(): string {
|
||||||
return `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
return `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
}
|
}
|
||||||
@@ -262,6 +278,11 @@ export class GroupQueue {
|
|||||||
const tempPath = `${filepath}.tmp`;
|
const tempPath = `${filepath}.tmp`;
|
||||||
fs.writeFileSync(tempPath, JSON.stringify({ type: 'message', text }));
|
fs.writeFileSync(tempPath, JSON.stringify({ type: 'message', text }));
|
||||||
fs.renameSync(tempPath, filepath);
|
fs.renameSync(tempPath, filepath);
|
||||||
|
state.activityTouch?.({
|
||||||
|
source: 'follow-up',
|
||||||
|
textLength: text.length,
|
||||||
|
filename,
|
||||||
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
groupJid,
|
groupJid,
|
||||||
@@ -390,6 +411,7 @@ export class GroupQueue {
|
|||||||
state.processName = null;
|
state.processName = null;
|
||||||
state.groupFolder = null;
|
state.groupFolder = null;
|
||||||
state.currentRunId = null;
|
state.currentRunId = null;
|
||||||
|
state.activityTouch = null;
|
||||||
this.activeCount--;
|
this.activeCount--;
|
||||||
this.drainGroup(groupJid);
|
this.drainGroup(groupJid);
|
||||||
}
|
}
|
||||||
@@ -424,6 +446,7 @@ export class GroupQueue {
|
|||||||
state.process = null;
|
state.process = null;
|
||||||
state.processName = null;
|
state.processName = null;
|
||||||
state.groupFolder = null;
|
state.groupFolder = null;
|
||||||
|
state.activityTouch = null;
|
||||||
this.activeCount--;
|
this.activeCount--;
|
||||||
this.drainGroup(groupJid);
|
this.drainGroup(groupJid);
|
||||||
}
|
}
|
||||||
|
|||||||
83
src/index.test.ts
Normal file
83
src/index.test.ts
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import {
|
||||||
|
editFormattedTrackedChannelMessage,
|
||||||
|
sendFormattedChannelMessage,
|
||||||
|
sendFormattedTrackedChannelMessage,
|
||||||
|
} from './index.js';
|
||||||
|
import { Channel } from './types.js';
|
||||||
|
|
||||||
|
function makeChannel(overrides: Partial<Channel> = {}): Channel {
|
||||||
|
return {
|
||||||
|
name: 'test',
|
||||||
|
connect: async () => {},
|
||||||
|
sendMessage: async () => {},
|
||||||
|
isConnected: () => true,
|
||||||
|
ownsJid: (jid: string) => jid === 'dc:test',
|
||||||
|
disconnect: async () => {},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('index scheduler messaging helpers', () => {
|
||||||
|
it('sends formatted tracked messages through sendAndTrack', async () => {
|
||||||
|
const sendAndTrack = vi.fn(async () => 'msg-123');
|
||||||
|
const channel = makeChannel({ sendAndTrack });
|
||||||
|
|
||||||
|
const messageId = await sendFormattedTrackedChannelMessage(
|
||||||
|
[channel],
|
||||||
|
'dc:test',
|
||||||
|
'<internal>hidden</internal>watching ci',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(messageId).toBe('msg-123');
|
||||||
|
expect(sendAndTrack).toHaveBeenCalledWith('dc:test', 'watching ci');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('edits formatted tracked messages through editMessage', async () => {
|
||||||
|
const editMessage = vi.fn(async () => {});
|
||||||
|
const channel = makeChannel({ editMessage });
|
||||||
|
|
||||||
|
await editFormattedTrackedChannelMessage(
|
||||||
|
[channel],
|
||||||
|
'dc:test',
|
||||||
|
'msg-123',
|
||||||
|
'<internal>hidden</internal>still watching',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(editMessage).toHaveBeenCalledWith(
|
||||||
|
'dc:test',
|
||||||
|
'msg-123',
|
||||||
|
'still watching',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips empty messages after formatting', async () => {
|
||||||
|
const sendMessage = vi.fn(async () => {});
|
||||||
|
const sendAndTrack = vi.fn(async () => 'msg-123');
|
||||||
|
const editMessage = vi.fn(async () => {});
|
||||||
|
const channel = makeChannel({ sendMessage, sendAndTrack, editMessage });
|
||||||
|
|
||||||
|
const trackedResult = await sendFormattedTrackedChannelMessage(
|
||||||
|
[channel],
|
||||||
|
'dc:test',
|
||||||
|
'<internal>only hidden</internal>',
|
||||||
|
);
|
||||||
|
await editFormattedTrackedChannelMessage(
|
||||||
|
[channel],
|
||||||
|
'dc:test',
|
||||||
|
'msg-123',
|
||||||
|
'<internal>only hidden</internal>',
|
||||||
|
);
|
||||||
|
await sendFormattedChannelMessage(
|
||||||
|
[channel],
|
||||||
|
'dc:test',
|
||||||
|
'<internal>only hidden</internal>',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(trackedResult).toBeNull();
|
||||||
|
expect(sendAndTrack).not.toHaveBeenCalled();
|
||||||
|
expect(editMessage).not.toHaveBeenCalled();
|
||||||
|
expect(sendMessage).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
908
src/index.ts
908
src/index.ts
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,7 @@ vi.mock('./agent-runner.js', () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
|
DATA_DIR: '/tmp/nanoclaw-test-data',
|
||||||
isSessionCommandSenderAllowed: vi.fn(() => false),
|
isSessionCommandSenderAllowed: vi.fn(() => false),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -27,6 +28,20 @@ vi.mock('./logger.js', () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./provider-fallback.js', () => ({
|
||||||
|
detectFallbackTrigger: vi.fn(() => ({ shouldFallback: false, reason: '' })),
|
||||||
|
getActiveProvider: vi.fn(() => 'claude'),
|
||||||
|
getFallbackEnvOverrides: vi.fn(() => ({
|
||||||
|
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
||||||
|
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
|
||||||
|
ANTHROPIC_MODEL: 'kimi-k2.5',
|
||||||
|
})),
|
||||||
|
getFallbackProviderName: vi.fn(() => 'kimi'),
|
||||||
|
hasGroupProviderOverride: vi.fn(() => false),
|
||||||
|
isFallbackEnabled: vi.fn(() => true),
|
||||||
|
markPrimaryCooldown: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock('./sender-allowlist.js', () => ({
|
vi.mock('./sender-allowlist.js', () => ({
|
||||||
isTriggerAllowed: vi.fn(() => true),
|
isTriggerAllowed: vi.fn(() => true),
|
||||||
loadSenderAllowlist: vi.fn(() => ({})),
|
loadSenderAllowlist: vi.fn(() => ({})),
|
||||||
@@ -42,6 +57,7 @@ vi.mock('./session-commands.js', () => ({
|
|||||||
import * as agentRunner from './agent-runner.js';
|
import * as agentRunner from './agent-runner.js';
|
||||||
import * as db from './db.js';
|
import * as db from './db.js';
|
||||||
import { createMessageRuntime } from './message-runtime.js';
|
import { createMessageRuntime } from './message-runtime.js';
|
||||||
|
import * as providerFallback from './provider-fallback.js';
|
||||||
import type { Channel, RegisteredGroup } from './types.js';
|
import type { Channel, RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
function makeGroup(agentType: 'claude-code' | 'codex'): RegisteredGroup {
|
function makeGroup(agentType: 'claude-code' | 'codex'): RegisteredGroup {
|
||||||
@@ -72,6 +88,21 @@ function makeChannel(chatJid: string): Channel {
|
|||||||
describe('createMessageRuntime', () => {
|
describe('createMessageRuntime', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(providerFallback.getActiveProvider).mockReturnValue('claude');
|
||||||
|
vi.mocked(providerFallback.getFallbackProviderName).mockReturnValue('kimi');
|
||||||
|
vi.mocked(providerFallback.getFallbackEnvOverrides).mockReturnValue({
|
||||||
|
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
||||||
|
ANTHROPIC_AUTH_TOKEN: 'test-kimi-key',
|
||||||
|
ANTHROPIC_MODEL: 'kimi-k2.5',
|
||||||
|
});
|
||||||
|
vi.mocked(providerFallback.hasGroupProviderOverride).mockReturnValue(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
vi.mocked(providerFallback.isFallbackEnabled).mockReturnValue(true);
|
||||||
|
vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({
|
||||||
|
shouldFallback: false,
|
||||||
|
reason: '',
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('clears Claude sessions and closes stdin immediately on poisoned output', async () => {
|
it('clears Claude sessions and closes stdin immediately on poisoned output', async () => {
|
||||||
@@ -318,6 +349,114 @@ describe('createMessageRuntime', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('resets the idle timeout when follow-up Codex progress keeps arriving', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('codex');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
const closeStdin = vi.fn();
|
||||||
|
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
||||||
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
result: '초기 응답입니다.',
|
||||||
|
newSessionId: 'session-follow-up',
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(800);
|
||||||
|
expect(closeStdin).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'progress',
|
||||||
|
result: '후속 작업 진행 중입니다.',
|
||||||
|
newSessionId: 'session-follow-up',
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(800);
|
||||||
|
expect(closeStdin).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'progress',
|
||||||
|
result: '후속 작업 계속 진행 중입니다.',
|
||||||
|
newSessionId: 'session-follow-up',
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(800);
|
||||||
|
expect(closeStdin).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
newSessionId: 'session-follow-up',
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
newSessionId: 'session-follow-up',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin,
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => ({}),
|
||||||
|
saveState: vi.fn(),
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-follow-up-progress',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(closeStdin).not.toHaveBeenCalled();
|
||||||
|
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||||
|
chatJid,
|
||||||
|
'초기 응답입니다.',
|
||||||
|
);
|
||||||
|
expect(channel.sendAndTrack).toHaveBeenCalledWith(
|
||||||
|
chatJid,
|
||||||
|
'후속 작업 진행 중입니다.\n\n0초',
|
||||||
|
);
|
||||||
|
expect(channel.editMessage).toHaveBeenCalledWith(
|
||||||
|
chatJid,
|
||||||
|
'progress-1',
|
||||||
|
'후속 작업 계속 진행 중입니다.\n\n0초',
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('formats longer Codex progress durations with minutes and hours', async () => {
|
it('formats longer Codex progress durations with minutes and hours', async () => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
@@ -1019,4 +1158,172 @@ describe('createMessageRuntime', () => {
|
|||||||
expect(lastAgentTimestamps[chatJid]).toBe('2026-03-19T00:00:00.000Z');
|
expect(lastAgentTimestamps[chatJid]).toBe('2026-03-19T00:00:00.000Z');
|
||||||
expect(saveState).toHaveBeenCalled();
|
expect(saveState).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('retries with the fallback provider when Claude returns a 429 error before any output', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('claude-code');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(providerFallback.detectFallbackTrigger).mockReturnValue({
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: '429',
|
||||||
|
retryAfterMs: 60_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
status: 'error',
|
||||||
|
result: null,
|
||||||
|
error: '429 rate limited retry after 60',
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'fallback 응답입니다.',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => ({}),
|
||||||
|
saveState: vi.fn(),
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-fallback-429',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenNthCalledWith(
|
||||||
|
2,
|
||||||
|
expect.anything(),
|
||||||
|
expect.objectContaining({ sessionId: undefined }),
|
||||||
|
expect.any(Function),
|
||||||
|
expect.any(Function),
|
||||||
|
expect.objectContaining({
|
||||||
|
ANTHROPIC_BASE_URL: 'https://api.kimi.com/coding/',
|
||||||
|
ANTHROPIC_MODEL: 'kimi-k2.5',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
||||||
|
'429',
|
||||||
|
60_000,
|
||||||
|
);
|
||||||
|
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||||
|
chatJid,
|
||||||
|
'fallback 응답입니다.',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('retries with the fallback provider when Claude ends with success-null-result before any output', async () => {
|
||||||
|
const chatJid = 'group@test';
|
||||||
|
const group = makeGroup('claude-code');
|
||||||
|
const channel = makeChannel(chatJid);
|
||||||
|
|
||||||
|
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 'msg-1',
|
||||||
|
chat_jid: chatJid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: 'hello',
|
||||||
|
timestamp: '2026-03-19T00:00:00.000Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(agentRunner.runAgentProcess)
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||||
|
await onOutput?.({
|
||||||
|
status: 'success',
|
||||||
|
phase: 'final',
|
||||||
|
result: 'success-null-result 폴백 응답입니다.',
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'success',
|
||||||
|
result: null,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const runtime = createMessageRuntime({
|
||||||
|
assistantName: 'Andy',
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
pollInterval: 1_000,
|
||||||
|
timezone: 'UTC',
|
||||||
|
triggerPattern: /^@Andy\b/i,
|
||||||
|
channels: [channel],
|
||||||
|
queue: {
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
closeStdin: vi.fn(),
|
||||||
|
notifyIdle: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||||
|
getSessions: () => ({}),
|
||||||
|
getLastTimestamp: () => '',
|
||||||
|
setLastTimestamp: vi.fn(),
|
||||||
|
getLastAgentTimestamps: () => ({}),
|
||||||
|
saveState: vi.fn(),
|
||||||
|
persistSession: vi.fn(),
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runtime.processGroupMessages(chatJid, {
|
||||||
|
runId: 'run-fallback-success-null',
|
||||||
|
reason: 'messages',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toBe(true);
|
||||||
|
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||||
|
expect(providerFallback.markPrimaryCooldown).toHaveBeenCalledWith(
|
||||||
|
'success-null-result',
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||||
|
chatJid,
|
||||||
|
'success-null-result 폴백 응답입니다.',
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,12 +8,20 @@ import {
|
|||||||
import {
|
import {
|
||||||
getAllChats,
|
getAllChats,
|
||||||
getAllTasks,
|
getAllTasks,
|
||||||
getLastHumanMessageTimestamp,
|
|
||||||
getMessagesSince,
|
getMessagesSince,
|
||||||
getNewMessages,
|
getNewMessages,
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
import { isSessionCommandSenderAllowed } from './config.js';
|
import { DATA_DIR, isSessionCommandSenderAllowed } from './config.js';
|
||||||
import { GroupQueue, GroupRunContext } from './group-queue.js';
|
import { GroupQueue, GroupRunContext } from './group-queue.js';
|
||||||
|
import {
|
||||||
|
detectFallbackTrigger,
|
||||||
|
getActiveProvider,
|
||||||
|
getFallbackEnvOverrides,
|
||||||
|
getFallbackProviderName,
|
||||||
|
hasGroupProviderOverride,
|
||||||
|
isFallbackEnabled,
|
||||||
|
markPrimaryCooldown,
|
||||||
|
} from './provider-fallback.js';
|
||||||
import { findChannel, formatMessages, formatOutbound } from './router.js';
|
import { findChannel, formatMessages, formatOutbound } from './router.js';
|
||||||
import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js';
|
import { isTriggerAllowed, loadSenderAllowlist } from './sender-allowlist.js';
|
||||||
import {
|
import {
|
||||||
@@ -26,6 +34,7 @@ import { shouldResetSessionOnAgentFailure } from './session-recovery.js';
|
|||||||
import { isTaskStatusControlMessage } from './task-scheduler.js';
|
import { isTaskStatusControlMessage } from './task-scheduler.js';
|
||||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
export interface MessageRuntimeDeps {
|
export interface MessageRuntimeDeps {
|
||||||
assistantName: string;
|
assistantName: string;
|
||||||
@@ -121,63 +130,298 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
|
|
||||||
let resetSessionRequested = false;
|
let resetSessionRequested = false;
|
||||||
|
|
||||||
const wrappedOnOutput = onOutput
|
const settingsPath = path.join(
|
||||||
? async (output: AgentOutput) => {
|
DATA_DIR,
|
||||||
if (output.newSessionId) {
|
'sessions',
|
||||||
deps.persistSession(group.folder, output.newSessionId);
|
group.folder,
|
||||||
|
'.claude',
|
||||||
|
'settings.json',
|
||||||
|
);
|
||||||
|
const groupHasOverride = hasGroupProviderOverride(settingsPath);
|
||||||
|
const canFallback =
|
||||||
|
isClaudeCodeAgent && isFallbackEnabled() && !groupHasOverride;
|
||||||
|
|
||||||
|
const agentInput = {
|
||||||
|
prompt,
|
||||||
|
sessionId,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
chatJid,
|
||||||
|
runId,
|
||||||
|
isMain,
|
||||||
|
assistantName: deps.assistantName,
|
||||||
|
};
|
||||||
|
|
||||||
|
const runAttempt = async (provider: string): Promise<{
|
||||||
|
output?: AgentOutput;
|
||||||
|
error?: unknown;
|
||||||
|
sawOutput: boolean;
|
||||||
|
sawSuccessNullResultWithoutOutput: boolean;
|
||||||
|
streamedTriggerReason?: {
|
||||||
|
reason: string;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
};
|
||||||
|
}> => {
|
||||||
|
const persistSessionIds = provider === 'claude';
|
||||||
|
let sawOutput = false;
|
||||||
|
let sawSuccessNullResultWithoutOutput = false;
|
||||||
|
let streamedTriggerReason:
|
||||||
|
| {
|
||||||
|
reason: string;
|
||||||
|
retryAfterMs?: number;
|
||||||
}
|
}
|
||||||
if (isClaudeCodeAgent && shouldResetSessionOnAgentFailure(output)) {
|
| undefined;
|
||||||
resetSessionRequested = true;
|
|
||||||
|
const wrappedOnOutput = onOutput
|
||||||
|
? async (output: AgentOutput) => {
|
||||||
|
if (persistSessionIds && output.newSessionId) {
|
||||||
|
deps.persistSession(group.folder, output.newSessionId);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
persistSessionIds &&
|
||||||
|
isClaudeCodeAgent &&
|
||||||
|
shouldResetSessionOnAgentFailure(output)
|
||||||
|
) {
|
||||||
|
resetSessionRequested = true;
|
||||||
|
}
|
||||||
|
if (output.result !== null && output.result !== undefined) {
|
||||||
|
sawOutput = true;
|
||||||
|
} else if (
|
||||||
|
provider === 'claude' &&
|
||||||
|
output.status === 'success' &&
|
||||||
|
!sawOutput
|
||||||
|
) {
|
||||||
|
sawSuccessNullResultWithoutOutput = true;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
provider === 'claude' &&
|
||||||
|
output.status === 'error' &&
|
||||||
|
!sawOutput &&
|
||||||
|
!streamedTriggerReason
|
||||||
|
) {
|
||||||
|
const trigger = detectFallbackTrigger(output.error);
|
||||||
|
if (trigger.shouldFallback) {
|
||||||
|
streamedTriggerReason = {
|
||||||
|
reason: trigger.reason,
|
||||||
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await onOutput(output);
|
||||||
}
|
}
|
||||||
await onOutput(output);
|
: undefined;
|
||||||
}
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
try {
|
if (provider !== 'claude') {
|
||||||
const output = await runAgentProcess(
|
logger.info(
|
||||||
group,
|
{ chatJid, group: group.name, groupFolder: group.folder, runId, provider },
|
||||||
{
|
`Claude provider in cooldown, routing request to ${provider}`,
|
||||||
prompt,
|
|
||||||
sessionId,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
chatJid,
|
|
||||||
runId,
|
|
||||||
isMain,
|
|
||||||
assistantName: deps.assistantName,
|
|
||||||
},
|
|
||||||
(proc, processName) =>
|
|
||||||
deps.queue.registerProcess(chatJid, proc, processName, group.folder),
|
|
||||||
wrappedOnOutput,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (output.newSessionId) {
|
|
||||||
deps.persistSession(group.folder, output.newSessionId);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
isClaudeCodeAgent &&
|
|
||||||
(resetSessionRequested || shouldResetSessionOnAgentFailure(output))
|
|
||||||
) {
|
|
||||||
deps.clearSession(group.folder);
|
|
||||||
logger.warn(
|
|
||||||
{ group: group.name, chatJid, runId },
|
|
||||||
'Cleared poisoned agent session after unrecoverable error',
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (output.status === 'error') {
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
provider,
|
||||||
|
canFallback,
|
||||||
|
groupHasOverride,
|
||||||
|
},
|
||||||
|
`Using provider: ${provider}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const output = await runAgentProcess(
|
||||||
|
group,
|
||||||
|
{
|
||||||
|
...agentInput,
|
||||||
|
sessionId: persistSessionIds ? sessionId : undefined,
|
||||||
|
},
|
||||||
|
(proc, processName) =>
|
||||||
|
deps.queue.registerProcess(chatJid, proc, processName, group.folder),
|
||||||
|
wrappedOnOutput,
|
||||||
|
provider === 'claude' ? undefined : getFallbackEnvOverrides(),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (persistSessionIds && output.newSessionId) {
|
||||||
|
deps.persistSession(group.folder, output.newSessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
provider,
|
||||||
|
status: output.status,
|
||||||
|
sawOutput,
|
||||||
|
},
|
||||||
|
`Provider response completed (provider: ${provider})`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
output,
|
||||||
|
sawOutput,
|
||||||
|
sawSuccessNullResultWithoutOutput,
|
||||||
|
streamedTriggerReason,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
error,
|
||||||
|
sawOutput,
|
||||||
|
sawSuccessNullResultWithoutOutput,
|
||||||
|
streamedTriggerReason,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const runFallbackAttempt = async (
|
||||||
|
reason: string,
|
||||||
|
retryAfterMs?: number,
|
||||||
|
): Promise<'success' | 'error'> => {
|
||||||
|
const fallbackName = getFallbackProviderName();
|
||||||
|
markPrimaryCooldown(reason, retryAfterMs);
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
reason,
|
||||||
|
retryAfterMs,
|
||||||
|
fallbackProvider: fallbackName,
|
||||||
|
},
|
||||||
|
`Falling back to provider: ${fallbackName} (reason: ${reason})`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const fallbackAttempt = await runAttempt(fallbackName);
|
||||||
|
if (fallbackAttempt.error) {
|
||||||
logger.error(
|
logger.error(
|
||||||
{ group: group.name, chatJid, runId, error: output.error },
|
{
|
||||||
'Agent process error',
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
provider: fallbackName,
|
||||||
|
err: fallbackAttempt.error,
|
||||||
|
},
|
||||||
|
'Fallback provider also threw',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fallbackAttempt.output?.status === 'error') {
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
provider: fallbackName,
|
||||||
|
error: fallbackAttempt.output.error,
|
||||||
|
},
|
||||||
|
`Fallback provider (${fallbackName}) also failed`,
|
||||||
);
|
);
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
return 'success';
|
return 'success';
|
||||||
} catch (err) {
|
};
|
||||||
logger.error({ group: group.name, chatJid, runId, err }, 'Agent error');
|
|
||||||
|
const provider = canFallback ? getActiveProvider() : 'claude';
|
||||||
|
const primaryAttempt = await runAttempt(provider);
|
||||||
|
|
||||||
|
if (primaryAttempt.error) {
|
||||||
|
if (canFallback && provider === 'claude' && !primaryAttempt.sawOutput) {
|
||||||
|
const errMsg =
|
||||||
|
primaryAttempt.error instanceof Error
|
||||||
|
? primaryAttempt.error.message
|
||||||
|
: String(primaryAttempt.error);
|
||||||
|
const trigger = primaryAttempt.streamedTriggerReason
|
||||||
|
? {
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
}
|
||||||
|
: detectFallbackTrigger(errMsg);
|
||||||
|
if (trigger.shouldFallback) {
|
||||||
|
return runFallbackAttempt(trigger.reason, trigger.retryAfterMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
provider,
|
||||||
|
err: primaryAttempt.error,
|
||||||
|
},
|
||||||
|
'Agent error',
|
||||||
|
);
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const output = primaryAttempt.output;
|
||||||
|
if (!output) {
|
||||||
|
logger.error(
|
||||||
|
{ chatJid, group: group.name, groupFolder: group.folder, runId, provider },
|
||||||
|
'Agent produced no output object',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
canFallback &&
|
||||||
|
provider === 'claude' &&
|
||||||
|
!primaryAttempt.sawOutput &&
|
||||||
|
primaryAttempt.sawSuccessNullResultWithoutOutput
|
||||||
|
) {
|
||||||
|
return runFallbackAttempt('success-null-result');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
isClaudeCodeAgent &&
|
||||||
|
(resetSessionRequested || shouldResetSessionOnAgentFailure(output))
|
||||||
|
) {
|
||||||
|
deps.clearSession(group.folder);
|
||||||
|
logger.warn(
|
||||||
|
{ group: group.name, chatJid, runId },
|
||||||
|
'Cleared poisoned agent session after unrecoverable error',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (output.status === 'error') {
|
||||||
|
if (canFallback && provider === 'claude' && !primaryAttempt.sawOutput) {
|
||||||
|
const trigger = primaryAttempt.streamedTriggerReason
|
||||||
|
? {
|
||||||
|
shouldFallback: true,
|
||||||
|
reason: primaryAttempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: primaryAttempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
}
|
||||||
|
: detectFallbackTrigger(output.error);
|
||||||
|
if (trigger.shouldFallback) {
|
||||||
|
return runFallbackAttempt(trigger.reason, trigger.retryAfterMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
group: group.name,
|
||||||
|
chatJid,
|
||||||
|
runId,
|
||||||
|
provider,
|
||||||
|
error: output.error,
|
||||||
|
},
|
||||||
|
'Agent process error',
|
||||||
|
);
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'success';
|
||||||
};
|
};
|
||||||
|
|
||||||
const processGroupMessages = async (
|
const processGroupMessages = async (
|
||||||
@@ -323,6 +567,21 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
const resetIdleTimer = () => {
|
const resetIdleTimer = () => {
|
||||||
if (idleTimer) clearTimeout(idleTimer);
|
if (idleTimer) clearTimeout(idleTimer);
|
||||||
idleTimer = setTimeout(() => {
|
idleTimer = setTimeout(() => {
|
||||||
|
if (followUpQueuedAt !== null) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
followUpQueuedAt: new Date(followUpQueuedAt).toISOString(),
|
||||||
|
followUpQueuedTextLength,
|
||||||
|
followUpQueuedFilename,
|
||||||
|
followUpWaitMs: Date.now() - followUpQueuedAt,
|
||||||
|
},
|
||||||
|
'Idle timeout reached while a queued follow-up still had no agent output',
|
||||||
|
);
|
||||||
|
}
|
||||||
logger.info(
|
logger.info(
|
||||||
{ chatJid, group: group.name, groupFolder: group.folder, runId },
|
{ chatJid, group: group.name, groupFolder: group.folder, runId },
|
||||||
'Idle timeout reached, closing active agent stdin',
|
'Idle timeout reached, closing active agent stdin',
|
||||||
@@ -333,6 +592,111 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
});
|
});
|
||||||
}, deps.idleTimeout);
|
}, deps.idleTimeout);
|
||||||
};
|
};
|
||||||
|
let followUpQueuedAt: number | null = null;
|
||||||
|
let followUpQueuedTextLength: number | null = null;
|
||||||
|
let followUpQueuedFilename: string | null = null;
|
||||||
|
let followUpNoOutputWarnTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
const FOLLOW_UP_NO_OUTPUT_WARN_MS = 10_000;
|
||||||
|
|
||||||
|
const clearFollowUpNoOutputWarnTimer = () => {
|
||||||
|
if (followUpNoOutputWarnTimer) {
|
||||||
|
clearTimeout(followUpNoOutputWarnTimer);
|
||||||
|
followUpNoOutputWarnTimer = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const clearPendingFollowUpDiagnostics = () => {
|
||||||
|
clearFollowUpNoOutputWarnTimer();
|
||||||
|
followUpQueuedAt = null;
|
||||||
|
followUpQueuedTextLength = null;
|
||||||
|
followUpQueuedFilename = null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const scheduleFollowUpNoOutputWarning = () => {
|
||||||
|
clearFollowUpNoOutputWarnTimer();
|
||||||
|
followUpNoOutputWarnTimer = setTimeout(() => {
|
||||||
|
if (followUpQueuedAt === null) return;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
followUpQueuedAt: new Date(followUpQueuedAt).toISOString(),
|
||||||
|
followUpQueuedTextLength,
|
||||||
|
followUpQueuedFilename,
|
||||||
|
elapsedMs: Date.now() - followUpQueuedAt,
|
||||||
|
},
|
||||||
|
'No agent output observed within 10s of queuing a follow-up message',
|
||||||
|
);
|
||||||
|
}, FOLLOW_UP_NO_OUTPUT_WARN_MS);
|
||||||
|
};
|
||||||
|
|
||||||
|
const noteFollowUpQueued = (meta?: {
|
||||||
|
source: 'follow-up';
|
||||||
|
textLength: number;
|
||||||
|
filename: string;
|
||||||
|
}) => {
|
||||||
|
if (meta?.source !== 'follow-up') return;
|
||||||
|
followUpQueuedAt = Date.now();
|
||||||
|
followUpQueuedTextLength = meta.textLength;
|
||||||
|
followUpQueuedFilename = meta.filename;
|
||||||
|
scheduleFollowUpNoOutputWarning();
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
followUpQueuedTextLength,
|
||||||
|
followUpQueuedFilename,
|
||||||
|
},
|
||||||
|
'Registered follow-up activity for active agent',
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const noteAgentOutputObserved = (phase?: string) => {
|
||||||
|
if (followUpQueuedAt === null) return;
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
followUpQueuedAt: new Date(followUpQueuedAt).toISOString(),
|
||||||
|
followUpQueuedTextLength,
|
||||||
|
followUpQueuedFilename,
|
||||||
|
followUpWaitMs: Date.now() - followUpQueuedAt,
|
||||||
|
resultPhase: phase,
|
||||||
|
},
|
||||||
|
'Agent produced output after a queued follow-up',
|
||||||
|
);
|
||||||
|
clearPendingFollowUpDiagnostics();
|
||||||
|
};
|
||||||
|
|
||||||
|
const warnFollowUpEndedWithoutOutput = (reason: string) => {
|
||||||
|
if (followUpQueuedAt === null) return;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
runId,
|
||||||
|
followUpQueuedAt: new Date(followUpQueuedAt).toISOString(),
|
||||||
|
followUpQueuedTextLength,
|
||||||
|
followUpQueuedFilename,
|
||||||
|
followUpWaitMs: Date.now() - followUpQueuedAt,
|
||||||
|
reason,
|
||||||
|
},
|
||||||
|
'Active agent ended a turn without any output after a queued follow-up',
|
||||||
|
);
|
||||||
|
clearPendingFollowUpDiagnostics();
|
||||||
|
};
|
||||||
|
|
||||||
|
deps.queue.setActivityTouch?.(chatJid, (meta) => {
|
||||||
|
noteFollowUpQueued(meta);
|
||||||
|
resetIdleTimer();
|
||||||
|
});
|
||||||
|
|
||||||
let hadError = false;
|
let hadError = false;
|
||||||
let finalOutputSentToUser = false;
|
let finalOutputSentToUser = false;
|
||||||
@@ -463,7 +827,15 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (channel.sendAndTrack) {
|
if (channel.sendAndTrack) {
|
||||||
progressMessageId = await channel.sendAndTrack(chatJid, rendered);
|
try {
|
||||||
|
progressMessageId = await channel.sendAndTrack(chatJid, rendered);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ chatJid, group: group.name, groupFolder: group.folder, runId, err },
|
||||||
|
'Failed to send tracked progress message',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (progressMessageId) {
|
if (progressMessageId) {
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
@@ -532,6 +904,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
},
|
},
|
||||||
`Agent output: ${raw.slice(0, 200)}`,
|
`Agent output: ${raw.slice(0, 200)}`,
|
||||||
);
|
);
|
||||||
|
noteAgentOutputObserved(result.phase);
|
||||||
if (result.phase === 'progress') {
|
if (result.phase === 'progress') {
|
||||||
// Detect new logical turn (follow-up IPC): if a final was already
|
// Detect new logical turn (follow-up IPC): if a final was already
|
||||||
// sent in this run, a new progress output means a follow-up turn
|
// sent in this run, a new progress output means a follow-up turn
|
||||||
@@ -557,6 +930,9 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
if (text) {
|
if (text) {
|
||||||
await sendProgressMessage(text);
|
await sendProgressMessage(text);
|
||||||
}
|
}
|
||||||
|
if (!poisonedSessionDetected) {
|
||||||
|
resetIdleTimer();
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -584,6 +960,9 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
latestModelProgressTextForFinalFallback = null;
|
latestModelProgressTextForFinalFallback = null;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
if (result.status === 'success') {
|
||||||
|
warnFollowUpEndedWithoutOutput('success-null-result');
|
||||||
|
}
|
||||||
await finalizeProgressMessage();
|
await finalizeProgressMessage();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -636,6 +1015,8 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
clearProgressTicker();
|
clearProgressTicker();
|
||||||
|
|
||||||
if (idleTimer) clearTimeout(idleTimer);
|
if (idleTimer) clearTimeout(idleTimer);
|
||||||
|
clearPendingFollowUpDiagnostics();
|
||||||
|
deps.queue.setActivityTouch?.(chatJid, null);
|
||||||
|
|
||||||
if (hadError) {
|
if (hadError) {
|
||||||
if (finalOutputSentToUser || progressOutputSentToUser) {
|
if (finalOutputSentToUser || progressOutputSentToUser) {
|
||||||
@@ -723,23 +1104,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const isMainGroup = group.isMain === true;
|
const isMainGroup = group.isMain === true;
|
||||||
const allFromBots = groupMessages.every(
|
|
||||||
(msg) => msg.is_from_me || !!msg.is_bot_message,
|
|
||||||
);
|
|
||||||
if (allFromBots) {
|
|
||||||
const lastHuman = getLastHumanMessageTimestamp(chatJid);
|
|
||||||
if (
|
|
||||||
!lastHuman ||
|
|
||||||
Date.now() - new Date(lastHuman).getTime() > 12 * 60 * 60 * 1000
|
|
||||||
) {
|
|
||||||
logger.info(
|
|
||||||
{ chatJid, lastHuman },
|
|
||||||
'Bot-collaboration timeout: no human message within 12h, skipping',
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const loopCmdMsg = groupMessages.find(
|
const loopCmdMsg = groupMessages.find(
|
||||||
(msg) =>
|
(msg) =>
|
||||||
extractSessionCommand(msg.content, deps.triggerPattern) !==
|
extractSessionCommand(msg.content, deps.triggerPattern) !==
|
||||||
|
|||||||
297
src/provider-fallback.ts
Normal file
297
src/provider-fallback.ts
Normal file
@@ -0,0 +1,297 @@
|
|||||||
|
/**
|
||||||
|
* Provider Fallback Module
|
||||||
|
*
|
||||||
|
* Manages automatic fallback from the primary provider (Claude) to a
|
||||||
|
* fallback provider (e.g. Kimi K2.5) when 429/rate-limit or network
|
||||||
|
* errors are detected.
|
||||||
|
*
|
||||||
|
* Cooldown-based recovery:
|
||||||
|
* Claude 429 → immediate Kimi retry for that turn
|
||||||
|
* Claude enters cooldown (retry-after header or default 10 min)
|
||||||
|
* During cooldown → skip Claude, route directly to fallback
|
||||||
|
* After cooldown → try Claude first again
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
|
||||||
|
import { readEnvFile } from './env.js';
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
|
||||||
|
// ── Types ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export type ProviderName = 'claude' | string; // fallback name is configurable
|
||||||
|
|
||||||
|
export interface FallbackTriggerResult {
|
||||||
|
shouldFallback: boolean;
|
||||||
|
reason: string;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CooldownState {
|
||||||
|
startedAt: number;
|
||||||
|
expiresAt: number;
|
||||||
|
reason: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FallbackConfig {
|
||||||
|
enabled: boolean;
|
||||||
|
providerName: string; // e.g. "kimi"
|
||||||
|
baseUrl: string;
|
||||||
|
authToken: string;
|
||||||
|
model: string;
|
||||||
|
smallModel: string;
|
||||||
|
defaultCooldownMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── State ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
let cooldown: CooldownState | null = null;
|
||||||
|
|
||||||
|
// ── Config ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
let _config: FallbackConfig | null = null;
|
||||||
|
|
||||||
|
function loadConfig(): FallbackConfig {
|
||||||
|
if (_config) return _config;
|
||||||
|
|
||||||
|
const env = readEnvFile([
|
||||||
|
'FALLBACK_PROVIDER_NAME',
|
||||||
|
'FALLBACK_BASE_URL',
|
||||||
|
'FALLBACK_AUTH_TOKEN',
|
||||||
|
'FALLBACK_MODEL',
|
||||||
|
'FALLBACK_SMALL_MODEL',
|
||||||
|
'FALLBACK_COOLDOWN_MS',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const baseUrl =
|
||||||
|
process.env.FALLBACK_BASE_URL || env.FALLBACK_BASE_URL || '';
|
||||||
|
const authToken =
|
||||||
|
process.env.FALLBACK_AUTH_TOKEN || env.FALLBACK_AUTH_TOKEN || '';
|
||||||
|
const model = process.env.FALLBACK_MODEL || env.FALLBACK_MODEL || '';
|
||||||
|
|
||||||
|
_config = {
|
||||||
|
enabled: Boolean(baseUrl && authToken && model),
|
||||||
|
providerName:
|
||||||
|
process.env.FALLBACK_PROVIDER_NAME ||
|
||||||
|
env.FALLBACK_PROVIDER_NAME ||
|
||||||
|
'kimi',
|
||||||
|
baseUrl,
|
||||||
|
authToken,
|
||||||
|
model,
|
||||||
|
smallModel:
|
||||||
|
process.env.FALLBACK_SMALL_MODEL || env.FALLBACK_SMALL_MODEL || model,
|
||||||
|
defaultCooldownMs: parseInt(
|
||||||
|
process.env.FALLBACK_COOLDOWN_MS || env.FALLBACK_COOLDOWN_MS || '600000',
|
||||||
|
10,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (_config.enabled) {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
provider: _config.providerName,
|
||||||
|
model: _config.model,
|
||||||
|
cooldownMs: _config.defaultCooldownMs,
|
||||||
|
},
|
||||||
|
'Provider fallback configured',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return _config;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Force re-read of config (useful after .env changes). */
|
||||||
|
export function resetFallbackConfig(): void {
|
||||||
|
_config = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Public API ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Check whether the fallback system is configured and available. */
|
||||||
|
export function isFallbackEnabled(): boolean {
|
||||||
|
return loadConfig().enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get the display name of the fallback provider (e.g. "kimi"). */
|
||||||
|
export function getFallbackProviderName(): string {
|
||||||
|
return loadConfig().providerName;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine which provider should be used for the next request.
|
||||||
|
* Returns 'claude' when Claude is healthy or cooldown has expired,
|
||||||
|
* or the fallback provider name during an active cooldown.
|
||||||
|
*/
|
||||||
|
export function getActiveProvider(): string {
|
||||||
|
const config = loadConfig();
|
||||||
|
if (!config.enabled) return 'claude';
|
||||||
|
|
||||||
|
if (cooldown) {
|
||||||
|
if (Date.now() < cooldown.expiresAt) {
|
||||||
|
return config.providerName;
|
||||||
|
}
|
||||||
|
// Cooldown expired — try Claude again
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
provider: 'claude',
|
||||||
|
cooldownDurationMs: cooldown.expiresAt - cooldown.startedAt,
|
||||||
|
reason: cooldown.reason,
|
||||||
|
},
|
||||||
|
'Claude cooldown expired, retrying primary provider',
|
||||||
|
);
|
||||||
|
cooldown = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'claude';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark Claude as rate-limited. All subsequent requests will route to
|
||||||
|
* the fallback provider until the cooldown expires.
|
||||||
|
*/
|
||||||
|
export function markPrimaryCooldown(
|
||||||
|
reason: string,
|
||||||
|
retryAfterMs?: number,
|
||||||
|
): void {
|
||||||
|
const config = loadConfig();
|
||||||
|
const durationMs = retryAfterMs || config.defaultCooldownMs;
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
cooldown = {
|
||||||
|
startedAt: now,
|
||||||
|
expiresAt: now + durationMs,
|
||||||
|
reason,
|
||||||
|
};
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
reason,
|
||||||
|
cooldownMs: durationMs,
|
||||||
|
expiresAt: new Date(cooldown.expiresAt).toISOString(),
|
||||||
|
fallbackProvider: config.providerName,
|
||||||
|
},
|
||||||
|
`Falling back to provider: ${config.providerName} (reason: ${reason}, cooldownMs: ${durationMs})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Manually clear cooldown (e.g. after a successful Claude response). */
|
||||||
|
export function clearPrimaryCooldown(): void {
|
||||||
|
if (cooldown) {
|
||||||
|
logger.info(
|
||||||
|
{ reason: cooldown.reason },
|
||||||
|
'Claude cooldown cleared manually',
|
||||||
|
);
|
||||||
|
cooldown = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get current cooldown info (for diagnostics / status dashboard). */
|
||||||
|
export function getCooldownInfo(): {
|
||||||
|
active: boolean;
|
||||||
|
reason?: string;
|
||||||
|
expiresAt?: string;
|
||||||
|
remainingMs?: number;
|
||||||
|
} {
|
||||||
|
if (!cooldown || Date.now() >= cooldown.expiresAt) {
|
||||||
|
return { active: false };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
active: true,
|
||||||
|
reason: cooldown.reason,
|
||||||
|
expiresAt: new Date(cooldown.expiresAt).toISOString(),
|
||||||
|
remainingMs: cooldown.expiresAt - Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the env-var overrides that make Claude Code SDK talk to
|
||||||
|
* the fallback provider instead of Claude.
|
||||||
|
*/
|
||||||
|
export function getFallbackEnvOverrides(): Record<string, string> {
|
||||||
|
const config = loadConfig();
|
||||||
|
return {
|
||||||
|
ANTHROPIC_BASE_URL: config.baseUrl,
|
||||||
|
ANTHROPIC_AUTH_TOKEN: config.authToken,
|
||||||
|
ANTHROPIC_MODEL: config.model,
|
||||||
|
ANTHROPIC_SMALL_FAST_MODEL: config.smallModel,
|
||||||
|
// Disable non-essential traffic (usage telemetry etc.) on fallback
|
||||||
|
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
|
||||||
|
// Generous timeout for third-party APIs
|
||||||
|
API_TIMEOUT_MS: '3000000',
|
||||||
|
// Disable tool search (not supported by most fallback providers)
|
||||||
|
ENABLE_TOOL_SEARCH: 'false',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inspect an agent error string and decide whether it warrants
|
||||||
|
* a provider fallback.
|
||||||
|
*
|
||||||
|
* Triggers:
|
||||||
|
* - 429 / rate limit / too many requests
|
||||||
|
* - 503 / overloaded (transient provider issue)
|
||||||
|
* - Network / connection errors
|
||||||
|
*
|
||||||
|
* Does NOT trigger for:
|
||||||
|
* - Poisoned sessions
|
||||||
|
* - Prompt / tool failures
|
||||||
|
* - Timeouts (agent took too long, not a provider issue)
|
||||||
|
*/
|
||||||
|
export function detectFallbackTrigger(
|
||||||
|
error?: string | null,
|
||||||
|
): FallbackTriggerResult {
|
||||||
|
if (!error) return { shouldFallback: false, reason: '' };
|
||||||
|
|
||||||
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
|
// 429 Rate Limit
|
||||||
|
if (
|
||||||
|
lower.includes('429') ||
|
||||||
|
lower.includes('rate limit') ||
|
||||||
|
lower.includes('too many requests') ||
|
||||||
|
lower.includes('rate_limit')
|
||||||
|
) {
|
||||||
|
// Try to extract retry-after value (seconds → ms)
|
||||||
|
const retryMatch = error.match(/retry[\s_-]*after[:\s]*(\d+)/i);
|
||||||
|
const retryAfterMs = retryMatch
|
||||||
|
? parseInt(retryMatch[1], 10) * 1000
|
||||||
|
: undefined;
|
||||||
|
return { shouldFallback: true, reason: '429', retryAfterMs };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 503 Overloaded
|
||||||
|
if (lower.includes('503') || lower.includes('overloaded')) {
|
||||||
|
return { shouldFallback: true, reason: 'overloaded' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Network / connection errors
|
||||||
|
if (
|
||||||
|
lower.includes('econnrefused') ||
|
||||||
|
lower.includes('econnreset') ||
|
||||||
|
lower.includes('etimedout') ||
|
||||||
|
lower.includes('enotfound') ||
|
||||||
|
lower.includes('fetch failed') ||
|
||||||
|
lower.includes('network error')
|
||||||
|
) {
|
||||||
|
return { shouldFallback: true, reason: 'network-error' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { shouldFallback: false, reason: '' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether a per-group settings.json already overrides the
|
||||||
|
* provider (e.g. the Kimi test channel). If so, we should NOT
|
||||||
|
* apply fallback env overrides on top — the channel already has
|
||||||
|
* its own provider configuration.
|
||||||
|
*/
|
||||||
|
export function hasGroupProviderOverride(settingsJsonPath: string): boolean {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(settingsJsonPath, 'utf-8');
|
||||||
|
const settings = JSON.parse(raw);
|
||||||
|
const env = settings?.env || {};
|
||||||
|
return Boolean(env.ANTHROPIC_BASE_URL || env.ANTHROPIC_MODEL);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
70
src/restart-context-cli.ts
Normal file
70
src/restart-context-cli.ts
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
import { SERVICE_ID } from './config.js';
|
||||||
|
import { writeRestartContext } from './restart-context.js';
|
||||||
|
|
||||||
|
function printUsageAndExit(): never {
|
||||||
|
console.error(
|
||||||
|
[
|
||||||
|
'Usage:',
|
||||||
|
' tsx src/restart-context-cli.ts write --chat-jid <jid> --summary <text> [--verify <text> ...] [--service-id <id> ...]',
|
||||||
|
].join('\n'),
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const [, , command, ...args] = process.argv;
|
||||||
|
if (command !== 'write') {
|
||||||
|
printUsageAndExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
let chatJid = '';
|
||||||
|
let summary = '';
|
||||||
|
const verify: string[] = [];
|
||||||
|
const serviceIds: string[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < args.length; i += 1) {
|
||||||
|
const arg = args[i];
|
||||||
|
const value = args[i + 1];
|
||||||
|
if (!value) {
|
||||||
|
printUsageAndExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (arg === '--chat-jid' || arg === '--jid') {
|
||||||
|
chatJid = value;
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (arg === '--summary') {
|
||||||
|
summary = value;
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (arg === '--verify') {
|
||||||
|
verify.push(value);
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (arg === '--service-id') {
|
||||||
|
serviceIds.push(value);
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
printUsageAndExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!chatJid || !summary) {
|
||||||
|
printUsageAndExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
const written = writeRestartContext(
|
||||||
|
{
|
||||||
|
chatJid,
|
||||||
|
summary,
|
||||||
|
verify,
|
||||||
|
},
|
||||||
|
serviceIds.length > 0 ? serviceIds : [SERVICE_ID],
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const filePath of written) {
|
||||||
|
console.log(filePath);
|
||||||
|
}
|
||||||
258
src/restart-context.ts
Normal file
258
src/restart-context.ts
Normal file
@@ -0,0 +1,258 @@
|
|||||||
|
import { execSync } from 'child_process';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { DATA_DIR, SERVICE_ID, TIMEZONE } from './config.js';
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
import type { RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
|
export interface RestartInterruptedGroup {
|
||||||
|
chatJid: string;
|
||||||
|
groupName: string;
|
||||||
|
status: 'processing' | 'idle' | 'waiting';
|
||||||
|
elapsedMs: number | null;
|
||||||
|
pendingMessages: boolean;
|
||||||
|
pendingTasks: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RestartContext {
|
||||||
|
chatJid: string;
|
||||||
|
summary: string;
|
||||||
|
verify: string[];
|
||||||
|
writtenAt: string;
|
||||||
|
source?: 'explicit' | 'shutdown-snapshot';
|
||||||
|
signal?: string;
|
||||||
|
interruptedGroups?: RestartInterruptedGroup[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InferredRestartContext {
|
||||||
|
chatJid: string;
|
||||||
|
lines: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const INFER_WINDOW_MS = 3 * 60 * 1000;
|
||||||
|
|
||||||
|
function getRestartContextPath(serviceId: string = SERVICE_ID): string {
|
||||||
|
return path.join(DATA_DIR, `restart-context.${serviceId}.json`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatKoreanTimestamp(timestamp: string | number | Date): string {
|
||||||
|
return new Date(timestamp).toLocaleString('ko-KR', {
|
||||||
|
timeZone: TIMEZONE,
|
||||||
|
hour12: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMainGroupJid(
|
||||||
|
registeredGroups: Record<string, RegisteredGroup>,
|
||||||
|
): string | null {
|
||||||
|
const mainEntry = Object.entries(registeredGroups).find(
|
||||||
|
([, group]) => group.isMain === true,
|
||||||
|
);
|
||||||
|
return mainEntry?.[0] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readRestartContextFile(filePath: string): RestartContext | null {
|
||||||
|
if (!fs.existsSync(filePath)) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(fs.readFileSync(filePath, 'utf8')) as RestartContext;
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, filePath },
|
||||||
|
'Failed to parse restart context file; ignoring invalid content',
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getLatestDistBuildTime(): number | null {
|
||||||
|
const distDir = path.join(process.cwd(), 'dist');
|
||||||
|
if (!fs.existsSync(distDir)) return null;
|
||||||
|
|
||||||
|
let latestMtime = 0;
|
||||||
|
for (const entry of fs.readdirSync(distDir)) {
|
||||||
|
if (!entry.endsWith('.js')) continue;
|
||||||
|
const stat = fs.statSync(path.join(distDir, entry));
|
||||||
|
latestMtime = Math.max(latestMtime, stat.mtimeMs);
|
||||||
|
}
|
||||||
|
return latestMtime > 0 ? latestMtime : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getRecentCommit(processStartedAtMs: number): string | null {
|
||||||
|
try {
|
||||||
|
const sinceSeconds = Math.max(
|
||||||
|
60,
|
||||||
|
Math.ceil(INFER_WINDOW_MS / 1000) + 30,
|
||||||
|
).toString();
|
||||||
|
const output = execSync(
|
||||||
|
`git log --since='${sinceSeconds} seconds ago' -1 --format=%h%x20%s`,
|
||||||
|
{
|
||||||
|
cwd: process.cwd(),
|
||||||
|
stdio: ['ignore', 'pipe', 'ignore'],
|
||||||
|
encoding: 'utf8',
|
||||||
|
},
|
||||||
|
).trim();
|
||||||
|
if (!output) return null;
|
||||||
|
|
||||||
|
const commitTime = execSync('git log -1 --format=%ct', {
|
||||||
|
cwd: process.cwd(),
|
||||||
|
stdio: ['ignore', 'pipe', 'ignore'],
|
||||||
|
encoding: 'utf8',
|
||||||
|
}).trim();
|
||||||
|
const commitTimeMs = Number(commitTime) * 1000;
|
||||||
|
if (
|
||||||
|
Number.isFinite(commitTimeMs) &&
|
||||||
|
commitTimeMs >= processStartedAtMs - INFER_WINDOW_MS
|
||||||
|
) {
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore git inference failures on deployed environments without .git.
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeRestartContext(
|
||||||
|
context: Omit<RestartContext, 'writtenAt'>,
|
||||||
|
serviceIds: string[] = [SERVICE_ID],
|
||||||
|
): string[] {
|
||||||
|
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||||
|
const payload: RestartContext = {
|
||||||
|
...context,
|
||||||
|
verify: context.verify,
|
||||||
|
writtenAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
const writtenPaths: string[] = [];
|
||||||
|
for (const serviceId of serviceIds) {
|
||||||
|
const filePath = getRestartContextPath(serviceId);
|
||||||
|
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), 'utf8');
|
||||||
|
writtenPaths.push(filePath);
|
||||||
|
}
|
||||||
|
return writtenPaths;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeShutdownRestartContext(
|
||||||
|
registeredGroups: Record<string, RegisteredGroup>,
|
||||||
|
interruptedGroups: RestartInterruptedGroup[],
|
||||||
|
signal: string,
|
||||||
|
serviceIds: string[] = [SERVICE_ID],
|
||||||
|
): string[] {
|
||||||
|
if (interruptedGroups.length === 0) return [];
|
||||||
|
|
||||||
|
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||||
|
const mainChatJid =
|
||||||
|
getMainGroupJid(registeredGroups) ?? interruptedGroups[0].chatJid;
|
||||||
|
const writtenPaths: string[] = [];
|
||||||
|
|
||||||
|
for (const serviceId of serviceIds) {
|
||||||
|
const filePath = getRestartContextPath(serviceId);
|
||||||
|
const existing = readRestartContextFile(filePath);
|
||||||
|
const mergedInterrupted = [
|
||||||
|
...(existing?.interruptedGroups ?? []),
|
||||||
|
...interruptedGroups,
|
||||||
|
].filter(
|
||||||
|
(group, index, all) =>
|
||||||
|
all.findIndex((candidate) => candidate.chatJid === group.chatJid) ===
|
||||||
|
index,
|
||||||
|
);
|
||||||
|
|
||||||
|
const payload: RestartContext = {
|
||||||
|
chatJid: existing?.chatJid || mainChatJid,
|
||||||
|
summary:
|
||||||
|
existing?.summary || '서비스 재시작으로 진행 중인 작업이 중단됨',
|
||||||
|
verify: existing?.verify ?? [],
|
||||||
|
writtenAt: new Date().toISOString(),
|
||||||
|
source: existing?.source || 'shutdown-snapshot',
|
||||||
|
signal,
|
||||||
|
interruptedGroups: mergedInterrupted,
|
||||||
|
};
|
||||||
|
|
||||||
|
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), 'utf8');
|
||||||
|
writtenPaths.push(filePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
return writtenPaths;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildInterruptedRestartAnnouncement(
|
||||||
|
interrupted: RestartInterruptedGroup,
|
||||||
|
): string {
|
||||||
|
const lines = ['서비스 재시작으로 이전 작업이 중단됐습니다.'];
|
||||||
|
lines.push(`- 직전 상태: ${interrupted.status}`);
|
||||||
|
if (interrupted.elapsedMs !== null) {
|
||||||
|
lines.push(`- 직전 실행 시간: ${Math.round(interrupted.elapsedMs / 1000)}초`);
|
||||||
|
}
|
||||||
|
if (interrupted.pendingMessages) {
|
||||||
|
lines.push('- 미처리 메시지가 남아 있었음');
|
||||||
|
}
|
||||||
|
if (interrupted.pendingTasks > 0) {
|
||||||
|
lines.push(`- 대기 태스크: ${interrupted.pendingTasks}개`);
|
||||||
|
}
|
||||||
|
lines.push('- 필요하면 이어서 요청해 주세요.');
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function consumeRestartContext(): RestartContext | null {
|
||||||
|
const filePath = getRestartContextPath();
|
||||||
|
if (!fs.existsSync(filePath)) return null;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(
|
||||||
|
fs.readFileSync(filePath, 'utf8'),
|
||||||
|
) as RestartContext;
|
||||||
|
fs.unlinkSync(filePath);
|
||||||
|
return parsed;
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, filePath },
|
||||||
|
'Failed to read restart context; removing invalid file',
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(filePath);
|
||||||
|
} catch {
|
||||||
|
// Ignore cleanup failure.
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildRestartAnnouncement(
|
||||||
|
context: RestartContext,
|
||||||
|
): string {
|
||||||
|
const lines = ['재시작 완료.', `- 변경: ${context.summary}`];
|
||||||
|
if (context.interruptedGroups && context.interruptedGroups.length > 0) {
|
||||||
|
lines.push(`- 중단 작업 감지: ${context.interruptedGroups.length}개`);
|
||||||
|
}
|
||||||
|
if (context.verify.length > 0) {
|
||||||
|
lines.push(`- 검증: ${context.verify.join(', ')}`);
|
||||||
|
}
|
||||||
|
lines.push(`- 기록 시각: ${formatKoreanTimestamp(context.writtenAt)}`);
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inferRecentRestartContext(
|
||||||
|
registeredGroups: Record<string, RegisteredGroup>,
|
||||||
|
processStartedAtMs: number,
|
||||||
|
): InferredRestartContext | null {
|
||||||
|
const chatJid = getMainGroupJid(registeredGroups);
|
||||||
|
if (!chatJid) return null;
|
||||||
|
|
||||||
|
const latestBuildTime = getLatestDistBuildTime();
|
||||||
|
const recentCommit = getRecentCommit(processStartedAtMs);
|
||||||
|
const buildLooksRecent =
|
||||||
|
latestBuildTime !== null &&
|
||||||
|
latestBuildTime >= processStartedAtMs - INFER_WINDOW_MS;
|
||||||
|
|
||||||
|
if (!buildLooksRecent && !recentCommit) return null;
|
||||||
|
|
||||||
|
const lines = ['재시작 감지.', '- 명시적 재시작 힌트는 없어 추론 기반임.'];
|
||||||
|
if (recentCommit) {
|
||||||
|
lines.push(`- 최근 커밋: ${recentCommit}`);
|
||||||
|
}
|
||||||
|
if (latestBuildTime !== null) {
|
||||||
|
lines.push(`- 최근 빌드: ${formatKoreanTimestamp(latestBuildTime)}`);
|
||||||
|
}
|
||||||
|
lines.push(`- 시작 시각: ${formatKoreanTimestamp(processStartedAtMs)}`);
|
||||||
|
|
||||||
|
return { chatJid, lines };
|
||||||
|
}
|
||||||
@@ -61,6 +61,7 @@ describe('task scheduler', () => {
|
|||||||
id: 'task-claude',
|
id: 'task-claude',
|
||||||
group_folder: 'shared-group',
|
group_folder: 'shared-group',
|
||||||
chat_jid: 'shared@g.us',
|
chat_jid: 'shared@g.us',
|
||||||
|
agent_type: 'claude-code',
|
||||||
prompt: 'claude task',
|
prompt: 'claude task',
|
||||||
schedule_type: 'once',
|
schedule_type: 'once',
|
||||||
schedule_value: dueAt,
|
schedule_value: dueAt,
|
||||||
@@ -126,7 +127,11 @@ Check the run.
|
|||||||
expect(extractWatchCiTarget(prompt)).toBe('GitHub Actions run 123456');
|
expect(extractWatchCiTarget(prompt)).toBe('GitHub Actions run 123456');
|
||||||
|
|
||||||
const rendered = renderWatchCiStatusMessage({
|
const rendered = renderWatchCiStatusMessage({
|
||||||
task: { prompt },
|
task: {
|
||||||
|
prompt,
|
||||||
|
schedule_type: 'interval',
|
||||||
|
schedule_value: '60000',
|
||||||
|
} as any,
|
||||||
phase: 'waiting',
|
phase: 'waiting',
|
||||||
checkedAt: '2026-03-19T07:02:10.000Z',
|
checkedAt: '2026-03-19T07:02:10.000Z',
|
||||||
nextRun: '2026-03-19T07:04:10.000Z',
|
nextRun: '2026-03-19T07:04:10.000Z',
|
||||||
@@ -134,9 +139,11 @@ Check the run.
|
|||||||
|
|
||||||
expect(rendered).toContain('CI 감시 중: GitHub Actions run 123456');
|
expect(rendered).toContain('CI 감시 중: GitHub Actions run 123456');
|
||||||
expect(rendered).toContain('- 상태: 대기 중');
|
expect(rendered).toContain('- 상태: 대기 중');
|
||||||
expect(rendered).toContain('- 마지막 확인:');
|
expect(rendered).toContain('- 마지막 확인: 16시 02분 10초');
|
||||||
expect(rendered).toContain('- 다음 확인:');
|
expect(rendered).toContain('- 확인 간격: 1분');
|
||||||
expect(rendered).not.toContain('2분 10초');
|
expect(rendered).toContain('- 다음 확인: 16시 04분 10초');
|
||||||
|
expect(rendered).not.toContain('16:02:10');
|
||||||
|
expect(rendered).not.toContain('16:04:10');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('computeNextRun anchors interval tasks to scheduled time to prevent drift', () => {
|
it('computeNextRun anchors interval tasks to scheduled time to prevent drift', () => {
|
||||||
|
|||||||
@@ -113,11 +113,33 @@ function formatTimeLabel(timestampIso: string): string {
|
|||||||
second: '2-digit',
|
second: '2-digit',
|
||||||
hour12: false,
|
hour12: false,
|
||||||
timeZone: TIMEZONE,
|
timeZone: TIMEZONE,
|
||||||
}).format(new Date(timestampIso));
|
})
|
||||||
|
.format(new Date(timestampIso))
|
||||||
|
.replace(/:/g, '시 ')
|
||||||
|
.replace(/시 (\d{2})$/, '분 $1초');
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatWatchIntervalLabel(task: Pick<ScheduledTask, 'schedule_type' | 'schedule_value'>): string | null {
|
||||||
|
if (task.schedule_type !== 'interval') return null;
|
||||||
|
const ms = parseInt(task.schedule_value, 10);
|
||||||
|
if (!Number.isFinite(ms) || ms <= 0) return null;
|
||||||
|
|
||||||
|
const totalSeconds = Math.floor(ms / 1000);
|
||||||
|
if (totalSeconds < 60) return `${totalSeconds}초`;
|
||||||
|
|
||||||
|
const totalMinutes = Math.floor(totalSeconds / 60);
|
||||||
|
if (totalMinutes < 60) {
|
||||||
|
const seconds = totalSeconds % 60;
|
||||||
|
return seconds > 0 ? `${totalMinutes}분 ${seconds}초` : `${totalMinutes}분`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hours = Math.floor(totalMinutes / 60);
|
||||||
|
const minutes = totalMinutes % 60;
|
||||||
|
return minutes > 0 ? `${hours}시간 ${minutes}분` : `${hours}시간`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function renderWatchCiStatusMessage(args: {
|
export function renderWatchCiStatusMessage(args: {
|
||||||
task: Pick<ScheduledTask, 'prompt'>;
|
task: Pick<ScheduledTask, 'prompt' | 'schedule_type' | 'schedule_value'>;
|
||||||
phase: WatcherStatusPhase;
|
phase: WatcherStatusPhase;
|
||||||
checkedAt: string;
|
checkedAt: string;
|
||||||
nextRun?: string | null;
|
nextRun?: string | null;
|
||||||
@@ -141,6 +163,10 @@ export function renderWatchCiStatusMessage(args: {
|
|||||||
`- 상태: ${statusLabel}`,
|
`- 상태: ${statusLabel}`,
|
||||||
`- 마지막 확인: ${formatTimeLabel(args.checkedAt)}`,
|
`- 마지막 확인: ${formatTimeLabel(args.checkedAt)}`,
|
||||||
];
|
];
|
||||||
|
const intervalLabel = formatWatchIntervalLabel(args.task);
|
||||||
|
if (intervalLabel) {
|
||||||
|
lines.push(`- 확인 간격: ${intervalLabel}`);
|
||||||
|
}
|
||||||
|
|
||||||
if (args.nextRun) {
|
if (args.nextRun) {
|
||||||
lines.push(`- 다음 확인: ${formatTimeLabel(args.nextRun)}`);
|
lines.push(`- 다음 확인: ${formatTimeLabel(args.nextRun)}`);
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ export interface NewMessage {
|
|||||||
sender_name: string;
|
sender_name: string;
|
||||||
content: string;
|
content: string;
|
||||||
timestamp: string;
|
timestamp: string;
|
||||||
|
seq?: number;
|
||||||
is_from_me?: boolean;
|
is_from_me?: boolean;
|
||||||
is_bot_message?: boolean;
|
is_bot_message?: boolean;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user