backup current stable ejclaw state
This commit is contained in:
Codex
@@ -55,6 +55,17 @@ describe('agent-error-detection', () => {
|
||||
expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded');
|
||||
});
|
||||
|
||||
it('classifies Claude 500 API banners as overloaded provider failures', () => {
|
||||
const message =
|
||||
'API Error: 500 Internal server error. This is a server-side issue, usually temporary — try again in a moment. If it persists, check status.claude.com.';
|
||||
|
||||
expect(classifyAgentError(message)).toEqual({
|
||||
category: 'overloaded',
|
||||
reason: 'overloaded',
|
||||
});
|
||||
expect(detectClaudeProviderFailureMessage(message)).toBe('overloaded');
|
||||
});
|
||||
|
||||
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
||||
expect(shouldRotateClaudeToken('429')).toBe(true);
|
||||
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
||||
|
||||
@@ -260,8 +260,11 @@ export function classifyAgentError(
|
||||
return { category: 'rate-limit', reason: '429', retryAfterMs };
|
||||
}
|
||||
|
||||
// 503 / Overloaded
|
||||
const hasApi5xx = /\bapi error:\s*5\d\d\b/i.test(error);
|
||||
|
||||
// 5xx / Overloaded
|
||||
if (
|
||||
hasApi5xx ||
|
||||
lower.includes('503') ||
|
||||
lower.includes('overloaded') ||
|
||||
((lower.includes('502') || lower.includes('bad gateway')) &&
|
||||
|
||||
@@ -368,6 +368,67 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
||||
|
||||
expect(segments).toEqual(['platform prompt']);
|
||||
});
|
||||
|
||||
it('applies per-role codex model overrides when roomRole is supplied', () => {
|
||||
mockReadEnvFile.mockReturnValue({});
|
||||
|
||||
const groupWithByRole: RegisteredGroup = {
|
||||
...group,
|
||||
agentType: 'codex',
|
||||
agentConfig: {
|
||||
codexModel: 'gpt-5-sonnet',
|
||||
codexEffort: 'medium',
|
||||
codexModelByRole: { arbiter: 'gpt-5-opus' },
|
||||
codexEffortByRole: { arbiter: 'high' },
|
||||
},
|
||||
};
|
||||
|
||||
const arbiter = prepareGroupEnvironment(groupWithByRole, false, 'dc:test', {
|
||||
roomRole: 'arbiter',
|
||||
});
|
||||
expect(arbiter.env.CODEX_MODEL).toBe('gpt-5-opus');
|
||||
expect(arbiter.env.CODEX_EFFORT).toBe('high');
|
||||
|
||||
const owner = prepareGroupEnvironment(groupWithByRole, false, 'dc:test', {
|
||||
roomRole: 'owner',
|
||||
});
|
||||
expect(owner.env.CODEX_MODEL).toBe('gpt-5-sonnet');
|
||||
expect(owner.env.CODEX_EFFORT).toBe('medium');
|
||||
|
||||
const noRole = prepareGroupEnvironment(groupWithByRole, false, 'dc:test');
|
||||
expect(noRole.env.CODEX_MODEL).toBe('gpt-5-sonnet');
|
||||
expect(noRole.env.CODEX_EFFORT).toBe('medium');
|
||||
});
|
||||
|
||||
it('applies per-role claude model overrides when roomRole is supplied', () => {
|
||||
mockReadEnvFile.mockReturnValue({});
|
||||
|
||||
const groupWithByRole: RegisteredGroup = {
|
||||
...group,
|
||||
agentType: 'claude-code',
|
||||
agentConfig: {
|
||||
claudeModel: 'claude-sonnet-4-6',
|
||||
claudeEffort: 'medium',
|
||||
claudeModelByRole: { arbiter: 'claude-opus-4-7' },
|
||||
claudeEffortByRole: { arbiter: 'high' },
|
||||
},
|
||||
};
|
||||
|
||||
const arbiter = prepareGroupEnvironment(groupWithByRole, false, 'dc:test', {
|
||||
roomRole: 'arbiter',
|
||||
});
|
||||
expect(arbiter.env.CLAUDE_MODEL).toBe('claude-opus-4-7');
|
||||
expect(arbiter.env.CLAUDE_EFFORT).toBe('high');
|
||||
|
||||
const reviewer = prepareGroupEnvironment(
|
||||
groupWithByRole,
|
||||
false,
|
||||
'dc:test',
|
||||
{ roomRole: 'reviewer' },
|
||||
);
|
||||
expect(reviewer.env.CLAUDE_MODEL).toBe('claude-sonnet-4-6');
|
||||
expect(reviewer.env.CLAUDE_EFFORT).toBe('medium');
|
||||
});
|
||||
});
|
||||
|
||||
describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
||||
|
||||
@@ -32,7 +32,7 @@ import {
|
||||
getEffectiveChannelLease,
|
||||
hasReviewerLease,
|
||||
} from './service-routing.js';
|
||||
import type { AgentType, RegisteredGroup } from './types.js';
|
||||
import type { AgentType, PairedRoomRole, RegisteredGroup } from './types.js';
|
||||
|
||||
// writeCodexApiKeyAuth removed — Codex uses OAuth only.
|
||||
// API key auth caused unintended billing.
|
||||
@@ -232,6 +232,7 @@ function prepareClaudeEnvironment(args: {
|
||||
env: Record<string, string>;
|
||||
envVars: Record<string, string>;
|
||||
group: RegisteredGroup;
|
||||
roomRole?: PairedRoomRole;
|
||||
}): void {
|
||||
if (args.envVars.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY) {
|
||||
args.env.ANTHROPIC_API_KEY =
|
||||
@@ -279,6 +280,15 @@ function prepareClaudeEnvironment(args: {
|
||||
if (args.group.agentConfig?.claudeEffort) {
|
||||
args.env.CLAUDE_EFFORT = args.group.agentConfig.claudeEffort;
|
||||
}
|
||||
// Per-role overrides (paired room): take precedence over the flat values.
|
||||
if (args.roomRole) {
|
||||
const modelByRole =
|
||||
args.group.agentConfig?.claudeModelByRole?.[args.roomRole];
|
||||
if (modelByRole) args.env.CLAUDE_MODEL = modelByRole;
|
||||
const effortByRole =
|
||||
args.group.agentConfig?.claudeEffortByRole?.[args.roomRole];
|
||||
if (effortByRole) args.env.CLAUDE_EFFORT = effortByRole;
|
||||
}
|
||||
if (args.group.agentConfig?.claudeThinking) {
|
||||
args.env.CLAUDE_THINKING = args.group.agentConfig.claudeThinking;
|
||||
}
|
||||
@@ -301,19 +311,29 @@ function prepareCodexSessionEnvironment(args: {
|
||||
isPairedRoom: boolean;
|
||||
useFailoverPromptPack: boolean;
|
||||
memoryBriefing?: string;
|
||||
roomRole?: PairedRoomRole;
|
||||
}): void {
|
||||
// API key auth intentionally removed — Codex uses OAuth only.
|
||||
// Never pass any API key to Codex child process to prevent API billing.
|
||||
delete args.env.OPENAI_API_KEY;
|
||||
delete args.env.CODEX_OPENAI_API_KEY;
|
||||
|
||||
// Per-role overrides (paired room) win over flat config which wins over env.
|
||||
const codexModelByRole = args.roomRole
|
||||
? args.group.agentConfig?.codexModelByRole?.[args.roomRole]
|
||||
: undefined;
|
||||
const codexModel =
|
||||
codexModelByRole ||
|
||||
args.group.agentConfig?.codexModel ||
|
||||
args.envVars.CODEX_MODEL ||
|
||||
process.env.CODEX_MODEL;
|
||||
if (codexModel) args.env.CODEX_MODEL = codexModel;
|
||||
|
||||
const codexEffortByRole = args.roomRole
|
||||
? args.group.agentConfig?.codexEffortByRole?.[args.roomRole]
|
||||
: undefined;
|
||||
const codexEffort =
|
||||
codexEffortByRole ||
|
||||
args.group.agentConfig?.codexEffort ||
|
||||
args.envVars.CODEX_EFFORT ||
|
||||
process.env.CODEX_EFFORT;
|
||||
@@ -338,6 +358,23 @@ function prepareCodexSessionEnvironment(args: {
|
||||
}
|
||||
|
||||
const sessionAgentsPath = path.join(sessionCodexDir, 'AGENTS.md');
|
||||
const codexGlobalDir = path.join(GROUPS_DIR, 'global');
|
||||
const codexGlobalClaudeMdPath = path.join(codexGlobalDir, 'CLAUDE.md');
|
||||
const codexGlobalMemory =
|
||||
!args.isMain && fs.existsSync(codexGlobalClaudeMdPath)
|
||||
? fs.readFileSync(codexGlobalClaudeMdPath, 'utf-8').trim()
|
||||
: undefined;
|
||||
const codexGroupClaudeMdPath = path.join(
|
||||
GROUPS_DIR,
|
||||
args.group.folder,
|
||||
'CLAUDE.md',
|
||||
);
|
||||
const codexGroupMemory =
|
||||
args.group.folder !== 'global' &&
|
||||
args.group.folder !== 'main' &&
|
||||
fs.existsSync(codexGroupClaudeMdPath)
|
||||
? fs.readFileSync(codexGroupClaudeMdPath, 'utf-8').trim()
|
||||
: undefined;
|
||||
const sessionAgents = (
|
||||
args.useFailoverPromptPack
|
||||
? [
|
||||
@@ -352,9 +389,12 @@ function prepareCodexSessionEnvironment(args: {
|
||||
'owner-common-paired-room.md',
|
||||
)
|
||||
: undefined,
|
||||
codexGlobalMemory,
|
||||
codexGroupMemory,
|
||||
args.memoryBriefing,
|
||||
]
|
||||
: [
|
||||
readOptionalPromptFile(args.projectRoot, 'owner-common-platform.md'),
|
||||
readPlatformPrompt('codex', args.projectRoot),
|
||||
args.isPairedRoom
|
||||
? readOptionalPromptFile(
|
||||
@@ -362,6 +402,8 @@ function prepareCodexSessionEnvironment(args: {
|
||||
'owner-common-paired-room.md',
|
||||
)
|
||||
: undefined,
|
||||
codexGlobalMemory,
|
||||
codexGroupMemory,
|
||||
args.memoryBriefing,
|
||||
]
|
||||
)
|
||||
@@ -429,6 +471,13 @@ export function prepareGroupEnvironment(
|
||||
memoryBriefing?: string;
|
||||
runtimeTaskId?: string;
|
||||
useTaskScopedSession?: boolean;
|
||||
/**
|
||||
* When provided, the runner consults `agentConfig.{claudeModel,claudeEffort,
|
||||
* codexModel,codexEffort}ByRole[roomRole]` and lets those entries override
|
||||
* the flat per-group values. Used by paired rooms to spend the expensive
|
||||
* model on a specific role (typically arbiter) only.
|
||||
*/
|
||||
roomRole?: PairedRoomRole;
|
||||
},
|
||||
): PreparedGroupEnvironment {
|
||||
const projectRoot = process.cwd();
|
||||
@@ -494,6 +543,13 @@ export function prepareGroupEnvironment(
|
||||
!isMain && fs.existsSync(globalClaudeMdPath)
|
||||
? fs.readFileSync(globalClaudeMdPath, 'utf-8').trim()
|
||||
: undefined;
|
||||
const groupClaudeMdPath = path.join(GROUPS_DIR, group.folder, 'CLAUDE.md');
|
||||
const groupClaudeMemory =
|
||||
group.folder !== 'global' &&
|
||||
group.folder !== 'main' &&
|
||||
fs.existsSync(groupClaudeMdPath)
|
||||
? fs.readFileSync(groupClaudeMdPath, 'utf-8').trim()
|
||||
: undefined;
|
||||
// Owner CLAUDE.md: platform rules + owner paired room rules.
|
||||
// Reviewer paired room rules are NOT included — those belong to the
|
||||
// Read-only reviewer/arbiter session only (via prepareReadonlySessionEnvironment).
|
||||
@@ -502,6 +558,7 @@ export function prepareGroupEnvironment(
|
||||
claudePlatformPrompt,
|
||||
ownerCommonPairedRoomPrompt,
|
||||
globalClaudeMemory,
|
||||
groupClaudeMemory,
|
||||
options?.memoryBriefing,
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
@@ -559,9 +616,15 @@ export function prepareGroupEnvironment(
|
||||
isPairedRoom,
|
||||
useFailoverPromptPack: useCodexReviewFailoverPromptPack,
|
||||
memoryBriefing: options?.memoryBriefing,
|
||||
roomRole: options?.roomRole,
|
||||
});
|
||||
} else {
|
||||
prepareClaudeEnvironment({ env, envVars, group });
|
||||
prepareClaudeEnvironment({
|
||||
env,
|
||||
envVars,
|
||||
group,
|
||||
roomRole: options?.roomRole,
|
||||
});
|
||||
}
|
||||
|
||||
return { env, groupDir, runnerDir };
|
||||
@@ -626,11 +689,19 @@ export function prepareReadonlySessionEnvironment(args: {
|
||||
!isMain && fs.existsSync(globalClaudeMdPath)
|
||||
? fs.readFileSync(globalClaudeMdPath, 'utf-8').trim()
|
||||
: undefined;
|
||||
const groupClaudeMdPath = path.join(GROUPS_DIR, groupFolder, 'CLAUDE.md');
|
||||
const groupClaudeMemory =
|
||||
groupFolder !== 'global' &&
|
||||
groupFolder !== 'main' &&
|
||||
fs.existsSync(groupClaudeMdPath)
|
||||
? fs.readFileSync(groupClaudeMdPath, 'utf-8').trim()
|
||||
: undefined;
|
||||
|
||||
const sessionClaudeMd = [
|
||||
claudePlatformPrompt,
|
||||
claudePairedRoomPrompt,
|
||||
globalClaudeMemory,
|
||||
groupClaudeMemory,
|
||||
memoryBriefing,
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
|
||||
@@ -61,6 +61,11 @@ export function runSpawnedAgentProcess(
|
||||
let stdoutTruncated = false;
|
||||
let stderrTruncated = false;
|
||||
|
||||
// Use UTF-8 string decoding on streams to avoid splitting multi-byte
|
||||
// characters (e.g. Korean) at chunk boundaries, which produces U+FFFD.
|
||||
stdoutStream.setEncoding('utf8');
|
||||
stderrStream.setEncoding('utf8');
|
||||
|
||||
// Streaming output: parse OUTPUT_START/END marker pairs as they arrive.
|
||||
let parseBuffer = '';
|
||||
let newSessionId: string | undefined;
|
||||
|
||||
@@ -73,6 +73,7 @@ export async function runAgentProcess(
|
||||
memoryBriefing: input.memoryBriefing,
|
||||
runtimeTaskId: input.runtimeTaskId,
|
||||
useTaskScopedSession: input.useTaskScopedSession,
|
||||
roomRole: input.roomRoleContext?.role,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
179
src/auto-paired-mode.ts
Normal file
179
src/auto-paired-mode.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Auto paired-mode degrade/restore.
|
||||
*
|
||||
* When all configured Claude accounts are at/over the 95% exhaustion threshold,
|
||||
* paired (tribunal) rooms cannot meaningfully run their reviewer/arbiter
|
||||
* because the gate blocks Claude turns. To keep the user-facing rooms
|
||||
* responsive, we temporarily flip selected rooms to `single` mode and notify
|
||||
* each affected channel. When usage recovers we flip them back to `tribunal`.
|
||||
*
|
||||
* Selection (Option B agreed with the user):
|
||||
* - The most-recently-active room (per chats.last_message_time)
|
||||
* - All rooms that are currently in-flight (queue status === 'processing')
|
||||
* - Deduplicated; rooms whose owner is `claude-code` are skipped because
|
||||
* degrading them does not help — the 95% gate blocks them either way.
|
||||
*
|
||||
* Restore behavior:
|
||||
* - Only restore a room if it is still in 'single' (don't override a manual
|
||||
* user change made during the degrade window).
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { getClaudeUsageExhaustion } from './claude-usage.js';
|
||||
import { DATA_DIR } from './config.js';
|
||||
import {
|
||||
getAllChats,
|
||||
getAllRoomBindings,
|
||||
getEffectiveRoomMode,
|
||||
getStoredRoomSettings,
|
||||
setExplicitRoomMode,
|
||||
} from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||
|
||||
const STATE_PATH = path.join(DATA_DIR, 'auto-paired-mode-state.json');
|
||||
|
||||
const DEGRADE_NOTICE =
|
||||
'클로드 사용량이 95%를 넘어 잠시 single 모드로 전환했어요. 사용량이 회복되면 paired 모드로 다시 켤게요.';
|
||||
const RESTORE_NOTICE = '클로드 사용량이 회복돼서 paired 모드로 다시 켰어요.';
|
||||
|
||||
interface DowngradeRecord {
|
||||
chatJid: string;
|
||||
atIso: string;
|
||||
}
|
||||
|
||||
interface AutoPairedModeState {
|
||||
exhausted: boolean;
|
||||
downgraded: DowngradeRecord[];
|
||||
}
|
||||
|
||||
export interface AutoPairedModeDeps {
|
||||
queue?: {
|
||||
getStatuses(jids: string[]): Array<{ jid: string; status: string }>;
|
||||
};
|
||||
sendNotice: (chatJid: string, text: string) => Promise<void>;
|
||||
}
|
||||
|
||||
function loadState(): AutoPairedModeState {
|
||||
const raw = readJsonFile<AutoPairedModeState>(STATE_PATH);
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
return { exhausted: false, downgraded: [] };
|
||||
}
|
||||
return {
|
||||
exhausted: Boolean(raw.exhausted),
|
||||
downgraded: Array.isArray(raw.downgraded) ? raw.downgraded : [],
|
||||
};
|
||||
}
|
||||
|
||||
function saveState(state: AutoPairedModeState): void {
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(STATE_PATH), { recursive: true });
|
||||
writeJsonFile(STATE_PATH, state);
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'auto-paired-mode: failed to persist state');
|
||||
}
|
||||
}
|
||||
|
||||
function pickCandidates(deps: AutoPairedModeDeps): string[] {
|
||||
const bindings = getAllRoomBindings();
|
||||
const allJids = Object.keys(bindings);
|
||||
if (allJids.length === 0) return [];
|
||||
|
||||
const candidates = new Set<string>();
|
||||
|
||||
// Most-recently-active room first (chats ordered DESC by last_message_time).
|
||||
const chats = getAllChats();
|
||||
const recent = chats.find((c) => allJids.includes(c.jid));
|
||||
if (recent) candidates.add(recent.jid);
|
||||
|
||||
// Currently in-flight rooms.
|
||||
if (deps.queue) {
|
||||
for (const status of deps.queue.getStatuses(allJids)) {
|
||||
if (status.status === 'processing') candidates.add(status.jid);
|
||||
}
|
||||
}
|
||||
|
||||
// Skip claude-owned rooms (Option B).
|
||||
const result: string[] = [];
|
||||
for (const jid of candidates) {
|
||||
const settings = getStoredRoomSettings(jid);
|
||||
if (settings?.ownerAgentType === 'claude-code') continue;
|
||||
if (getEffectiveRoomMode(jid) !== 'tribunal') continue;
|
||||
result.push(jid);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function evaluateAndApplyAutoPairedMode(
|
||||
deps: AutoPairedModeDeps,
|
||||
): Promise<void> {
|
||||
const exhaustion = getClaudeUsageExhaustion();
|
||||
const state = loadState();
|
||||
|
||||
if (exhaustion.exhausted && !state.exhausted) {
|
||||
// Transition: healthy → exhausted. Downgrade selected rooms.
|
||||
const targets = pickCandidates(deps);
|
||||
const downgraded: DowngradeRecord[] = [];
|
||||
for (const jid of targets) {
|
||||
try {
|
||||
setExplicitRoomMode(jid, 'single');
|
||||
downgraded.push({ chatJid: jid, atIso: new Date().toISOString() });
|
||||
logger.info(
|
||||
{ jid },
|
||||
'auto-paired-mode: downgraded room to single (claude exhausted)',
|
||||
);
|
||||
try {
|
||||
await deps.sendNotice(jid, DEGRADE_NOTICE);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, jid },
|
||||
'auto-paired-mode: failed to send degrade notice',
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ err, jid }, 'auto-paired-mode: failed to downgrade room');
|
||||
}
|
||||
}
|
||||
saveState({ exhausted: true, downgraded });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!exhaustion.exhausted && state.exhausted) {
|
||||
// Transition: exhausted → healthy. Restore previously-downgraded rooms.
|
||||
for (const record of state.downgraded) {
|
||||
try {
|
||||
if (getEffectiveRoomMode(record.chatJid) !== 'single') {
|
||||
// Respect manual user changes during the degrade window.
|
||||
logger.info(
|
||||
{ jid: record.chatJid },
|
||||
'auto-paired-mode: skip restore (room is no longer single)',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
setExplicitRoomMode(record.chatJid, 'tribunal');
|
||||
logger.info(
|
||||
{ jid: record.chatJid },
|
||||
'auto-paired-mode: restored room to tribunal',
|
||||
);
|
||||
try {
|
||||
await deps.sendNotice(record.chatJid, RESTORE_NOTICE);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, jid: record.chatJid },
|
||||
'auto-paired-mode: failed to send restore notice',
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, jid: record.chatJid },
|
||||
'auto-paired-mode: failed to restore room',
|
||||
);
|
||||
}
|
||||
}
|
||||
saveState({ exhausted: false, downgraded: [] });
|
||||
return;
|
||||
}
|
||||
|
||||
// Steady state — nothing to do.
|
||||
}
|
||||
@@ -23,7 +23,11 @@ import { extractImageTagPaths } from '../agent-protocol.js';
|
||||
import { validateOutboundAttachments } from '../outbound-attachments.js';
|
||||
import { formatOutbound } from '../router.js';
|
||||
import { hasReviewerLease } from '../service-routing.js';
|
||||
import type { OutboundAttachment, SendMessageOptions } from '../types.js';
|
||||
import type {
|
||||
EditMessageOptions,
|
||||
OutboundAttachment,
|
||||
SendMessageOptions,
|
||||
} from '../types.js';
|
||||
|
||||
const ATTACHMENTS_DIR = path.join(DATA_DIR, 'attachments');
|
||||
const TRANSCRIPTION_CACHE_DIR = path.join(CACHE_DIR, 'transcriptions');
|
||||
@@ -514,7 +518,7 @@ export class DiscordChannel implements Channel {
|
||||
for (const [name, id] of Object.entries(mentionMap)) {
|
||||
cleaned = cleaned.replace(new RegExp(`@${name}`, 'g'), `<@${id}>`);
|
||||
}
|
||||
cleaned = formatOutbound(cleaned);
|
||||
cleaned = options.rawMarkdown ? cleaned : formatOutbound(cleaned);
|
||||
|
||||
// Discord has a 2000 character limit per message and 10 attachments per message
|
||||
const MAX_LENGTH = 2000;
|
||||
@@ -561,9 +565,21 @@ export class DiscordChannel implements Channel {
|
||||
}
|
||||
} else {
|
||||
// Send text in chunks, attach first batch to the first chunk
|
||||
// Use a helper that avoids splitting surrogate pairs (emoji, etc.)
|
||||
let fileBatchIndex = 0;
|
||||
for (let i = 0; i < cleaned.length; i += MAX_LENGTH) {
|
||||
const chunk = cleaned.slice(i, i + MAX_LENGTH);
|
||||
for (let i = 0; i < cleaned.length; ) {
|
||||
let end = Math.min(i + MAX_LENGTH, cleaned.length);
|
||||
// If we'd split a surrogate pair, back up one code unit
|
||||
if (
|
||||
end < cleaned.length &&
|
||||
end > i &&
|
||||
cleaned.charCodeAt(end - 1) >= 0xd800 &&
|
||||
cleaned.charCodeAt(end - 1) <= 0xdbff
|
||||
) {
|
||||
end--;
|
||||
}
|
||||
const chunk = cleaned.slice(i, end);
|
||||
i = end;
|
||||
const batch = fileBatches[fileBatchIndex];
|
||||
recordSentMessage(
|
||||
await textChannel.send({
|
||||
@@ -678,13 +694,18 @@ export class DiscordChannel implements Channel {
|
||||
);
|
||||
}
|
||||
|
||||
async sendAndTrack(jid: string, text: string): Promise<string | null> {
|
||||
async sendAndTrack(
|
||||
jid: string,
|
||||
text: string,
|
||||
options: EditMessageOptions = {},
|
||||
): Promise<string | null> {
|
||||
if (!this.client) return null;
|
||||
try {
|
||||
const channelId = jid.replace(/^dc:/, '');
|
||||
const channel = await this.client.channels.fetch(channelId);
|
||||
if (!channel || !('send' in channel)) return null;
|
||||
const msg = await (channel as TextChannel).send(text);
|
||||
const cleaned = options.rawMarkdown ? text : formatOutbound(text);
|
||||
const msg = await (channel as TextChannel).send(cleaned);
|
||||
logger.info(
|
||||
{
|
||||
jid,
|
||||
@@ -693,7 +714,7 @@ export class DiscordChannel implements Channel {
|
||||
messageId: msg.id,
|
||||
botUserId: this.client.user?.id ?? null,
|
||||
botUsername: this.client.user?.username ?? null,
|
||||
length: text.length,
|
||||
length: cleaned.length,
|
||||
},
|
||||
'Discord tracked message sent',
|
||||
);
|
||||
@@ -814,6 +835,7 @@ export class DiscordChannel implements Channel {
|
||||
jid: string,
|
||||
messageId: string,
|
||||
text: string,
|
||||
options: EditMessageOptions = {},
|
||||
): Promise<void> {
|
||||
if (!this.client) return;
|
||||
try {
|
||||
@@ -821,14 +843,20 @@ export class DiscordChannel implements Channel {
|
||||
const channel = await this.client.channels.fetch(channelId);
|
||||
if (!channel || !('messages' in channel)) return;
|
||||
const msg = await (channel as TextChannel).messages.fetch(messageId);
|
||||
await msg.edit(text);
|
||||
// Run the same sanitization pipeline as sendMessage so streaming
|
||||
// edits do not bypass markdown neutralization (backticks, headings,
|
||||
// italics, etc. that Discord would otherwise render and mangle).
|
||||
// Dashboard callers opt out with rawMarkdown: true to preserve
|
||||
// intentional bold/italic section headers.
|
||||
const cleaned = options.rawMarkdown ? text : formatOutbound(text);
|
||||
await msg.edit(cleaned);
|
||||
logger.info(
|
||||
{
|
||||
jid,
|
||||
channelName: this.name,
|
||||
deliveryMode: 'edit',
|
||||
messageId,
|
||||
length: text.length,
|
||||
length: cleaned.length,
|
||||
botUserId: this.client.user?.id ?? null,
|
||||
botUsername: this.client.user?.username ?? null,
|
||||
},
|
||||
|
||||
@@ -112,12 +112,15 @@ export function getUsageCacheReadKeys(
|
||||
return keys;
|
||||
}
|
||||
|
||||
// Rate limit: at most one API call per token per 5 minutes
|
||||
const MIN_FETCH_INTERVAL_MS = 300_000;
|
||||
// Rate limit: at most one API call per token per 1 minute.
|
||||
// We poll usage every minute (was 5) so the 95% exhaustion gate has tight
|
||||
// resolution — at worst the gate fires ~60s after a window crosses 95%.
|
||||
const MIN_FETCH_INTERVAL_MS = 60_000;
|
||||
|
||||
async function fetchUsageForToken(
|
||||
token: string,
|
||||
accountIndex?: number,
|
||||
refreshAttempted = false,
|
||||
): Promise<ClaudeUsageData | null> {
|
||||
loadUsageDiskCache();
|
||||
|
||||
@@ -148,7 +151,11 @@ async function fetchUsageForToken(
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
const lastAttempt = cached?.lastAttemptAt ?? cached?.fetchedAt ?? 0;
|
||||
if (cached && Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS) {
|
||||
if (
|
||||
!refreshAttempted &&
|
||||
cached &&
|
||||
Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS
|
||||
) {
|
||||
return cached.usage;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
@@ -165,16 +172,47 @@ async function fetchUsageForToken(
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (res.status === 401) {
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
// 401 = token expired; 403 = token lacks `user:profile` scope.
|
||||
// Both are recoverable by exchanging refresh_token for a fresh access
|
||||
// token (mint includes current default scope set). Retry once.
|
||||
if (accountIndex != null && !refreshAttempted) {
|
||||
try {
|
||||
const { forceRefreshToken } = await import('./token-refresh.js');
|
||||
const newToken = await forceRefreshToken(accountIndex);
|
||||
if (newToken && newToken !== token) {
|
||||
logger.info(
|
||||
{
|
||||
account: accountIndex + 1,
|
||||
status: res.status,
|
||||
cacheKey: writeKey,
|
||||
},
|
||||
`Claude usage API: ${res.status} → refreshed access token, retrying`,
|
||||
);
|
||||
return fetchUsageForToken(newToken, accountIndex, true);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, account: accountIndex + 1 },
|
||||
'Claude usage API: force-refresh failed',
|
||||
);
|
||||
}
|
||||
}
|
||||
logger.warn(
|
||||
{
|
||||
account: accountIndex != null ? accountIndex + 1 : '?',
|
||||
tokenKey: legacyTokenCacheKey(token),
|
||||
cacheKey: writeKey,
|
||||
},
|
||||
'Claude usage API: token expired or invalid (401)',
|
||||
res.status === 401
|
||||
? 'Claude usage API: token expired or invalid (401)'
|
||||
: 'Claude usage API: forbidden — token missing required scope (403)',
|
||||
);
|
||||
return null;
|
||||
if (cached) {
|
||||
cached.lastAttemptAt = Date.now();
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
return cached?.usage ?? null;
|
||||
}
|
||||
if (res.status === 429) {
|
||||
const staleMs = cached ? Date.now() - cached.fetchedAt : 0;
|
||||
@@ -278,7 +316,15 @@ async function fetchUsageForToken(
|
||||
* Uses the current active token from rotation.
|
||||
*/
|
||||
export async function fetchClaudeUsage(): Promise<ClaudeUsageData | null> {
|
||||
const token = getCurrentToken() || getConfiguredClaudeTokens()[0];
|
||||
// Prefer the access token in ~/.claude/.credentials.json when present.
|
||||
// The static .env token (CLAUDE_CODE_OAUTH_TOKEN) was issued before the
|
||||
// `user:profile` scope was required for /api/oauth/usage and so always 403s.
|
||||
// The credentials file is the canonical source written by `claude auth
|
||||
// login` and kept fresh by token-refresh.ts — its accessToken carries the
|
||||
// full scope set including user:profile.
|
||||
const credsToken = readCredentialsAccessToken(0);
|
||||
const token =
|
||||
credsToken || getCurrentToken() || getConfiguredClaudeTokens()[0];
|
||||
if (!token) {
|
||||
logger.debug('No Claude OAuth token available for usage check');
|
||||
return null;
|
||||
@@ -435,7 +481,8 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
const allTokens = getAllTokens();
|
||||
logger.debug({ tokenCount: allTokens.length }, 'fetchAllClaudeUsage called');
|
||||
if (allTokens.length === 0) {
|
||||
const token = getConfiguredClaudeTokens()[0];
|
||||
const credsToken = readCredentialsAccessToken(0);
|
||||
const token = credsToken || getConfiguredClaudeTokens()[0];
|
||||
if (!token) return [];
|
||||
const usage = await fetchUsageForToken(token, 0);
|
||||
return [
|
||||
@@ -451,7 +498,14 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
|
||||
const results: ClaudeAccountUsage[] = [];
|
||||
for (const t of allTokens) {
|
||||
const usage = await fetchUsageForToken(t.token, t.index);
|
||||
// Prefer the per-account credentials.json access token when present.
|
||||
// The static .env CLAUDE_CODE_OAUTH_TOKEN(S) values were issued without
|
||||
// the `user:profile` scope and so are rejected by /api/oauth/usage.
|
||||
// The credentials file is rewritten by `claude auth login` and refreshed
|
||||
// by token-refresh.ts, so it always carries the full scope set.
|
||||
const credsToken = readCredentialsAccessToken(t.index);
|
||||
const tokenForFetch = credsToken || t.token;
|
||||
const usage = await fetchUsageForToken(tokenForFetch, t.index);
|
||||
results.push({
|
||||
index: t.index,
|
||||
masked: t.masked,
|
||||
@@ -465,3 +519,103 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
|
||||
// Legacy alias
|
||||
export const fetchClaudeUsageViaCli = fetchClaudeUsage;
|
||||
|
||||
// ── Exhaustion gate ───────────────────────────────────────────────────────
|
||||
//
|
||||
// Threshold above which a Claude account is considered "out of budget" for
|
||||
// the purpose of admitting new turns. A turn is blocked only when *every*
|
||||
// Claude account is either at/over this threshold on either window or
|
||||
// already in a rate-limit cooldown.
|
||||
|
||||
export const CLAUDE_EXHAUSTION_THRESHOLD_PCT = 95;
|
||||
|
||||
export interface UsageExhaustionInfo {
|
||||
exhausted: boolean;
|
||||
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
||||
nextResetAt: string | null;
|
||||
}
|
||||
|
||||
/** Normalise the API's utilization field (which can be fractional or percent). */
|
||||
function normaliseUtilization(u: number | undefined): number {
|
||||
if (u == null || !Number.isFinite(u)) return 0;
|
||||
return u >= 0 && u < 1 ? u * 100 : u;
|
||||
}
|
||||
|
||||
function parseResetMs(s: string | undefined | null): number | null {
|
||||
if (!s) return null;
|
||||
const t = Date.parse(s);
|
||||
return Number.isFinite(t) ? t : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether every configured Claude account is either ≥ 95% on a
|
||||
* window or in a rate-limit cooldown, and — when so — the soonest moment at
|
||||
* which any one account is expected to recover.
|
||||
*
|
||||
* Reads the on-disk usage cache (refreshed every minute by the dashboard
|
||||
* renderer); performs no network I/O. Returns `exhausted: false` when there
|
||||
* are no configured tokens, or when usage data is missing for any account
|
||||
* (err on letting work through).
|
||||
*/
|
||||
export function getClaudeUsageExhaustion(): UsageExhaustionInfo {
|
||||
loadUsageDiskCache();
|
||||
const allTokens = getAllTokens();
|
||||
if (allTokens.length === 0) return { exhausted: false, nextResetAt: null };
|
||||
|
||||
let earliestRecoveryMs = Infinity;
|
||||
|
||||
for (const t of allTokens) {
|
||||
if (t.isRateLimited) {
|
||||
// Rate-limit recovery time is not exposed on the token snapshot here;
|
||||
// we treat it as "unknown but eventually". Skip from the min calc.
|
||||
continue;
|
||||
}
|
||||
const credsToken = readCredentialsAccessToken(t.index);
|
||||
const readKeys = getUsageCacheReadKeys(t.token, t.index, credsToken);
|
||||
let entry: UsageCacheEntry | undefined;
|
||||
for (const key of readKeys) {
|
||||
if (usageDiskCache[key]) {
|
||||
entry = usageDiskCache[key];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!entry) return { exhausted: false, nextResetAt: null };
|
||||
|
||||
const h5 = normaliseUtilization(entry.usage.five_hour?.utilization);
|
||||
const d7 = normaliseUtilization(entry.usage.seven_day?.utilization);
|
||||
if (Math.max(h5, d7) < CLAUDE_EXHAUSTION_THRESHOLD_PCT) {
|
||||
return { exhausted: false, nextResetAt: null };
|
||||
}
|
||||
|
||||
// This account is exhausted. It becomes available again when whichever
|
||||
// window(s) are over the threshold drop back below it. We approximate
|
||||
// that as max(reset of each window currently ≥ threshold).
|
||||
const h5ResetMs =
|
||||
h5 >= CLAUDE_EXHAUSTION_THRESHOLD_PCT
|
||||
? parseResetMs(entry.usage.five_hour?.resets_at)
|
||||
: null;
|
||||
const d7ResetMs =
|
||||
d7 >= CLAUDE_EXHAUSTION_THRESHOLD_PCT
|
||||
? parseResetMs(entry.usage.seven_day?.resets_at)
|
||||
: null;
|
||||
const candidates = [h5ResetMs, d7ResetMs].filter(
|
||||
(v): v is number => v != null,
|
||||
);
|
||||
if (candidates.length > 0) {
|
||||
const accountRecovery = Math.max(...candidates);
|
||||
if (accountRecovery < earliestRecoveryMs) {
|
||||
earliestRecoveryMs = accountRecovery;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
||||
? new Date(earliestRecoveryMs).toISOString()
|
||||
: null;
|
||||
return { exhausted: true, nextResetAt };
|
||||
}
|
||||
|
||||
/** Backwards-compat boolean wrapper. */
|
||||
export function isClaudeUsageExhausted(): boolean {
|
||||
return getClaudeUsageExhaustion().exhausted;
|
||||
}
|
||||
|
||||
@@ -31,10 +31,120 @@ export interface CodexUsageRefreshResult {
|
||||
/** Full scan interval — exported so the orchestrator can schedule it. */
|
||||
export const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
||||
|
||||
/**
|
||||
* Threshold above which a Codex account is considered "out of budget" for
|
||||
* the purpose of admitting new turns. A turn is blocked only when *every*
|
||||
* Codex account is either at/over this threshold on either window or
|
||||
* already in a rate-limit cooldown.
|
||||
*/
|
||||
export const CODEX_EXHAUSTION_THRESHOLD_PCT = 95;
|
||||
|
||||
export interface UsageExhaustionInfo {
|
||||
exhausted: boolean;
|
||||
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
||||
nextResetAt: string | null;
|
||||
}
|
||||
|
||||
function parseResetMs(s: string | undefined | null): number | null {
|
||||
if (!s) return null;
|
||||
const t = Date.parse(s);
|
||||
return Number.isFinite(t) ? t : null;
|
||||
}
|
||||
|
||||
/** Coerce a string-or-number resetsAt (numeric epoch in seconds) to ISO. */
|
||||
function toIsoMaybe(value: string | number | undefined): string | undefined {
|
||||
if (value == null) return undefined;
|
||||
if (typeof value === 'number') {
|
||||
return new Date(value * 1000).toISOString();
|
||||
}
|
||||
// Already a string — pass through if it parses, else drop.
|
||||
const ms = Date.parse(value);
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether every configured Codex account is either ≥ 95% on a
|
||||
* window or in a rate-limit cooldown, and — when so — the soonest moment
|
||||
* at which any one account is expected to recover.
|
||||
*
|
||||
* Reads the in-process rotation snapshot; performs no I/O. Returns
|
||||
* `exhausted: false` when there are no configured accounts, or when usage
|
||||
* data is missing for any account (err on letting work through).
|
||||
*/
|
||||
export function getCodexUsageExhaustion(): UsageExhaustionInfo {
|
||||
const accounts = getAllCodexAccounts();
|
||||
if (accounts.length === 0) return { exhausted: false, nextResetAt: null };
|
||||
|
||||
let earliestRecoveryMs = Infinity;
|
||||
|
||||
for (const a of accounts) {
|
||||
if (a.isRateLimited) continue; // counts as exhausted, recovery unknown here
|
||||
const h5 = a.cachedUsagePct;
|
||||
const d7 = a.cachedUsageD7Pct;
|
||||
// -1 / undefined means "unknown" → treat as headroom
|
||||
if (h5 == null || h5 < 0 || d7 == null || d7 < 0) {
|
||||
return { exhausted: false, nextResetAt: null };
|
||||
}
|
||||
if (Math.max(h5, d7) < CODEX_EXHAUSTION_THRESHOLD_PCT) {
|
||||
return { exhausted: false, nextResetAt: null };
|
||||
}
|
||||
|
||||
const h5ResetMs =
|
||||
h5 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetAt) : null;
|
||||
const d7ResetMs =
|
||||
d7 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetD7At) : null;
|
||||
const candidates = [h5ResetMs, d7ResetMs].filter(
|
||||
(v): v is number => v != null,
|
||||
);
|
||||
if (candidates.length > 0) {
|
||||
const accountRecovery = Math.max(...candidates);
|
||||
if (accountRecovery < earliestRecoveryMs) {
|
||||
earliestRecoveryMs = accountRecovery;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
||||
? new Date(earliestRecoveryMs).toISOString()
|
||||
: null;
|
||||
return { exhausted: true, nextResetAt };
|
||||
}
|
||||
|
||||
/** Backwards-compat boolean wrapper. */
|
||||
export function isCodexUsageExhausted(): boolean {
|
||||
return getCodexUsageExhaustion().exhausted;
|
||||
}
|
||||
|
||||
function getFnmCodexBinDirs(): string[] {
|
||||
const fnmRoot = path.join(os.homedir(), '.local', 'share', 'fnm');
|
||||
const dirs: string[] = [];
|
||||
// Prefer the alias `default` first if it exists.
|
||||
const defaultBin = path.join(fnmRoot, 'aliases', 'default', 'bin');
|
||||
if (fs.existsSync(defaultBin)) dirs.push(defaultBin);
|
||||
// Then fall back to scanning every installed node version.
|
||||
const versionsRoot = path.join(fnmRoot, 'node-versions');
|
||||
try {
|
||||
if (fs.existsSync(versionsRoot)) {
|
||||
for (const entry of fs.readdirSync(versionsRoot)) {
|
||||
const bin = path.join(versionsRoot, entry, 'installation', 'bin');
|
||||
if (fs.existsSync(bin)) dirs.push(bin);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
function getPreferredCodexPathEntries(): string[] {
|
||||
const entries = [
|
||||
path.dirname(process.execPath),
|
||||
path.join(os.homedir(), '.npm-global', 'bin'),
|
||||
// fnm-managed node installs (where `npm i -g @openai/codex` actually
|
||||
// lands when the host uses fnm). Without these, ejclaw running under
|
||||
// bun/systemd cannot find the `codex` binary even though the user has
|
||||
// it installed via fnm's default node.
|
||||
...getFnmCodexBinDirs(),
|
||||
];
|
||||
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
||||
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
||||
@@ -42,6 +152,18 @@ function getPreferredCodexPathEntries(): string[] {
|
||||
return [...new Set(entries)];
|
||||
}
|
||||
|
||||
function findCodexBinary(): string {
|
||||
const candidates = [
|
||||
path.join(os.homedir(), '.npm-global', 'bin', 'codex'),
|
||||
...getFnmCodexBinDirs().map((dir) => path.join(dir, 'codex')),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) return candidate;
|
||||
}
|
||||
// Last resort: rely on PATH (we extend it via getPreferredCodexPathEntries).
|
||||
return 'codex';
|
||||
}
|
||||
|
||||
function getCodexHomeForAccount(accountIndex?: number): string | null {
|
||||
const authPath = getCodexAuthPath(accountIndex);
|
||||
if (!authPath || !fs.existsSync(authPath)) return null;
|
||||
@@ -51,8 +173,7 @@ function getCodexHomeForAccount(accountIndex?: number): string | null {
|
||||
export async function fetchCodexUsage(
|
||||
codexHomeOverride?: string,
|
||||
): Promise<CodexRateLimit[] | null> {
|
||||
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
||||
const codexBin = fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
||||
const codexBin = findCodexBinary();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
@@ -197,20 +318,19 @@ export function applyCodexUsageToAccount(
|
||||
|
||||
const pct = Math.round(effective.primary.usedPercent);
|
||||
const d7Pct = Math.round(effective.secondary.usedPercent);
|
||||
const resetStr = effective.primary.resetsAt
|
||||
? formatResetRemaining(effective.primary.resetsAt)
|
||||
: undefined;
|
||||
const resetD7Str = effective.secondary.resetsAt
|
||||
? formatResetRemaining(effective.secondary.resetsAt)
|
||||
: undefined;
|
||||
updateCodexAccountUsage(pct, resetStr, accountIndex, d7Pct, resetD7Str);
|
||||
// Store raw ISO timestamps (not pre-formatted strings) so the exhaustion
|
||||
// gate can compute "minutes until reset" later. The dashboard render path
|
||||
// formats these at display time via `formatResetRemaining`.
|
||||
const resetIso = toIsoMaybe(effective.primary.resetsAt);
|
||||
const resetD7Iso = toIsoMaybe(effective.secondary.resetsAt);
|
||||
updateCodexAccountUsage(pct, resetIso, accountIndex, d7Pct, resetD7Iso);
|
||||
logger.info(
|
||||
{
|
||||
account: accountIndex + 1,
|
||||
bucket: effective.limitId,
|
||||
h5: pct,
|
||||
d7: d7Pct,
|
||||
reset: resetStr,
|
||||
reset: resetIso,
|
||||
},
|
||||
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
||||
);
|
||||
@@ -233,9 +353,9 @@ export function buildCodexUsageRowsFromState(): UsageRow[] {
|
||||
return {
|
||||
name: label,
|
||||
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
||||
h5reset: acct.resetAt || '',
|
||||
h5reset: acct.resetAt ? formatResetRemaining(acct.resetAt) : '',
|
||||
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
||||
d7reset: acct.resetD7At || '',
|
||||
d7reset: acct.resetD7At ? formatResetRemaining(acct.resetD7At) : '',
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -253,7 +253,7 @@ export function loadConfig(): AppConfig {
|
||||
status: {
|
||||
channelId: readText('STATUS_CHANNEL_ID') ?? '',
|
||||
updateInterval: 10000,
|
||||
usageUpdateInterval: 300000,
|
||||
usageUpdateInterval: 60000,
|
||||
showRooms: readBooleanUnlessFalse('STATUS_SHOW_ROOMS', true),
|
||||
showRoomDetails: readBooleanUnlessFalse('STATUS_SHOW_ROOM_DETAILS', true),
|
||||
usageDashboardEnabled: readText('USAGE_DASHBOARD') === 'true',
|
||||
|
||||
@@ -28,6 +28,7 @@ describe('listUnexpectedDataStateFiles', () => {
|
||||
fs.writeFileSync(path.join(dataDir, 'codex-warmup-state.json'), '{}');
|
||||
fs.writeFileSync(path.join(dataDir, 'claude-usage-cache.json'), '{}');
|
||||
fs.writeFileSync(path.join(dataDir, 'restart-context.abc.json'), '{}');
|
||||
fs.writeFileSync(path.join(dataDir, 'auto-paired-mode-state.json'), '{}');
|
||||
|
||||
expect(listUnexpectedDataStateFiles(dataDir)).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ const ALLOWED_DATA_JSON_PATTERNS = [
|
||||
/^codex-warmup-state\.json$/,
|
||||
/^claude-usage-cache\.json$/,
|
||||
/^restart-context\..+\.json$/,
|
||||
/^auto-paired-mode-state\.json$/,
|
||||
];
|
||||
|
||||
function isAllowedDataStateJson(filename: string): boolean {
|
||||
|
||||
@@ -2725,7 +2725,10 @@ describe('room assignment writes', () => {
|
||||
modeSource: 'inferred',
|
||||
name: 'Legacy SQL Room',
|
||||
folder: 'legacy-sql-room',
|
||||
ownerAgentType: 'codex',
|
||||
// Owner inference picks OWNER_AGENT_TYPE (env-driven, default codex but
|
||||
// currently claude-code in this deployment) when both agent types are
|
||||
// registered. See inferOwnerAgentTypeFromRegisteredAgentTypes.
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
expect(getEffectiveRoomMode('dc:legacy-sql')).toBe('tribunal');
|
||||
expect(getEffectiveRuntimeRoomMode('dc:legacy-sql')).toBe('tribunal');
|
||||
@@ -3341,7 +3344,8 @@ describe('paired room registration', () => {
|
||||
name: 'Room Settings Test',
|
||||
folder: 'room-settings-test',
|
||||
trigger: '@Codex',
|
||||
ownerAgentType: 'codex',
|
||||
// Both agent types registered → owner inference picks OWNER_AGENT_TYPE.
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
|
||||
setExplicitRoomMode('dc:room-settings', 'single');
|
||||
@@ -3353,7 +3357,7 @@ describe('paired room registration', () => {
|
||||
name: 'Room Settings Test',
|
||||
folder: 'room-settings-test',
|
||||
trigger: '@Codex',
|
||||
ownerAgentType: 'codex',
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
|
||||
updateRegisteredGroupName('dc:room-settings', 'Room Settings Renamed');
|
||||
@@ -3365,7 +3369,7 @@ describe('paired room registration', () => {
|
||||
name: 'Room Settings Renamed',
|
||||
folder: 'room-settings-test',
|
||||
trigger: '@Codex',
|
||||
ownerAgentType: 'codex',
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ export {
|
||||
enforceMemoryBounds,
|
||||
expireStaleMemories,
|
||||
getAllChats,
|
||||
getEarliestUnansweredHumanSeq,
|
||||
getLastHumanMessageContent,
|
||||
getLastHumanMessageSender,
|
||||
getLastHumanMessageTimestamp,
|
||||
@@ -22,6 +23,7 @@ export {
|
||||
getOpenWorkItem,
|
||||
getOpenWorkItemForChat,
|
||||
getRecentChatMessages,
|
||||
getRecentDeliveredOwnerWorkItemsForChat,
|
||||
hasRecentRestartAnnouncement,
|
||||
markWorkItemDelivered,
|
||||
markWorkItemDeliveryRetry,
|
||||
|
||||
@@ -332,6 +332,38 @@ export function getRecentChatMessagesFromDatabase(
|
||||
return rows.map(normalizeMessageRow);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the seq of the earliest unanswered human message in this chat —
|
||||
* i.e. the first user message that came after the most recent bot reply.
|
||||
* Returns null if no such message exists (the user is "caught up").
|
||||
*
|
||||
* Used by restart-recovery to rewind the agent cursor before re-running an
|
||||
* interrupted turn from the message that initiated it.
|
||||
*/
|
||||
export function getEarliestUnansweredHumanSeqFromDatabase(
|
||||
database: Database,
|
||||
chatJid: string,
|
||||
): number | null {
|
||||
const lastBot = database
|
||||
.prepare(
|
||||
`SELECT MAX(seq) AS seq FROM messages
|
||||
WHERE chat_jid = ? AND is_bot_message = 1`,
|
||||
)
|
||||
.get(chatJid) as { seq: number | null } | undefined;
|
||||
const lastBotSeq = lastBot?.seq ?? 0;
|
||||
const row = database
|
||||
.prepare(
|
||||
`SELECT MIN(seq) AS seq FROM messages
|
||||
WHERE chat_jid = ?
|
||||
AND seq > ?
|
||||
AND is_bot_message = 0
|
||||
AND is_from_me = 0
|
||||
AND content != '' AND content IS NOT NULL`,
|
||||
)
|
||||
.get(chatJid, lastBotSeq) as { seq: number | null } | undefined;
|
||||
return row?.seq ?? null;
|
||||
}
|
||||
|
||||
export function getLastHumanMessageTimestampFromDatabase(
|
||||
database: Database,
|
||||
chatJid: string,
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { Database } from 'bun:sqlite';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
import { parseVisibleVerdict } from '../paired-verdict.js';
|
||||
import {
|
||||
parseReviewerVisibleVerdict,
|
||||
parseVisibleVerdict,
|
||||
} from '../paired-verdict.js';
|
||||
import { PairedRoomRole, PairedTurnOutput } from '../types.js';
|
||||
|
||||
const MAX_TURN_OUTPUT_CHARS = 50_000;
|
||||
@@ -38,7 +41,9 @@ export function insertPairedTurnOutputInDatabase(
|
||||
turnNumber,
|
||||
role,
|
||||
outputText.slice(0, MAX_TURN_OUTPUT_CHARS),
|
||||
parseVisibleVerdict(outputText),
|
||||
role === 'reviewer'
|
||||
? parseReviewerVisibleVerdict(outputText)
|
||||
: parseVisibleVerdict(outputText),
|
||||
createdAt ?? new Date().toISOString(),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
import {
|
||||
type ChatInfo,
|
||||
getAllChatsFromDatabase,
|
||||
getEarliestUnansweredHumanSeqFromDatabase,
|
||||
getLastHumanMessageContentFromDatabase,
|
||||
getLastHumanMessageSenderFromDatabase,
|
||||
getLastHumanMessageTimestampFromDatabase,
|
||||
@@ -36,6 +37,7 @@ import {
|
||||
createProducedWorkItemInDatabase,
|
||||
getOpenWorkItemForChatFromDatabase,
|
||||
getOpenWorkItemFromDatabase,
|
||||
getRecentDeliveredOwnerWorkItemsForChatFromDatabase,
|
||||
markWorkItemDeliveredInDatabase,
|
||||
markWorkItemDeliveryRetryInDatabase,
|
||||
} from './work-items.js';
|
||||
@@ -223,6 +225,10 @@ export function getLastHumanMessageTimestamp(chatJid: string): string | null {
|
||||
return getLastHumanMessageTimestampFromDatabase(requireDatabase(), chatJid);
|
||||
}
|
||||
|
||||
export function getEarliestUnansweredHumanSeq(chatJid: string): number | null {
|
||||
return getEarliestUnansweredHumanSeqFromDatabase(requireDatabase(), chatJid);
|
||||
}
|
||||
|
||||
export function getLastHumanMessageSender(chatJid: string): string | null {
|
||||
return getLastHumanMessageSenderFromDatabase(requireDatabase(), chatJid);
|
||||
}
|
||||
@@ -266,6 +272,17 @@ export function getOpenWorkItemForChat(
|
||||
);
|
||||
}
|
||||
|
||||
export function getRecentDeliveredOwnerWorkItemsForChat(
|
||||
chatJid: string,
|
||||
limit: number,
|
||||
): WorkItem[] {
|
||||
return getRecentDeliveredOwnerWorkItemsForChatFromDatabase(
|
||||
requireDatabase(),
|
||||
chatJid,
|
||||
limit,
|
||||
);
|
||||
}
|
||||
|
||||
export function createProducedWorkItem(
|
||||
input: CreateProducedWorkItemInput,
|
||||
): WorkItem {
|
||||
|
||||
@@ -199,6 +199,34 @@ export function getOpenWorkItemFromDatabase(
|
||||
return row ? hydrateWorkItemRow(row) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recent delivered owner-side work items for a chat, newest last.
|
||||
* Used by single-mode prompt building to surface the bot's own prior
|
||||
* answers (which only live in work_items.result_payload) so a new turn
|
||||
* can reference what it previously said.
|
||||
*/
|
||||
export function getRecentDeliveredOwnerWorkItemsForChatFromDatabase(
|
||||
database: Database,
|
||||
chatJid: string,
|
||||
limit: number,
|
||||
): WorkItem[] {
|
||||
if (limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
const rows = database
|
||||
.prepare(
|
||||
`SELECT *
|
||||
FROM work_items
|
||||
WHERE chat_jid = ?
|
||||
AND status = 'delivered'
|
||||
AND (delivery_role = 'owner' OR delivery_role IS NULL)
|
||||
ORDER BY id DESC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(chatJid, limit) as StoredWorkItemRow[];
|
||||
return rows.map(hydrateWorkItemRow).reverse();
|
||||
}
|
||||
|
||||
export function getOpenWorkItemForChatFromDatabase(
|
||||
database: Database,
|
||||
chatJid: string,
|
||||
|
||||
66
src/index.ts
66
src/index.ts
@@ -21,6 +21,7 @@ import { writeGroupsSnapshot } from './agent-runner.js';
|
||||
import {
|
||||
type AssignRoomInput,
|
||||
getAllTasks,
|
||||
getEarliestUnansweredHumanSeq,
|
||||
hasRecentRestartAnnouncement,
|
||||
initDatabase,
|
||||
storeChatMetadata,
|
||||
@@ -55,6 +56,7 @@ import {
|
||||
import { createMessageRuntime } from './message-runtime.js';
|
||||
import { nudgeSchedulerLoop, startSchedulerLoop } from './task-scheduler.js';
|
||||
import { startUnifiedDashboard } from './unified-dashboard.js';
|
||||
import { startUsagePrimer } from './usage-primer.js';
|
||||
import { startWebDashboardServer } from './web-dashboard-server.js';
|
||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||
import { logger } from './logger.js';
|
||||
@@ -270,14 +272,29 @@ async function main(): Promise<void> {
|
||||
status: 'processing' | 'waiting';
|
||||
} => status.status !== 'inactive',
|
||||
)
|
||||
.map((status) => ({
|
||||
chatJid: status.jid,
|
||||
groupName: roomBindings[status.jid]?.name || status.jid,
|
||||
status: status.status,
|
||||
elapsedMs: status.elapsedMs,
|
||||
pendingMessages: status.pendingMessages,
|
||||
pendingTasks: status.pendingTasks,
|
||||
}));
|
||||
.map((status) => {
|
||||
// Capture the seq of the earliest unanswered human message for this
|
||||
// chat so the next startup can rewind the agent cursor and re-run
|
||||
// the interrupted turn automatically.
|
||||
let rewindToSeq: number | null = null;
|
||||
try {
|
||||
rewindToSeq = getEarliestUnansweredHumanSeq(status.jid);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, chatJid: status.jid },
|
||||
'Failed to capture rewind seq for shutdown snapshot',
|
||||
);
|
||||
}
|
||||
return {
|
||||
chatJid: status.jid,
|
||||
groupName: roomBindings[status.jid]?.name || status.jid,
|
||||
status: status.status,
|
||||
elapsedMs: status.elapsedMs,
|
||||
pendingMessages: status.pendingMessages,
|
||||
pendingTasks: status.pendingTasks,
|
||||
rewindToSeq,
|
||||
};
|
||||
});
|
||||
const writtenPaths = writeShutdownRestartContext(
|
||||
roomBindings,
|
||||
interruptedGroups,
|
||||
@@ -488,6 +505,37 @@ async function main(): Promise<void> {
|
||||
restartContext,
|
||||
roomBindings,
|
||||
)) {
|
||||
// Auto-resume: rewind the agent cursor to just before the earliest
|
||||
// unanswered human message so the missed-message processor picks it up
|
||||
// again. Claude/Codex sessions are persisted by group folder, so the
|
||||
// agent will resume mid-turn rather than restarting from scratch.
|
||||
if (candidate.rewindToSeq != null && candidate.rewindToSeq > 0) {
|
||||
const cursors = runtimeState.getLastAgentTimestamps();
|
||||
const previousCursor = cursors[candidate.chatJid];
|
||||
const targetCursor = String(candidate.rewindToSeq - 1);
|
||||
// Only rewind if the current cursor is past the target — never advance.
|
||||
const previousNum = previousCursor
|
||||
? Number.parseInt(previousCursor, 10)
|
||||
: 0;
|
||||
if (
|
||||
Number.isFinite(previousNum) &&
|
||||
previousNum >= candidate.rewindToSeq
|
||||
) {
|
||||
cursors[candidate.chatJid] = targetCursor;
|
||||
runtimeState.saveState();
|
||||
logger.info(
|
||||
{
|
||||
chatJid: candidate.chatJid,
|
||||
groupFolder: candidate.groupFolder,
|
||||
previousCursor,
|
||||
rewoundTo: targetCursor,
|
||||
rewindToSeq: candidate.rewindToSeq,
|
||||
},
|
||||
'Rewound agent cursor for interrupted-turn auto-resume',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
queue.enqueueMessageCheck(
|
||||
candidate.chatJid,
|
||||
resolveGroupIpcPath(candidate.groupFolder),
|
||||
@@ -499,6 +547,7 @@ async function main(): Promise<void> {
|
||||
status: candidate.status,
|
||||
pendingMessages: candidate.pendingMessages,
|
||||
pendingTasks: candidate.pendingTasks,
|
||||
rewindToSeq: candidate.rewindToSeq,
|
||||
},
|
||||
'Queued interrupted group for restart recovery',
|
||||
);
|
||||
@@ -522,6 +571,7 @@ async function main(): Promise<void> {
|
||||
purgeOnStart: true,
|
||||
});
|
||||
webDashboardServer = startWebDashboardServer(WEB_DASHBOARD);
|
||||
startUsagePrimer();
|
||||
|
||||
leaseRecoveryTimer = setInterval(() => {
|
||||
const failover = getGlobalFailoverInfo();
|
||||
|
||||
@@ -200,6 +200,12 @@ export async function executeMessageAgentAttemptLifecycle(args: {
|
||||
|
||||
const freshAttempt = await runTrackedAttempt('claude');
|
||||
if (!isRetryableClaudeSessionFailure(freshAttempt)) {
|
||||
// The session-poisoning that triggered this recovery was handled by the
|
||||
// clearStoredSession() above, and the fresh attempt has already
|
||||
// persisted its own session_id via onPersistSession. Reset the outer
|
||||
// flag so that finalizePrimaryAttempt won't wipe the freshly persisted
|
||||
// session based on the previous attempt's stale failure state.
|
||||
resetSessionRequested = freshAttempt.resetSessionRequested === true;
|
||||
return { attempt: freshAttempt, resolved: null };
|
||||
}
|
||||
|
||||
@@ -251,6 +257,11 @@ export async function executeMessageAgentAttemptLifecycle(args: {
|
||||
|
||||
const freshAttempt = await runTrackedAttempt('codex');
|
||||
if (!isRetryableCodexSessionFailure(freshAttempt)) {
|
||||
// See the matching Claude recovery branch above: clear the outer
|
||||
// reset-session flag so finalizePrimaryAttempt doesn't wipe the
|
||||
// freshly persisted Codex session based on the previous attempt's
|
||||
// stale failure state.
|
||||
resetSessionRequested = freshAttempt.resetSessionRequested === true;
|
||||
return { attempt: freshAttempt, resolved: null };
|
||||
}
|
||||
|
||||
|
||||
@@ -355,7 +355,10 @@ export function createPairedExecutionLifecycle(args: {
|
||||
requiresVisibleVerdict &&
|
||||
(!pairedFinalOutput || pairedFinalOutput.length === 0);
|
||||
if (missingVisibleVerdict) {
|
||||
pairedExecutionSummary = missingVisibleVerdictSummary;
|
||||
const previousSummary = pairedExecutionSummary?.trim();
|
||||
pairedExecutionSummary = previousSummary
|
||||
? `${previousSummary}\n\n${missingVisibleVerdictSummary}`
|
||||
: missingVisibleVerdictSummary;
|
||||
log.warn(
|
||||
{
|
||||
pairedTaskId: pairedExecutionContext?.task.id ?? null,
|
||||
|
||||
@@ -1190,7 +1190,9 @@ describe('runAgentForGroup room memory', () => {
|
||||
taskId: 'paired-task-review-no-verdict',
|
||||
role: 'reviewer',
|
||||
status: 'failed',
|
||||
summary: 'Execution completed without a visible terminal verdict.',
|
||||
summary: expect.stringContaining(
|
||||
'Execution completed without a visible terminal verdict.',
|
||||
),
|
||||
}),
|
||||
);
|
||||
expect(db.failPairedTurn).toHaveBeenCalledWith({
|
||||
@@ -1202,7 +1204,9 @@ describe('runAgentForGroup room memory', () => {
|
||||
intentKind: 'reviewer-turn',
|
||||
role: 'reviewer',
|
||||
},
|
||||
error: 'Execution completed without a visible terminal verdict.',
|
||||
error: expect.stringContaining(
|
||||
'Execution completed without a visible terminal verdict.',
|
||||
),
|
||||
});
|
||||
expect(db.completePairedTurn).not.toHaveBeenCalled();
|
||||
expect(db.insertPairedTurnOutput).not.toHaveBeenCalled();
|
||||
@@ -3459,6 +3463,14 @@ describe('runAgentForGroup Claude rotation', () => {
|
||||
expect(result).toBe('success');
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||
expect(deps.clearSession).toHaveBeenCalledWith('test-claude');
|
||||
// The session must be cleared exactly once (before the fresh retry).
|
||||
// The fresh retry's newSessionId must end up persisted so the NEXT
|
||||
// user turn can resume that session.
|
||||
expect(deps.clearSession).toHaveBeenCalledTimes(1);
|
||||
expect(deps.persistSession).toHaveBeenCalledWith(
|
||||
'test-claude',
|
||||
'fresh-session-id',
|
||||
);
|
||||
});
|
||||
|
||||
it('drops a poisoned Codex session id before retrying a fresh session after remote compaction failure', async () => {
|
||||
@@ -3487,9 +3499,13 @@ describe('runAgentForGroup Claude rotation', () => {
|
||||
vi.mocked(sessionRecovery.shouldRetryFreshCodexSessionOnAgentFailure)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(false);
|
||||
vi.mocked(sessionRecovery.shouldResetCodexSessionOnAgentFailure)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(false);
|
||||
// The first attempt throws synchronously, so streaming evaluation never
|
||||
// runs on its (non-existent) output — only the fresh retry's success
|
||||
// output reaches shouldResetCodexSessionOnAgentFailure, and that output
|
||||
// does not match any reset pattern.
|
||||
vi.mocked(
|
||||
sessionRecovery.shouldResetCodexSessionOnAgentFailure,
|
||||
).mockReturnValue(false);
|
||||
|
||||
vi.mocked(agentRunner.runAgentProcess)
|
||||
.mockImplementationOnce(async (_group, input) => {
|
||||
@@ -3523,6 +3539,13 @@ describe('runAgentForGroup Claude rotation', () => {
|
||||
expect(result).toBe('success');
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||
expect(deps.clearSession).toHaveBeenCalledWith('test-codex');
|
||||
// Same invariant as Claude: clearSession runs exactly once and the
|
||||
// fresh retry's newSessionId is persisted for the next turn.
|
||||
expect(deps.clearSession).toHaveBeenCalledTimes(1);
|
||||
expect(deps.persistSession).toHaveBeenCalledWith(
|
||||
'test-codex',
|
||||
'fresh-codex-session-id',
|
||||
);
|
||||
});
|
||||
|
||||
it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => {
|
||||
|
||||
@@ -9,9 +9,14 @@ vi.mock('./session-commands.js', () => ({
|
||||
handleSessionCommand: handleSessionCommandMock,
|
||||
}));
|
||||
|
||||
import { handleQueuedRunGates } from './message-runtime-gating.js';
|
||||
import {
|
||||
exhaustionChecks,
|
||||
handleQueuedRunGates,
|
||||
} from './message-runtime-gating.js';
|
||||
|
||||
describe('message-runtime-gating', () => {
|
||||
const sendMessage = vi.fn();
|
||||
|
||||
const baseArgs = {
|
||||
chatJid: 'room-1',
|
||||
group: {
|
||||
@@ -20,17 +25,21 @@ describe('message-runtime-gating', () => {
|
||||
isMain: false,
|
||||
trigger: '코덱스',
|
||||
added_at: new Date().toISOString(),
|
||||
agentType: 'claude-code' as const,
|
||||
} satisfies RegisteredGroup,
|
||||
runId: 'run-1',
|
||||
missedMessages: [],
|
||||
triggerPattern: /^코덱스/,
|
||||
timezone: 'Asia/Seoul',
|
||||
hasImplicitContinuationWindow: () => false,
|
||||
sessionCommandDeps: {} as never,
|
||||
sessionCommandDeps: { sendMessage } as never,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
handleSessionCommandMock.mockReset();
|
||||
sendMessage.mockReset();
|
||||
exhaustionChecks.claude = () => ({ exhausted: false, nextResetAt: null });
|
||||
exhaustionChecks.codex = () => ({ exhausted: false, nextResetAt: null });
|
||||
});
|
||||
|
||||
it('returns session-command results directly when a command is handled', async () => {
|
||||
@@ -42,13 +51,79 @@ describe('message-runtime-gating', () => {
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
|
||||
expect(result).toEqual({ handled: true, success: false });
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls through when no session command is handled', async () => {
|
||||
it('falls through when no session command is handled and accounts have headroom', async () => {
|
||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
|
||||
expect(result).toEqual({ handled: false });
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('blocks the turn and notifies the channel when Claude is exhausted', async () => {
|
||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||
const resetAt = new Date(Date.now() + 90 * 60_000).toISOString(); // ~1h30m
|
||||
exhaustionChecks.claude = () => ({
|
||||
exhausted: true,
|
||||
nextResetAt: resetAt,
|
||||
});
|
||||
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
|
||||
expect(result).toEqual({ handled: true, success: true });
|
||||
expect(sendMessage).toHaveBeenCalledTimes(1);
|
||||
const text = sendMessage.mock.calls[0][0];
|
||||
expect(text).toMatch(/토큰 부족으로 종료됨/);
|
||||
expect(text).toMatch(/Claude/);
|
||||
expect(text).toMatch(/초기화 예정/);
|
||||
expect(text).toMatch(/1시간/);
|
||||
});
|
||||
|
||||
it('blocks the turn and notifies the channel when Codex is exhausted', async () => {
|
||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||
const resetAt = new Date(Date.now() + 25 * 60_000).toISOString(); // ~25m
|
||||
exhaustionChecks.codex = () => ({
|
||||
exhausted: true,
|
||||
nextResetAt: resetAt,
|
||||
});
|
||||
|
||||
const result = await handleQueuedRunGates({
|
||||
...baseArgs,
|
||||
group: { ...baseArgs.group, agentType: 'codex' },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ handled: true, success: true });
|
||||
expect(sendMessage).toHaveBeenCalledTimes(1);
|
||||
const text = sendMessage.mock.calls[0][0];
|
||||
expect(text).toMatch(/Codex/);
|
||||
expect(text).toMatch(/약 \d+분 뒤 초기화 예정/);
|
||||
});
|
||||
|
||||
it('omits countdown when no reset time is known', async () => {
|
||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||
exhaustionChecks.claude = () => ({ exhausted: true, nextResetAt: null });
|
||||
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
|
||||
expect(result).toEqual({ handled: true, success: true });
|
||||
const text = sendMessage.mock.calls[0][0];
|
||||
expect(text).toMatch(/토큰 부족으로 종료됨/);
|
||||
expect(text).not.toMatch(/초기화 예정/);
|
||||
});
|
||||
|
||||
it('does not cross-block: Codex exhaustion does not stop a Claude room', async () => {
|
||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||
exhaustionChecks.codex = () => ({
|
||||
exhausted: true,
|
||||
nextResetAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
});
|
||||
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
|
||||
expect(result).toEqual({ handled: false });
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,80 @@
|
||||
import {
|
||||
CLAUDE_EXHAUSTION_THRESHOLD_PCT,
|
||||
getClaudeUsageExhaustion,
|
||||
} from './claude-usage.js';
|
||||
import {
|
||||
CODEX_EXHAUSTION_THRESHOLD_PCT,
|
||||
getCodexUsageExhaustion,
|
||||
} from './codex-usage-collector.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
handleSessionCommand,
|
||||
type SessionCommandDeps,
|
||||
} from './session-commands.js';
|
||||
import type { NewMessage, RegisteredGroup } from './types.js';
|
||||
import type { AgentType, NewMessage, RegisteredGroup } from './types.js';
|
||||
|
||||
/**
|
||||
* Test seam — overridable from unit tests so we don't need a real disk
|
||||
* cache or rotation state to exercise the exhaustion gate.
|
||||
*/
|
||||
export const exhaustionChecks = {
|
||||
claude: getClaudeUsageExhaustion,
|
||||
codex: getCodexUsageExhaustion,
|
||||
};
|
||||
|
||||
/** Render an ISO timestamp as "약 N분 뒤" / "약 Nh Nm 뒤" relative to now. */
|
||||
function formatResetCountdown(iso: string | null): string | null {
|
||||
if (!iso) return null;
|
||||
const target = Date.parse(iso);
|
||||
if (!Number.isFinite(target)) return null;
|
||||
const diffMs = target - Date.now();
|
||||
if (diffMs <= 0) return '곧';
|
||||
const totalMinutes = Math.ceil(diffMs / 60_000);
|
||||
if (totalMinutes < 60) return `약 ${totalMinutes}분 뒤`;
|
||||
const hours = Math.floor(totalMinutes / 60);
|
||||
const minutes = totalMinutes % 60;
|
||||
if (hours < 24) {
|
||||
return minutes > 0
|
||||
? `약 ${hours}시간 ${minutes}분 뒤`
|
||||
: `약 ${hours}시간 뒤`;
|
||||
}
|
||||
const days = Math.floor(hours / 24);
|
||||
const remH = hours % 24;
|
||||
return remH > 0 ? `약 ${days}일 ${remH}시간 뒤` : `약 ${days}일 뒤`;
|
||||
}
|
||||
|
||||
function exhaustionMessageFor(
|
||||
agentType: AgentType,
|
||||
nextResetAt: string | null,
|
||||
): string {
|
||||
const label = agentType === 'codex' ? 'Codex' : 'Claude';
|
||||
const pct =
|
||||
agentType === 'codex'
|
||||
? CODEX_EXHAUSTION_THRESHOLD_PCT
|
||||
: CLAUDE_EXHAUSTION_THRESHOLD_PCT;
|
||||
const countdown = formatResetCountdown(nextResetAt);
|
||||
const head = `토큰 부족으로 종료됨 (${label} 모든 계정 사용량 ${pct}% 초과)`;
|
||||
return countdown ? `${head} — ${countdown} 초기화 예정` : head;
|
||||
}
|
||||
|
||||
// ── Usage-window alignment gap ──
|
||||
// Bot primer fires at 08/13/18/23 KST to anchor 5h windows. The 04:00–08:00
|
||||
// KST gap before the day's first primer must be free of inference calls,
|
||||
// otherwise an off-cycle window starts and the daily reset alignment drifts.
|
||||
// During the gap, send an auto-reply instead of running the model.
|
||||
// Urgent bypass: any message containing an @-mention (bot's trigger pattern)
|
||||
// is processed anyway — user explicitly tagged the bot, accept the drift.
|
||||
const GAP_START_HOUR_KST = 4;
|
||||
const GAP_END_HOUR_KST = 8;
|
||||
const KST_OFFSET_MS = 9 * 60 * 60 * 1000;
|
||||
|
||||
export function isInUsageAlignmentGap(nowMs: number = Date.now()): boolean {
|
||||
const kstHour = new Date(nowMs + KST_OFFSET_MS).getUTCHours();
|
||||
return kstHour >= GAP_START_HOUR_KST && kstHour < GAP_END_HOUR_KST;
|
||||
}
|
||||
|
||||
const GAP_AUTO_REPLY =
|
||||
'사용량 정렬 시간(04~08 KST)입니다. 08시 이후 다시 보내주세요. 긴급한 경우 @태그하면 처리됩니다.';
|
||||
|
||||
export async function handleQueuedRunGates(args: {
|
||||
chatJid: string;
|
||||
@@ -31,5 +102,57 @@ export async function handleQueuedRunGates(args: {
|
||||
return cmdResult;
|
||||
}
|
||||
|
||||
// ── Usage-window alignment gap gate ──
|
||||
if (isInUsageAlignmentGap()) {
|
||||
const hasUrgentMention = args.missedMessages.some((m) =>
|
||||
args.triggerPattern.test(m.content),
|
||||
);
|
||||
if (!hasUrgentMention) {
|
||||
logger.info(
|
||||
{
|
||||
chatJid: args.chatJid,
|
||||
groupName: args.group.name,
|
||||
runId: args.runId,
|
||||
},
|
||||
'Blocking new turn — usage alignment gap (04-08 KST)',
|
||||
);
|
||||
try {
|
||||
await args.sessionCommandDeps.sendMessage(GAP_AUTO_REPLY);
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Failed to send alignment gap notice');
|
||||
}
|
||||
return { handled: true, success: true };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Usage-exhaustion gate ──
|
||||
// Block new turns when the agent provider that would handle this room has
|
||||
// every account at ≥95% on either window (or rate-limited). In-flight
|
||||
// runs are not affected; this only stops *new* turns from starting.
|
||||
const agentType: AgentType = args.group.agentType ?? 'claude-code';
|
||||
const info =
|
||||
agentType === 'codex'
|
||||
? exhaustionChecks.codex()
|
||||
: exhaustionChecks.claude();
|
||||
if (info.exhausted) {
|
||||
const text = exhaustionMessageFor(agentType, info.nextResetAt);
|
||||
logger.warn(
|
||||
{
|
||||
chatJid: args.chatJid,
|
||||
groupName: args.group.name,
|
||||
agentType,
|
||||
runId: args.runId,
|
||||
nextResetAt: info.nextResetAt,
|
||||
},
|
||||
'Blocking new turn — all accounts exhausted',
|
||||
);
|
||||
try {
|
||||
await args.sessionCommandDeps.sendMessage(text);
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Failed to send exhaustion notice');
|
||||
}
|
||||
return { handled: true, success: true };
|
||||
}
|
||||
|
||||
return { handled: false };
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { getPairedTurnOutputs, getRecentChatMessages } from './db.js';
|
||||
import {
|
||||
getPairedTurnOutputs,
|
||||
getRecentChatMessages,
|
||||
getRecentDeliveredOwnerWorkItemsForChat,
|
||||
} from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
buildArbiterPromptForTask,
|
||||
@@ -270,10 +274,31 @@ export async function runQueuedGroupTurn(args: {
|
||||
turnOutputs,
|
||||
});
|
||||
} else {
|
||||
prompt = args.formatMessages(
|
||||
// Single-mode (or paired-not-yet-activated) fallback.
|
||||
// The messages table only stores human messages, so the bot's own prior
|
||||
// answers (which live in work_items.result_payload) are otherwise invisible
|
||||
// to the next turn. Prepend the most recent delivered owner answers so the
|
||||
// model can reference what it previously said (e.g. "translate your last
|
||||
// answer to Korean").
|
||||
const recentBotAnswers = getRecentDeliveredOwnerWorkItemsForChat(
|
||||
chatJid,
|
||||
3,
|
||||
);
|
||||
const formattedNew = args.formatMessages(
|
||||
args.labelPairedSenders(chatJid, missedMessages),
|
||||
args.timezone,
|
||||
);
|
||||
if (recentBotAnswers.length === 0) {
|
||||
prompt = formattedNew;
|
||||
} else {
|
||||
const priorBlock = recentBotAnswers
|
||||
.map((item, idx) => {
|
||||
const label = `Prior bot answer ${idx + 1}/${recentBotAnswers.length} (delivered ${item.delivered_at ?? item.updated_at})`;
|
||||
return `${label}:\n${item.result_payload}`;
|
||||
})
|
||||
.join('\n\n');
|
||||
prompt = `[Recent prior answers from you in this channel — for context only, do not re-send]\n${priorBlock}\n\n[New messages to respond to]\n${formattedNew}`;
|
||||
}
|
||||
}
|
||||
|
||||
const startSeq = missedMessages[0].seq ?? null;
|
||||
|
||||
@@ -56,7 +56,27 @@ import {
|
||||
} from './types.js';
|
||||
import { createScopedLogger, logger } from './logger.js';
|
||||
import { hasReviewerLease } from './service-routing.js';
|
||||
import { resolveGroupFolderPath } from './group-folder.js';
|
||||
import type { WorkItem } from './db/work-items.js';
|
||||
|
||||
/**
|
||||
* Build the allow-list of base directories under which an outgoing message's
|
||||
* attachments may live. Each room is sandboxed to its own group folder
|
||||
* (`groups/<folder>`) by default; if an explicit `workDir` is registered, that
|
||||
* is added too. This way a screenshot the agent saved under
|
||||
* `groups/<folder>/repo/.artifacts/...` is allowed even when the room has no
|
||||
* workDir configured, while remaining isolated from sibling rooms.
|
||||
*/
|
||||
function getGroupAttachmentBaseDirs(group: RegisteredGroup): string[] {
|
||||
const dirs: string[] = [];
|
||||
try {
|
||||
dirs.push(resolveGroupFolderPath(group.folder));
|
||||
} catch {
|
||||
// Invalid folder names should not block delivery; fall through.
|
||||
}
|
||||
if (group.workDir) dirs.push(group.workDir);
|
||||
return dirs;
|
||||
}
|
||||
export {
|
||||
resolveHandoffCursorKey,
|
||||
resolveHandoffRoleOverride,
|
||||
@@ -255,7 +275,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
channel,
|
||||
item: workItem,
|
||||
log: logger,
|
||||
attachmentBaseDirs: group.workDir ? [group.workDir] : undefined,
|
||||
attachmentBaseDirs: getGroupAttachmentBaseDirs(group),
|
||||
replaceMessageId,
|
||||
isDuplicateOfLastBotFinal: checkDuplicateOfLastBotFinal,
|
||||
openContinuation: (targetChatJid) =>
|
||||
@@ -458,7 +478,7 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
channel,
|
||||
roleToChannel,
|
||||
log,
|
||||
attachmentBaseDirs: group.workDir ? [group.workDir] : undefined,
|
||||
attachmentBaseDirs: getGroupAttachmentBaseDirs(group),
|
||||
isPairedRoom: hasReviewerLease(chatJid),
|
||||
getMissingRoleChannelMessage,
|
||||
isDuplicateOfLastBotFinal: checkDuplicateOfLastBotFinal,
|
||||
|
||||
@@ -802,4 +802,30 @@ describe('MessageTurnController outbound audit logging', () => {
|
||||
replaceMessageId: 'progress-1',
|
||||
});
|
||||
});
|
||||
|
||||
it('publishes a failure final when an error finishes before any visible output', async () => {
|
||||
const channel = makeChannel();
|
||||
const deliverFinalText = vi.fn().mockResolvedValue(true);
|
||||
const controller = new MessageTurnController({
|
||||
chatJid: 'dc:test-room',
|
||||
group: makeGroup(),
|
||||
runId: 'run-silent-error-final',
|
||||
channel,
|
||||
idleTimeout: 1_000,
|
||||
failureFinalText: '실패',
|
||||
isClaudeCodeAgent: true,
|
||||
clearSession: vi.fn(),
|
||||
requestClose: vi.fn(),
|
||||
deliverFinalText,
|
||||
deliveryRole: 'owner',
|
||||
});
|
||||
|
||||
await controller.start();
|
||||
const finishResult = await controller.finish('error');
|
||||
|
||||
expect(finishResult.visiblePhase).toBe('final');
|
||||
expect(deliverFinalText).toHaveBeenCalledWith('실패', {
|
||||
replaceMessageId: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -379,6 +379,12 @@ export class MessageTurnController {
|
||||
this.hadError
|
||||
) {
|
||||
await this.publishFailureFinal();
|
||||
} else if (
|
||||
outputStatus === 'error' &&
|
||||
this.visiblePhase === 'silent' &&
|
||||
!this.terminalObserved()
|
||||
) {
|
||||
await this.publishFailureFinal();
|
||||
}
|
||||
|
||||
this.clearProgressTicker();
|
||||
|
||||
@@ -12,12 +12,14 @@ import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
||||
import {
|
||||
applyPairedTaskPatch,
|
||||
hasCodeChangesSinceRef,
|
||||
isUserInputWaitVerdict,
|
||||
parseVisibleVerdict,
|
||||
requestArbiterOrEscalate,
|
||||
resolveOwnerCompletionSignal,
|
||||
resolveCanonicalSourceRef,
|
||||
transitionPairedTaskStatus,
|
||||
} from './paired-execution-context-shared.js';
|
||||
import { shouldSkipReviewerForTrivialTurn } from './paired-trivial-turn-detector.js';
|
||||
import type { PairedTask } from './types.js';
|
||||
|
||||
type OwnerFinalizeOutcome = 'stop' | 're_review';
|
||||
@@ -110,6 +112,29 @@ function handleOwnerFinalizeCompletion(args: {
|
||||
ownerVerdict === 'step_done' && hasNewChanges === false
|
||||
? (task.empty_step_done_streak ?? 0) + 1
|
||||
: 0;
|
||||
|
||||
if (isUserInputWaitVerdict(summary)) {
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
completion_reason: 'escalated',
|
||||
owner_failure_count: 0,
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: nextFinalizeStepDoneCount,
|
||||
empty_step_done_streak: nextEmptyStepDoneStreak,
|
||||
},
|
||||
});
|
||||
logger.info(
|
||||
{ taskId, ownerVerdict, summary: summary?.slice(0, 100) },
|
||||
'Owner finalize is waiting for user input — completing paired task to stop follow-up loop',
|
||||
);
|
||||
return 'stop';
|
||||
}
|
||||
|
||||
const signal = resolveOwnerCompletionSignal({
|
||||
phase: 'finalize',
|
||||
visibleVerdict: ownerVerdict,
|
||||
@@ -344,6 +369,27 @@ export function handleOwnerCompletion(args: {
|
||||
const ownerVerdict = parseVisibleVerdict(summary);
|
||||
const nextOwnerStepDoneStreak =
|
||||
ownerVerdict === 'step_done' ? (task.owner_step_done_streak ?? 0) + 1 : 0;
|
||||
|
||||
if (isUserInputWaitVerdict(summary)) {
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
completion_reason: 'escalated',
|
||||
owner_failure_count: 0,
|
||||
owner_step_done_streak: nextOwnerStepDoneStreak,
|
||||
},
|
||||
});
|
||||
logger.info(
|
||||
{ taskId, ownerVerdict, summary: summary?.slice(0, 100) },
|
||||
'Owner is waiting for user input — completing paired task to stop follow-up loop',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const signal = resolveOwnerCompletionSignal({
|
||||
phase: 'normal',
|
||||
visibleVerdict: ownerVerdict,
|
||||
@@ -381,6 +427,31 @@ export function handleOwnerCompletion(args: {
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Skip reviewer for clearly conversational turns (no code changes,
|
||||
// short non-technical reply, no code-work keywords from the user).
|
||||
// The detector is best-effort optimization — if anything goes wrong
|
||||
// we fall back to running the reviewer (the safe pre-change behavior).
|
||||
let skip = false;
|
||||
let skipReason = 'evaluation-error';
|
||||
try {
|
||||
const decision = shouldSkipReviewerForTrivialTurn({ task, summary });
|
||||
skip = decision.skip;
|
||||
skipReason = decision.reason;
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, taskId },
|
||||
'paired-trivial-turn-detector: unexpected evaluation error — running reviewer',
|
||||
);
|
||||
}
|
||||
if (skip) {
|
||||
logger.info(
|
||||
{ taskId, reason: skipReason, summary: summary?.slice(0, 100) },
|
||||
'Skipping reviewer for trivial conversational turn',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
maybeAutoTriggerReviewerAfterOwnerCompletion({
|
||||
task,
|
||||
taskId,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
|
||||
import { ARBITER_DEADLOCK_THRESHOLD, isArbiterEnabled } from './config.js';
|
||||
import { getPairedWorkspace } from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
parseVisibleVerdict,
|
||||
parseReviewerVisibleVerdict,
|
||||
isUserInputWaitVerdict,
|
||||
requestArbiterOrEscalate,
|
||||
resolveReviewerCompletionSignal,
|
||||
resolveReviewerFailureSignal,
|
||||
@@ -11,6 +12,12 @@ import {
|
||||
} from './paired-execution-context-shared.js';
|
||||
import type { PairedTask } from './types.js';
|
||||
|
||||
function isReviewerUnavailableFailure(summary: string | null | undefined) {
|
||||
return /(?:\b429\b|usage limit|rate limit|try again at|purchase more credits)/i.test(
|
||||
summary ?? '',
|
||||
);
|
||||
}
|
||||
|
||||
export function handleFailedReviewerExecution(args: {
|
||||
task: PairedTask;
|
||||
taskId: string;
|
||||
@@ -20,7 +27,7 @@ export function handleFailedReviewerExecution(args: {
|
||||
const now = new Date().toISOString();
|
||||
|
||||
if (summary) {
|
||||
const verdict = parseVisibleVerdict(summary);
|
||||
const verdict = parseReviewerVisibleVerdict(summary);
|
||||
const signal = resolveReviewerFailureSignal({
|
||||
visibleVerdict: verdict,
|
||||
});
|
||||
@@ -67,6 +74,28 @@ export function handleFailedReviewerExecution(args: {
|
||||
}
|
||||
}
|
||||
|
||||
if (isReviewerUnavailableFailure(summary)) {
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
completion_reason: 'escalated',
|
||||
},
|
||||
});
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role: 'reviewer',
|
||||
summary: summary?.slice(0, 160),
|
||||
},
|
||||
'Reviewer unavailable due to rate/usage limit — stopping retry loop',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const fallbackStatus =
|
||||
task.status === 'in_review' || task.status === 'review_ready'
|
||||
? 'review_ready'
|
||||
@@ -98,11 +127,33 @@ export function handleReviewerCompletion(args: {
|
||||
}): void {
|
||||
const { task, taskId, summary } = args;
|
||||
const now = new Date().toISOString();
|
||||
const verdict = parseVisibleVerdict(summary);
|
||||
const verdict = parseReviewerVisibleVerdict(summary);
|
||||
const deadlockThreshold = isArbiterEnabled()
|
||||
? ARBITER_DEADLOCK_THRESHOLD
|
||||
: Number.POSITIVE_INFINITY;
|
||||
|
||||
if (isUserInputWaitVerdict(summary)) {
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
completion_reason: 'escalated',
|
||||
},
|
||||
});
|
||||
logger.info(
|
||||
{ taskId, verdict, summary: summary?.slice(0, 100) },
|
||||
'Reviewer is waiting for user input — completing paired task to stop follow-up loop',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const signal = resolveReviewerCompletionSignal({
|
||||
visibleVerdict: verdict,
|
||||
roundTripCount: task.round_trip_count,
|
||||
deadlockThreshold: ARBITER_DEADLOCK_THRESHOLD,
|
||||
deadlockThreshold,
|
||||
});
|
||||
|
||||
switch (signal.kind) {
|
||||
|
||||
@@ -2,6 +2,8 @@ import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
parseVisibleVerdict,
|
||||
isUserInputWaitVerdict,
|
||||
parseReviewerVisibleVerdict,
|
||||
resolveOwnerCompletionSignal,
|
||||
resolveReviewerCompletionSignal,
|
||||
resolveReviewerFailureSignal,
|
||||
@@ -11,6 +13,8 @@ describe('paired execution context shared verdict helpers', () => {
|
||||
it('parses visible verdicts from the first summary line only', () => {
|
||||
expect(parseVisibleVerdict('STEP_DONE\nmore to do')).toBe('step_done');
|
||||
expect(parseVisibleVerdict('TASK_DONE\nall done')).toBe('task_done');
|
||||
expect(parseVisibleVerdict('APPROVE\nlooks good')).toBe('task_done');
|
||||
expect(parseVisibleVerdict('APPROVED\nlooks good')).toBe('task_done');
|
||||
expect(
|
||||
parseVisibleVerdict(
|
||||
'DONE_WITH_CONCERNS\n\nfollow-up detail that should not affect parsing',
|
||||
@@ -18,6 +22,31 @@ describe('paired execution context shared verdict helpers', () => {
|
||||
).toBe('done_with_concerns');
|
||||
expect(parseVisibleVerdict('BLOCKED\nextra detail')).toBe('blocked');
|
||||
expect(parseVisibleVerdict('random prose')).toBe('continue');
|
||||
expect(parseVisibleVerdict('PROCEED — arbiter-style text')).toBe(
|
||||
'continue',
|
||||
);
|
||||
expect(
|
||||
parseReviewerVisibleVerdict('PROCEED — review approved, no work left'),
|
||||
).toBe('task_done');
|
||||
});
|
||||
|
||||
it('detects STEP_DONE summaries that are only waiting for user input', () => {
|
||||
expect(
|
||||
isUserInputWaitVerdict(
|
||||
'STEP_DONE — 대기 유지. whoami 결과를 보내주면 바로 SSH 로그인 테스트합니다.',
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isUserInputWaitVerdict(
|
||||
'STEP_DONE\n키 등록 끝나면 알려주세요. 그 전까지 더 안 보냅니다.',
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isUserInputWaitVerdict('STEP_DONE\n1단계 완료, 다음 단계 진행합니다.'),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isUserInputWaitVerdict('TASK_DONE\n사용자 입력 대기 없이 완료'),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('maps normal owner completion verdicts to reviewer or arbiter signals', () => {
|
||||
@@ -93,6 +122,13 @@ describe('paired execution context shared verdict helpers', () => {
|
||||
});
|
||||
|
||||
it('maps reviewer completion verdicts to finalize, owner changes, or arbiter', () => {
|
||||
expect(
|
||||
resolveReviewerCompletionSignal({
|
||||
visibleVerdict: parseVisibleVerdict('APPROVE\nverified'),
|
||||
roundTripCount: 2,
|
||||
deadlockThreshold: 2,
|
||||
}),
|
||||
).toEqual({ kind: 'request_owner_finalize' });
|
||||
expect(
|
||||
resolveReviewerCompletionSignal({
|
||||
visibleVerdict: 'task_done',
|
||||
|
||||
@@ -3,7 +3,11 @@ import { execFileSync } from 'child_process';
|
||||
import { isArbiterEnabled } from './config.js';
|
||||
import { updatePairedTaskIfUnchanged } from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import { parseVisibleVerdict, type VisibleVerdict } from './paired-verdict.js';
|
||||
import {
|
||||
parseReviewerVisibleVerdict,
|
||||
parseVisibleVerdict,
|
||||
type VisibleVerdict,
|
||||
} from './paired-verdict.js';
|
||||
import type { PairedTaskStatus } from './types.js';
|
||||
|
||||
export type CompletionSignal =
|
||||
@@ -14,9 +18,36 @@ export type CompletionSignal =
|
||||
| { kind: 'complete'; completionReason: 'done' | 'escalated' }
|
||||
| { kind: 'preserve_review_ready' };
|
||||
|
||||
export { parseVisibleVerdict };
|
||||
export { parseReviewerVisibleVerdict, parseVisibleVerdict };
|
||||
export type { VisibleVerdict };
|
||||
|
||||
export function isUserInputWaitVerdict(
|
||||
summary: string | null | undefined,
|
||||
): boolean {
|
||||
if (parseVisibleVerdict(summary) !== 'step_done') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const text = (summary ?? '')
|
||||
.replace(/<internal>[\s\S]*?<\/internal>/g, '')
|
||||
.trim();
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const asksForUserInput =
|
||||
/(대기|기다리|보내\s*주|알려\s*주|주시면|끝나면|완료되면|그 전까지|더 안 보냅니다|추가 메시지 보류|whoami|사용자명|키\s*등록|공개키|인증 정보 부족)/i.test(
|
||||
text,
|
||||
);
|
||||
if (!asksForUserInput) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !/(수정|고쳐|고치|변경|구현|빌드|커밋|푸시|계속\s*진행|다음\s*단계\s*진행|후속\s*작업\s*계속|남은\s*(?:범위|작업))/i.test(
|
||||
text,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveOwnerCompletionSignal(args: {
|
||||
phase: 'normal' | 'finalize';
|
||||
visibleVerdict: VisibleVerdict;
|
||||
|
||||
@@ -875,6 +875,34 @@ describe('paired execution context', () => {
|
||||
).toHaveBeenCalledWith('task-1');
|
||||
});
|
||||
|
||||
it('completes instead of reviewing when owner STEP_DONE is waiting for user input', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'active',
|
||||
round_trip_count: 1,
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'succeeded',
|
||||
summary:
|
||||
'STEP_DONE — 대기 유지. whoami 결과를 보내주면 바로 SSH 로그인 테스트합니다.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
completion_reason: 'escalated',
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
pairedWorkspaceManager.markPairedTaskReviewReady,
|
||||
).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('requests review instead of arbiter when active STEP_DONE repeats without code changes', () => {
|
||||
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(true);
|
||||
|
||||
@@ -1323,6 +1351,98 @@ describe('paired execution context', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('treats reviewer PROCEED as approval instead of owner-change feedback', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_review',
|
||||
source_ref: 'reviewed-ref',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'reviewer',
|
||||
status: 'succeeded',
|
||||
summary: 'PROCEED — v0.4.27 종료 상태입니다. 추가 작업 없습니다.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'merge_ready',
|
||||
source_ref: 'reviewed-ref',
|
||||
}),
|
||||
);
|
||||
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'active',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('completes instead of returning to owner when reviewer STEP_DONE is waiting for user input', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_review',
|
||||
source_ref: 'reviewed-ref',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'reviewer',
|
||||
status: 'succeeded',
|
||||
summary:
|
||||
'STEP_DONE\n\n대기 중입니다. whoami 결과를 보내주면 바로 SSH 접속 테스트합니다.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
completion_reason: 'escalated',
|
||||
}),
|
||||
);
|
||||
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'active',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns reviewer change requests to owner when arbiter is disabled at the deadlock threshold', () => {
|
||||
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(false);
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_review',
|
||||
round_trip_count: config.ARBITER_DEADLOCK_THRESHOLD,
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'reviewer',
|
||||
status: 'succeeded',
|
||||
summary: 'REVISE\nowner needs another concrete change',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'active',
|
||||
}),
|
||||
);
|
||||
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
completion_reason: 'escalated',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps reviewer tasks review_ready when reviewer execution fails without a terminal verdict', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
@@ -1346,6 +1466,30 @@ describe('paired execution context', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('escalates reviewer usage-limit failures instead of retrying forever', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_review',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'reviewer',
|
||||
status: 'failed',
|
||||
summary:
|
||||
"You've hit your usage limit. Upgrade to Pro or try again later.\n\nExecution completed without a visible terminal verdict.",
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
completion_reason: 'escalated',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps arbiter tasks arbiter_requested when arbiter execution fails without a terminal verdict', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
|
||||
183
src/paired-trivial-turn-detector.test.ts
Normal file
183
src/paired-trivial-turn-detector.test.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { shouldSkipReviewerForTrivialTurn } from './paired-trivial-turn-detector.js';
|
||||
import type { NewMessage, PairedTask } from './types.js';
|
||||
|
||||
function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||
return {
|
||||
id: 'task-1',
|
||||
chat_jid: 'dc:111',
|
||||
group_folder: 'general',
|
||||
owner_service_id: 'svc-owner',
|
||||
reviewer_service_id: 'svc-reviewer',
|
||||
title: null,
|
||||
source_ref: 'abc123',
|
||||
plan_notes: null,
|
||||
review_requested_at: null,
|
||||
round_trip_count: 0,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-05-06T00:00:00Z',
|
||||
updated_at: '2026-05-06T00:00:00Z',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function userMsg(content: string): NewMessage {
|
||||
return {
|
||||
id: 'm1',
|
||||
chat_jid: 'dc:111',
|
||||
sender: 'user',
|
||||
sender_name: '사용자',
|
||||
content,
|
||||
timestamp: '2026-05-06T00:00:00Z',
|
||||
is_bot_message: false,
|
||||
};
|
||||
}
|
||||
|
||||
const cleanWorkspace = () => false;
|
||||
|
||||
describe('shouldSkipReviewerForTrivialTurn', () => {
|
||||
it('skips review for a short conversational reply with no code changes and a chit-chat user message', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: '오늘 서울 날씨는 맑음입니다.' },
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('서울 날씨 어때?')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(true);
|
||||
expect(decision.reason).toBe('trivial-turn');
|
||||
});
|
||||
|
||||
it('runs reviewer when the workspace has actual code changes', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: '날씨는 맑음' },
|
||||
{
|
||||
detectCodeChanges: () => true,
|
||||
recentMessages: () => [userMsg('서울 날씨 어때?')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('has-code-changes');
|
||||
});
|
||||
|
||||
it('runs reviewer when the owner reply contains a fenced code block', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{
|
||||
task: makeTask(),
|
||||
summary: '예시:\n```ts\nconst x = 1;\n```',
|
||||
},
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('아무거나')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('has-code-block');
|
||||
});
|
||||
|
||||
it('runs reviewer when the owner reply mentions a source file', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{
|
||||
task: makeTask(),
|
||||
summary: 'src/db.ts 에서 처리합니다.',
|
||||
},
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('아무거나')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('mentions-code-file');
|
||||
});
|
||||
|
||||
it('runs reviewer when the user message contains a code-work keyword (Korean)', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: '확인했습니다.' },
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('이거 코드 좀 수정해줘')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toContain('user-keyword');
|
||||
});
|
||||
|
||||
it('runs reviewer when the user message contains a code-work keyword (English)', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: 'OK' },
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('please fix the bug')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toContain('user-keyword');
|
||||
});
|
||||
|
||||
it('runs reviewer when the verdict is DONE_WITH_CONCERNS', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{
|
||||
task: makeTask(),
|
||||
summary: 'DONE_WITH_CONCERNS\nminor issue noticed',
|
||||
},
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('서울 날씨 어때?')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('verdict-done-with-concerns');
|
||||
});
|
||||
|
||||
it('runs reviewer when the owner reply is unusually long', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: 'a'.repeat(900) },
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('서울 날씨 어때?')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('long-summary');
|
||||
});
|
||||
|
||||
it('runs reviewer when the owner reply is empty', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: '' },
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [userMsg('서울 날씨 어때?')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('empty-summary');
|
||||
});
|
||||
|
||||
it('runs reviewer when there is no recent user message (conservative fallback)', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: '안녕하세요!' },
|
||||
{
|
||||
detectCodeChanges: cleanWorkspace,
|
||||
recentMessages: () => [],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('no-user-message');
|
||||
});
|
||||
|
||||
it('runs reviewer when code-change status cannot be determined', () => {
|
||||
const decision = shouldSkipReviewerForTrivialTurn(
|
||||
{ task: makeTask(), summary: '오늘 서울 날씨는 맑음입니다.' },
|
||||
{
|
||||
detectCodeChanges: () => null,
|
||||
recentMessages: () => [userMsg('서울 날씨 어때?')],
|
||||
},
|
||||
);
|
||||
expect(decision.skip).toBe(false);
|
||||
expect(decision.reason).toBe('code-change-status-unknown');
|
||||
});
|
||||
});
|
||||
214
src/paired-trivial-turn-detector.ts
Normal file
214
src/paired-trivial-turn-detector.ts
Normal file
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* Trivial-turn detector for paired (tribunal) mode.
|
||||
*
|
||||
* Goal: skip the reviewer for owner turns that are obviously conversational
|
||||
* (asking the time, the weather, "what does X mean", chit-chat) so we don't
|
||||
* burn an extra Claude turn on every trivial reply.
|
||||
*
|
||||
* A turn is considered "trivial" only when ALL of the following hold:
|
||||
* 1. There are no actual code changes in the owner workspace
|
||||
* (`git diff --quiet <source_ref> HEAD` is clean).
|
||||
* 2. The owner's reply itself looks conversational — short, no fenced
|
||||
* code block, no source-file mentions.
|
||||
* 3. The most recent user message contains no "code-work" keywords
|
||||
* (수정/구현/빌드/푸시/fix/implement/build/...).
|
||||
* 4. The owner verdict is not `done_with_concerns` (which explicitly
|
||||
* asks the reviewer to look again).
|
||||
*
|
||||
* If any of these fails, fall back to the normal behavior (run the
|
||||
* reviewer). Bias toward running the reviewer when uncertain — skipping
|
||||
* incorrectly could hide a real issue, while reviewing unnecessarily only
|
||||
* costs tokens.
|
||||
*/
|
||||
import { getPairedWorkspace, getRecentChatMessages } from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
hasCodeChangesSinceRef,
|
||||
parseVisibleVerdict,
|
||||
} from './paired-execution-context-shared.js';
|
||||
import type { NewMessage, PairedTask } from './types.js';
|
||||
|
||||
export interface TrivialTurnDeps {
|
||||
/** Returns true if there are uncommitted/committed code changes since
|
||||
* the task source ref. Defaults to running `git diff` against the
|
||||
* paired workspace. Override in tests. */
|
||||
detectCodeChanges?: (task: PairedTask) => boolean | null;
|
||||
/** Returns recent messages for the chat (newest at the end). Defaults
|
||||
* to `getRecentChatMessages`. Override in tests. */
|
||||
recentMessages?: (chatJid: string, limit: number) => NewMessage[];
|
||||
}
|
||||
|
||||
const MAX_TRIVIAL_SUMMARY_LENGTH = 800;
|
||||
|
||||
const CODE_WORK_KEYWORDS: readonly string[] = [
|
||||
// Korean — code/work intents
|
||||
'수정',
|
||||
'변경',
|
||||
'추가',
|
||||
'제거',
|
||||
'삭제',
|
||||
'만들어',
|
||||
'구현',
|
||||
'빌드',
|
||||
'배포',
|
||||
'푸시',
|
||||
'커밋',
|
||||
'리팩토',
|
||||
'리팩터',
|
||||
'리팩',
|
||||
'버그',
|
||||
'에러',
|
||||
'오류',
|
||||
'고쳐',
|
||||
'고치',
|
||||
'등록',
|
||||
'적용',
|
||||
'재시작',
|
||||
'설치',
|
||||
'테스트',
|
||||
'실행',
|
||||
'디버그',
|
||||
'분석',
|
||||
'리뷰',
|
||||
'검증',
|
||||
'롤백',
|
||||
'머지',
|
||||
'코드',
|
||||
'파일',
|
||||
'함수',
|
||||
'모듈',
|
||||
'커서',
|
||||
'스크립트',
|
||||
'의존성',
|
||||
'패키지',
|
||||
'컴파일',
|
||||
'타입',
|
||||
'인터페이스',
|
||||
// English
|
||||
'fix',
|
||||
'implement',
|
||||
'create ',
|
||||
'build',
|
||||
'deploy',
|
||||
'push',
|
||||
'commit',
|
||||
'refactor',
|
||||
'bug',
|
||||
'error',
|
||||
'debug',
|
||||
'install',
|
||||
'restart',
|
||||
'rollback',
|
||||
'merge',
|
||||
'review',
|
||||
'compile',
|
||||
'typecheck',
|
||||
'lint',
|
||||
'pull request',
|
||||
' pr ',
|
||||
'test ',
|
||||
'tests',
|
||||
' run ',
|
||||
'script',
|
||||
'function',
|
||||
'module',
|
||||
'package',
|
||||
];
|
||||
|
||||
const CODE_FILE_EXT_REGEX =
|
||||
/\b\w+\.(ts|tsx|js|jsx|mjs|cjs|json|py|go|rs|cpp|cc|c|h|hpp|java|kt|swift|rb|php|sh|yml|yaml|toml|md|html|css|scss|sql)\b/i;
|
||||
const SOURCE_PATH_REGEX = /(?:^|[\s`'"(])(?:src|tests?|scripts?)\/[\w./-]+/;
|
||||
|
||||
export interface TrivialTurnDecision {
|
||||
skip: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export function shouldSkipReviewerForTrivialTurn(
|
||||
args: {
|
||||
task: PairedTask;
|
||||
summary?: string | null;
|
||||
},
|
||||
deps: TrivialTurnDeps = {},
|
||||
): TrivialTurnDecision {
|
||||
const verdict = parseVisibleVerdict(args.summary);
|
||||
if (verdict === 'done_with_concerns') {
|
||||
return { skip: false, reason: 'verdict-done-with-concerns' };
|
||||
}
|
||||
|
||||
// Real code changes always require review.
|
||||
const detectCodeChanges = deps.detectCodeChanges ?? defaultDetectCodeChanges;
|
||||
let codeChanges: boolean | null = null;
|
||||
try {
|
||||
codeChanges = detectCodeChanges(args.task);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, taskId: args.task.id },
|
||||
'paired-trivial-turn-detector: workspace diff lookup failed',
|
||||
);
|
||||
return { skip: false, reason: 'diff-lookup-failed' };
|
||||
}
|
||||
if (codeChanges === true) {
|
||||
return { skip: false, reason: 'has-code-changes' };
|
||||
}
|
||||
if (codeChanges === null) {
|
||||
// Couldn't determine — be conservative and run the reviewer.
|
||||
return { skip: false, reason: 'code-change-status-unknown' };
|
||||
}
|
||||
|
||||
const summary = (args.summary ?? '').trim();
|
||||
if (summary.length === 0) {
|
||||
return { skip: false, reason: 'empty-summary' };
|
||||
}
|
||||
if (summary.length > MAX_TRIVIAL_SUMMARY_LENGTH) {
|
||||
return { skip: false, reason: 'long-summary' };
|
||||
}
|
||||
if (summary.includes('```')) {
|
||||
return { skip: false, reason: 'has-code-block' };
|
||||
}
|
||||
if (CODE_FILE_EXT_REGEX.test(summary)) {
|
||||
return { skip: false, reason: 'mentions-code-file' };
|
||||
}
|
||||
if (SOURCE_PATH_REGEX.test(summary)) {
|
||||
return { skip: false, reason: 'mentions-source-path' };
|
||||
}
|
||||
|
||||
// Look at the most recent non-bot message (the user request that
|
||||
// triggered this owner turn). If it contains code-work keywords, run
|
||||
// the reviewer.
|
||||
const recentMessages = deps.recentMessages ?? getRecentChatMessages;
|
||||
let userText = '';
|
||||
try {
|
||||
const recent = recentMessages(args.task.chat_jid, 10);
|
||||
for (let i = recent.length - 1; i >= 0; i -= 1) {
|
||||
const msg = recent[i];
|
||||
if (!msg.is_bot_message) {
|
||||
userText = (msg.content ?? '').toLowerCase();
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, taskId: args.task.id },
|
||||
'paired-trivial-turn-detector: recent message lookup failed',
|
||||
);
|
||||
return { skip: false, reason: 'recent-message-lookup-failed' };
|
||||
}
|
||||
if (!userText) {
|
||||
// No identifiable user message — be conservative and run the reviewer.
|
||||
return { skip: false, reason: 'no-user-message' };
|
||||
}
|
||||
for (const keyword of CODE_WORK_KEYWORDS) {
|
||||
if (userText.includes(keyword.toLowerCase())) {
|
||||
return { skip: false, reason: `user-keyword:${keyword.trim()}` };
|
||||
}
|
||||
}
|
||||
|
||||
return { skip: true, reason: 'trivial-turn' };
|
||||
}
|
||||
|
||||
function defaultDetectCodeChanges(task: PairedTask): boolean | null {
|
||||
const workspace = getPairedWorkspace(task.id, 'owner');
|
||||
if (!workspace?.workspace_dir) return null;
|
||||
return hasCodeChangesSinceRef(workspace.workspace_dir, task.source_ref);
|
||||
}
|
||||
@@ -7,17 +7,23 @@ export type VisibleVerdict =
|
||||
| 'needs_context'
|
||||
| 'continue';
|
||||
|
||||
function firstVisibleLine(summary: string | null | undefined): string | null {
|
||||
if (!summary) return null;
|
||||
const cleaned = summary.replace(/<internal>[\s\S]*?<\/internal>/g, '').trim();
|
||||
if (!cleaned) return null;
|
||||
return cleaned.split('\n')[0].trim();
|
||||
}
|
||||
|
||||
export function parseVisibleVerdict(
|
||||
summary: string | null | undefined,
|
||||
): VisibleVerdict {
|
||||
if (!summary) return 'continue';
|
||||
const cleaned = summary.replace(/<internal>[\s\S]*?<\/internal>/g, '').trim();
|
||||
if (!cleaned) return 'continue';
|
||||
const firstLine = cleaned.split('\n')[0].trim();
|
||||
const firstLine = firstVisibleLine(summary);
|
||||
if (!firstLine) return 'continue';
|
||||
if (/^\*{0,2}BLOCKED\*{0,2}\b/i.test(firstLine)) return 'blocked';
|
||||
if (/^\*{0,2}NEEDS_CONTEXT\*{0,2}\b/i.test(firstLine)) return 'needs_context';
|
||||
if (/^\*{0,2}STEP_DONE\*{0,2}\b/i.test(firstLine)) return 'step_done';
|
||||
if (/^\*{0,2}TASK_DONE\*{0,2}\b/i.test(firstLine)) return 'task_done';
|
||||
if (/^\*{0,2}APPROV(?:E|ED)\.?\*{0,2}\b/i.test(firstLine)) return 'task_done';
|
||||
if (/^\*{0,2}DONE_WITH_CONCERNS\*{0,2}\b/i.test(firstLine))
|
||||
return 'done_with_concerns';
|
||||
if (/^\*{0,2}DONE\*{0,2}\b/i.test(firstLine)) return 'done';
|
||||
@@ -26,6 +32,20 @@ export function parseVisibleVerdict(
|
||||
return 'continue';
|
||||
}
|
||||
|
||||
export function parseReviewerVisibleVerdict(
|
||||
summary: string | null | undefined,
|
||||
): VisibleVerdict {
|
||||
const verdict = parseVisibleVerdict(summary);
|
||||
if (verdict !== 'continue') return verdict;
|
||||
|
||||
const firstLine = firstVisibleLine(summary);
|
||||
if (/^\*{0,2}PROCEED\.?\*{0,2}\b/i.test(firstLine ?? '')) {
|
||||
return 'task_done';
|
||||
}
|
||||
|
||||
return verdict;
|
||||
}
|
||||
|
||||
export function resolveStoredVisibleVerdict(args: {
|
||||
verdict?: VisibleVerdict | null;
|
||||
outputText?: string | null;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
buildInterruptedRestartAnnouncement,
|
||||
getInterruptedRecoveryCandidates,
|
||||
type RestartContext,
|
||||
} from './restart-context.js';
|
||||
@@ -36,6 +37,7 @@ describe('getInterruptedRecoveryCandidates', () => {
|
||||
elapsedMs: 1000,
|
||||
pendingMessages: true,
|
||||
pendingTasks: 0,
|
||||
rewindToSeq: 42,
|
||||
},
|
||||
{
|
||||
chatJid: 'dc:1',
|
||||
@@ -71,6 +73,7 @@ describe('getInterruptedRecoveryCandidates', () => {
|
||||
status: 'processing',
|
||||
pendingMessages: true,
|
||||
pendingTasks: 0,
|
||||
rewindToSeq: 42,
|
||||
},
|
||||
{
|
||||
chatJid: 'dc:2',
|
||||
@@ -78,6 +81,7 @@ describe('getInterruptedRecoveryCandidates', () => {
|
||||
status: 'idle',
|
||||
pendingMessages: false,
|
||||
pendingTasks: 0,
|
||||
rewindToSeq: null,
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -86,3 +90,33 @@ describe('getInterruptedRecoveryCandidates', () => {
|
||||
expect(getInterruptedRecoveryCandidates(null, {})).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildInterruptedRestartAnnouncement', () => {
|
||||
it('promises auto-resume when a rewind seq is captured', () => {
|
||||
const message = buildInterruptedRestartAnnouncement({
|
||||
chatJid: 'dc:1',
|
||||
groupName: 'room',
|
||||
status: 'processing',
|
||||
elapsedMs: 12000,
|
||||
pendingMessages: true,
|
||||
pendingTasks: 0,
|
||||
rewindToSeq: 100,
|
||||
});
|
||||
expect(message).toContain('서비스 재시작으로 이전 작업이 중단됐습니다.');
|
||||
expect(message).toContain('자동으로 이어갑니다');
|
||||
expect(message).not.toContain('필요하면 이어서');
|
||||
});
|
||||
|
||||
it('falls back to manual prompt when no rewind seq exists', () => {
|
||||
const message = buildInterruptedRestartAnnouncement({
|
||||
chatJid: 'dc:1',
|
||||
groupName: 'room',
|
||||
status: 'waiting',
|
||||
elapsedMs: null,
|
||||
pendingMessages: false,
|
||||
pendingTasks: 0,
|
||||
});
|
||||
expect(message).toContain('필요하면 이어서');
|
||||
expect(message).not.toContain('자동으로 이어갑니다');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,6 +14,12 @@ export interface RestartInterruptedGroup {
|
||||
elapsedMs: number | null;
|
||||
pendingMessages: boolean;
|
||||
pendingTasks: number;
|
||||
/**
|
||||
* Seq of the earliest unanswered human message at shutdown time. If
|
||||
* present, startup will rewind the agent cursor to (this seq − 1) so the
|
||||
* interrupted turn re-runs automatically. Null when nothing was waiting.
|
||||
*/
|
||||
rewindToSeq?: number | null;
|
||||
}
|
||||
|
||||
export interface RestartContext {
|
||||
@@ -37,6 +43,7 @@ export interface RestartRecoveryCandidate {
|
||||
status: RestartInterruptedGroup['status'];
|
||||
pendingMessages: boolean;
|
||||
pendingTasks: number;
|
||||
rewindToSeq: number | null;
|
||||
}
|
||||
|
||||
const INFER_WINDOW_MS = 3 * 60 * 1000;
|
||||
@@ -197,7 +204,11 @@ export function buildInterruptedRestartAnnouncement(
|
||||
if (interrupted.pendingTasks > 0) {
|
||||
lines.push(`- 대기 태스크: ${interrupted.pendingTasks}개`);
|
||||
}
|
||||
lines.push('- 필요하면 이어서 요청해 주세요.');
|
||||
if (interrupted.rewindToSeq != null) {
|
||||
lines.push('- 중단된 작업을 자동으로 이어갑니다.');
|
||||
} else {
|
||||
lines.push('- 필요하면 이어서 요청해 주세요.');
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
@@ -251,6 +262,7 @@ export function getInterruptedRecoveryCandidates(
|
||||
status: interrupted.status,
|
||||
pendingMessages: interrupted.pendingMessages,
|
||||
pendingTasks: interrupted.pendingTasks,
|
||||
rewindToSeq: interrupted.rewindToSeq ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@ import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
findChannelForDeliveryRole,
|
||||
formatOutbound,
|
||||
neutralizeStrayBackticks,
|
||||
neutralizeStrayMarkdown,
|
||||
resolveChannelForDeliveryRole,
|
||||
} from './router.js';
|
||||
import { type Channel } from './types.js';
|
||||
@@ -92,3 +95,134 @@ describe('findChannelForDeliveryRole', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('neutralizeStrayMarkdown', () => {
|
||||
it('strips inline backticks', () => {
|
||||
expect(neutralizeStrayMarkdown('use `foo` and `bar` here')).toBe(
|
||||
'use foo and bar here',
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves well-formed fenced code blocks', () => {
|
||||
const input = 'before\n```ts\nconst x = `tpl`;\n```\nafter `inline`';
|
||||
expect(neutralizeStrayMarkdown(input)).toBe(
|
||||
'before\n```ts\nconst x = `tpl`;\n```\nafter inline',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips emphasis in an unterminated fence', () => {
|
||||
expect(neutralizeStrayMarkdown('oops ```code without close')).toBe(
|
||||
'oops code without close',
|
||||
);
|
||||
});
|
||||
|
||||
it('passes through plain text unchanged', () => {
|
||||
const s = 'plain text with no markup';
|
||||
expect(neutralizeStrayMarkdown(s)).toBe(s);
|
||||
});
|
||||
|
||||
it('strips italic asterisks', () => {
|
||||
expect(neutralizeStrayMarkdown('이건 *기울임* 처리')).toBe(
|
||||
'이건 기울임 처리',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips bold asterisks', () => {
|
||||
expect(neutralizeStrayMarkdown('이건 **강조** 처리')).toBe(
|
||||
'이건 강조 처리',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips bold-italic triple asterisks', () => {
|
||||
expect(neutralizeStrayMarkdown('***hybrid***')).toBe('hybrid');
|
||||
});
|
||||
|
||||
it('strips italic underscores at word boundaries', () => {
|
||||
expect(neutralizeStrayMarkdown('이건 _italic_ 처리')).toBe(
|
||||
'이건 italic 처리',
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves single underscores in snake_case identifiers', () => {
|
||||
expect(neutralizeStrayMarkdown('use foo_bar_baz here')).toBe(
|
||||
'use foo_bar_baz here',
|
||||
);
|
||||
});
|
||||
|
||||
it('escapes underscores in absolute paths so Discord does not italicize them', () => {
|
||||
expect(
|
||||
neutralizeStrayMarkdown(
|
||||
'경로: /home/claude/EJClaw/groups/mc_the_scene_plugin/src/main_file.ts',
|
||||
),
|
||||
).toBe(
|
||||
'경로: /home/claude/EJClaw/groups/mc\\_the\\_scene\\_plugin/src/main\\_file.ts',
|
||||
);
|
||||
});
|
||||
|
||||
it('escapes underscores in Windows paths', () => {
|
||||
expect(
|
||||
neutralizeStrayMarkdown(
|
||||
'경로: C:\\Custom_vscode\\minecraft_launcher\\src\\main_file.ts',
|
||||
),
|
||||
).toBe(
|
||||
'경로: C:\\Custom\\_vscode\\minecraft\\_launcher\\src\\main\\_file.ts',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips double-underscore underline markers', () => {
|
||||
expect(neutralizeStrayMarkdown('__bold__ text')).toBe('bold text');
|
||||
});
|
||||
|
||||
it('strips heading hashes at line start', () => {
|
||||
expect(neutralizeStrayMarkdown('## 빌드 타임 메타\n본문')).toBe(
|
||||
'빌드 타임 메타\n본문',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips block-quote markers', () => {
|
||||
expect(neutralizeStrayMarkdown('> quoted line\n> another')).toBe(
|
||||
'quoted line\nanother',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips strikethrough', () => {
|
||||
expect(neutralizeStrayMarkdown('이건 ~~취소선~~ 입니다')).toBe(
|
||||
'이건 취소선 입니다',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips spoiler markers', () => {
|
||||
expect(neutralizeStrayMarkdown('||비밀||')).toBe('비밀');
|
||||
});
|
||||
|
||||
it('preserves Discord mention syntax', () => {
|
||||
expect(neutralizeStrayMarkdown('<@123> hi <#456>')).toBe(
|
||||
'<@123> hi <#456>',
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves code fence content while stripping prose emphasis', () => {
|
||||
const input = '**bold** outside\n```py\nx = *2\n```\nand *more*';
|
||||
expect(neutralizeStrayMarkdown(input)).toBe(
|
||||
'bold outside\n```py\nx = *2\n```\nand more',
|
||||
);
|
||||
});
|
||||
|
||||
it('back-compat alias still exported', () => {
|
||||
expect(neutralizeStrayBackticks('*x*')).toBe('x');
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatOutbound', () => {
|
||||
it('runs the full pipeline and strips stray backticks', () => {
|
||||
const raw = '<internal>note</internal>Use `foo` in your code.';
|
||||
expect(formatOutbound(raw)).toBe('Use foo in your code.');
|
||||
});
|
||||
|
||||
it('preserves fenced code blocks through the pipeline', () => {
|
||||
const raw = 'See:\n```js\nconst a = 1;\n```\nor `inline`.';
|
||||
expect(formatOutbound(raw)).toBe(
|
||||
'See:\n```js\nconst a = 1;\n```\nor inline.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -78,12 +78,90 @@ export function stripToolCallLeaks(text: string): string {
|
||||
return stripped.replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
function escapePathMarkdownUnderscores(segment: string): string {
|
||||
return segment.replace(
|
||||
/(^|[\s[{<])((?:[A-Za-z]:[\\/]|~?[\\/]|\.{1,2}[\\/])\S*_\S*_\S*)/g,
|
||||
(_match, prefix: string, filePath: string) =>
|
||||
`${prefix}${filePath.replace(/_/g, '\\_')}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip Discord markdown emphasis/heading markers that mangle plain-prose
|
||||
* output when wrapping across lines. Used inside non-code segments only —
|
||||
* callers should preserve fenced code blocks separately.
|
||||
*
|
||||
* Discord rendering reference:
|
||||
* *italic*, _italic_, **bold**, ***bold-italic***
|
||||
* __underline__ ~~strike~~ ||spoiler||
|
||||
* # H1 ## H2 ### H3 (at line start)
|
||||
* > quote, >>> multi-line quote (at line start)
|
||||
*
|
||||
* `<@id>`/`<#id>`/`<:emoji:id>` mentions and `[text](url)` links are left
|
||||
* untouched (handled elsewhere or harmless as raw text).
|
||||
*/
|
||||
function stripMarkdownInProse(segment: string): string {
|
||||
return (
|
||||
escapePathMarkdownUnderscores(segment)
|
||||
// Inline backtick code — agents abuse this to highlight identifiers
|
||||
// and Discord then fragments coloring across line wraps.
|
||||
.replace(/`/g, '')
|
||||
// Strikethrough and spoiler wrappers (always paired punctuation)
|
||||
.replace(/~~/g, '')
|
||||
.replace(/\|\|/g, '')
|
||||
// Heading markers at the start of a line (after optional whitespace)
|
||||
.replace(/^[ \t]*#{1,3}[ \t]+/gm, '')
|
||||
// Block quote markers at the start of a line
|
||||
.replace(/^[ \t]*>{1,3}[ \t]?/gm, '')
|
||||
// Bold/italic asterisks — Discord italicizes even mid-word (a*b*c),
|
||||
// so just remove every '*'. Math/code uses live in fenced blocks.
|
||||
.replace(/\*/g, '')
|
||||
// Underline/bold uses '__'; remove paired runs of 2+ underscores.
|
||||
.replace(/_{2,}/g, '')
|
||||
// Italic '_word_' — only strip when the underscores are at word
|
||||
// boundaries (so snake_case identifiers survive intact).
|
||||
.replace(/(^|[^A-Za-z0-9_])_([^_\n]+?)_(?=$|[^A-Za-z0-9_])/g, '$1$2')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Neutralize stray Discord markdown in prose while preserving well-formed
|
||||
* triple-backtick fenced code blocks (legit code snippets).
|
||||
*/
|
||||
export function neutralizeStrayMarkdown(text: string): string {
|
||||
if (!text) return text;
|
||||
const parts: string[] = [];
|
||||
let i = 0;
|
||||
while (i < text.length) {
|
||||
const fenceStart = text.indexOf('```', i);
|
||||
if (fenceStart === -1) {
|
||||
parts.push(stripMarkdownInProse(text.slice(i)));
|
||||
break;
|
||||
}
|
||||
parts.push(stripMarkdownInProse(text.slice(i, fenceStart)));
|
||||
const fenceEnd = text.indexOf('```', fenceStart + 3);
|
||||
if (fenceEnd === -1) {
|
||||
// Unterminated fence — not a real code block; treat as prose.
|
||||
parts.push(stripMarkdownInProse(text.slice(fenceStart)));
|
||||
break;
|
||||
}
|
||||
// Preserve the entire fenced block including delimiters.
|
||||
parts.push(text.slice(fenceStart, fenceEnd + 3));
|
||||
i = fenceEnd + 3;
|
||||
}
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
/** @deprecated Kept for back-compat; use neutralizeStrayMarkdown. */
|
||||
export const neutralizeStrayBackticks = neutralizeStrayMarkdown;
|
||||
|
||||
export function formatOutbound(rawText: string): string {
|
||||
let text = stripInternalTags(rawText);
|
||||
if (!text) return '';
|
||||
text = stripToolCallLeaks(text);
|
||||
if (!text) return '';
|
||||
return redactSecrets(text);
|
||||
text = redactSecrets(text);
|
||||
return neutralizeStrayMarkdown(text);
|
||||
}
|
||||
|
||||
export function findChannel(
|
||||
|
||||
36
src/types.ts
36
src/types.ts
@@ -9,6 +9,14 @@ export interface AgentConfig {
|
||||
claudeEffort?: string;
|
||||
claudeThinking?: 'adaptive' | 'enabled' | 'disabled';
|
||||
claudeThinkingBudget?: number;
|
||||
// Per-role overrides for paired rooms. When a roomRoleContext is supplied
|
||||
// these take precedence over the flat claudeModel/claudeEffort/codexModel/
|
||||
// codexEffort fields. Useful for spending Opus only on arbiter judgments
|
||||
// while keeping owner/reviewer on Sonnet.
|
||||
claudeModelByRole?: Partial<Record<PairedRoomRole, string>>;
|
||||
claudeEffortByRole?: Partial<Record<PairedRoomRole, string>>;
|
||||
codexModelByRole?: Partial<Record<PairedRoomRole, string>>;
|
||||
codexEffortByRole?: Partial<Record<PairedRoomRole, string>>;
|
||||
}
|
||||
|
||||
export type AgentType = 'claude-code' | 'codex';
|
||||
@@ -40,6 +48,21 @@ export interface SendMessageOptions {
|
||||
* the global Discord attachment allowlist.
|
||||
*/
|
||||
attachmentBaseDirs?: string[];
|
||||
/**
|
||||
* If true, skip Discord markdown neutralization (used by the status
|
||||
* dashboard and other callers that build pre-formatted content with
|
||||
* intentional bold/italic/heading markup). Default: false.
|
||||
*/
|
||||
rawMarkdown?: boolean;
|
||||
}
|
||||
|
||||
export interface EditMessageOptions {
|
||||
/**
|
||||
* If true, skip Discord markdown neutralization for this edit. Default:
|
||||
* false — the streaming edit path will sanitize agent prose the same way
|
||||
* as the initial send.
|
||||
*/
|
||||
rawMarkdown?: boolean;
|
||||
}
|
||||
|
||||
export type PairedRoomRole = 'owner' | 'reviewer' | 'arbiter';
|
||||
@@ -264,9 +287,18 @@ export interface Channel {
|
||||
// Optional: typing indicator. Channels that support it implement it.
|
||||
setTyping?(jid: string, isTyping: boolean): Promise<void>;
|
||||
// Optional: edit/delete messages (used by status dashboard and tracked progress cleanup).
|
||||
editMessage?(jid: string, messageId: string, text: string): Promise<void>;
|
||||
editMessage?(
|
||||
jid: string,
|
||||
messageId: string,
|
||||
text: string,
|
||||
options?: EditMessageOptions,
|
||||
): Promise<void>;
|
||||
deleteMessage?(jid: string, messageId: string): Promise<void>;
|
||||
sendAndTrack?(jid: string, text: string): Promise<string | null>;
|
||||
sendAndTrack?(
|
||||
jid: string,
|
||||
text: string,
|
||||
options?: EditMessageOptions,
|
||||
): Promise<string | null>;
|
||||
// Optional: sync group/chat names from the platform.
|
||||
syncGroups?(force: boolean): Promise<void>;
|
||||
// Optional: get channel metadata (position, category) for ordering.
|
||||
|
||||
@@ -89,23 +89,101 @@ describe('renderUsageTable', () => {
|
||||
expect(codexIdx).toBeGreaterThan(sepIdx);
|
||||
});
|
||||
|
||||
it('omits separator when only Claude rows exist', () => {
|
||||
it('shows "Codex: 조회 불가" when only Claude rows exist', () => {
|
||||
const lines = renderUsageTable([claudeRow], []);
|
||||
|
||||
expect(lines.some((l) => /^─+$/.test(l))).toBe(false);
|
||||
expect(lines.some((l) => l.includes('Claude'))).toBe(true);
|
||||
expect(
|
||||
lines.some((l) => l.includes('Codex') && l.includes('조회 불가')),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('omits separator when only Codex rows exist', () => {
|
||||
it('shows "Claude: 조회 불가" when only Codex rows exist', () => {
|
||||
const lines = renderUsageTable([], [codexRow]);
|
||||
|
||||
expect(lines.some((l) => /^─+$/.test(l))).toBe(false);
|
||||
expect(
|
||||
lines.some((l) => l.includes('Claude') && l.includes('조회 불가')),
|
||||
).toBe(true);
|
||||
expect(lines.some((l) => l.includes('Codex'))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns fallback text when both groups are empty', () => {
|
||||
it('shows separate 조회 불가 for both when both groups are empty', () => {
|
||||
const lines = renderUsageTable([], []);
|
||||
expect(lines).toEqual(['_조회 불가_']);
|
||||
expect(
|
||||
lines.some((l) => l.includes('Claude') && l.includes('조회 불가')),
|
||||
).toBe(true);
|
||||
expect(
|
||||
lines.some((l) => l.includes('Codex') && l.includes('조회 불가')),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('shows "Claude: 사용량 없음" when all Claude rows are at >=99% in a window', () => {
|
||||
const exhaustedClaude: UsageRow = {
|
||||
name: 'Claude pro',
|
||||
h5pct: 99,
|
||||
h5reset: '',
|
||||
d7pct: 30,
|
||||
d7reset: '',
|
||||
};
|
||||
const lines = renderUsageTable([exhaustedClaude], [codexRow]);
|
||||
|
||||
expect(
|
||||
lines.some((l) => l.includes('Claude') && l.includes('사용량 없음')),
|
||||
).toBe(true);
|
||||
// Codex side should still render normally
|
||||
expect(lines.some((l) => l.includes('Codex') && /\d+%/.test(l))).toBe(true);
|
||||
});
|
||||
|
||||
it('shows "Codex: 사용량 없음" when all Codex rows are at 100% in either window', () => {
|
||||
const exhaustedCodex: UsageRow = {
|
||||
name: 'Codex',
|
||||
h5pct: 100,
|
||||
h5reset: '',
|
||||
d7pct: 50,
|
||||
d7reset: '',
|
||||
};
|
||||
const lines = renderUsageTable([claudeRow], [exhaustedCodex]);
|
||||
|
||||
expect(
|
||||
lines.some((l) => l.includes('Codex') && l.includes('사용량 없음')),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('treats <99% rows as having capacity remaining', () => {
|
||||
const almostExhausted: UsageRow = {
|
||||
name: 'Claude pro',
|
||||
h5pct: 98,
|
||||
h5reset: '',
|
||||
d7pct: 50,
|
||||
d7reset: '',
|
||||
};
|
||||
const lines = renderUsageTable([almostExhausted], [codexRow]);
|
||||
|
||||
// Should render the row as data, not the "사용량 없음" status line
|
||||
expect(lines.some((l) => l.includes('사용량 없음'))).toBe(false);
|
||||
});
|
||||
|
||||
it('treats source as exhausted only when ALL its rows are exhausted', () => {
|
||||
const exhausted: UsageRow = {
|
||||
name: 'Claude max',
|
||||
h5pct: 100,
|
||||
h5reset: '',
|
||||
d7pct: 100,
|
||||
d7reset: '',
|
||||
};
|
||||
const fresh: UsageRow = {
|
||||
name: 'Claude pro',
|
||||
h5pct: 10,
|
||||
h5reset: '',
|
||||
d7pct: 5,
|
||||
d7reset: '',
|
||||
};
|
||||
const lines = renderUsageTable([exhausted, fresh], [codexRow]);
|
||||
|
||||
// Mixed exhausted/fresh → render rows normally, no 사용량 없음 banner
|
||||
expect(lines.some((l) => l.includes('사용량 없음'))).toBe(false);
|
||||
expect(lines.some((l) => l.includes('Claude max'))).toBe(true);
|
||||
expect(lines.some((l) => l.includes('Claude pro'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
refreshAllCodexAccountUsage,
|
||||
} from './codex-usage-collector.js';
|
||||
import { runCodexWarmupCycle } from './codex-warmup.js';
|
||||
import { evaluateAndApplyAutoPairedMode } from './auto-paired-mode.js';
|
||||
import {
|
||||
composeDashboardContent,
|
||||
formatElapsed,
|
||||
@@ -448,12 +449,44 @@ function buildStatusContent(): string {
|
||||
* Ordering: Claude rows → separator → Codex rows.
|
||||
* Exported for testing.
|
||||
*/
|
||||
/**
|
||||
* A row is considered "exhausted" (1% or less remaining in any window) when
|
||||
* either the 5h or 7d utilization is at or above 99%. Rows where a window's
|
||||
* percentage is unknown (h5pct/d7pct < 0) aren't counted toward exhaustion
|
||||
* for that window.
|
||||
*/
|
||||
function isExhaustedRow(row: UsageRow): boolean {
|
||||
const h5Out = row.h5pct >= 99;
|
||||
const d7Out = row.d7pct >= 99;
|
||||
return h5Out || d7Out;
|
||||
}
|
||||
|
||||
function allRowsExhausted(rows: UsageRow[]): boolean {
|
||||
if (rows.length === 0) return false;
|
||||
return rows.every(isExhaustedRow);
|
||||
}
|
||||
|
||||
export function renderUsageTable(
|
||||
claudeBotRows: UsageRow[],
|
||||
codexBotRows: UsageRow[],
|
||||
): string[] {
|
||||
const allRows = [...claudeBotRows, ...codexBotRows];
|
||||
if (allRows.length === 0) return ['_조회 불가_'];
|
||||
// Per-source status: 조회 불가 (no rows) vs 사용량 없음 (all exhausted)
|
||||
const claudeStatus: 'ok' | 'unavailable' | 'exhausted' =
|
||||
claudeBotRows.length === 0
|
||||
? 'unavailable'
|
||||
: allRowsExhausted(claudeBotRows)
|
||||
? 'exhausted'
|
||||
: 'ok';
|
||||
const codexStatus: 'ok' | 'unavailable' | 'exhausted' =
|
||||
codexBotRows.length === 0
|
||||
? 'unavailable'
|
||||
: allRowsExhausted(codexBotRows)
|
||||
? 'exhausted'
|
||||
: 'ok';
|
||||
|
||||
const renderableClaude = claudeStatus === 'ok' ? claudeBotRows : [];
|
||||
const renderableCodex = codexStatus === 'ok' ? codexBotRows : [];
|
||||
const allRenderable = [...renderableClaude, ...renderableCodex];
|
||||
|
||||
const bar = (pct: number) => {
|
||||
const filled = Math.max(0, Math.min(5, Math.round(pct / 20)));
|
||||
@@ -463,12 +496,15 @@ export function renderUsageTable(
|
||||
const visualWidth = (s: string) =>
|
||||
[...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0);
|
||||
const maxNameWidth =
|
||||
Math.max(8, ...allRows.map((r) => visualWidth(r.name))) + 1;
|
||||
Math.max(8, ...allRenderable.map((r) => visualWidth(r.name))) + 1;
|
||||
const padName = (s: string) =>
|
||||
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
|
||||
const compactReset = (s: string) =>
|
||||
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
|
||||
|
||||
const statusLabel = (status: 'unavailable' | 'exhausted'): string =>
|
||||
status === 'unavailable' ? '조회 불가' : '사용량 없음';
|
||||
|
||||
const lines: string[] = [];
|
||||
|
||||
const renderRows = (rows: UsageRow[]) => {
|
||||
@@ -498,14 +534,22 @@ export function renderUsageTable(
|
||||
lines.push('```');
|
||||
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
|
||||
|
||||
renderRows(claudeBotRows);
|
||||
|
||||
if (claudeBotRows.length > 0 && codexBotRows.length > 0) {
|
||||
const separatorWidth = maxNameWidth + 20;
|
||||
lines.push('─'.repeat(separatorWidth));
|
||||
// Claude section
|
||||
if (claudeStatus === 'ok') {
|
||||
renderRows(claudeBotRows);
|
||||
} else {
|
||||
lines.push(`${padName('Claude')}${statusLabel(claudeStatus)}`);
|
||||
}
|
||||
|
||||
renderRows(codexBotRows);
|
||||
const separatorWidth = maxNameWidth + 20;
|
||||
lines.push('─'.repeat(separatorWidth));
|
||||
|
||||
// Codex section
|
||||
if (codexStatus === 'ok') {
|
||||
renderRows(codexBotRows);
|
||||
} else {
|
||||
lines.push(`${padName('Codex')}${statusLabel(codexStatus)}`);
|
||||
}
|
||||
|
||||
lines.push('```');
|
||||
|
||||
@@ -751,9 +795,13 @@ export async function startUnifiedDashboard(
|
||||
}
|
||||
|
||||
if (statusMessageId && channel.editMessage) {
|
||||
await channel.editMessage(statusJid, statusMessageId, content);
|
||||
await channel.editMessage(statusJid, statusMessageId, content, {
|
||||
rawMarkdown: true,
|
||||
});
|
||||
} else if (channel.sendAndTrack) {
|
||||
const id = await channel.sendAndTrack(statusJid, content);
|
||||
const id = await channel.sendAndTrack(statusJid, content, {
|
||||
rawMarkdown: true,
|
||||
});
|
||||
if (id) statusMessageId = id;
|
||||
}
|
||||
if (!dashboardUpdateLogged) {
|
||||
@@ -773,7 +821,29 @@ export async function startUnifiedDashboard(
|
||||
await updateStatus();
|
||||
|
||||
if (isRenderer) {
|
||||
setInterval(refreshUsageCache, RENDERER_USAGE_REFRESH_MS);
|
||||
const evaluatePairedMode = async () => {
|
||||
try {
|
||||
await evaluateAndApplyAutoPairedMode({
|
||||
queue: opts.queue,
|
||||
sendNotice: async (jid, text) => {
|
||||
const ch = opts.channels.find(
|
||||
(c) =>
|
||||
c.name.startsWith('discord') &&
|
||||
c.isConnected() &&
|
||||
c.ownsJid(jid),
|
||||
);
|
||||
if (ch) await ch.sendMessage(jid, text);
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'auto-paired-mode evaluation failed');
|
||||
}
|
||||
};
|
||||
setInterval(async () => {
|
||||
await refreshUsageCache();
|
||||
await evaluatePairedMode();
|
||||
}, RENDERER_USAGE_REFRESH_MS);
|
||||
await evaluatePairedMode();
|
||||
}
|
||||
|
||||
// Codex usage collection — runs in unified service regardless of renderer role.
|
||||
|
||||
336
src/usage-exhaustion.test.ts
Normal file
336
src/usage-exhaustion.test.ts
Normal file
@@ -0,0 +1,336 @@
|
||||
/**
|
||||
* Unit tests for the 95% usage-exhaustion helpers (Claude + Codex).
|
||||
*
|
||||
* These do not exercise the real disk cache or rotation state; they mock
|
||||
* just enough to verify the gate's truth-table:
|
||||
*
|
||||
* - no accounts → false (don't block)
|
||||
* - missing data on any → false (don't block)
|
||||
* - rate-limited account → counts as exhausted
|
||||
* - any account < 95% → false (route there instead)
|
||||
* - all accounts ≥ 95% → true, with earliest expected recovery time
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./logger.js', () => ({
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('./config.js', () => ({
|
||||
DATA_DIR: '/tmp/ejclaw-usage-exhaustion-data',
|
||||
}));
|
||||
|
||||
const { getAllTokensMock } = vi.hoisted(() => ({
|
||||
getAllTokensMock: vi.fn(),
|
||||
}));
|
||||
vi.mock('./token-rotation.js', () => ({
|
||||
getAllTokens: getAllTokensMock,
|
||||
getCurrentToken: () => null,
|
||||
getConfiguredClaudeTokens: () => [],
|
||||
}));
|
||||
|
||||
const { getAllCodexAccountsMock } = vi.hoisted(() => ({
|
||||
getAllCodexAccountsMock: vi.fn(),
|
||||
}));
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
getAllCodexAccounts: getAllCodexAccountsMock,
|
||||
getCodexAuthPath: () => null,
|
||||
updateCodexAccountUsage: vi.fn(),
|
||||
}));
|
||||
|
||||
const { readJsonFileMock } = vi.hoisted(() => ({
|
||||
readJsonFileMock: vi.fn(),
|
||||
}));
|
||||
vi.mock('./utils.js', async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import('./utils.js')>('./utils.js');
|
||||
return {
|
||||
...actual,
|
||||
readJsonFile: readJsonFileMock,
|
||||
writeJsonFile: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('getClaudeUsageExhaustion', () => {
|
||||
beforeEach(() => {
|
||||
readJsonFileMock.mockReturnValue({});
|
||||
});
|
||||
|
||||
it('returns exhausted=false when no tokens are configured', async () => {
|
||||
getAllTokensMock.mockReturnValue([]);
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
expect(getClaudeUsageExhaustion()).toEqual({
|
||||
exhausted: false,
|
||||
nextResetAt: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns exhausted=false when an account has no cached usage', async () => {
|
||||
getAllTokensMock.mockReturnValue([
|
||||
{
|
||||
token: 'sk-ant-oat01-aaaaaaaa',
|
||||
index: 0,
|
||||
masked: 'aaa',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
},
|
||||
]);
|
||||
readJsonFileMock.mockReturnValue({}); // empty cache
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
expect(getClaudeUsageExhaustion().exhausted).toBe(false);
|
||||
});
|
||||
|
||||
it('returns exhausted=true with the earliest recovery when all accounts are exhausted', async () => {
|
||||
const reset0 = new Date('2026-04-28T01:00:00Z').toISOString();
|
||||
const reset1 = new Date('2026-04-28T03:00:00Z').toISOString();
|
||||
getAllTokensMock.mockReturnValue([
|
||||
{
|
||||
token: 'sk-ant-oat01-aaaaaaaa',
|
||||
index: 0,
|
||||
masked: 'aaa',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
},
|
||||
{
|
||||
token: 'sk-ant-oat01-bbbbbbbb',
|
||||
index: 1,
|
||||
masked: 'bbb',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
},
|
||||
]);
|
||||
readJsonFileMock.mockReturnValue({
|
||||
'account-0': {
|
||||
usage: {
|
||||
five_hour: { utilization: 96, resets_at: reset0 },
|
||||
seven_day: { utilization: 50, resets_at: '' },
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
'account-1': {
|
||||
usage: {
|
||||
five_hour: { utilization: 30, resets_at: '' },
|
||||
seven_day: { utilization: 99, resets_at: reset1 },
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
const info = getClaudeUsageExhaustion();
|
||||
expect(info.exhausted).toBe(true);
|
||||
// Earliest recovery is account-0 (5h reset at reset0).
|
||||
expect(info.nextResetAt).toBe(reset0);
|
||||
});
|
||||
|
||||
it('returns exhausted=false when at least one account still has headroom', async () => {
|
||||
getAllTokensMock.mockReturnValue([
|
||||
{
|
||||
token: 'sk-ant-oat01-aaaaaaaa',
|
||||
index: 0,
|
||||
masked: 'aaa',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
},
|
||||
{
|
||||
token: 'sk-ant-oat01-bbbbbbbb',
|
||||
index: 1,
|
||||
masked: 'bbb',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
},
|
||||
]);
|
||||
readJsonFileMock.mockReturnValue({
|
||||
'account-0': {
|
||||
usage: {
|
||||
five_hour: { utilization: 99 },
|
||||
seven_day: { utilization: 99 },
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
'account-1': {
|
||||
usage: {
|
||||
five_hour: { utilization: 80 },
|
||||
seven_day: { utilization: 70 },
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
expect(getClaudeUsageExhaustion().exhausted).toBe(false);
|
||||
});
|
||||
|
||||
it('treats rate-limited accounts as exhausted', async () => {
|
||||
getAllTokensMock.mockReturnValue([
|
||||
{
|
||||
token: 'sk-ant-oat01-aaaaaaaa',
|
||||
index: 0,
|
||||
masked: 'aaa',
|
||||
isActive: false,
|
||||
isRateLimited: true,
|
||||
},
|
||||
]);
|
||||
readJsonFileMock.mockReturnValue({});
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
expect(getClaudeUsageExhaustion().exhausted).toBe(true);
|
||||
});
|
||||
|
||||
it('handles utilization on the 0–1 scale (legacy fractional form)', async () => {
|
||||
getAllTokensMock.mockReturnValue([
|
||||
{
|
||||
token: 'sk-ant-oat01-aaaaaaaa',
|
||||
index: 0,
|
||||
masked: 'aaa',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
},
|
||||
]);
|
||||
readJsonFileMock.mockReturnValue({
|
||||
'account-0': {
|
||||
usage: {
|
||||
five_hour: { utilization: 0.97 },
|
||||
seven_day: { utilization: 0.4 },
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
expect(getClaudeUsageExhaustion().exhausted).toBe(true);
|
||||
});
|
||||
|
||||
it('treats percent-scale utilization=1 as 1%, not 100%', async () => {
|
||||
getAllTokensMock.mockReturnValue([
|
||||
{
|
||||
token: 'sk-ant-oat01-aaaaaaaa',
|
||||
index: 0,
|
||||
masked: 'aaa',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
},
|
||||
]);
|
||||
readJsonFileMock.mockReturnValue({
|
||||
'account-0': {
|
||||
usage: {
|
||||
five_hour: { utilization: 1 },
|
||||
seven_day: { utilization: 40 },
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const { getClaudeUsageExhaustion } = await import('./claude-usage.js');
|
||||
expect(getClaudeUsageExhaustion().exhausted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCodexUsageExhaustion', () => {
|
||||
it('returns exhausted=false when no Codex accounts are configured', async () => {
|
||||
getAllCodexAccountsMock.mockReturnValue([]);
|
||||
const { getCodexUsageExhaustion } =
|
||||
await import('./codex-usage-collector.js');
|
||||
expect(getCodexUsageExhaustion()).toEqual({
|
||||
exhausted: false,
|
||||
nextResetAt: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns exhausted=false when an account has unknown usage (-1)', async () => {
|
||||
getAllCodexAccountsMock.mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'a',
|
||||
planType: 'pro',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: -1,
|
||||
cachedUsageD7Pct: -1,
|
||||
},
|
||||
]);
|
||||
const { getCodexUsageExhaustion } =
|
||||
await import('./codex-usage-collector.js');
|
||||
expect(getCodexUsageExhaustion().exhausted).toBe(false);
|
||||
});
|
||||
|
||||
it('returns exhausted=true with earliest recovery when all accounts are exhausted', async () => {
|
||||
const reset0 = new Date('2026-04-28T02:00:00Z').toISOString();
|
||||
const reset1 = new Date('2026-04-28T05:00:00Z').toISOString();
|
||||
getAllCodexAccountsMock.mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'a',
|
||||
planType: 'pro',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 96,
|
||||
cachedUsageD7Pct: 40,
|
||||
resetAt: reset0,
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
accountId: 'b',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 30,
|
||||
cachedUsageD7Pct: 99,
|
||||
resetD7At: reset1,
|
||||
},
|
||||
]);
|
||||
const { getCodexUsageExhaustion } =
|
||||
await import('./codex-usage-collector.js');
|
||||
const info = getCodexUsageExhaustion();
|
||||
expect(info.exhausted).toBe(true);
|
||||
expect(info.nextResetAt).toBe(reset0);
|
||||
});
|
||||
|
||||
it('returns exhausted=false when at least one Codex account has headroom', async () => {
|
||||
getAllCodexAccountsMock.mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'a',
|
||||
planType: 'pro',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 99,
|
||||
cachedUsageD7Pct: 99,
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
accountId: 'b',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 50,
|
||||
cachedUsageD7Pct: 80,
|
||||
},
|
||||
]);
|
||||
const { getCodexUsageExhaustion } =
|
||||
await import('./codex-usage-collector.js');
|
||||
expect(getCodexUsageExhaustion().exhausted).toBe(false);
|
||||
});
|
||||
|
||||
it('treats rate-limited Codex accounts as exhausted', async () => {
|
||||
getAllCodexAccountsMock.mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'a',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: true,
|
||||
cachedUsagePct: 10,
|
||||
cachedUsageD7Pct: 10,
|
||||
},
|
||||
]);
|
||||
const { getCodexUsageExhaustion } =
|
||||
await import('./codex-usage-collector.js');
|
||||
expect(getCodexUsageExhaustion().exhausted).toBe(true);
|
||||
});
|
||||
});
|
||||
188
src/usage-primer.ts
Normal file
188
src/usage-primer.ts
Normal file
@@ -0,0 +1,188 @@
|
||||
import { spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { CODEX_WARMUP_CONFIG } from './config.js';
|
||||
import { runCodexWarmupCycle } from './codex-warmup.js';
|
||||
import { logger } from './logger.js';
|
||||
import { getAllTokens } from './token-rotation.js';
|
||||
|
||||
/**
|
||||
* Usage-window alignment primer.
|
||||
*
|
||||
* Anthropic/OpenAI enforce 5h rolling usage windows that start on first
|
||||
* inference call. We anchor windows to fixed KST times by firing a tiny
|
||||
* primer call at 08, 13, 18, 23 KST (4 windows × 5h = 20h, leaving a
|
||||
* 04:00–08:00 gap blocked by message-runtime-gating).
|
||||
*
|
||||
* Daily reset point ends up being 08:00 KST (start of first slot after gap).
|
||||
*/
|
||||
|
||||
const PRIMER_HOURS_KST = [8, 13, 18, 23];
|
||||
const KST_OFFSET_MS = 9 * 60 * 60 * 1000;
|
||||
const CLAUDE_PRIMER_TIMEOUT_MS = 60_000;
|
||||
|
||||
function resolveClaudeBinary(): string {
|
||||
const candidates = [
|
||||
path.resolve(
|
||||
process.cwd(),
|
||||
'runners/agent-runner/node_modules/@anthropic-ai/claude-agent-sdk-linux-x64/claude',
|
||||
),
|
||||
path.resolve(
|
||||
process.cwd(),
|
||||
'runners/agent-runner/node_modules/@anthropic-ai/claude-agent-sdk-linux-x64-musl/claude',
|
||||
),
|
||||
];
|
||||
for (const p of candidates) {
|
||||
if (fs.existsSync(p)) return p;
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
function runClaudePrimerOnce(
|
||||
token: string,
|
||||
): Promise<{ ok: boolean; reason: string }> {
|
||||
const binary = resolveClaudeBinary();
|
||||
if (!fs.existsSync(binary)) {
|
||||
return Promise.resolve({ ok: false, reason: 'binary_missing' });
|
||||
}
|
||||
const args = [
|
||||
'-p',
|
||||
'ok',
|
||||
'--model',
|
||||
'haiku',
|
||||
'--output-format',
|
||||
'text',
|
||||
'--max-turns',
|
||||
'1',
|
||||
'--dangerously-skip-permissions',
|
||||
];
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
let stderr = '';
|
||||
const finish = (result: { ok: boolean; reason: string }): void => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
resolve(result);
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
try {
|
||||
proc.kill();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
finish({ ok: false, reason: 'timeout' });
|
||||
}, CLAUDE_PRIMER_TIMEOUT_MS);
|
||||
|
||||
const proc = spawn(binary, args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: {
|
||||
...process.env,
|
||||
CLAUDE_CODE_OAUTH_TOKEN: token,
|
||||
},
|
||||
});
|
||||
proc.stderr?.on('data', (chunk: Buffer) => {
|
||||
stderr += chunk.toString();
|
||||
if (stderr.length > 2000) stderr = stderr.slice(-2000);
|
||||
});
|
||||
proc.on('error', (err) => {
|
||||
logger.warn({ err }, 'Claude primer spawn error');
|
||||
finish({ ok: false, reason: 'spawn_error' });
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
finish({ ok: true, reason: 'ok' });
|
||||
} else {
|
||||
logger.warn(
|
||||
{ exitCode: code, stderr: stderr.trim().slice(-500) || undefined },
|
||||
'Claude primer command failed',
|
||||
);
|
||||
finish({ ok: false, reason: `exit_${code ?? 'unknown'}` });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runClaudePrimerCycle(): Promise<void> {
|
||||
const tokens = getAllTokens();
|
||||
const candidates = tokens.filter((t) => !t.isRateLimited);
|
||||
if (candidates.length === 0) {
|
||||
logger.info('Claude primer skipped: no eligible tokens');
|
||||
return;
|
||||
}
|
||||
const target = candidates[0];
|
||||
logger.info(
|
||||
{ tokenIndex: target.index },
|
||||
'Starting Claude primer (haiku, prompt=ok)',
|
||||
);
|
||||
const result = await runClaudePrimerOnce(target.token);
|
||||
if (result.ok) {
|
||||
logger.info({ tokenIndex: target.index }, 'Claude primer completed');
|
||||
} else {
|
||||
logger.warn(
|
||||
{ tokenIndex: target.index, reason: result.reason },
|
||||
'Claude primer failed',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function runCodexPrimerCycle(): Promise<void> {
|
||||
try {
|
||||
const result = await runCodexWarmupCycle(CODEX_WARMUP_CONFIG);
|
||||
logger.info({ result }, 'Codex primer cycle finished');
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Codex primer cycle threw');
|
||||
}
|
||||
}
|
||||
|
||||
async function runPrimerCycle(): Promise<void> {
|
||||
logger.info('Running scheduled usage primer cycle');
|
||||
await Promise.allSettled([runClaudePrimerCycle(), runCodexPrimerCycle()]);
|
||||
}
|
||||
|
||||
export function msUntilNextPrimerSlotKST(nowMs: number = Date.now()): number {
|
||||
// KST = UTC+9 (no DST). Shift so KST clock can be read via getUTC*.
|
||||
const kstNowShifted = nowMs + KST_OFFSET_MS;
|
||||
const kstDate = new Date(kstNowShifted);
|
||||
const kstH = kstDate.getUTCHours();
|
||||
|
||||
let nextH = PRIMER_HOURS_KST.find((h) => h > kstH);
|
||||
let dayOffset = 0;
|
||||
if (nextH === undefined) {
|
||||
nextH = PRIMER_HOURS_KST[0];
|
||||
dayOffset = 1;
|
||||
}
|
||||
|
||||
const target = new Date(kstNowShifted);
|
||||
target.setUTCHours(nextH, 0, 0, 0);
|
||||
if (dayOffset === 1) target.setUTCDate(target.getUTCDate() + 1);
|
||||
|
||||
return target.getTime() - kstNowShifted;
|
||||
}
|
||||
|
||||
let primerTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
export function startUsagePrimer(): void {
|
||||
if (primerTimer) return;
|
||||
const scheduleNext = (): void => {
|
||||
const delay = msUntilNextPrimerSlotKST();
|
||||
primerTimer = setTimeout(() => {
|
||||
void runPrimerCycle().finally(scheduleNext);
|
||||
}, delay);
|
||||
const fireAt = new Date(Date.now() + delay).toISOString();
|
||||
logger.info(
|
||||
{ delayMinutes: Math.round(delay / 60_000), fireAt },
|
||||
'Next usage primer scheduled',
|
||||
);
|
||||
};
|
||||
scheduleNext();
|
||||
}
|
||||
|
||||
export function stopUsagePrimer(): void {
|
||||
if (primerTimer) {
|
||||
clearTimeout(primerTimer);
|
||||
primerTimer = null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user