feat: pause room work on the Claude usage limit and resume after reset
Before, a Claude run that hit the account usage limit closed the open paired task as claude_usage_limit, posted the reset time and consumed the user's messages, so the work was lost and had to be re-requested. Now the room is paused instead: - The failed turn is released without touching the paired task (no failure count, no status change, no follow-up) and the queue keeps the human messages unconsumed, so the same turn can run again. - The room is recorded in router_state `claude_usage_pause` with the reset time (error text, else cached usage API reset, else a 10 min retry). A banner time that just passed is read as "now", not tomorrow. - While the limit lasts, paused rooms run nothing and any other room's Claude turn pauses that room without spawning a run (the limit is account-wide). A stop request still goes through and drops the room from the resume list. - The scheduler tick clears the pause 1 min after the reset, posts a resume notice and re-queues the rooms; the pending turn or the kept messages then run again. The state survives restarts. Safety refusals keep closing the task as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -13,10 +13,10 @@ vi.mock('./claude-usage.js', () => ({
|
||||
}));
|
||||
|
||||
import {
|
||||
buildClaudeUsageLimitMessage,
|
||||
formatClaudeUsageLimitMessage,
|
||||
formatClaudeUsagePauseMessage,
|
||||
isClaudeUsageLimitReason,
|
||||
parseClaudeLimitResetTime,
|
||||
resolveClaudeUsageResume,
|
||||
} from './claude-usage-limit.js';
|
||||
|
||||
// 2026-10-01 19:00 KST
|
||||
@@ -67,39 +67,73 @@ describe('parseClaudeLimitResetTime', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Claude usage limit message', () => {
|
||||
describe('Claude usage limit pause', () => {
|
||||
it('shows HH:mm for a reset later today and the date otherwise', () => {
|
||||
expect(
|
||||
formatClaudeUsageLimitMessage(new Date('2026-10-01T12:40:00Z'), NOW),
|
||||
).toBe('Claude 사용 한도에 도달했습니다. 21:40에 재개할 수 있습니다.');
|
||||
expect(
|
||||
formatClaudeUsageLimitMessage(new Date('2026-10-01T19:00:00Z'), NOW),
|
||||
formatClaudeUsagePauseMessage(
|
||||
{ resumeAt: new Date('2026-10-01T12:40:00Z'), resetKnown: true },
|
||||
NOW,
|
||||
),
|
||||
).toBe(
|
||||
'Claude 사용 한도에 도달했습니다. 10월 2일 04:00에 재개할 수 있습니다.',
|
||||
'Claude 사용 한도에 도달해서 이 방의 작업을 일시정지했습니다. 21:40에 사용량이 초기화되면 자동으로 이어서 진행합니다. 그 전에 멈추려면 "중단"이라고 보내 주세요.',
|
||||
);
|
||||
expect(
|
||||
formatClaudeUsagePauseMessage(
|
||||
{ resumeAt: new Date('2026-10-01T19:00:00Z'), resetKnown: true },
|
||||
NOW,
|
||||
),
|
||||
).toContain('10월 2일 04:00에 사용량이 초기화되면');
|
||||
});
|
||||
|
||||
it('asks to retry later without a reset time', () => {
|
||||
expect(formatClaudeUsageLimitMessage(null, NOW)).toBe(
|
||||
'Claude 사용 한도에 도달했습니다. 잠시 후 다시 시도해 주세요.',
|
||||
);
|
||||
it('says it will retry when the reset time is unknown', () => {
|
||||
expect(
|
||||
formatClaudeUsagePauseMessage(
|
||||
{ resumeAt: new Date('2026-10-01T10:10:00Z'), resetKnown: false },
|
||||
NOW,
|
||||
),
|
||||
).toContain('초기화 시각을 알 수 없어서 19:10에 다시 시도합니다.');
|
||||
});
|
||||
|
||||
it('retries after the fallback delay without a reset time', () => {
|
||||
expect(resolveClaudeUsageResume('API Error: 429', NOW)).toEqual({
|
||||
resumeAt: new Date('2026-10-01T10:10:00Z'),
|
||||
resetKnown: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to the cached usage reset time', () => {
|
||||
cachedResetAt.mockReturnValueOnce(new Date('2026-10-01T11:15:00Z'));
|
||||
expect(buildClaudeUsageLimitMessage('API Error: 429', NOW)).toBe(
|
||||
'Claude 사용 한도에 도달했습니다. 20:15에 재개할 수 있습니다.',
|
||||
);
|
||||
expect(resolveClaudeUsageResume('API Error: 429', NOW)).toEqual({
|
||||
resumeAt: new Date('2026-10-01T11:15:00Z'),
|
||||
resetKnown: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('prefers the reset time in the error text', () => {
|
||||
cachedResetAt.mockReturnValueOnce(new Date('2026-10-01T11:15:00Z'));
|
||||
expect(
|
||||
buildClaudeUsageLimitMessage(
|
||||
resolveClaudeUsageResume(
|
||||
"You've hit your limit · resets 9:40pm (Asia/Seoul)",
|
||||
NOW,
|
||||
),
|
||||
).toBe('Claude 사용 한도에 도달했습니다. 21:40에 재개할 수 있습니다.');
|
||||
).toEqual({
|
||||
resumeAt: new Date('2026-10-01T12:40:00Z'),
|
||||
resetKnown: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not push a banner time that just passed to tomorrow', () => {
|
||||
// 21:41 KST, the banner still says 9:40pm: retry shortly, not in 24 hours.
|
||||
const justAfter = Date.parse('2026-10-01T12:41:00Z');
|
||||
expect(
|
||||
resolveClaudeUsageResume(
|
||||
"You've hit your limit · resets 9:40pm (Asia/Seoul)",
|
||||
justAfter,
|
||||
),
|
||||
).toEqual({
|
||||
resumeAt: new Date(justAfter + 10 * 60_000),
|
||||
resetKnown: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('treats only rate-limit and usage-exhausted triggers as usage limits', () => {
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
/**
|
||||
* User-facing notice for a Claude run that hit the account usage limit.
|
||||
* Reset time and user-facing notice for a Claude run that hit the account
|
||||
* usage limit.
|
||||
*
|
||||
* There is a single Claude account and no fallback provider, so the room is
|
||||
* told when Claude becomes available again instead of the work being handed
|
||||
* off or retried silently.
|
||||
* There is a single Claude account and no fallback provider, so the room's
|
||||
* work is paused until the limit resets and then resumed (see
|
||||
* claude-usage-pause.ts).
|
||||
*/
|
||||
|
||||
import type { AgentTriggerReason } from './agent-error-detection.js';
|
||||
import { getCachedClaudeUsageLimitResetAt } from './claude-usage.js';
|
||||
import { TIMEZONE } from './config.js';
|
||||
|
||||
const LIMIT_PREFIX = 'Claude 사용 한도에 도달했습니다.';
|
||||
const RETRY_LATER = '잠시 후 다시 시도해 주세요.';
|
||||
const PAUSE_PREFIX =
|
||||
'Claude 사용 한도에 도달해서 이 방의 작업을 일시정지했습니다.';
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
jan: 1,
|
||||
@@ -198,26 +199,48 @@ function formatResetTime(resetAt: Date, nowMs: number): string {
|
||||
return sameDay ? time : `${reset.month}월 ${reset.day}일 ${time}`;
|
||||
}
|
||||
|
||||
export function formatClaudeUsageLimitMessage(
|
||||
resetAt: Date | null,
|
||||
nowMs: number = Date.now(),
|
||||
): string {
|
||||
if (!resetAt || resetAt.getTime() <= nowMs) {
|
||||
return `${LIMIT_PREFIX} ${RETRY_LATER}`;
|
||||
}
|
||||
return `${LIMIT_PREFIX} ${formatResetTime(resetAt, nowMs)}에 재개할 수 있습니다.`;
|
||||
/** A banner time that passed less than this long ago means "resetting now", not tomorrow. */
|
||||
const JUST_PASSED_RESET_GRACE_MS = 15 * 60_000;
|
||||
/** Retry delay when the limit error names no usable reset time. */
|
||||
export const CLAUDE_USAGE_RETRY_FALLBACK_MS = 10 * 60_000;
|
||||
|
||||
export interface ClaudeUsageResume {
|
||||
resumeAt: Date;
|
||||
/** False when no reset time was found and resumeAt is only a retry time. */
|
||||
resetKnown: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Notice for the room: the reset time comes from the error text, else from
|
||||
* the cached usage API data, else the user is asked to retry later.
|
||||
* When paused Claude work may resume: the reset time in the error text, else
|
||||
* the cached usage API reset, else a short retry delay.
|
||||
*
|
||||
* A banner such as "resets 9:40pm" read at 21:41 is treated as "now" instead
|
||||
* of tomorrow 21:40, so a limit that outlives its reset by a minute is not
|
||||
* paused for a whole day.
|
||||
*/
|
||||
export function buildClaudeUsageLimitMessage(
|
||||
export function resolveClaudeUsageResume(
|
||||
errorText: string | null | undefined,
|
||||
nowMs: number = Date.now(),
|
||||
): string {
|
||||
): ClaudeUsageResume {
|
||||
const resetAt =
|
||||
parseClaudeLimitResetTime(errorText, nowMs) ??
|
||||
parseClaudeLimitResetTime(errorText, nowMs - JUST_PASSED_RESET_GRACE_MS) ??
|
||||
getCachedClaudeUsageLimitResetAt(nowMs);
|
||||
return formatClaudeUsageLimitMessage(resetAt, nowMs);
|
||||
if (resetAt && resetAt.getTime() > nowMs) {
|
||||
return { resumeAt: resetAt, resetKnown: true };
|
||||
}
|
||||
return {
|
||||
resumeAt: new Date(nowMs + CLAUDE_USAGE_RETRY_FALLBACK_MS),
|
||||
resetKnown: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatClaudeUsagePauseMessage(
|
||||
resume: ClaudeUsageResume,
|
||||
nowMs: number = Date.now(),
|
||||
): string {
|
||||
const at = formatResetTime(resume.resumeAt, nowMs);
|
||||
const when = resume.resetKnown
|
||||
? `${at}에 사용량이 초기화되면 자동으로 이어서 진행합니다.`
|
||||
: `초기화 시각을 알 수 없어서 ${at}에 다시 시도합니다.`;
|
||||
return `${PAUSE_PREFIX} ${when} 그 전에 멈추려면 "중단"이라고 보내 주세요.`;
|
||||
}
|
||||
|
||||
140
src/claude-usage-pause.test.ts
Normal file
140
src/claude-usage-pause.test.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const { routerState } = vi.hoisted(() => ({
|
||||
routerState: new Map<string, string>(),
|
||||
}));
|
||||
vi.mock('./db.js', () => ({
|
||||
getRouterState: (key: string) => routerState.get(key),
|
||||
setRouterState: (key: string, value: string) => {
|
||||
routerState.set(key, value);
|
||||
},
|
||||
}));
|
||||
vi.mock('./logger.js', () => ({
|
||||
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
|
||||
}));
|
||||
|
||||
import {
|
||||
CLAUDE_USAGE_PAUSE_KEY,
|
||||
CLAUDE_USAGE_RESUME_MARGIN_MS,
|
||||
consumeRunPausedForClaudeUsage,
|
||||
getActiveClaudeUsagePause,
|
||||
isRoomPausedForClaudeUsage,
|
||||
markRunPausedForClaudeUsage,
|
||||
pauseRoomForClaudeUsage,
|
||||
releaseRoomFromClaudeUsagePause,
|
||||
takeRoomsReadyToResume,
|
||||
} from './claude-usage-pause.js';
|
||||
import { isSupportedRouterStateKey } from './db/router-state.js';
|
||||
|
||||
const NOW = new Date('2026-10-13T03:00:00Z');
|
||||
const RESET = new Date('2026-10-13T05:00:00Z');
|
||||
|
||||
beforeEach(() => {
|
||||
routerState.clear();
|
||||
});
|
||||
|
||||
describe('Claude usage pause', () => {
|
||||
it('uses a router_state key that startup accepts', () => {
|
||||
expect(isSupportedRouterStateKey(CLAUDE_USAGE_PAUSE_KEY)).toBe(true);
|
||||
});
|
||||
|
||||
it('pauses rooms until the reset and keeps the later reset time', () => {
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-a',
|
||||
resumeAt: RESET,
|
||||
resetKnown: true,
|
||||
now: NOW,
|
||||
});
|
||||
const pause = pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-b',
|
||||
resumeAt: new Date(NOW.getTime() + 10 * 60_000),
|
||||
resetKnown: false,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(pause).toEqual({ until: RESET, resetKnown: true });
|
||||
expect(isRoomPausedForClaudeUsage('room-a')).toBe(true);
|
||||
expect(isRoomPausedForClaudeUsage('room-b')).toBe(true);
|
||||
expect(isRoomPausedForClaudeUsage('room-c')).toBe(false);
|
||||
expect(getActiveClaudeUsagePause(NOW.getTime())).toEqual({
|
||||
until: RESET,
|
||||
resetKnown: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('moves the pause later when a later reset is reported', () => {
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-a',
|
||||
resumeAt: RESET,
|
||||
resetKnown: false,
|
||||
now: NOW,
|
||||
});
|
||||
const later = new Date(RESET.getTime() + 60 * 60_000);
|
||||
expect(
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-a',
|
||||
resumeAt: later,
|
||||
resetKnown: true,
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ until: later, resetKnown: true });
|
||||
});
|
||||
|
||||
it('holds the rooms until the reset plus a margin, then hands them back once', () => {
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-a',
|
||||
resumeAt: RESET,
|
||||
resetKnown: true,
|
||||
now: NOW,
|
||||
});
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-b',
|
||||
resumeAt: RESET,
|
||||
resetKnown: true,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(takeRoomsReadyToResume(RESET.getTime())).toEqual([]);
|
||||
expect(getActiveClaudeUsagePause(RESET.getTime())).toBeNull();
|
||||
expect(isRoomPausedForClaudeUsage('room-a')).toBe(true);
|
||||
|
||||
const resumeAt = RESET.getTime() + CLAUDE_USAGE_RESUME_MARGIN_MS;
|
||||
expect(takeRoomsReadyToResume(resumeAt)).toEqual(['room-a', 'room-b']);
|
||||
expect(isRoomPausedForClaudeUsage('room-a')).toBe(false);
|
||||
expect(takeRoomsReadyToResume(resumeAt)).toEqual([]);
|
||||
});
|
||||
|
||||
it('releases one room when its work is stopped', () => {
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-a',
|
||||
resumeAt: RESET,
|
||||
resetKnown: true,
|
||||
now: NOW,
|
||||
});
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'room-b',
|
||||
resumeAt: RESET,
|
||||
resetKnown: true,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(releaseRoomFromClaudeUsagePause('room-a')).toBe(true);
|
||||
expect(releaseRoomFromClaudeUsagePause('room-a')).toBe(false);
|
||||
expect(isRoomPausedForClaudeUsage('room-a')).toBe(false);
|
||||
expect(isRoomPausedForClaudeUsage('room-b')).toBe(true);
|
||||
});
|
||||
|
||||
it('ignores an unreadable stored state', () => {
|
||||
routerState.set(CLAUDE_USAGE_PAUSE_KEY, '{not json');
|
||||
expect(getActiveClaudeUsagePause(NOW.getTime())).toBeNull();
|
||||
expect(isRoomPausedForClaudeUsage('room-a')).toBe(false);
|
||||
expect(takeRoomsReadyToResume(NOW.getTime())).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports a paused run once', () => {
|
||||
markRunPausedForClaudeUsage('run-1');
|
||||
expect(consumeRunPausedForClaudeUsage('run-1')).toBe(true);
|
||||
expect(consumeRunPausedForClaudeUsage('run-1')).toBe(false);
|
||||
expect(consumeRunPausedForClaudeUsage('run-2')).toBe(false);
|
||||
});
|
||||
});
|
||||
139
src/claude-usage-pause.ts
Normal file
139
src/claude-usage-pause.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Rooms whose Claude work is paused because the Claude account hit its usage
|
||||
* limit.
|
||||
*
|
||||
* There is one Claude account, so the limit is global: while it lasts, every
|
||||
* Claude run would fail the same way. The first run that hits it (or that
|
||||
* would start while it lasts) pauses its room: the paired task is left as it
|
||||
* was, the human messages stay unconsumed, and the room is recorded here.
|
||||
* Paused rooms run nothing until the reset time; the scheduler tick then
|
||||
* clears the pause and re-queues them, so the interrupted turn runs again.
|
||||
*
|
||||
* Stored in router_state so a restart keeps the pause and still resumes.
|
||||
*/
|
||||
|
||||
import { getRouterState, setRouterState } from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
|
||||
export const CLAUDE_USAGE_PAUSE_KEY = 'claude_usage_pause';
|
||||
|
||||
/** Wait this long past the reset time before resuming, to not race it. */
|
||||
export const CLAUDE_USAGE_RESUME_MARGIN_MS = 60_000;
|
||||
|
||||
export const CLAUDE_USAGE_RESUME_NOTICE =
|
||||
'Claude 사용량이 초기화되어서 일시정지했던 작업을 이어서 진행합니다.';
|
||||
|
||||
interface ClaudeUsagePauseState {
|
||||
/** ISO time the limit resets (or the next retry when unknown). */
|
||||
until: string;
|
||||
resetKnown: boolean;
|
||||
/** chatJid -> ISO time the room was paused. */
|
||||
rooms: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ActiveClaudeUsagePause {
|
||||
until: Date;
|
||||
resetKnown: boolean;
|
||||
}
|
||||
|
||||
function readState(): ClaudeUsagePauseState | null {
|
||||
const raw = getRouterState(CLAUDE_USAGE_PAUSE_KEY);
|
||||
if (!raw) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as ClaudeUsagePauseState | null;
|
||||
if (!parsed || !Number.isFinite(Date.parse(parsed.until))) return null;
|
||||
return { ...parsed, rooms: parsed.rooms ?? {} };
|
||||
} catch {
|
||||
logger.warn({ raw }, 'Ignoring unreadable Claude usage pause state');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function writeState(state: ClaudeUsagePauseState | null): void {
|
||||
setRouterState(CLAUDE_USAGE_PAUSE_KEY, JSON.stringify(state));
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause `chatJid` until `resumeAt` (kept if a later pause already exists).
|
||||
* Returns the pause now in effect.
|
||||
*/
|
||||
export function pauseRoomForClaudeUsage(args: {
|
||||
chatJid: string;
|
||||
resumeAt: Date;
|
||||
resetKnown: boolean;
|
||||
now?: Date;
|
||||
}): ActiveClaudeUsagePause {
|
||||
const now = args.now ?? new Date();
|
||||
const state = readState();
|
||||
const keepExisting =
|
||||
state !== null && Date.parse(state.until) >= args.resumeAt.getTime();
|
||||
const next: ClaudeUsagePauseState = {
|
||||
until: keepExisting ? state.until : args.resumeAt.toISOString(),
|
||||
resetKnown: keepExisting ? state.resetKnown : args.resetKnown,
|
||||
rooms: { ...(state?.rooms ?? {}) },
|
||||
};
|
||||
next.rooms[args.chatJid] ??= now.toISOString();
|
||||
writeState(next);
|
||||
logger.warn(
|
||||
{ chatJid: args.chatJid, until: next.until, resetKnown: next.resetKnown },
|
||||
'Paused room work until the Claude usage limit resets',
|
||||
);
|
||||
return { until: new Date(next.until), resetKnown: next.resetKnown };
|
||||
}
|
||||
|
||||
/** The Claude usage limit still in effect, or null. */
|
||||
export function getActiveClaudeUsagePause(
|
||||
nowMs: number = Date.now(),
|
||||
): ActiveClaudeUsagePause | null {
|
||||
const state = readState();
|
||||
if (!state) return null;
|
||||
const until = new Date(state.until);
|
||||
return until.getTime() > nowMs
|
||||
? { until, resetKnown: state.resetKnown }
|
||||
: null;
|
||||
}
|
||||
|
||||
export function isRoomPausedForClaudeUsage(chatJid: string): boolean {
|
||||
return Boolean(readState()?.rooms[chatJid]);
|
||||
}
|
||||
|
||||
/** Drop a room from the pause (the user stopped its work). */
|
||||
export function releaseRoomFromClaudeUsagePause(chatJid: string): boolean {
|
||||
const state = readState();
|
||||
if (!state?.rooms[chatJid]) return false;
|
||||
const rooms = { ...state.rooms };
|
||||
delete rooms[chatJid];
|
||||
writeState({ ...state, rooms });
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Once the reset time (plus a margin) has passed, clear the pause and return
|
||||
* the rooms to resume. Returns [] while the pause still holds.
|
||||
*/
|
||||
export function takeRoomsReadyToResume(nowMs: number = Date.now()): string[] {
|
||||
const state = readState();
|
||||
if (!state) return [];
|
||||
if (Date.parse(state.until) + CLAUDE_USAGE_RESUME_MARGIN_MS > nowMs) {
|
||||
return [];
|
||||
}
|
||||
writeState(null);
|
||||
return Object.keys(state.rooms);
|
||||
}
|
||||
|
||||
// Runs that ended paused, so the queue keeps their messages unconsumed.
|
||||
const pausedRunIds = new Set<string>();
|
||||
const MAX_TRACKED_PAUSED_RUNS = 100;
|
||||
|
||||
export function markRunPausedForClaudeUsage(runId: string): void {
|
||||
pausedRunIds.add(runId);
|
||||
if (pausedRunIds.size > MAX_TRACKED_PAUSED_RUNS) {
|
||||
const oldest = pausedRunIds.values().next().value;
|
||||
if (oldest !== undefined) pausedRunIds.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
/** True once for a run that ended paused by the Claude usage limit. */
|
||||
export function consumeRunPausedForClaudeUsage(runId: string): boolean {
|
||||
return pausedRunIds.delete(runId);
|
||||
}
|
||||
@@ -36,7 +36,12 @@ export function getRouterStateFromDatabase(
|
||||
* the same list, so an unregistered key fails on its first write and in tests
|
||||
* instead of at the next startup.
|
||||
*/
|
||||
const SUPPORTED_ROUTER_STATE_KEYS = new Set(['last_seq', 'last_agent_seq']);
|
||||
const SUPPORTED_ROUTER_STATE_KEYS = new Set([
|
||||
'last_seq',
|
||||
'last_agent_seq',
|
||||
// claude-usage-pause.ts
|
||||
'claude_usage_pause',
|
||||
]);
|
||||
export const PAIRED_SESSION_TASK_KEY_PREFIX = 'paired_session_task:';
|
||||
const SUPPORTED_ROUTER_STATE_KEY_PREFIXES = [PAIRED_SESSION_TASK_KEY_PREFIX];
|
||||
|
||||
|
||||
@@ -44,17 +44,18 @@ function releaseInterruptedPairedExecution(
|
||||
taskId: string,
|
||||
runId: string,
|
||||
log: ExecutorLog,
|
||||
cause = 'a human message interrupted the turn',
|
||||
): void {
|
||||
log.info(
|
||||
{ pairedTaskId: taskId, runId },
|
||||
'Released paired execution lease without counting a failure because a human message interrupted the turn',
|
||||
`Released paired execution lease without counting a failure because ${cause}`,
|
||||
);
|
||||
try {
|
||||
releasePairedTaskExecutionLease({ taskId, runId });
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
{ pairedTaskId: taskId, runId, err },
|
||||
'Failed to release paired execution lease after human interruption',
|
||||
{ pairedTaskId: taskId, runId, err, cause },
|
||||
'Failed to release paired execution lease without counting a failure',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -178,6 +179,11 @@ export interface PairedExecutionLifecycle {
|
||||
/** The text to deliver for an accepted final output. */
|
||||
resolveFinalDeliveryText(outputText: string): string;
|
||||
completeImmediately(args: { status: 'succeeded' | 'failed' }): void;
|
||||
/**
|
||||
* End the turn without touching the paired task (no failure count, no
|
||||
* status change, no follow-up) so the same turn can run again later.
|
||||
*/
|
||||
releaseForClaudeUsagePause(): void;
|
||||
markStatus(status: 'succeeded' | 'failed'): void;
|
||||
markSawOutput(sawOutput: boolean): void;
|
||||
getSummary(): string | null;
|
||||
@@ -208,6 +214,7 @@ interface FinalizeState {
|
||||
|
||||
class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
private pairedExecutionStatus: PairedExecutionStatus = 'failed';
|
||||
private pausedForClaudeUsage = false;
|
||||
private pairedExecutionSummary: string | null = null;
|
||||
private pairedFinalOutput: string | null = null;
|
||||
private pairedFinalAttachments: OutboundAttachment[] = [];
|
||||
@@ -330,6 +337,23 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
this.pairedExecutionCompleted = true;
|
||||
}
|
||||
|
||||
releaseForClaudeUsagePause(): void {
|
||||
const { pairedExecutionContext, runId, log } = this.args;
|
||||
this.pausedForClaudeUsage = true;
|
||||
this.pairedExecutionStatus = 'failed';
|
||||
if (!pairedExecutionContext || this.pairedExecutionCompleted) {
|
||||
return;
|
||||
}
|
||||
this.clearLeaseHeartbeat();
|
||||
releaseInterruptedPairedExecution(
|
||||
pairedExecutionContext.task.id,
|
||||
runId,
|
||||
log,
|
||||
'Claude hit its usage limit and the room is paused',
|
||||
);
|
||||
this.pairedExecutionCompleted = true;
|
||||
}
|
||||
|
||||
markStatus(status: PairedExecutionStatus): void {
|
||||
this.pairedExecutionStatus = status;
|
||||
}
|
||||
@@ -680,7 +704,11 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
): Promise<void> {
|
||||
const { chatJid, completedRole, onOutput, pairedExecutionContext } =
|
||||
this.args;
|
||||
if (!pairedExecutionContext || state.interruptedByHumanMessage) {
|
||||
if (
|
||||
!pairedExecutionContext ||
|
||||
state.interruptedByHumanMessage ||
|
||||
this.pausedForClaudeUsage
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const finishedTask = getPairedTaskById(pairedExecutionContext.task.id);
|
||||
|
||||
@@ -225,6 +225,7 @@ import * as sessionRecovery from './session-recovery.js';
|
||||
import * as serviceRouting from './service-routing.js';
|
||||
import * as tokenRefresh from './token-refresh.js';
|
||||
import * as claudeOauthToken from './claude-oauth-token.js';
|
||||
import { consumeRunPausedForClaudeUsage } from './claude-usage-pause.js';
|
||||
import type { RegisteredGroup } from './types.js';
|
||||
|
||||
function makeGroup(): RegisteredGroup {
|
||||
@@ -2754,8 +2755,23 @@ describe('runAgentForGroup room memory', () => {
|
||||
});
|
||||
|
||||
describe('runAgentForGroup Claude failure recovery', () => {
|
||||
let routerState: Map<string, string>;
|
||||
const readUsagePause = () =>
|
||||
JSON.parse(routerState.get('claude_usage_pause') ?? 'null') as {
|
||||
until: string;
|
||||
resetKnown: boolean;
|
||||
rooms: Record<string, string>;
|
||||
} | null;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
routerState = new Map();
|
||||
vi.mocked(db.getRouterState).mockImplementation((key) =>
|
||||
routerState.get(key),
|
||||
);
|
||||
vi.mocked(db.setRouterState).mockImplementation((key, value) => {
|
||||
routerState.set(key, value);
|
||||
});
|
||||
resetPairedFollowUpScheduleState();
|
||||
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
|
||||
vi.mocked(claudeOauthToken.getCurrentClaudeToken).mockReturnValue(
|
||||
@@ -3039,7 +3055,7 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
||||
expect(outputs).toEqual([]);
|
||||
});
|
||||
|
||||
it('tells the room when Claude resets instead of retrying on usage exhaustion', async () => {
|
||||
it('pauses the room until Claude resets instead of retrying on usage exhaustion', async () => {
|
||||
const outputs: string[] = [];
|
||||
|
||||
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
|
||||
@@ -3077,11 +3093,17 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
||||
expect(outputs).toHaveLength(1);
|
||||
expect(outputs[0]).toMatch(
|
||||
/^Claude 사용 한도에 도달했습니다\. (\d+월 \d+일 )?04:00에 재개할 수 있습니다\.$/,
|
||||
/^Claude 사용 한도에 도달해서 이 방의 작업을 일시정지했습니다\. (\d+월 \d+일 )?04:00에 사용량이 초기화되면 자동으로 이어서 진행합니다\./,
|
||||
);
|
||||
const pause = readUsagePause();
|
||||
expect(pause?.resetKnown).toBe(true);
|
||||
expect(Object.keys(pause?.rooms ?? {})).toEqual(['group@test']);
|
||||
expect(consumeRunPausedForClaudeUsage('run-all-exhausted-error')).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('closes the paired task when the Claude owner hits its usage limit', async () => {
|
||||
it('keeps the paired task as it was when the Claude owner hits its usage limit', async () => {
|
||||
const outputs: string[] = [];
|
||||
const pairedTask = {
|
||||
id: 'paired-task-usage-limit',
|
||||
@@ -3137,25 +3159,87 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
||||
|
||||
expect(result).toBe('success');
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
||||
expect(db.updatePairedTaskIfUnchanged).toHaveBeenCalledWith(
|
||||
'paired-task-usage-limit',
|
||||
'2026-04-06T00:00:00.000Z',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
completion_reason: 'claude_usage_limit',
|
||||
}),
|
||||
);
|
||||
// No status change and no failure count: the same turn runs on resume.
|
||||
expect(db.updatePairedTaskIfUnchanged).not.toHaveBeenCalled();
|
||||
expect(
|
||||
pairedExecutionContext.completePairedExecutionContext,
|
||||
).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
taskId: 'paired-task-usage-limit',
|
||||
status: 'failed',
|
||||
).not.toHaveBeenCalled();
|
||||
expect(db.releasePairedTaskExecutionLease).toHaveBeenCalledWith({
|
||||
taskId: 'paired-task-usage-limit',
|
||||
runId: 'run-paired-usage-limit',
|
||||
});
|
||||
expect(outputs).toEqual([
|
||||
expect.stringMatching(
|
||||
/^Claude 사용 한도에 도달해서 이 방의 작업을 일시정지했습니다\. .*다시 시도합니다\./,
|
||||
),
|
||||
]);
|
||||
expect(readUsagePause()?.rooms).toHaveProperty('group@test');
|
||||
expect(consumeRunPausedForClaudeUsage('run-paired-usage-limit')).toBe(true);
|
||||
});
|
||||
|
||||
it('pauses a Claude turn without running it while the usage limit lasts', async () => {
|
||||
const outputs: string[] = [];
|
||||
const until = new Date(Date.now() + 60 * 60_000);
|
||||
routerState.set(
|
||||
'claude_usage_pause',
|
||||
JSON.stringify({
|
||||
until: until.toISOString(),
|
||||
resetKnown: true,
|
||||
rooms: { 'other@test': new Date().toISOString() },
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runAgentForGroup(makeDeps(), {
|
||||
group: makeGroup(),
|
||||
prompt: 'hello',
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-during-usage-pause',
|
||||
onOutput: async (output) => {
|
||||
const text =
|
||||
output.output && 'text' in output.output
|
||||
? output.output.text
|
||||
: output.result;
|
||||
if (typeof text === 'string') outputs.push(text);
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBe('success');
|
||||
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
|
||||
expect(outputs).toEqual([
|
||||
expect.stringMatching(/^Claude 사용 한도에 도달했습니다\. /),
|
||||
expect.stringMatching(/사용량이 초기화되면 자동으로 이어서 진행합니다/),
|
||||
]);
|
||||
const pause = readUsagePause();
|
||||
expect(pause?.until).toBe(until.toISOString());
|
||||
expect(Object.keys(pause?.rooms ?? {}).sort()).toEqual([
|
||||
'group@test',
|
||||
'other@test',
|
||||
]);
|
||||
expect(consumeRunPausedForClaudeUsage('run-during-usage-pause')).toBe(true);
|
||||
});
|
||||
|
||||
it('runs a Claude turn normally once the usage limit has reset', async () => {
|
||||
routerState.set(
|
||||
'claude_usage_pause',
|
||||
JSON.stringify({
|
||||
until: new Date(Date.now() - 1_000).toISOString(),
|
||||
resetKnown: true,
|
||||
rooms: { 'group@test': new Date().toISOString() },
|
||||
}),
|
||||
);
|
||||
vi.mocked(agentRunner.runAgentProcess).mockResolvedValueOnce({
|
||||
status: 'success',
|
||||
result: 'done',
|
||||
});
|
||||
|
||||
await runAgentForGroup(makeDeps(), {
|
||||
group: makeGroup(),
|
||||
prompt: 'hello',
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-after-usage-reset',
|
||||
});
|
||||
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
||||
expect(consumeRunPausedForClaudeUsage('run-after-usage-reset')).toBe(false);
|
||||
});
|
||||
|
||||
it('closes the task, clears the session and posts a notice on a safeguards refusal', async () => {
|
||||
@@ -3420,7 +3504,9 @@ describe('runAgentForGroup Claude failure recovery', () => {
|
||||
expect(result).toBe('success');
|
||||
expect(outputs).toEqual([
|
||||
'대화 요약 중...',
|
||||
expect.stringMatching(/^Claude 사용 한도에 도달했습니다\. .*02:00에/),
|
||||
expect.stringMatching(
|
||||
/^Claude 사용 한도에 도달해서 이 방의 작업을 일시정지했습니다\. .*02:00에/,
|
||||
),
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -9,9 +9,16 @@ import type { PairedTurnIdentity } from './paired-turn-identity.js';
|
||||
import { executeMessageAgentAttemptLifecycle } from './message-agent-executor-lifecycle.js';
|
||||
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
|
||||
import {
|
||||
buildClaudeUsageLimitMessage,
|
||||
formatClaudeUsagePauseMessage,
|
||||
isClaudeUsageLimitReason,
|
||||
resolveClaudeUsageResume,
|
||||
type ClaudeUsageResume,
|
||||
} from './claude-usage-limit.js';
|
||||
import {
|
||||
getActiveClaudeUsagePause,
|
||||
markRunPausedForClaudeUsage,
|
||||
pauseRoomForClaudeUsage,
|
||||
} from './claude-usage-pause.js';
|
||||
import { getPairedTaskById } from './db.js';
|
||||
import { completePairedTaskWithReason } from './paired-task-status.js';
|
||||
import { prepareMessageAgentExecutionTarget } from './message-agent-executor-target.js';
|
||||
@@ -30,9 +37,6 @@ import {
|
||||
|
||||
// ── Main executor ─────────────────────────────────────────────────
|
||||
|
||||
/** completion_reason of a paired task closed because Claude ran out of usage. */
|
||||
export const CLAUDE_USAGE_LIMIT_COMPLETION_REASON = 'claude_usage_limit';
|
||||
|
||||
/** completion_reason of a paired task closed because Claude's safeguards refused it. */
|
||||
export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal';
|
||||
|
||||
@@ -74,54 +78,133 @@ interface ClaudeUsageLimitFailure {
|
||||
errorText: string | null;
|
||||
}
|
||||
|
||||
type PauseLifecycle = Pick<
|
||||
PairedExecutionLifecycle,
|
||||
'updateSummary' | 'releaseForClaudeUsagePause'
|
||||
>;
|
||||
|
||||
/**
|
||||
* When a Claude run failed because the account is out of usage, or because
|
||||
* the model's safeguards refused the request, close the paired task (retrying
|
||||
* cannot help; the next user message starts a new task) and tell the room.
|
||||
* The refusal is never handed to the reviewer/arbiter as if it were an answer.
|
||||
* Returns false for any other failure.
|
||||
* Claude is out of usage: pause the room instead of failing the turn. The
|
||||
* paired task is left untouched and the run is marked paused so the queue
|
||||
* keeps its messages; the scheduler re-queues the room after the reset and the
|
||||
* same turn runs again.
|
||||
*/
|
||||
async function pauseRunForClaudeUsage(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
resume: ClaudeUsageResume;
|
||||
summary: string;
|
||||
pairedExecutionLifecycle: PauseLifecycle;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
}): Promise<void> {
|
||||
const pause = pauseRoomForClaudeUsage({
|
||||
chatJid: args.chatJid,
|
||||
resumeAt: args.resume.resumeAt,
|
||||
resetKnown: args.resume.resetKnown,
|
||||
});
|
||||
markRunPausedForClaudeUsage(args.runId);
|
||||
args.pairedExecutionLifecycle.updateSummary({ outputText: args.summary });
|
||||
args.pairedExecutionLifecycle.releaseForClaudeUsagePause();
|
||||
const notice = formatClaudeUsagePauseMessage({
|
||||
resumeAt: pause.until,
|
||||
resetKnown: pause.resetKnown,
|
||||
});
|
||||
await args.onOutput?.({
|
||||
status: 'success',
|
||||
result: null,
|
||||
output: { visibility: 'public', text: notice },
|
||||
phase: 'final',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The Claude limit is account-wide: while it lasts, a Claude turn in a room
|
||||
* that is not paused yet pauses it instead of starting a run that would fail.
|
||||
* Returns false when no limit is in effect.
|
||||
*/
|
||||
async function pauseWhileClaudeUsageLimited(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
activeRole: PairedRoomRole;
|
||||
pairedExecutionLifecycle: PauseLifecycle &
|
||||
Pick<PairedExecutionLifecycle, 'asyncFinalize'>;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||
}): Promise<boolean> {
|
||||
const pause = getActiveClaudeUsagePause();
|
||||
if (!pause) return false;
|
||||
args.log.warn(
|
||||
{ until: pause.until.toISOString(), role: args.activeRole },
|
||||
'Claude usage limit still in effect; pausing the room without starting a run',
|
||||
);
|
||||
try {
|
||||
await pauseRunForClaudeUsage({
|
||||
chatJid: args.chatJid,
|
||||
runId: args.runId,
|
||||
resume: { resumeAt: pause.until, resetKnown: pause.resetKnown },
|
||||
summary: 'Claude usage limit still in effect',
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
} finally {
|
||||
await args.pairedExecutionLifecycle.asyncFinalize();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* When a Claude run failed because the account is out of usage, pause the
|
||||
* room until the limit resets. When the model's safeguards refused the
|
||||
* request, close the paired task (retrying cannot help; the next user message
|
||||
* starts a new task) and tell the room; the refusal is never handed to the
|
||||
* reviewer/arbiter as if it were an answer. Returns false for any other
|
||||
* failure.
|
||||
*/
|
||||
async function reportClaudeUsageLimitToRoom(
|
||||
args: ClaudeUsageLimitFailure & {
|
||||
isClaudeCodeAgent: boolean;
|
||||
activeRole: PairedRoomRole;
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
pairedTaskId?: string;
|
||||
pairedExecutionLifecycle: Pick<
|
||||
PairedExecutionLifecycle,
|
||||
'updateSummary' | 'completeImmediately'
|
||||
'updateSummary' | 'completeImmediately' | 'releaseForClaudeUsagePause'
|
||||
>;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||
},
|
||||
): Promise<boolean> {
|
||||
if (!args.isClaudeCodeAgent) return false;
|
||||
const isSafetyRefusal = args.reason === 'safety-refusal';
|
||||
if (!isSafetyRefusal && !isClaudeUsageLimitReason(args.reason)) {
|
||||
return false;
|
||||
if (isClaudeUsageLimitReason(args.reason)) {
|
||||
args.log.warn(
|
||||
{ reason: args.reason, role: args.activeRole, errorText: args.errorText },
|
||||
'Claude usage limit reached; pausing the room until it resets',
|
||||
);
|
||||
await pauseRunForClaudeUsage({
|
||||
chatJid: args.chatJid,
|
||||
runId: args.runId,
|
||||
resume: resolveClaudeUsageResume(args.errorText),
|
||||
summary: args.errorText ?? 'Claude usage limit reached',
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
const notice = isSafetyRefusal
|
||||
? CLAUDE_SAFETY_REFUSAL_NOTICE
|
||||
: buildClaudeUsageLimitMessage(args.errorText);
|
||||
if (args.reason !== 'safety-refusal') return false;
|
||||
const task = args.pairedTaskId
|
||||
? getPairedTaskById(args.pairedTaskId)
|
||||
: undefined;
|
||||
if (task) {
|
||||
completePairedTaskWithReason(
|
||||
task,
|
||||
isSafetyRefusal
|
||||
? CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON
|
||||
: CLAUDE_USAGE_LIMIT_COMPLETION_REASON,
|
||||
);
|
||||
completePairedTaskWithReason(task, CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON);
|
||||
}
|
||||
args.log.warn(
|
||||
{ reason: args.reason, role: args.activeRole, errorText: args.errorText },
|
||||
isSafetyRefusal
|
||||
? 'Claude safeguards refused the request; closed the task and told the room'
|
||||
: 'Claude usage limit reached; told the room when it resets',
|
||||
'Claude safeguards refused the request; closed the task and told the room',
|
||||
);
|
||||
await deliverNoticeAndFailExecution({
|
||||
notice,
|
||||
summary: isSafetyRefusal ? notice : (args.errorText ?? notice),
|
||||
notice: CLAUDE_SAFETY_REFUSAL_NOTICE,
|
||||
summary: CLAUDE_SAFETY_REFUSAL_NOTICE,
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
@@ -286,6 +369,8 @@ export async function runAgentForGroup(
|
||||
...limit,
|
||||
isClaudeCodeAgent,
|
||||
activeRole,
|
||||
chatJid,
|
||||
runId,
|
||||
pairedTaskId: pairedExecutionContext?.task.id,
|
||||
pairedExecutionLifecycle,
|
||||
onOutput,
|
||||
@@ -342,6 +427,20 @@ export async function runAgentForGroup(
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
isClaudeCodeAgent &&
|
||||
(await pauseWhileClaudeUsageLimited({
|
||||
chatJid,
|
||||
runId,
|
||||
activeRole,
|
||||
pairedExecutionLifecycle,
|
||||
onOutput,
|
||||
log,
|
||||
}))
|
||||
) {
|
||||
return 'success';
|
||||
}
|
||||
|
||||
const runAttempt = (currentProvider: 'claude' | 'codex') =>
|
||||
runMessageAgentAttempt({
|
||||
provider: currentProvider,
|
||||
|
||||
@@ -36,7 +36,12 @@ import {
|
||||
RELOGIN_REQUIRED_MESSAGE,
|
||||
} from './token-refresh.js';
|
||||
import { findChannel, formatMessages } from './router.js';
|
||||
import { stopRoomWorkIfRequested, type RoomStopResult } from './room-stop.js';
|
||||
import {
|
||||
findStopRequest,
|
||||
stopRoomWorkIfRequested,
|
||||
type RoomStopResult,
|
||||
} from './room-stop.js';
|
||||
import { isRoomPausedForClaudeUsage } from './claude-usage-pause.js';
|
||||
import { reopenReviewSkippedTaskForReview } from './review-request-gate.js';
|
||||
import { createScopedLogger, logger } from './logger.js';
|
||||
import type { GroupQueue, GroupRunContext } from './group-queue.js';
|
||||
@@ -269,6 +274,18 @@ async function processMissedMessages(
|
||||
args.failureFinalText,
|
||||
);
|
||||
|
||||
// A stop request still goes through, so the user can drop paused work.
|
||||
if (
|
||||
isRoomPausedForClaudeUsage(runtime.chatJid) &&
|
||||
!findStopRequest(missedMessages)
|
||||
) {
|
||||
runtime.log.info(
|
||||
{ pendingMessages: missedMessages.length },
|
||||
'Room is paused by the Claude usage limit; leaving its work for the resume',
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (missedMessages.length === 0) {
|
||||
return handleNoMissedMessages(args, runtime, rawMissedMessages);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { _initTestDatabase, createPairedTask } from './db.js';
|
||||
import { markRunPausedForClaudeUsage } from './claude-usage-pause.js';
|
||||
import { runQueuedGroupTurn } from './message-runtime-queue.js';
|
||||
import { resetPairedFollowUpScheduleState } from './paired-follow-up-scheduler.js';
|
||||
import type {
|
||||
@@ -154,6 +155,39 @@ describe('message-runtime queued cursor handling', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps the queued human cursor when the Claude usage limit paused the turn', async () => {
|
||||
const task = makeTask();
|
||||
createPairedTask(task);
|
||||
const executeTurn: RunQueuedGroupTurnArgs['executeTurn'] = vi.fn(
|
||||
async () => {
|
||||
// The executor posted the pause notice and marked the run paused.
|
||||
markRunPausedForClaudeUsage('run-owner-queued-cursor');
|
||||
return {
|
||||
outputStatus: 'success' as const,
|
||||
deliverySucceeded: true,
|
||||
visiblePhase: 'final',
|
||||
};
|
||||
},
|
||||
);
|
||||
const lastAgentTimestamps: Record<string, string> = {};
|
||||
const saveState = vi.fn(() => undefined);
|
||||
|
||||
const outcome = await runQueuedGroupTurn(
|
||||
makeQueuedTurnArgs({
|
||||
task,
|
||||
executeTurn,
|
||||
lastAgentTimestamps,
|
||||
saveState,
|
||||
}),
|
||||
);
|
||||
|
||||
// Handled (no queue retry backoff), but the message stays unconsumed so
|
||||
// the same request runs again when the room resumes.
|
||||
expect(outcome).toBe(true);
|
||||
expect(lastAgentTimestamps).toEqual({});
|
||||
expect(saveState).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('advances the queued human cursor when a silent owner turn succeeds', async () => {
|
||||
const task = makeTask({
|
||||
owner_service_id: 'codex',
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { getPairedTurnOutputs } from './db.js';
|
||||
import { consumeRunPausedForClaudeUsage } from './claude-usage-pause.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
buildArbiterPromptForTask,
|
||||
@@ -442,6 +443,9 @@ export async function runQueuedGroupTurn(args: {
|
||||
pairedTurnIdentity,
|
||||
});
|
||||
|
||||
// Paused by the Claude usage limit: keep the messages for the resume.
|
||||
if (consumeRunPausedForClaudeUsage(runId)) return true;
|
||||
|
||||
if (
|
||||
!finalizeQueuedRunCursor({
|
||||
outputStatus,
|
||||
|
||||
@@ -104,6 +104,8 @@ vi.mock('./db.js', () => {
|
||||
getLastHumanMessageContent: vi.fn(() => null),
|
||||
getMessagesSince,
|
||||
getNewMessages,
|
||||
getRouterState: vi.fn(() => undefined),
|
||||
setRouterState: vi.fn(),
|
||||
getLatestMessageSeqAtOrBefore: vi.fn(() => 0),
|
||||
getMessagesSinceSeq: vi.fn(
|
||||
(
|
||||
@@ -375,6 +377,68 @@ describe('createMessageRuntime', () => {
|
||||
expect(saveState).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves the messages of a room paused by the Claude usage limit for the resume', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('claude-code');
|
||||
const channel = makeChannel(chatJid);
|
||||
const saveState = vi.fn();
|
||||
const lastAgentTimestamps: Record<string, string> = {};
|
||||
|
||||
vi.mocked(db.getRouterState).mockImplementation((key) =>
|
||||
key === 'claude_usage_pause'
|
||||
? JSON.stringify({
|
||||
until: new Date(Date.now() + 60 * 60_000).toISOString(),
|
||||
resetKnown: true,
|
||||
rooms: { [chatJid]: new Date().toISOString() },
|
||||
})
|
||||
: undefined,
|
||||
);
|
||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||
{
|
||||
id: 'msg-paused-1',
|
||||
chat_jid: chatJid,
|
||||
sender: 'user@test',
|
||||
sender_name: 'User',
|
||||
content: '이것도 해줘',
|
||||
timestamp: '2026-10-13T03:00:00.000Z',
|
||||
is_bot_message: false,
|
||||
},
|
||||
]);
|
||||
|
||||
const runtime = createMessageRuntime({
|
||||
assistantName: 'Andy',
|
||||
idleTimeout: 1_000,
|
||||
pollInterval: 1_000,
|
||||
timezone: 'UTC',
|
||||
channels: [channel],
|
||||
queue: {
|
||||
enqueueMessageCheck: vi.fn(),
|
||||
registerProcess: vi.fn(),
|
||||
closeStdin: vi.fn(),
|
||||
notifyIdle: vi.fn(),
|
||||
} as any,
|
||||
getRoomBindings: () => ({ [chatJid]: group }),
|
||||
getSessions: () => ({}),
|
||||
getLastTimestamp: () => '',
|
||||
setLastTimestamp: vi.fn(),
|
||||
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||
saveState,
|
||||
persistSession: vi.fn(),
|
||||
clearSession: vi.fn(),
|
||||
});
|
||||
|
||||
const result = await runtime.processGroupMessages(chatJid, {
|
||||
runId: 'run-claude-usage-paused-room',
|
||||
reason: 'messages',
|
||||
});
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
|
||||
expect(channel.sendMessage).not.toHaveBeenCalled();
|
||||
expect(lastAgentTimestamps).toEqual({});
|
||||
expect(saveState).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps mentionless substantive bot messages in paired rooms', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('codex');
|
||||
|
||||
@@ -17,6 +17,10 @@ import {
|
||||
storeMessage,
|
||||
} from './db.js';
|
||||
import { requireDatabase } from './db/runtime-database.js';
|
||||
import {
|
||||
isRoomPausedForClaudeUsage,
|
||||
pauseRoomForClaudeUsage,
|
||||
} from './claude-usage-pause.js';
|
||||
import { processLoopGroupMessages } from './message-runtime-dispatch.js';
|
||||
import { advanceLastAgentCursor } from './message-runtime-rules.js';
|
||||
import { hasReviewerLease } from './service-routing.js';
|
||||
@@ -224,6 +228,31 @@ describe('natural-language stop in the message loop', () => {
|
||||
expect(args.enqueueMessageCheck).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('drops a room paused by the Claude usage limit from the resume list', async () => {
|
||||
const request = postHumanMessage('배포 스크립트 고쳐줘');
|
||||
const stop = postHumanMessage('중단');
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid,
|
||||
resumeAt: new Date(Date.now() + 60 * 60_000),
|
||||
resetKnown: true,
|
||||
});
|
||||
const controls = makeControls(false);
|
||||
const lastAgentTimestamps: Record<string, string> = {
|
||||
[chatJid]: String(request.seq! - 1),
|
||||
};
|
||||
|
||||
await processLoopGroupMessages(
|
||||
makeArgs({
|
||||
groupMessages: [request, stop],
|
||||
lastAgentTimestamps,
|
||||
controls,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(controls.sendReply).toHaveBeenCalledWith(chatJid, USER_STOP_REPLY);
|
||||
expect(isRoomPausedForClaudeUsage(chatJid)).toBe(false);
|
||||
});
|
||||
|
||||
it('still processes a new instruction sent after the stop', async () => {
|
||||
const stop = postHumanMessage('중단');
|
||||
const next = postHumanMessage('대신 테스트만 돌려줘');
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
getPairedTurnsForTask,
|
||||
releasePairedTaskExecutionLeasesForTask,
|
||||
} from './db.js';
|
||||
import { releaseRoomFromClaudeUsagePause } from './claude-usage-pause.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
advanceLastAgentCursor,
|
||||
@@ -185,6 +186,7 @@ export async function stopRoomWorkIfRequested(args: {
|
||||
saveState: args.saveState,
|
||||
});
|
||||
const closedTask = openTask ? closePairedTaskForStop(openTask) : null;
|
||||
const releasedUsagePause = releaseRoomFromClaudeUsagePause(chatJid);
|
||||
const killedProcess = controls.stopRunningAgent(chatJid);
|
||||
|
||||
logger.info(
|
||||
@@ -198,6 +200,7 @@ export async function stopRoomWorkIfRequested(args: {
|
||||
hadPendingMessages: hasPendingMessages,
|
||||
pairedTaskId: openTask?.id ?? null,
|
||||
pairedTaskStatus: openTask?.status ?? null,
|
||||
releasedUsagePause,
|
||||
...(closedTask ?? {}),
|
||||
},
|
||||
'Stopped room work on user request',
|
||||
|
||||
@@ -125,7 +125,14 @@ import {
|
||||
createPairedTask,
|
||||
createTask,
|
||||
getTaskById,
|
||||
setRouterState,
|
||||
} from './db.js';
|
||||
import {
|
||||
CLAUDE_USAGE_PAUSE_KEY,
|
||||
CLAUDE_USAGE_RESUME_NOTICE,
|
||||
isRoomPausedForClaudeUsage,
|
||||
pauseRoomForClaudeUsage,
|
||||
} from './claude-usage-pause.js';
|
||||
import { TIMEZONE } from './config.js';
|
||||
import * as serviceRouting from './service-routing.js';
|
||||
import * as tokenRefresh from './token-refresh.js';
|
||||
@@ -321,6 +328,59 @@ describe('task scheduler', () => {
|
||||
expect(enqueueMessageCheck).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resumes rooms paused by the Claude usage limit once it has reset', async () => {
|
||||
pauseRoomForClaudeUsage({
|
||||
chatJid: 'paused@g.us',
|
||||
resumeAt: new Date(Date.now() + 60 * 60_000),
|
||||
resetKnown: true,
|
||||
});
|
||||
const enqueueMessageCheck = vi.fn();
|
||||
const sendMessage = vi.fn(async () => {});
|
||||
const deps = {
|
||||
serviceAgentType: 'claude-code' as const,
|
||||
roomBindings: () => ({
|
||||
'paused@g.us': {
|
||||
name: 'Paused',
|
||||
folder: 'paused-group',
|
||||
trigger: '@Andy',
|
||||
added_at: '2026-10-13T00:00:00.000Z',
|
||||
},
|
||||
}),
|
||||
getSessions: () => ({}),
|
||||
queue: { enqueueTask: vi.fn(), enqueueMessageCheck } as any,
|
||||
onProcess: () => {},
|
||||
sendMessage,
|
||||
};
|
||||
|
||||
await runSchedulerTickOnce(deps);
|
||||
expect(enqueueMessageCheck).not.toHaveBeenCalled();
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
|
||||
// The limit has reset (the stored reset time is now in the past).
|
||||
setRouterState(
|
||||
CLAUDE_USAGE_PAUSE_KEY,
|
||||
JSON.stringify({
|
||||
until: new Date(Date.now() - 2 * 60_000).toISOString(),
|
||||
resetKnown: true,
|
||||
rooms: { 'paused@g.us': new Date().toISOString() },
|
||||
}),
|
||||
);
|
||||
|
||||
await runSchedulerTickOnce(deps);
|
||||
expect(sendMessage).toHaveBeenCalledWith(
|
||||
'paused@g.us',
|
||||
CLAUDE_USAGE_RESUME_NOTICE,
|
||||
);
|
||||
expect(enqueueMessageCheck).toHaveBeenCalledWith(
|
||||
'paused@g.us',
|
||||
expect.stringContaining('paused-group'),
|
||||
);
|
||||
expect(isRoomPausedForClaudeUsage('paused@g.us')).toBe(false);
|
||||
|
||||
await runSchedulerTickOnce(deps);
|
||||
expect(enqueueMessageCheck).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('can execute one scheduler tick without starting the timer loop', async () => {
|
||||
const dueAt = new Date(Date.now() - 60_000).toISOString();
|
||||
createTask({
|
||||
|
||||
@@ -34,6 +34,10 @@ import { getTaskQueueJid, isGitHubCiTask } from './task-watch-status.js';
|
||||
import { normalizeAgentOutputPhase, ScheduledTask } from './types.js';
|
||||
import { resolveGroupIpcPath } from './group-folder.js';
|
||||
import { schedulePairedFollowUpOnce } from './paired-follow-up-scheduler.js';
|
||||
import {
|
||||
CLAUDE_USAGE_RESUME_NOTICE,
|
||||
takeRoomsReadyToResume,
|
||||
} from './claude-usage-pause.js';
|
||||
import {
|
||||
hasTaskExceededMaxDuration,
|
||||
resolveTaskExecutionContext,
|
||||
@@ -429,6 +433,7 @@ export async function runSchedulerTickOnce(
|
||||
): Promise<void> {
|
||||
// Unified service: process all agent types, not just the service default.
|
||||
const nowMs = Date.now();
|
||||
await resumeClaudeUsagePausedRooms(deps, nowMs);
|
||||
reconcileOrphanedPairedReviewReadyTasks(deps);
|
||||
const activeTasks = getAllTasks().filter((task) => task.status === 'active');
|
||||
|
||||
@@ -474,6 +479,39 @@ export async function runSchedulerTickOnce(
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-queue rooms paused by the Claude usage limit once it has reset. */
|
||||
async function resumeClaudeUsagePausedRooms(
|
||||
deps: SchedulerDependencies,
|
||||
nowMs: number,
|
||||
): Promise<void> {
|
||||
const chatJids = takeRoomsReadyToResume(nowMs);
|
||||
if (chatJids.length === 0) return;
|
||||
const bindings = deps.roomBindings();
|
||||
for (const chatJid of chatJids) {
|
||||
const binding = bindings[chatJid];
|
||||
if (!binding) {
|
||||
logger.warn(
|
||||
{ chatJid },
|
||||
'Skipped resuming a Claude-usage-paused room without a room binding',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await deps.sendMessage(chatJid, CLAUDE_USAGE_RESUME_NOTICE);
|
||||
} catch (err) {
|
||||
logger.warn({ chatJid, err }, 'Failed to send the Claude resume notice');
|
||||
}
|
||||
deps.queue.enqueueMessageCheck(
|
||||
chatJid,
|
||||
resolveGroupIpcPath(binding.folder),
|
||||
);
|
||||
logger.info(
|
||||
{ chatJid, groupFolder: binding.folder },
|
||||
'Resumed room work paused by the Claude usage limit',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function reconcileOrphanedPairedReviewReadyTasks(
|
||||
deps: SchedulerDependencies,
|
||||
): void {
|
||||
|
||||
Reference in New Issue
Block a user