fix: keep usage resets on the slot grid when Claude hits 100%

Evidence (journal): 10/04 and 10/05 08:00 and 10/12 23:00 the primer
failed (exit 1) because the account was at 100% in an off-slot window
(resets 9am / 3am); the first request after that reset then opened the
next window off the slot too, and the drift carried on.

- Usage limit that resets off the slot grid (not ~04/08/13/18/23:00 KST):
  extend the alignment hold to the next window start (08/13/18/23), so
  nothing opens a window between the reset and the primer. The pause
  notice names both times. Rooms paused by the limit stay paused
  (silently, now alignment-only, @-mention bypass works) until the hold
  ends; an expired real limit no longer counts as a limit.
- A real limit takes precedence over the hold in the pre-run gate.
- Primer: when the current 5h window (cached usage) ends within 10 min
  after the slot, wait until 5 s after that reset before firing, so the
  primer is not counted in, or rejected by, the old window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-10-16 06:03:38 +09:00
parent f944f97e54
commit a2b9898c0b
11 changed files with 334 additions and 10 deletions

View File

@@ -160,6 +160,32 @@ describe('Claude usage pause', () => {
expect(isRoomPausedOnlyForAlignment('room-c')).toBe(false);
});
it('treats a real limit that already reset as gone when the alignment hold pauses a room', () => {
pauseRoomForClaudeUsage({
chatJid: 'room-a',
resumeAt: RESET,
resetKnown: true,
now: NOW,
});
const afterReset = new Date(RESET.getTime() + 30_000);
const holdEnd = new Date(RESET.getTime() + 3 * 60 * 60_000);
pauseRoomForClaudeUsage({
chatJid: 'room-b',
resumeAt: holdEnd,
resetKnown: true,
alignment: true,
now: afterReset,
});
expect(isRoomPausedOnlyForAlignment('room-a')).toBe(true);
expect(isRoomPausedOnlyForAlignment('room-b')).toBe(true);
expect(getActiveClaudeUsagePause(afterReset.getTime())).toBeNull();
expect(takeRoomsReadyToResume(holdEnd.getTime() + 60_000)).toEqual([
'room-a',
'room-b',
]);
});
it('ignores an unreadable stored state', () => {
routerState.set(CLAUDE_USAGE_PAUSE_KEY, '{not json');
expect(getActiveClaudeUsagePause(NOW.getTime())).toBeNull();

View File

@@ -79,9 +79,13 @@ export function pauseRoomForClaudeUsage(args: {
until: keepExisting ? state.until : args.resumeAt.toISOString(),
resetKnown: keepExisting ? state.resetKnown : args.resetKnown,
rooms: { ...(state?.rooms ?? {}) },
// A real limit that has already reset (rooms not handed back yet) no
// longer counts as a limit.
alignmentOnly:
args.alignment === true &&
(state === null || state.alignmentOnly === true),
(state === null ||
state.alignmentOnly === true ||
Date.parse(state.until) <= now.getTime()),
};
next.rooms[args.chatJid] ??= now.toISOString();
writeState(next);

View File

@@ -139,6 +139,20 @@ export function getCachedClaudeUsageLimitResetAt(
return latest == null ? null : new Date(latest);
}
/**
* Reset time of the current 5h window from the last successful usage fetch,
* or null when none is cached or it has already passed.
*/
export function getCachedClaudeFiveHourResetAt(
now: number = Date.now(),
): Date | null {
loadUsageDiskCache();
const resetMs = Date.parse(
usageDiskCache[USAGE_CACHE_KEY]?.usage?.five_hour?.resets_at ?? '',
);
return Number.isFinite(resetMs) && resetMs > now ? new Date(resetMs) : null;
}
// Rate limit: at most one API call per 5 minutes.
// The 1-minute cadence existed for the 95% auto-paired-mode gate, which is no
// longer wired into the runtime. The dashboard only needs coarse numbers, and

View File

@@ -3105,6 +3105,61 @@ describe('runAgentForGroup Claude failure recovery', () => {
);
});
it('holds until the next slot when the usage limit resets off the slot grid', async () => {
const outputs: string[] = [];
const actualHold = await vi.importActual<
typeof import('./usage-alignment-hold.js')
>('./usage-alignment-hold.js');
vi.mocked(usageAlignmentHold.getUsageAlignmentHold).mockImplementation(
actualHold.getUsageAlignmentHold,
);
// Outside the 04-08 KST gap, so only the limit applies.
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-10-16T08:30:00+09:00'));
vi.mocked(agentRunner.runAgentProcess).mockImplementationOnce(
async (_group, _input, _onProcess, onOutput) => {
await onOutput?.({
status: 'success',
phase: 'final',
result:
"You've hit your session limit \u00b7 resets 9:20am (Asia/Seoul)",
});
return { status: 'success', result: null };
},
);
try {
await runAgentForGroup(makeDeps(), {
group: makeGroup(),
prompt: 'hello',
chatJid: 'group@test',
runId: 'run-off-slot-limit',
onOutput: async (output) => {
const text =
output.output && 'text' in output.output
? output.output.text
: output.result;
if (typeof text === 'string') outputs.push(text);
},
});
} finally {
vi.useRealTimers();
}
const pause = readUsagePause();
expect(pause?.until).toBe('2026-10-16T00:20:00.000Z');
expect(pause?.alignmentOnly).toBe(false);
expect(routerState.get('usage_alignment_hold_until')).toBe(
'2026-10-16T04:00:00.000Z',
);
expect(outputs).toEqual([
expect.stringMatching(
/사용량은 09:20에 초기화되지만, .*13:00까지 기다렸다가 자동으로 이어서 진행합니다/,
),
]);
expect(consumeRunPausedForClaudeUsage('run-off-slot-limit')).toBe(true);
});
it('keeps the paired task as it was when the Claude owner hits its usage limit', async () => {
const outputs: string[] = [];
const pairedTask = {

View File

@@ -22,9 +22,12 @@ import {
} from './claude-usage-pause.js';
import { getPairedTaskById } from './db.js';
import {
extendUsageAlignmentHold,
formatUsageAlignmentHoldMessage,
formatUsageLimitAlignedPauseMessage,
getUsageAlignmentHold,
isUsageAlignmentBypassed,
nextWindowStartAfterOffSlotReset,
} from './usage-alignment-hold.js';
import { completePairedTaskWithReason } from './paired-task-status.js';
import { prepareMessageAgentExecutionTarget } from './message-agent-executor-target.js';
@@ -114,12 +117,32 @@ async function pauseRunForClaudeUsage(args: {
markRunPausedForClaudeUsage(args.runId);
args.pairedExecutionLifecycle.updateSummary({ outputText: args.summary });
args.pairedExecutionLifecycle.releaseForClaudeUsagePause();
const notice = args.alignment
? formatUsageAlignmentHoldMessage(formatResetTime(pause.until, Date.now()))
: formatClaudeUsagePauseMessage({
resumeAt: pause.until,
resetKnown: pause.resetKnown,
});
const nowMs = Date.now();
let notice: string;
if (args.alignment) {
notice = formatUsageAlignmentHoldMessage(
formatResetTime(pause.until, nowMs),
);
} else {
// A limit that resets off the slot grid would open an off-slot window
// on the first request after the reset; hold until the next slot.
if (pause.resetKnown) {
const slot = nextWindowStartAfterOffSlotReset(pause.until);
if (slot) extendUsageAlignmentHold(slot);
}
const holdAtReset = pause.resetKnown
? getUsageAlignmentHold(pause.until.getTime())
: null;
notice = holdAtReset
? formatUsageLimitAlignedPauseMessage(
formatResetTime(pause.until, nowMs),
formatResetTime(holdAtReset.until, nowMs),
)
: formatClaudeUsagePauseMessage({
resumeAt: pause.until,
resetKnown: pause.resetKnown,
});
}
await args.onOutput?.({
status: 'success',
result: null,
@@ -148,7 +171,8 @@ async function pauseWhileClaudeUsageLimited(args: {
: getUsageAlignmentHold();
if (!pause && !hold) return false;
try {
if (pause && (!hold || pause.until >= hold.until)) {
// A real limit wins; its notice names any hold that follows the reset.
if (pause) {
args.log.warn(
{ until: pause.until.toISOString(), role: args.activeRole },
'Claude usage limit still in effect; pausing the room without starting a run',

View File

@@ -131,6 +131,7 @@ import {
CLAUDE_USAGE_PAUSE_KEY,
CLAUDE_USAGE_RESUME_NOTICE,
isRoomPausedForClaudeUsage,
isRoomPausedOnlyForAlignment,
pauseRoomForClaudeUsage,
} from './claude-usage-pause.js';
import { TIMEZONE } from './config.js';
@@ -434,6 +435,44 @@ describe('task scheduler', () => {
);
});
it('keeps limit-paused rooms paused, silently, while an alignment hold follows the reset', async () => {
setRouterState(
CLAUDE_USAGE_PAUSE_KEY,
JSON.stringify({
until: new Date(Date.now() - 2 * 60_000).toISOString(),
resetKnown: true,
rooms: { 'paused@g.us': new Date().toISOString() },
}),
);
const holdUntil = new Date(Date.now() + 60 * 60_000);
vi.mocked(usageAlignmentHold.getUsageAlignmentHold).mockReturnValueOnce({
until: holdUntil,
});
const enqueueMessageCheck = vi.fn();
const sendMessage = vi.fn(async () => {});
await runSchedulerTickOnce({
serviceAgentType: 'claude-code' as const,
roomBindings: () => ({
'paused@g.us': {
name: 'Paused',
folder: 'paused-group',
trigger: '@Andy',
added_at: '2026-10-13T00:00:00.000Z',
},
}),
getSessions: () => ({}),
queue: { enqueueTask: vi.fn(), enqueueMessageCheck } as any,
onProcess: () => {},
sendMessage,
});
expect(sendMessage).not.toHaveBeenCalled();
expect(enqueueMessageCheck).not.toHaveBeenCalled();
expect(isRoomPausedForClaudeUsage('paused@g.us')).toBe(true);
expect(isRoomPausedOnlyForAlignment('paused@g.us')).toBe(true);
});
it('can execute one scheduler tick without starting the timer loop', async () => {
const dueAt = new Date(Date.now() - 60_000).toISOString();
createTask({

View File

@@ -36,6 +36,7 @@ import { resolveGroupIpcPath } from './group-folder.js';
import { schedulePairedFollowUpOnce } from './paired-follow-up-scheduler.js';
import {
CLAUDE_USAGE_RESUME_NOTICE,
pauseRoomForClaudeUsage,
takeRoomsReadyToResume,
} from './claude-usage-pause.js';
import {
@@ -45,7 +46,10 @@ import {
writeTaskSnapshotForGroup,
} from './task-scheduler-runtime.js';
import { runGithubCiTask } from './task-scheduler-github.js';
import { getUsageAlignmentHold } from './usage-alignment-hold.js';
import {
getUsageAlignmentHold,
isUsageAlignmentBypassed,
} from './usage-alignment-hold.js';
import type {
SchedulerDependencies,
TaskExecutionContext,
@@ -510,7 +514,24 @@ async function resumeClaudeUsagePausedRooms(
const chatJids = takeRoomsReadyToResume(nowMs);
if (chatJids.length === 0) return;
const bindings = deps.roomBindings();
// The limit reset off the slot grid (or inside the dawn gap): keep the rooms
// paused, silently, until the alignment hold ends.
const alignmentHold = getUsageAlignmentHold(nowMs);
for (const chatJid of chatJids) {
if (alignmentHold && !isUsageAlignmentBypassed(chatJid, nowMs)) {
pauseRoomForClaudeUsage({
chatJid,
resumeAt: alignmentHold.until,
resetKnown: true,
alignment: true,
now: new Date(nowMs),
});
logger.info(
{ chatJid, until: alignmentHold.until.toISOString() },
'Claude usage limit reset; room stays paused until the alignment hold ends',
);
continue;
}
const binding = bindings[chatJid];
if (!binding) {
logger.warn(

View File

@@ -5,6 +5,9 @@ const { routerState } = vi.hoisted(() => ({
}));
vi.mock('./db.js', () => ({
getRouterState: (key: string) => routerState.get(key),
setRouterState: (key: string, value: string) => {
routerState.set(key, value);
},
}));
import type { NewMessage } from './types.js';
@@ -13,10 +16,12 @@ import { isSupportedRouterStateKey } from './db/router-state.js';
// test/vitest-env.ts stubs getUsageAlignmentHold for every other test file.
const {
USAGE_ALIGNMENT_HOLD_UNTIL_KEY,
extendUsageAlignmentHold,
getUsageAlignmentHold,
grantUsageAlignmentBypass,
hasUrgentMention,
isUsageAlignmentBypassed,
nextWindowStartAfterOffSlotReset,
} = await vi.importActual<typeof import('./usage-alignment-hold.js')>(
'./usage-alignment-hold.js',
);
@@ -104,4 +109,33 @@ describe('usage alignment hold', () => {
isUsageAlignmentBypassed('room@test', kst('2026-10-16T13:00:00')),
).toBe(false);
});
it('finds the next slot after a limit that resets off the slot grid', () => {
const next = (iso: string): string | undefined =>
nextWindowStartAfterOffSlotReset(new Date(kst(iso)))?.toISOString();
// On the grid (08/13/18/23 and the 23-04 window's 04), sub-second jitter
// and a minute either side included.
expect(next('2026-10-16T13:00:00.369')).toBeUndefined();
expect(next('2026-10-16T12:59:00')).toBeUndefined();
expect(next('2026-10-16T04:00:00')).toBeUndefined();
expect(next('2026-10-16T23:01:00')).toBeUndefined();
// Off the grid: wait for the next window start.
expect(next('2026-10-16T09:20:00')).toBe('2026-10-16T04:00:00.000Z');
expect(next('2026-10-16T03:20:00')).toBe('2026-10-15T23:00:00.000Z');
expect(next('2026-10-16T05:00:00')).toBe('2026-10-15T23:00:00.000Z');
expect(next('2026-10-16T18:20:00')).toBe('2026-10-16T14:00:00.000Z');
expect(next('2026-10-16T23:40:00')).toBe('2026-10-16T23:00:00.000Z');
});
it('only ever extends the one-off hold', () => {
extendUsageAlignmentHold(new Date(kst('2026-10-16T13:00:00')));
extendUsageAlignmentHold(new Date(kst('2026-10-16T08:00:00')));
expect(routerState.get(USAGE_ALIGNMENT_HOLD_UNTIL_KEY)).toBe(
'2026-10-16T04:00:00.000Z',
);
extendUsageAlignmentHold(new Date(kst('2026-10-16T18:00:00')));
expect(routerState.get(USAGE_ALIGNMENT_HOLD_UNTIL_KEY)).toBe(
'2026-10-16T09:00:00.000Z',
);
});
});

View File

@@ -17,9 +17,14 @@
*
* Urgent bypass: a human message that @-mentions someone lets that room run
* until the hold ends (the user accepts the drift).
*
* Usage limit: when the account hits 100% and the window resets off the slot
* grid (e.g. 09:20), the first request after the reset would open an off-slot
* window. The one-off hold is then extended to the next slot so the primer
* opens the next window on the grid (see claude-usage-pause flow).
*/
import { getRouterState } from './db.js';
import { getRouterState, setRouterState } from './db.js';
import type { NewMessage } from './types.js';
export const USAGE_ALIGNMENT_HOLD_UNTIL_KEY = 'usage_alignment_hold_until';
@@ -27,6 +32,13 @@ export const USAGE_ALIGNMENT_HOLD_UNTIL_KEY = 'usage_alignment_hold_until';
const GAP_START_HOUR_KST = 4;
const GAP_END_HOUR_KST = 8;
const KST_OFFSET_MS = 9 * 60 * 60 * 1000;
const HOUR_MS = 60 * 60 * 1000;
const DAY_MS = 24 * HOUR_MS;
/** Primer slots: the hours a usage window should start. */
const WINDOW_START_HOURS_KST = [8, 13, 18, 23];
/** Hours an aligned window ends (the 23-04 window ends at 04). */
const ALIGNED_RESET_HOURS_KST = [4, 8, 13, 18, 23];
const ALIGNED_RESET_TOLERANCE_MS = 2 * 60_000;
export interface UsageAlignmentHold {
until: Date;
@@ -61,6 +73,44 @@ export function getUsageAlignmentHold(
return { until: new Date(until) };
}
/** Epoch ms (UTC) of `hour`:00 KST on the KST day `dayOffset` from `ms`. */
function kstHourAt(ms: number, dayOffset: number, hour: number): number {
const kstDayStart = Math.floor((ms + KST_OFFSET_MS) / DAY_MS) * DAY_MS;
return kstDayStart + dayOffset * DAY_MS + hour * HOUR_MS - KST_OFFSET_MS;
}
/**
* The slot the next window should start at when a usage window resets at
* `resetAt`, or null when `resetAt` is already on the slot grid.
*/
export function nextWindowStartAfterOffSlotReset(resetAt: Date): Date | null {
const resetMs = resetAt.getTime();
for (const dayOffset of [-1, 0, 1]) {
for (const hour of ALIGNED_RESET_HOURS_KST) {
const boundary = kstHourAt(resetMs, dayOffset, hour);
if (Math.abs(resetMs - boundary) <= ALIGNED_RESET_TOLERANCE_MS) {
return null;
}
}
}
for (const dayOffset of [0, 1]) {
for (const hour of WINDOW_START_HOURS_KST) {
const start = kstHourAt(resetMs, dayOffset, hour);
if (start > resetMs) return new Date(start);
}
}
return null;
}
/** Make the one-off hold last at least until `until` (never shortens it). */
export function extendUsageAlignmentHold(until: Date): void {
const current = Date.parse(
getRouterState(USAGE_ALIGNMENT_HOLD_UNTIL_KEY) ?? '',
);
if (Number.isFinite(current) && current >= until.getTime()) return;
setRouterState(USAGE_ALIGNMENT_HOLD_UNTIL_KEY, until.toISOString());
}
const MENTION_PATTERN = /<@!?\d+>|(^|\s)@\S/;
/** True when a human message @-mentions someone (urgent bypass). */
@@ -94,3 +144,10 @@ export function isUsageAlignmentBypassed(
export function formatUsageAlignmentHoldMessage(at: string): string {
return `사용량 초기화 시각(08·13·18·23시)을 맞추려고 ${at}까지 이 방의 Claude 작업을 보류합니다. ${at} 이후 자동으로 이어서 진행합니다. 급하면 @태그해서 보내 주시고, 멈추려면 "중단"이라고 보내 주세요.`;
}
export function formatUsageLimitAlignedPauseMessage(
resetAt: string,
resumeAt: string,
): string {
return `Claude 사용 한도에 도달해서 이 방의 작업을 일시정지했습니다. 사용량은 ${resetAt}에 초기화되지만, 초기화 시각(08·13·18·23시)을 맞추려고 ${resumeAt}까지 기다렸다가 자동으로 이어서 진행합니다. 그 전에 멈추려면 "중단"이라고 보내 주세요.`;
}

View File

@@ -38,6 +38,10 @@ vi.mock('./claude-oauth-token.js', () => ({
getCurrentClaudeToken: vi.fn(() => undefined),
}));
vi.mock('./claude-usage.js', () => ({
getCachedClaudeFiveHourResetAt: vi.fn(() => null),
}));
vi.mock('./codex-usage-collector.js', () => ({
refreshCodexUsage,
}));
@@ -67,4 +71,16 @@ describe('usage-primer', () => {
{ ignoreZeroUsageWindow: true, forceAttempt: true },
);
});
it('waits for a window that resets just after the slot, but not for an off-slot one', async () => {
const { claudePrimerDelayMs } = await import('./usage-primer.js');
const slot = Date.parse('2026-10-16T13:00:00+09:00');
// Window ends 13:00:00.369 -> fire 5s after it.
expect(claudePrimerDelayMs(slot, new Date(slot + 369))).toBe(369 + 5_000);
// Already reset, unknown, or an off-slot window far ahead: fire now.
expect(claudePrimerDelayMs(slot, new Date(slot - 1_000))).toBe(0);
expect(claudePrimerDelayMs(slot, null)).toBe(0);
expect(claudePrimerDelayMs(slot, new Date(slot + 20 * 60_000))).toBe(0);
});
});

View File

@@ -7,6 +7,7 @@ import { refreshCodexUsage } from './codex-usage-collector.js';
import { runCodexWarmupCycle } from './codex-warmup.js';
import { logger } from './logger.js';
import { getCurrentClaudeToken } from './claude-oauth-token.js';
import { getCachedClaudeFiveHourResetAt } from './claude-usage.js';
/**
* Usage-window warm-up primer.
@@ -123,12 +124,45 @@ function runClaudePrimerOnce(
});
}
/** Wait for a window that ends at most this long after the slot. */
const CLAUDE_PRIMER_MAX_RESET_WAIT_MS = 10 * 60_000;
/** Fire this long after the window's reset, never just before it. */
const CLAUDE_PRIMER_RESET_MARGIN_MS = 5_000;
/**
* How long the primer must wait so it lands after the current 5h window
* resets. A primer that reaches Claude a moment before the reset (the reset
* carries sub-second jitter, e.g. 13:00:00.37) is counted in the old window
* or rejected when that window is at 100%, and the next window then opens on
* whatever request comes later, off the slot.
*/
export function claudePrimerDelayMs(
nowMs: number,
windowResetAt: Date | null,
): number {
if (!windowResetAt) return 0;
const untilReset = windowResetAt.getTime() - nowMs;
if (untilReset <= 0 || untilReset > CLAUDE_PRIMER_MAX_RESET_WAIT_MS) {
return 0;
}
return untilReset + CLAUDE_PRIMER_RESET_MARGIN_MS;
}
async function runClaudePrimerCycle(): Promise<void> {
const token = getCurrentClaudeToken();
if (!token) {
logger.info('Claude primer skipped: no Claude OAuth token configured');
return;
}
const windowResetAt = getCachedClaudeFiveHourResetAt();
const delayMs = claudePrimerDelayMs(Date.now(), windowResetAt);
if (delayMs > 0) {
logger.info(
{ windowResetAt: windowResetAt?.toISOString(), delayMs },
'Delaying Claude primer until the current usage window resets',
);
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
logger.info('Starting Claude primer (haiku, prompt=ok)');
const result = await runClaudePrimerOnce(token);
if (result.ok) {