feat: pause room work on the Claude usage limit and resume after reset
Before, a Claude run that hit the account usage limit closed the open paired task as claude_usage_limit, posted the reset time and consumed the user's messages, so the work was lost and had to be re-requested. Now the room is paused instead: - The failed turn is released without touching the paired task (no failure count, no status change, no follow-up) and the queue keeps the human messages unconsumed, so the same turn can run again. - The room is recorded in router_state `claude_usage_pause` with the reset time (error text, else cached usage API reset, else a 10 min retry). A banner time that just passed is read as "now", not tomorrow. - While the limit lasts, paused rooms run nothing and any other room's Claude turn pauses that room without spawning a run (the limit is account-wide). A stop request still goes through and drops the room from the resume list. - The scheduler tick clears the pause 1 min after the reset, posts a resume notice and re-queues the rooms; the pending turn or the kept messages then run again. The state survives restarts. Safety refusals keep closing the task as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,9 +9,16 @@ import type { PairedTurnIdentity } from './paired-turn-identity.js';
|
||||
import { executeMessageAgentAttemptLifecycle } from './message-agent-executor-lifecycle.js';
|
||||
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
|
||||
import {
|
||||
buildClaudeUsageLimitMessage,
|
||||
formatClaudeUsagePauseMessage,
|
||||
isClaudeUsageLimitReason,
|
||||
resolveClaudeUsageResume,
|
||||
type ClaudeUsageResume,
|
||||
} from './claude-usage-limit.js';
|
||||
import {
|
||||
getActiveClaudeUsagePause,
|
||||
markRunPausedForClaudeUsage,
|
||||
pauseRoomForClaudeUsage,
|
||||
} from './claude-usage-pause.js';
|
||||
import { getPairedTaskById } from './db.js';
|
||||
import { completePairedTaskWithReason } from './paired-task-status.js';
|
||||
import { prepareMessageAgentExecutionTarget } from './message-agent-executor-target.js';
|
||||
@@ -30,9 +37,6 @@ import {
|
||||
|
||||
// ── Main executor ─────────────────────────────────────────────────
|
||||
|
||||
/** completion_reason of a paired task closed because Claude ran out of usage. */
|
||||
export const CLAUDE_USAGE_LIMIT_COMPLETION_REASON = 'claude_usage_limit';
|
||||
|
||||
/** completion_reason of a paired task closed because Claude's safeguards refused it. */
|
||||
export const CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON = 'claude_safety_refusal';
|
||||
|
||||
@@ -74,54 +78,133 @@ interface ClaudeUsageLimitFailure {
|
||||
errorText: string | null;
|
||||
}
|
||||
|
||||
type PauseLifecycle = Pick<
|
||||
PairedExecutionLifecycle,
|
||||
'updateSummary' | 'releaseForClaudeUsagePause'
|
||||
>;
|
||||
|
||||
/**
|
||||
* When a Claude run failed because the account is out of usage, or because
|
||||
* the model's safeguards refused the request, close the paired task (retrying
|
||||
* cannot help; the next user message starts a new task) and tell the room.
|
||||
* The refusal is never handed to the reviewer/arbiter as if it were an answer.
|
||||
* Returns false for any other failure.
|
||||
* Claude is out of usage: pause the room instead of failing the turn. The
|
||||
* paired task is left untouched and the run is marked paused so the queue
|
||||
* keeps its messages; the scheduler re-queues the room after the reset and the
|
||||
* same turn runs again.
|
||||
*/
|
||||
async function pauseRunForClaudeUsage(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
resume: ClaudeUsageResume;
|
||||
summary: string;
|
||||
pairedExecutionLifecycle: PauseLifecycle;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
}): Promise<void> {
|
||||
const pause = pauseRoomForClaudeUsage({
|
||||
chatJid: args.chatJid,
|
||||
resumeAt: args.resume.resumeAt,
|
||||
resetKnown: args.resume.resetKnown,
|
||||
});
|
||||
markRunPausedForClaudeUsage(args.runId);
|
||||
args.pairedExecutionLifecycle.updateSummary({ outputText: args.summary });
|
||||
args.pairedExecutionLifecycle.releaseForClaudeUsagePause();
|
||||
const notice = formatClaudeUsagePauseMessage({
|
||||
resumeAt: pause.until,
|
||||
resetKnown: pause.resetKnown,
|
||||
});
|
||||
await args.onOutput?.({
|
||||
status: 'success',
|
||||
result: null,
|
||||
output: { visibility: 'public', text: notice },
|
||||
phase: 'final',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The Claude limit is account-wide: while it lasts, a Claude turn in a room
|
||||
* that is not paused yet pauses it instead of starting a run that would fail.
|
||||
* Returns false when no limit is in effect.
|
||||
*/
|
||||
async function pauseWhileClaudeUsageLimited(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
activeRole: PairedRoomRole;
|
||||
pairedExecutionLifecycle: PauseLifecycle &
|
||||
Pick<PairedExecutionLifecycle, 'asyncFinalize'>;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||
}): Promise<boolean> {
|
||||
const pause = getActiveClaudeUsagePause();
|
||||
if (!pause) return false;
|
||||
args.log.warn(
|
||||
{ until: pause.until.toISOString(), role: args.activeRole },
|
||||
'Claude usage limit still in effect; pausing the room without starting a run',
|
||||
);
|
||||
try {
|
||||
await pauseRunForClaudeUsage({
|
||||
chatJid: args.chatJid,
|
||||
runId: args.runId,
|
||||
resume: { resumeAt: pause.until, resetKnown: pause.resetKnown },
|
||||
summary: 'Claude usage limit still in effect',
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
} finally {
|
||||
await args.pairedExecutionLifecycle.asyncFinalize();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* When a Claude run failed because the account is out of usage, pause the
|
||||
* room until the limit resets. When the model's safeguards refused the
|
||||
* request, close the paired task (retrying cannot help; the next user message
|
||||
* starts a new task) and tell the room; the refusal is never handed to the
|
||||
* reviewer/arbiter as if it were an answer. Returns false for any other
|
||||
* failure.
|
||||
*/
|
||||
async function reportClaudeUsageLimitToRoom(
|
||||
args: ClaudeUsageLimitFailure & {
|
||||
isClaudeCodeAgent: boolean;
|
||||
activeRole: PairedRoomRole;
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
pairedTaskId?: string;
|
||||
pairedExecutionLifecycle: Pick<
|
||||
PairedExecutionLifecycle,
|
||||
'updateSummary' | 'completeImmediately'
|
||||
'updateSummary' | 'completeImmediately' | 'releaseForClaudeUsagePause'
|
||||
>;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
log: { warn: (obj: Record<string, unknown>, msg: string) => void };
|
||||
},
|
||||
): Promise<boolean> {
|
||||
if (!args.isClaudeCodeAgent) return false;
|
||||
const isSafetyRefusal = args.reason === 'safety-refusal';
|
||||
if (!isSafetyRefusal && !isClaudeUsageLimitReason(args.reason)) {
|
||||
return false;
|
||||
if (isClaudeUsageLimitReason(args.reason)) {
|
||||
args.log.warn(
|
||||
{ reason: args.reason, role: args.activeRole, errorText: args.errorText },
|
||||
'Claude usage limit reached; pausing the room until it resets',
|
||||
);
|
||||
await pauseRunForClaudeUsage({
|
||||
chatJid: args.chatJid,
|
||||
runId: args.runId,
|
||||
resume: resolveClaudeUsageResume(args.errorText),
|
||||
summary: args.errorText ?? 'Claude usage limit reached',
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
const notice = isSafetyRefusal
|
||||
? CLAUDE_SAFETY_REFUSAL_NOTICE
|
||||
: buildClaudeUsageLimitMessage(args.errorText);
|
||||
if (args.reason !== 'safety-refusal') return false;
|
||||
const task = args.pairedTaskId
|
||||
? getPairedTaskById(args.pairedTaskId)
|
||||
: undefined;
|
||||
if (task) {
|
||||
completePairedTaskWithReason(
|
||||
task,
|
||||
isSafetyRefusal
|
||||
? CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON
|
||||
: CLAUDE_USAGE_LIMIT_COMPLETION_REASON,
|
||||
);
|
||||
completePairedTaskWithReason(task, CLAUDE_SAFETY_REFUSAL_COMPLETION_REASON);
|
||||
}
|
||||
args.log.warn(
|
||||
{ reason: args.reason, role: args.activeRole, errorText: args.errorText },
|
||||
isSafetyRefusal
|
||||
? 'Claude safeguards refused the request; closed the task and told the room'
|
||||
: 'Claude usage limit reached; told the room when it resets',
|
||||
'Claude safeguards refused the request; closed the task and told the room',
|
||||
);
|
||||
await deliverNoticeAndFailExecution({
|
||||
notice,
|
||||
summary: isSafetyRefusal ? notice : (args.errorText ?? notice),
|
||||
notice: CLAUDE_SAFETY_REFUSAL_NOTICE,
|
||||
summary: CLAUDE_SAFETY_REFUSAL_NOTICE,
|
||||
pairedExecutionLifecycle: args.pairedExecutionLifecycle,
|
||||
onOutput: args.onOutput,
|
||||
});
|
||||
@@ -286,6 +369,8 @@ export async function runAgentForGroup(
|
||||
...limit,
|
||||
isClaudeCodeAgent,
|
||||
activeRole,
|
||||
chatJid,
|
||||
runId,
|
||||
pairedTaskId: pairedExecutionContext?.task.id,
|
||||
pairedExecutionLifecycle,
|
||||
onOutput,
|
||||
@@ -342,6 +427,20 @@ export async function runAgentForGroup(
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
isClaudeCodeAgent &&
|
||||
(await pauseWhileClaudeUsageLimited({
|
||||
chatJid,
|
||||
runId,
|
||||
activeRole,
|
||||
pairedExecutionLifecycle,
|
||||
onOutput,
|
||||
log,
|
||||
}))
|
||||
) {
|
||||
return 'success';
|
||||
}
|
||||
|
||||
const runAttempt = (currentProvider: 'claude' | 'codex') =>
|
||||
runMessageAgentAttempt({
|
||||
provider: currentProvider,
|
||||
|
||||
Reference in New Issue
Block a user