refactor: add recovery mode + shared JSON/error/fetch utilities

- GroupQueue recovery mode: limit to 3 concurrent agents for 60s after restart
  to prevent API rate-limit storms (configurable via env vars)
- getErrorMessage: replace 14 occurrences of instanceof Error pattern
- readJsonFile/writeJsonFile: replace 19 occurrences of JSON+fs pattern
- fetchWithTimeout: replace 3 occurrences of AbortController pattern

354/354 tests passing
This commit is contained in:
Eyejoker
2026-03-25 07:08:49 +09:00
parent a576d623a1
commit b6d3f879ef
11 changed files with 123 additions and 37 deletions

View File

@@ -1,6 +1,10 @@
import { ChildProcess } from 'child_process';
import { MAX_CONCURRENT_AGENTS } from './config.js';
import {
MAX_CONCURRENT_AGENTS,
RECOVERY_CONCURRENT_AGENTS,
RECOVERY_DURATION_MS,
} from './config.js';
import { queueFollowUpMessage, writeCloseSentinel } from './group-queue-ipc.js';
import { logger } from './logger.js';
@@ -50,6 +54,8 @@ export class GroupQueue {
private activeCount = 0;
private activeTaskCount = 0;
private waitingGroups: string[] = [];
private recoveryMode = false;
private recoveryTimer: ReturnType<typeof setTimeout> | null = null;
private processMessagesFn:
| ((groupJid: string, context: GroupRunContext) => Promise<boolean>)
| null = null;
@@ -85,6 +91,30 @@ export class GroupQueue {
this.processMessagesFn = fn;
}
/** Limit concurrency after restart to avoid API rate-limit storms. */
enterRecoveryMode(): void {
this.recoveryMode = true;
logger.info(
{ maxConcurrent: RECOVERY_CONCURRENT_AGENTS, durationMs: RECOVERY_DURATION_MS },
'Entering recovery mode (staggered restart)',
);
this.recoveryTimer = setTimeout(() => {
this.recoveryMode = false;
this.recoveryTimer = null;
logger.info(
{ maxConcurrent: MAX_CONCURRENT_AGENTS },
'Recovery mode ended, full concurrency restored',
);
this.drainWaiting();
}, RECOVERY_DURATION_MS);
}
private get effectiveMaxConcurrent(): number {
return this.recoveryMode
? RECOVERY_CONCURRENT_AGENTS
: MAX_CONCURRENT_AGENTS;
}
private createRunId(): string {
return `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
}
@@ -121,13 +151,13 @@ export class GroupQueue {
return;
}
if (this.activeCount >= MAX_CONCURRENT_AGENTS) {
if (this.activeCount >= this.effectiveMaxConcurrent) {
state.pendingMessages = true;
if (!this.waitingGroups.includes(groupJid)) {
this.waitingGroups.push(groupJid);
}
logger.debug(
{ groupJid, activeCount: this.activeCount },
{ groupJid, activeCount: this.activeCount, max: this.effectiveMaxConcurrent },
'At concurrency limit, message queued',
);
return;
@@ -160,7 +190,7 @@ export class GroupQueue {
}
if (
this.activeCount >= MAX_CONCURRENT_AGENTS ||
this.activeCount >= this.effectiveMaxConcurrent ||
this.activeTaskCount >= MAX_CONCURRENT_TASKS
) {
state.pendingTasks.push({ id: taskId, groupJid, fn });
@@ -466,7 +496,7 @@ export class GroupQueue {
private drainWaiting(): void {
while (
this.waitingGroups.length > 0 &&
this.activeCount < MAX_CONCURRENT_AGENTS
this.activeCount < this.effectiveMaxConcurrent
) {
const nextMessageIndex = this.waitingGroups.findIndex((jid) => {
const state = this.getGroup(jid);