import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; import { EventEmitter } from 'events'; import fs from 'fs'; import { GroupQueue, type GroupRunContext } from './group-queue.js'; // Mock config to control concurrency limit vi.mock('./config.js', () => ({ ASSISTANT_NAME: 'Andy', DATA_DIR: '/tmp/ejclaw-test-data', IS_TEST_ENV: true, LOG_LEVEL: 'info', MAX_CONCURRENT_AGENTS: 2, })); // Mock fs operations used by closeStdin vi.mock('fs', async () => { const actual = await vi.importActual('fs'); return { ...actual, default: { ...actual, mkdirSync: vi.fn(), writeFileSync: vi.fn(), renameSync: vi.fn(), }, }; }); describe('GroupQueue', () => { let queue: GroupQueue; beforeEach(() => { vi.useFakeTimers(); vi.clearAllMocks(); queue = new GroupQueue(); }); afterEach(() => { vi.useRealTimers(); }); // --- Single group at a time --- it('only runs one agent per group at a time', async () => { let concurrentCount = 0; let maxConcurrent = 0; const processMessages = vi.fn(async (_groupJid: string) => { concurrentCount++; maxConcurrent = Math.max(maxConcurrent, concurrentCount); // Simulate async work await new Promise((resolve) => setTimeout(resolve, 100)); concurrentCount--; return true; }); queue.setProcessMessagesFn(processMessages); // Enqueue two messages for the same group queue.enqueueMessageCheck('group1@g.us'); queue.enqueueMessageCheck('group1@g.us'); // Advance timers to let the first process complete await vi.advanceTimersByTimeAsync(200); // Second enqueue should have been queued, not concurrent expect(maxConcurrent).toBe(1); }); it('pipes follow-up messages to an active non-task process', async () => { const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let releaseRun!: (value: boolean) => void; const blocker = new Promise((resolve) => { releaseRun = resolve; }); const processMessages = vi.fn(async () => await blocker); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); expect(queue.sendMessage('group1@g.us', '후속 메시지')).toBe(true); expect(fs.mkdirSync).toHaveBeenCalledWith(`${ipcDir}/input`, { recursive: true, }); expect(fs.writeFileSync).toHaveBeenCalled(); expect(fs.renameSync).toHaveBeenCalled(); releaseRun(true); await vi.advanceTimersByTimeAsync(10); }); it('does not pipe follow-up messages after stdin close was requested', async () => { const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let releaseRun!: (value: boolean) => void; const blocker = new Promise((resolve) => { releaseRun = resolve; }); const processMessages = vi.fn(async () => await blocker); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); vi.mocked(fs.mkdirSync).mockClear(); vi.mocked(fs.writeFileSync).mockClear(); vi.mocked(fs.renameSync).mockClear(); queue.closeStdin('group1@g.us'); vi.mocked(fs.mkdirSync).mockClear(); vi.mocked(fs.writeFileSync).mockClear(); vi.mocked(fs.renameSync).mockClear(); expect(queue.sendMessage('group1@g.us', '후속 메시지')).toBe(false); expect(fs.mkdirSync).not.toHaveBeenCalled(); expect(fs.writeFileSync).not.toHaveBeenCalled(); expect(fs.renameSync).not.toHaveBeenCalled(); releaseRun(true); await vi.advanceTimersByTimeAsync(10); }); it('stores direct terminal text for the active run, keeps a recent run record, and clears the active slot when the run ends', async () => { let releaseRun!: (value: boolean) => void; let runId: string | undefined; const blocker = new Promise((resolve) => { releaseRun = resolve; }); queue.setProcessMessagesFn( async (_groupJid: string, context: GroupRunContext) => { runId = context.runId; return await blocker; }, ); queue.enqueueMessageCheck('group1@g.us'); await vi.advanceTimersByTimeAsync(10); expect(runId).toEqual(expect.any(String)); queue.noteDirectTerminalDelivery( 'group1@g.us', 'reviewer', 'DONE_WITH_CONCERNS\n리뷰 완료', ); expect( queue.hasDirectTerminalDeliveryForRun('group1@g.us', runId!, 'reviewer'), ).toBe(true); expect( queue.getDirectTerminalDeliveryForRun('group1@g.us', runId!, 'reviewer'), ).toBe('DONE_WITH_CONCERNS\n리뷰 완료'); expect( queue.hasRecordedDirectTerminalDeliveryForRun( 'group1@g.us', runId!, 'reviewer', ), ).toBe(true); releaseRun(true); await vi.advanceTimersByTimeAsync(10); expect( queue.hasDirectTerminalDeliveryForRun('group1@g.us', runId!, 'reviewer'), ).toBe(false); expect( queue.getDirectTerminalDeliveryForRun('group1@g.us', runId!, 'reviewer'), ).toBeNull(); expect( queue.hasRecordedDirectTerminalDeliveryForRun( 'group1@g.us', runId!, 'reviewer', ), ).toBe(true); }); it('force-terminates a lingering process after output was delivered', async () => { class StubbornProcess extends EventEmitter { exitCode: number | null = null; signalCode: NodeJS.Signals | null = null; kill = vi.fn(() => true); } const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let releaseRun!: (value: boolean) => void; let runId: string | undefined; const blocker = new Promise((resolve) => { releaseRun = resolve; }); const processMessages = vi.fn( async (_groupJid: string, context: GroupRunContext) => { runId = context.runId; return await blocker; }, ); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); expect(runId).toEqual(expect.any(String)); const proc = new StubbornProcess(); queue.registerProcess( 'group1@g.us', proc as unknown as import('child_process').ChildProcess, 'proc-1', ipcDir, ); queue.closeStdin('group1@g.us', { runId, reason: 'output-delivered-close', }); await vi.advanceTimersByTimeAsync(60_000); expect(proc.kill).toHaveBeenCalledWith('SIGTERM'); await vi.advanceTimersByTimeAsync(15_000); expect(proc.kill).toHaveBeenCalledWith('SIGKILL'); releaseRun(true); await vi.advanceTimersByTimeAsync(10); }); it('clears post-close termination timers once the run exits', async () => { class FakeProcess extends EventEmitter { exitCode: number | null = null; signalCode: NodeJS.Signals | null = null; kill = vi.fn(() => true); } const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let releaseRun!: (value: boolean) => void; const blocker = new Promise((resolve) => { releaseRun = resolve; }); const proc = new FakeProcess(); const processMessages = vi.fn( async (_groupJid: string, context: GroupRunContext) => { queue.registerProcess( 'group1@g.us', proc as unknown as import('child_process').ChildProcess, 'proc-1', ipcDir, ); queue.closeStdin('group1@g.us', { runId: context.runId, reason: 'output-delivered-close', }); await blocker; return true; }, ); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); releaseRun(true); await vi.advanceTimersByTimeAsync(10); await vi.advanceTimersByTimeAsync(75_000); expect(proc.kill).not.toHaveBeenCalled(); }); // --- Global concurrency limit --- it('respects global concurrency limit', async () => { let activeCount = 0; let maxActive = 0; const completionCallbacks: Array<() => void> = []; const processMessages = vi.fn(async (_groupJid: string) => { activeCount++; maxActive = Math.max(maxActive, activeCount); await new Promise((resolve) => completionCallbacks.push(resolve)); activeCount--; return true; }); queue.setProcessMessagesFn(processMessages); // Enqueue 3 groups (limit is 2) queue.enqueueMessageCheck('group1@g.us'); queue.enqueueMessageCheck('group2@g.us'); queue.enqueueMessageCheck('group3@g.us'); // Let promises settle await vi.advanceTimersByTimeAsync(10); // Only 2 should be active (MAX_CONCURRENT_AGENTS = 2) expect(maxActive).toBe(2); expect(activeCount).toBe(2); // Complete one — third should start completionCallbacks[0](); await vi.advanceTimersByTimeAsync(10); expect(processMessages).toHaveBeenCalledTimes(3); }); it('reserves one foreground slot so queued tasks do not block new messages', async () => { let resolveTaskOne!: () => void; let resolveTaskTwo!: () => void; let messageStarted = false; const processMessages = vi.fn(async (groupJid: string) => { if (groupJid === 'group3@g.us') { messageStarted = true; } return true; }); queue.setProcessMessagesFn(processMessages); queue.enqueueTask('group1@g.us', 'task-1', async () => { await new Promise((resolve) => { resolveTaskOne = resolve; }); }); await vi.advanceTimersByTimeAsync(10); queue.enqueueTask('group2@g.us', 'task-2', async () => { await new Promise((resolve) => { resolveTaskTwo = resolve; }); }); queue.enqueueMessageCheck('group3@g.us'); await vi.advanceTimersByTimeAsync(10); expect(messageStarted).toBe(true); expect(processMessages).toHaveBeenCalledWith('group3@g.us', { runId: expect.any(String), reason: 'messages', }); resolveTaskOne!(); await vi.advanceTimersByTimeAsync(10); expect(messageStarted).toBe(true); resolveTaskTwo!(); await vi.advanceTimersByTimeAsync(10); }); // --- Tasks prioritized over messages --- it('drains tasks before messages for same group', async () => { const executionOrder: string[] = []; let resolveFirst: () => void; const processMessages = vi.fn(async (_groupJid: string) => { if (executionOrder.length === 0) { // First call: block until we release it await new Promise((resolve) => { resolveFirst = resolve; }); } executionOrder.push('messages'); return true; }); queue.setProcessMessagesFn(processMessages); // Start processing messages (takes the active slot) queue.enqueueMessageCheck('group1@g.us'); await vi.advanceTimersByTimeAsync(10); // While active, enqueue both a task and pending messages const taskFn = vi.fn(async () => { executionOrder.push('task'); }); queue.enqueueTask('group1@g.us', 'task-1', taskFn); queue.enqueueMessageCheck('group1@g.us'); // Release the first processing resolveFirst!(); await vi.advanceTimersByTimeAsync(10); // Task should have run before the second message check expect(executionOrder[0]).toBe('messages'); // first call expect(executionOrder[1]).toBe('task'); // task runs first in drain // Messages would run after task completes }); // --- Retry with backoff on failure --- it('retries with exponential backoff on failure', async () => { let callCount = 0; const processMessages = vi.fn(async () => { callCount++; return false; // failure }); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us'); // First call happens immediately await vi.advanceTimersByTimeAsync(10); expect(callCount).toBe(1); // First retry after 5000ms (BASE_RETRY_MS * 2^0) await vi.advanceTimersByTimeAsync(5000); await vi.advanceTimersByTimeAsync(10); expect(callCount).toBe(2); // Second retry after 10000ms (BASE_RETRY_MS * 2^1) await vi.advanceTimersByTimeAsync(10000); await vi.advanceTimersByTimeAsync(10); expect(callCount).toBe(3); }); it('does not bypass retry backoff when new messages arrive', async () => { let callCount = 0; const processMessages = vi.fn(async () => { callCount++; return false; }); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us'); await vi.advanceTimersByTimeAsync(10); expect(callCount).toBe(1); queue.enqueueMessageCheck('group1@g.us'); await vi.advanceTimersByTimeAsync(1000); expect(callCount).toBe(1); await vi.advanceTimersByTimeAsync(4000); await vi.advanceTimersByTimeAsync(10); expect(callCount).toBe(2); }); // --- Shutdown prevents new enqueues --- it('prevents new enqueues after shutdown', async () => { const processMessages = vi.fn(async () => true); queue.setProcessMessagesFn(processMessages); await queue.shutdown(1000); queue.enqueueMessageCheck('group1@g.us'); await vi.advanceTimersByTimeAsync(100); expect(processMessages).not.toHaveBeenCalled(); }); it('signals and terminates active agent processes during shutdown', async () => { class FakeProcess extends EventEmitter { exitCode: number | null = null; signalCode: NodeJS.Signals | null = null; kill = vi.fn((signal?: NodeJS.Signals) => { this.signalCode = signal ?? 'SIGTERM'; return true; }); } const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let releaseRun!: (value: boolean) => void; const blocker = new Promise((resolve) => { releaseRun = resolve; }); const processMessages = vi.fn(async () => await blocker); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); const proc = new FakeProcess(); queue.registerProcess( 'group1@g.us', proc as unknown as import('child_process').ChildProcess, 'proc-1', ipcDir, ); vi.mocked(fs.writeFileSync).mockClear(); const shutdownPromise = queue.shutdown(1_000); expect(fs.writeFileSync).toHaveBeenCalledWith( expect.stringContaining('/input/_close'), '', ); await vi.advanceTimersByTimeAsync(1_000); await vi.advanceTimersByTimeAsync(2_000); await shutdownPromise; expect(proc.kill).toHaveBeenCalledWith('SIGTERM'); releaseRun(true); await vi.advanceTimersByTimeAsync(10); }); // --- Max retries exceeded --- it('stops retrying after MAX_RETRIES and resets', async () => { let callCount = 0; const processMessages = vi.fn(async () => { callCount++; return false; // always fail }); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us'); // Run through all 5 retries (MAX_RETRIES = 5) // Initial call await vi.advanceTimersByTimeAsync(10); expect(callCount).toBe(1); // Retry 1: 5000ms, Retry 2: 10000ms, Retry 3: 20000ms, Retry 4: 40000ms, Retry 5: 80000ms const retryDelays = [5000, 10000, 20000, 40000, 80000]; for (let i = 0; i < retryDelays.length; i++) { await vi.advanceTimersByTimeAsync(retryDelays[i] + 10); expect(callCount).toBe(i + 2); } // After 5 retries (6 total calls), should stop — no more retries const countAfterMaxRetries = callCount; await vi.advanceTimersByTimeAsync(200000); // Wait a long time expect(callCount).toBe(countAfterMaxRetries); }); // --- Waiting groups get drained when slots free up --- it('drains waiting groups when active slots free up', async () => { const processed: string[] = []; const completionCallbacks: Array<() => void> = []; const processMessages = vi.fn(async (groupJid: string) => { processed.push(groupJid); await new Promise((resolve) => completionCallbacks.push(resolve)); return true; }); queue.setProcessMessagesFn(processMessages); // Fill both slots queue.enqueueMessageCheck('group1@g.us'); queue.enqueueMessageCheck('group2@g.us'); await vi.advanceTimersByTimeAsync(10); // Queue a third queue.enqueueMessageCheck('group3@g.us'); await vi.advanceTimersByTimeAsync(10); expect(processed).toEqual(['group1@g.us', 'group2@g.us']); // Free up a slot completionCallbacks[0](); await vi.advanceTimersByTimeAsync(10); expect(processed).toContain('group3@g.us'); }); // --- Running task dedup (Issue #138) --- it('rejects duplicate enqueue of a currently-running task', async () => { let resolveTask: () => void; let taskCallCount = 0; const taskFn = vi.fn(async () => { taskCallCount++; await new Promise((resolve) => { resolveTask = resolve; }); }); // Start the task (runs immediately — slot available) queue.enqueueTask('group1@g.us', 'task-1', taskFn); await vi.advanceTimersByTimeAsync(10); expect(taskCallCount).toBe(1); // Scheduler poll re-discovers the same task while it's running — // this must be silently dropped const dupFn = vi.fn(async () => {}); queue.enqueueTask('group1@g.us', 'task-1', dupFn); await vi.advanceTimersByTimeAsync(10); // Duplicate was NOT queued expect(dupFn).not.toHaveBeenCalled(); // Complete the original task resolveTask!(); await vi.advanceTimersByTimeAsync(10); // Only one execution total expect(taskCallCount).toBe(1); }); // --- Active runs queue work without preemption --- it('does not preempt an active agent when a task is queued', async () => { const fs = await import('fs'); let resolveProcess: () => void; const processMessages = vi.fn(async () => { await new Promise((resolve) => { resolveProcess = resolve; }); return true; }); queue.setProcessMessagesFn(processMessages); // Start processing (takes the active slot) queue.enqueueMessageCheck('group1@g.us'); await vi.advanceTimersByTimeAsync(10); // Register a process so closeStdin has a groupFolder queue.registerProcess('group1@g.us', {} as any, 'agent-1', 'test-group'); // Enqueue a task while agent is active but NOT idle const taskFn = vi.fn(async () => {}); queue.enqueueTask('group1@g.us', 'task-1', taskFn); // _close should NOT have been written (agent is working, not idle) const writeFileSync = vi.mocked(fs.default.writeFileSync); const closeWrites = writeFileSync.mock.calls.filter( (call) => typeof call[0] === 'string' && call[0].endsWith('_close'), ); expect(closeWrites).toHaveLength(0); resolveProcess!(); await vi.advanceTimersByTimeAsync(10); }); // --- Run phase transitions --- it('transitions running_messages → closing_messages → idle', async () => { const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let releaseRun!: (value: boolean) => void; const blocker = new Promise((resolve) => { releaseRun = resolve; }); const processMessages = vi.fn(async () => await blocker); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); // While running, phase should be running_messages const duringRun = queue.getStatuses(['group1@g.us']); expect(duringRun[0].runPhase).toBe('running_messages'); // closeStdin transitions to closing_messages queue.closeStdin('group1@g.us'); const afterClose = queue.getStatuses(['group1@g.us']); expect(afterClose[0].runPhase).toBe('closing_messages'); // sendMessage should fail in closing_messages expect(queue.sendMessage('group1@g.us', 'test')).toBe(false); // Complete the run — should go to idle releaseRun(true); await vi.advanceTimersByTimeAsync(10); const afterFinish = queue.getStatuses(['group1@g.us']); expect(afterFinish[0].runPhase).toBe('idle'); expect(afterFinish[0].status).toBe('inactive'); }); it('closeStdin does not change phase during running_task', async () => { const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; let resolveTask!: () => void; const blocker = new Promise((resolve) => { resolveTask = resolve; }); queue.enqueueTask('group1@g.us', 'task-1', async () => { // Register process so closeStdin has an ipcDir queue.registerProcess('group1@g.us', {} as any, 'agent-1', ipcDir); await blocker; }); await vi.advanceTimersByTimeAsync(10); const duringTask = queue.getStatuses(['group1@g.us']); expect(duringTask[0].runPhase).toBe('running_task'); // closeStdin during task — phase should stay running_task queue.closeStdin('group1@g.us'); const afterClose = queue.getStatuses(['group1@g.us']); expect(afterClose[0].runPhase).toBe('running_task'); resolveTask(); await vi.advanceTimersByTimeAsync(10); const afterFinish = queue.getStatuses(['group1@g.us']); expect(afterFinish[0].runPhase).toBe('idle'); }); it('sendMessage returns true only in running_messages phase', async () => { const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder'; // idle → false expect(queue.sendMessage('group1@g.us', 'test')).toBe(false); // running_task → false let resolveTask!: () => void; queue.enqueueTask('group1@g.us', 'task-1', async () => { queue.registerProcess('group1@g.us', {} as any, 'agent-1', ipcDir); await new Promise((resolve) => { resolveTask = resolve; }); }); await vi.advanceTimersByTimeAsync(10); expect(queue.sendMessage('group1@g.us', 'test')).toBe(false); resolveTask(); await vi.advanceTimersByTimeAsync(10); // running_messages → true let releaseRun!: (value: boolean) => void; const processMessages = vi.fn(async () => { return await new Promise((resolve) => { releaseRun = resolve; }); }); queue.setProcessMessagesFn(processMessages); queue.enqueueMessageCheck('group1@g.us', ipcDir); await vi.advanceTimersByTimeAsync(10); expect(queue.sendMessage('group1@g.us', 'msg')).toBe(true); // closing_messages → false queue.closeStdin('group1@g.us'); expect(queue.sendMessage('group1@g.us', 'msg')).toBe(false); releaseRun(true); await vi.advanceTimersByTimeAsync(10); }); });