Complete paired-room SSOT cleanup and logger singleton
This commit is contained in:
182
src/agent-attempt-orchestration.ts
Normal file
182
src/agent-attempt-orchestration.ts
Normal file
@@ -0,0 +1,182 @@
|
||||
import type { AgentOutput } from './agent-runner.js';
|
||||
import {
|
||||
resolveAttemptRetryAction,
|
||||
type AttemptRetryAction,
|
||||
type AttemptRetryState,
|
||||
} from './agent-attempt-retry.js';
|
||||
import type {
|
||||
AgentTriggerReason,
|
||||
CodexRotationReason,
|
||||
} from './agent-error-detection.js';
|
||||
import {
|
||||
runClaudeRotationLoop,
|
||||
runCodexRotationLoop,
|
||||
} from './provider-retry.js';
|
||||
|
||||
type RotationLogContext = Record<string, unknown>;
|
||||
|
||||
type ClaudeRetryAttemptState = Pick<
|
||||
AttemptRetryState,
|
||||
'sawOutput' | 'streamedTriggerReason'
|
||||
> & {
|
||||
output?: Pick<AgentOutput, 'status' | 'result' | 'output' | 'error'>;
|
||||
error?: unknown;
|
||||
sawSuccessNullResultWithoutOutput?: boolean;
|
||||
};
|
||||
|
||||
type CodexRetryAttemptState = Pick<
|
||||
AttemptRetryState,
|
||||
'sawOutput' | 'streamedTriggerReason'
|
||||
> & {
|
||||
output?: Pick<AgentOutput, 'status' | 'result' | 'output' | 'error'>;
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
export type ExecutedAttemptRetryAction =
|
||||
| { kind: 'none' }
|
||||
| {
|
||||
kind: 'claude';
|
||||
trigger: { reason: AgentTriggerReason; retryAfterMs?: number };
|
||||
rotationMessage?: string;
|
||||
result: 'success' | 'error';
|
||||
}
|
||||
| {
|
||||
kind: 'codex';
|
||||
trigger: { reason: CodexRotationReason };
|
||||
rotationMessage?: string;
|
||||
result: 'success' | 'error';
|
||||
};
|
||||
|
||||
export async function runClaudeAttemptWithRotation<
|
||||
TAttempt extends ClaudeRetryAttemptState,
|
||||
>(args: {
|
||||
initialTrigger: {
|
||||
reason: AgentTriggerReason;
|
||||
retryAfterMs?: number;
|
||||
};
|
||||
runAttempt: () => Promise<TAttempt>;
|
||||
logContext: RotationLogContext;
|
||||
rotationMessage?: string;
|
||||
afterAttempt?: (attempt: TAttempt) => Promise<void> | void;
|
||||
onSuccess?: (outcome: { sawOutput: boolean }) => Promise<void> | void;
|
||||
}): Promise<'success' | 'error'> {
|
||||
const outcome = await runClaudeRotationLoop(
|
||||
args.initialTrigger,
|
||||
async () => {
|
||||
const attempt = await args.runAttempt();
|
||||
await args.afterAttempt?.(attempt);
|
||||
return {
|
||||
output: attempt.output,
|
||||
thrownError: attempt.error,
|
||||
sawOutput: attempt.sawOutput,
|
||||
sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput,
|
||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||
};
|
||||
},
|
||||
args.logContext,
|
||||
args.rotationMessage,
|
||||
);
|
||||
|
||||
if (outcome.type === 'success') {
|
||||
await args.onSuccess?.({ sawOutput: outcome.sawOutput });
|
||||
return 'success';
|
||||
}
|
||||
|
||||
return 'error';
|
||||
}
|
||||
|
||||
export async function runCodexAttemptWithRotation<
|
||||
TAttempt extends CodexRetryAttemptState,
|
||||
>(args: {
|
||||
initialTrigger: { reason: CodexRotationReason };
|
||||
runAttempt: () => Promise<TAttempt>;
|
||||
logContext: RotationLogContext;
|
||||
rotationMessage?: string;
|
||||
afterAttempt?: (attempt: TAttempt) => Promise<void> | void;
|
||||
}): Promise<'success' | 'error'> {
|
||||
const outcome = await runCodexRotationLoop(
|
||||
args.initialTrigger,
|
||||
async () => {
|
||||
const attempt = await args.runAttempt();
|
||||
await args.afterAttempt?.(attempt);
|
||||
return {
|
||||
output: attempt.output,
|
||||
thrownError: attempt.error,
|
||||
sawOutput: attempt.sawOutput,
|
||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||
};
|
||||
},
|
||||
args.logContext,
|
||||
args.rotationMessage,
|
||||
);
|
||||
|
||||
return outcome.type === 'success' ? 'success' : 'error';
|
||||
}
|
||||
|
||||
export async function executeAttemptRetryAction(args: {
|
||||
provider: 'claude' | 'codex';
|
||||
canRetryClaudeCredentials: boolean;
|
||||
canRetryCodex: boolean;
|
||||
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||
rotationMessage?: string | null;
|
||||
runClaude: (
|
||||
trigger: { reason: AgentTriggerReason; retryAfterMs?: number },
|
||||
rotationMessage?: string,
|
||||
) => Promise<'success' | 'error'>;
|
||||
runCodex: (
|
||||
trigger: { reason: CodexRotationReason },
|
||||
rotationMessage?: string,
|
||||
) => Promise<'success' | 'error'>;
|
||||
}): Promise<ExecutedAttemptRetryAction> {
|
||||
const retryAction = resolveAttemptRetryAction({
|
||||
provider: args.provider,
|
||||
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
||||
canRetryCodex: args.canRetryCodex,
|
||||
attempt: args.attempt,
|
||||
rotationMessage: args.rotationMessage,
|
||||
});
|
||||
|
||||
return executeResolvedAttemptRetryAction({
|
||||
retryAction,
|
||||
runClaude: args.runClaude,
|
||||
runCodex: args.runCodex,
|
||||
});
|
||||
}
|
||||
|
||||
async function executeResolvedAttemptRetryAction(args: {
|
||||
retryAction: AttemptRetryAction;
|
||||
runClaude: (
|
||||
trigger: { reason: AgentTriggerReason; retryAfterMs?: number },
|
||||
rotationMessage?: string,
|
||||
) => Promise<'success' | 'error'>;
|
||||
runCodex: (
|
||||
trigger: { reason: CodexRotationReason },
|
||||
rotationMessage?: string,
|
||||
) => Promise<'success' | 'error'>;
|
||||
}): Promise<ExecutedAttemptRetryAction> {
|
||||
if (args.retryAction.kind === 'claude') {
|
||||
return {
|
||||
kind: 'claude',
|
||||
trigger: args.retryAction.trigger,
|
||||
rotationMessage: args.retryAction.rotationMessage,
|
||||
result: await args.runClaude(
|
||||
args.retryAction.trigger,
|
||||
args.retryAction.rotationMessage,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (args.retryAction.kind === 'codex') {
|
||||
return {
|
||||
kind: 'codex',
|
||||
trigger: args.retryAction.trigger,
|
||||
rotationMessage: args.retryAction.rotationMessage,
|
||||
result: await args.runCodex(
|
||||
args.retryAction.trigger,
|
||||
args.retryAction.rotationMessage,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { kind: 'none' };
|
||||
}
|
||||
@@ -121,4 +121,25 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
||||
expect(active).toBeDefined();
|
||||
expect(active!.index).toBe(1); // fallback picks next non-rate-limited
|
||||
});
|
||||
|
||||
it('warns when Codex rotation state cannot be persisted', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
const utils = await import('./utils.js');
|
||||
const { logger } = await import('./logger.js');
|
||||
|
||||
vi.mocked(utils.writeJsonFile).mockImplementation(() => {
|
||||
throw new Error('disk full');
|
||||
});
|
||||
|
||||
mod.initCodexTokenRotation();
|
||||
expect(mod.rotateCodexToken('rate limit')).toBe(true);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
stateFile: '/tmp/ejclaw-codex-rot-data/codex-rotation-state.json',
|
||||
err: expect.any(Error),
|
||||
}),
|
||||
'Failed to persist Codex rotation state',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -138,8 +138,11 @@ function saveCodexState(): void {
|
||||
resetD7Ats: accounts.map((a) => a.resetD7At ?? null),
|
||||
};
|
||||
writeJsonFile(STATE_FILE, state);
|
||||
} catch {
|
||||
/* best effort */
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ stateFile: STATE_FILE, err },
|
||||
'Failed to persist Codex rotation state',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
33
src/logger.test.ts
Normal file
33
src/logger.test.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
describe('logger singleton', () => {
|
||||
it('reuses the root logger and does not add duplicate process listeners across module reloads', async () => {
|
||||
vi.resetModules();
|
||||
|
||||
const beforeUncaught = process.listeners('uncaughtException').length;
|
||||
const beforeUnhandled = process.listeners('unhandledRejection').length;
|
||||
const beforeExit = process.listeners('exit').length;
|
||||
|
||||
const first = await import('./logger.js');
|
||||
|
||||
const afterFirstUncaught = process.listeners('uncaughtException').length;
|
||||
const afterFirstUnhandled = process.listeners('unhandledRejection').length;
|
||||
const afterFirstExit = process.listeners('exit').length;
|
||||
|
||||
vi.resetModules();
|
||||
|
||||
const second = await import('./logger.js');
|
||||
|
||||
const afterSecondUncaught = process.listeners('uncaughtException').length;
|
||||
const afterSecondUnhandled = process.listeners('unhandledRejection').length;
|
||||
const afterSecondExit = process.listeners('exit').length;
|
||||
|
||||
expect(second.logger).toBe(first.logger);
|
||||
expect(afterFirstUncaught).toBeGreaterThanOrEqual(beforeUncaught);
|
||||
expect(afterFirstUnhandled).toBeGreaterThanOrEqual(beforeUnhandled);
|
||||
expect(afterSecondUncaught).toBe(afterFirstUncaught);
|
||||
expect(afterSecondUnhandled).toBe(afterFirstUnhandled);
|
||||
expect(afterFirstExit).toBeGreaterThanOrEqual(beforeExit);
|
||||
expect(afterSecondExit).toBe(afterFirstExit);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,34 @@
|
||||
import pino, { type Logger } from 'pino';
|
||||
|
||||
const serviceName = (process.env.ASSISTANT_NAME || 'claude').toLowerCase();
|
||||
const isTestEnv =
|
||||
process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
|
||||
|
||||
export const logger = pino({
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
name: serviceName,
|
||||
transport: { target: 'pino-pretty', options: { colorize: true } },
|
||||
});
|
||||
type LoggerGlobalState = typeof globalThis & {
|
||||
__ejclawLogger?: Logger;
|
||||
__ejclawProcessHandlersInstalled?: boolean;
|
||||
};
|
||||
|
||||
const globalState = globalThis as LoggerGlobalState;
|
||||
|
||||
function createRootLogger(): Logger {
|
||||
const baseOptions = {
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
name: serviceName,
|
||||
};
|
||||
|
||||
if (isTestEnv) {
|
||||
return pino(baseOptions);
|
||||
}
|
||||
|
||||
return pino({
|
||||
...baseOptions,
|
||||
transport: { target: 'pino-pretty', options: { colorize: true } },
|
||||
});
|
||||
}
|
||||
|
||||
export const logger =
|
||||
globalState.__ejclawLogger ?? (globalState.__ejclawLogger = createRootLogger());
|
||||
|
||||
type LogBindings = Record<string, unknown>;
|
||||
|
||||
@@ -30,11 +52,17 @@ export function createScopedLogger(bindings: LogBindings): Logger {
|
||||
}
|
||||
|
||||
// Route uncaught errors through pino so they get timestamps in stderr
|
||||
process.on('uncaughtException', (err) => {
|
||||
function handleUncaughtException(err: unknown): void {
|
||||
logger.fatal({ err }, 'Uncaught exception');
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
function handleUnhandledRejection(reason: unknown): void {
|
||||
logger.error({ err: reason }, 'Unhandled rejection');
|
||||
});
|
||||
}
|
||||
|
||||
if (!globalState.__ejclawProcessHandlersInstalled) {
|
||||
process.on('uncaughtException', handleUncaughtException);
|
||||
process.on('unhandledRejection', handleUnhandledRejection);
|
||||
globalState.__ejclawProcessHandlersInstalled = true;
|
||||
}
|
||||
|
||||
@@ -6,9 +6,11 @@ import { getErrorMessage } from './utils.js';
|
||||
import { getAgentOutputText } from './agent-output.js';
|
||||
import { createEvaluatedOutputHandler } from './agent-attempt.js';
|
||||
import {
|
||||
isRetryableClaudeSessionFailureAttempt,
|
||||
resolveAttemptRetryAction,
|
||||
} from './agent-attempt-retry.js';
|
||||
executeAttemptRetryAction,
|
||||
runClaudeAttemptWithRotation,
|
||||
runCodexAttemptWithRotation,
|
||||
} from './agent-attempt-orchestration.js';
|
||||
import { isRetryableClaudeSessionFailureAttempt } from './agent-attempt-retry.js';
|
||||
import {
|
||||
AgentOutput,
|
||||
runAgentProcess,
|
||||
@@ -37,10 +39,6 @@ import { resolveExecutionTarget } from './message-runtime-rules.js';
|
||||
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
||||
import { buildRoomRoleContext } from './room-role-context.js';
|
||||
import { type AgentTriggerReason } from './agent-error-detection.js';
|
||||
import {
|
||||
runClaudeRotationLoop,
|
||||
runCodexRotationLoop,
|
||||
} from './provider-retry.js';
|
||||
import {
|
||||
shouldResetSessionOnAgentFailure,
|
||||
shouldRetryFreshSessionOnAgentFailure,
|
||||
@@ -649,27 +647,17 @@ export async function runAgentForGroup(
|
||||
initialTrigger: { reason: CodexRotationReason },
|
||||
rotationMessage?: string,
|
||||
): Promise<'success' | 'error'> => {
|
||||
const outcome = await runCodexRotationLoop(
|
||||
return runCodexAttemptWithRotation({
|
||||
initialTrigger,
|
||||
async () => {
|
||||
const attempt = await runAttempt('codex');
|
||||
return {
|
||||
output: attempt.output,
|
||||
thrownError: attempt.error,
|
||||
sawOutput: attempt.sawOutput,
|
||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||
};
|
||||
},
|
||||
{
|
||||
runAttempt: () => runAttempt('codex'),
|
||||
logContext: {
|
||||
chatJid,
|
||||
group: group.name,
|
||||
groupFolder: group.folder,
|
||||
runId,
|
||||
},
|
||||
rotationMessage,
|
||||
);
|
||||
|
||||
return outcome.type === 'success' ? 'success' : 'error';
|
||||
});
|
||||
};
|
||||
|
||||
type AgentAttempt = Awaited<ReturnType<typeof runAttempt>>;
|
||||
@@ -688,81 +676,63 @@ export async function runAgentForGroup(
|
||||
runId,
|
||||
};
|
||||
|
||||
const outcome = await runClaudeRotationLoop(
|
||||
return runClaudeAttemptWithRotation({
|
||||
initialTrigger,
|
||||
async () => {
|
||||
const attempt = await runAttempt('claude');
|
||||
return {
|
||||
output: attempt.output,
|
||||
thrownError: attempt.error,
|
||||
sawOutput: attempt.sawOutput,
|
||||
sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput,
|
||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||
};
|
||||
},
|
||||
logCtx,
|
||||
runAttempt: () => runAttempt('claude'),
|
||||
logContext: logCtx,
|
||||
rotationMessage,
|
||||
);
|
||||
|
||||
switch (outcome.type) {
|
||||
case 'success':
|
||||
pairedSawOutput = outcome.sawOutput;
|
||||
return 'success';
|
||||
case 'error':
|
||||
return 'error';
|
||||
}
|
||||
onSuccess: ({ sawOutput }) => {
|
||||
pairedSawOutput = sawOutput;
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const retryClaudeAttemptIfNeeded = async (
|
||||
attempt: AgentAttempt,
|
||||
rotationMessage?: string | null,
|
||||
): Promise<'success' | 'error' | null> => {
|
||||
const retryAction = resolveAttemptRetryAction({
|
||||
const retryAction = await executeAttemptRetryAction({
|
||||
provider,
|
||||
canRetryClaudeCredentials,
|
||||
canRetryCodex: false,
|
||||
attempt,
|
||||
rotationMessage,
|
||||
runClaude: retryClaudeWithRotation,
|
||||
runCodex: retryCodexWithRotation,
|
||||
});
|
||||
if (retryAction.kind !== 'claude') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result = await retryClaudeWithRotation(
|
||||
retryAction.trigger,
|
||||
retryAction.rotationMessage,
|
||||
);
|
||||
if (result === 'error') {
|
||||
if (retryAction.result === 'error') {
|
||||
return maybeHandoffAfterError(retryAction.trigger.reason, attempt);
|
||||
}
|
||||
|
||||
pairedExecutionStatus = 'succeeded';
|
||||
return result;
|
||||
return retryAction.result;
|
||||
};
|
||||
|
||||
const retryCodexAttemptIfNeeded = async (
|
||||
attempt: AgentAttempt,
|
||||
rotationMessage?: string | null,
|
||||
): Promise<'success' | 'error' | null> => {
|
||||
const retryAction = resolveAttemptRetryAction({
|
||||
const retryAction = await executeAttemptRetryAction({
|
||||
provider,
|
||||
canRetryClaudeCredentials: false,
|
||||
canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1,
|
||||
attempt,
|
||||
rotationMessage,
|
||||
runClaude: retryClaudeWithRotation,
|
||||
runCodex: retryCodexWithRotation,
|
||||
});
|
||||
if (retryAction.kind !== 'codex') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result = await retryCodexWithRotation(
|
||||
retryAction.trigger,
|
||||
retryAction.rotationMessage,
|
||||
);
|
||||
if (result === 'success') {
|
||||
if (retryAction.result === 'success') {
|
||||
pairedExecutionStatus = 'succeeded';
|
||||
}
|
||||
return result;
|
||||
return retryAction.result;
|
||||
};
|
||||
|
||||
const maybeHandoffAfterError = (
|
||||
|
||||
@@ -68,25 +68,6 @@ export function isBotOnlyPairedRoomTurn(
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveLastDeliveredBotRole(
|
||||
messages: NewMessage[],
|
||||
): PairedRoomRole | null {
|
||||
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
||||
const message = messages[index];
|
||||
if (!message?.is_bot_message) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
message.sender_name === 'owner' ||
|
||||
message.sender_name === 'reviewer' ||
|
||||
message.sender_name === 'arbiter'
|
||||
) {
|
||||
return message.sender_name;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function buildPendingPairedTurn(args: {
|
||||
chatJid: string;
|
||||
timezone: string;
|
||||
@@ -98,7 +79,6 @@ export function buildPendingPairedTurn(args: {
|
||||
labeledRecentMessages: Parameters<
|
||||
typeof buildArbiterPromptForTask
|
||||
>[0]['labeledRecentMessages'];
|
||||
lastDeliveredMessages?: NewMessage[];
|
||||
resolveChannel: (taskStatus?: string | null) => Channel | null;
|
||||
}): PendingPairedTurn {
|
||||
const {
|
||||
@@ -117,10 +97,7 @@ export function buildPendingPairedTurn(args: {
|
||||
const lastTurnOutput = turnOutputs[turnOutputs.length - 1];
|
||||
const nextTurnAction = resolveNextTurnAction({
|
||||
taskStatus,
|
||||
lastTurnOutputRole:
|
||||
resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ??
|
||||
lastTurnOutput?.role ??
|
||||
null,
|
||||
lastTurnOutputRole: lastTurnOutput?.role ?? null,
|
||||
});
|
||||
const recentMessages = getRecentChatMessages(chatJid, 20);
|
||||
const lastHumanMessage = getLastHumanMessageContent(chatJid);
|
||||
@@ -255,7 +232,6 @@ export function resolveBotOnlyPairedFollowUpAction(args: {
|
||||
chatJid: string;
|
||||
task: PairedTask | null | undefined;
|
||||
isBotOnlyPairedFollowUp: boolean;
|
||||
lastDeliveredMessages?: NewMessage[];
|
||||
pendingCursorSource:
|
||||
| { seq?: number | null; timestamp?: string | null }
|
||||
| undefined;
|
||||
@@ -270,10 +246,7 @@ export function resolveBotOnlyPairedFollowUpAction(args: {
|
||||
const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1);
|
||||
const nextTurnAction = resolveNextTurnAction({
|
||||
taskStatus: task.status,
|
||||
lastTurnOutputRole:
|
||||
resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ??
|
||||
lastTurnOutput?.role ??
|
||||
null,
|
||||
lastTurnOutputRole: lastTurnOutput?.role ?? null,
|
||||
});
|
||||
|
||||
if (nextTurnAction.kind === 'finalize-owner-turn') {
|
||||
@@ -440,7 +413,6 @@ export function buildQueuedTurnDispatch(args: {
|
||||
chatJid: args.chatJid,
|
||||
task: args.loopPendingTask,
|
||||
isBotOnlyPairedFollowUp,
|
||||
lastDeliveredMessages: args.labeledMessagesToSend,
|
||||
pendingCursorSource,
|
||||
});
|
||||
|
||||
|
||||
@@ -1827,124 +1827,6 @@ describe('createMessageRuntime', () => {
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('requeues owner follow-ups from reviewer bot-only messages even when paired turn output storage is stale', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('codex');
|
||||
const ownerChannel = makeChannel(chatJid);
|
||||
const reviewerChannel = {
|
||||
...makeChannel(chatJid, 'discord-review', false),
|
||||
isOwnMessage: vi.fn((message) => message.sender === 'reviewer-bot@test'),
|
||||
};
|
||||
const enqueueMessageCheck = vi.fn();
|
||||
const closeStdin = vi.fn();
|
||||
const sendMessage = vi.fn(() => false);
|
||||
const setLastTimestamp = vi.fn();
|
||||
const saveState = vi.fn();
|
||||
const lastAgentTimestamps: Record<string, string> = {};
|
||||
const stopLoop = new Error('stop-message-loop');
|
||||
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({
|
||||
id: 'task-active-owner-follow-up-stale-output',
|
||||
chat_jid: chatJid,
|
||||
group_folder: group.folder,
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: '2026-03-30T00:00:00.000Z',
|
||||
round_trip_count: 1,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-30T00:00:00.000Z',
|
||||
updated_at: '2026-03-30T00:00:00.000Z',
|
||||
});
|
||||
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
||||
{
|
||||
id: 1,
|
||||
task_id: 'task-active-owner-follow-up-stale-output',
|
||||
turn_number: 1,
|
||||
role: 'owner',
|
||||
output_text: 'owner 초안',
|
||||
created_at: '2026-03-30T00:00:01.000Z',
|
||||
},
|
||||
]);
|
||||
vi.mocked(db.getNewMessages).mockReturnValue({
|
||||
messages: [
|
||||
{
|
||||
id: 'reviewer-bot-message-owner-follow-up-stale-output',
|
||||
chat_jid: chatJid,
|
||||
sender: 'reviewer-bot@test',
|
||||
sender_name: '리뷰어',
|
||||
content: 'DONE_WITH_CONCERNS\n\nreviewer direct message',
|
||||
timestamp: '2026-03-30T00:00:04.000Z',
|
||||
seq: 42,
|
||||
is_bot_message: true,
|
||||
} as any,
|
||||
],
|
||||
newTimestamp: '42',
|
||||
});
|
||||
|
||||
const runtime = createMessageRuntime({
|
||||
assistantName: 'Andy',
|
||||
idleTimeout: 60_000,
|
||||
pollInterval: 123,
|
||||
timezone: 'UTC',
|
||||
triggerPattern: /^@Andy\b/i,
|
||||
channels: [ownerChannel, reviewerChannel],
|
||||
queue: {
|
||||
registerProcess: vi.fn(),
|
||||
closeStdin,
|
||||
enqueueMessageCheck,
|
||||
notifyIdle: vi.fn(),
|
||||
sendMessage,
|
||||
} as any,
|
||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||
getSessions: () => ({}),
|
||||
getLastTimestamp: () => '',
|
||||
setLastTimestamp,
|
||||
getLastAgentTimestamps: () => lastAgentTimestamps,
|
||||
saveState,
|
||||
persistSession: vi.fn(),
|
||||
clearSession: vi.fn(),
|
||||
});
|
||||
|
||||
const originalSetTimeout = global.setTimeout;
|
||||
const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation(((
|
||||
handler: any,
|
||||
timeout?: number,
|
||||
...args: any[]
|
||||
) => {
|
||||
if (timeout === 123) {
|
||||
throw stopLoop;
|
||||
}
|
||||
return (originalSetTimeout as any)(handler, timeout, ...args);
|
||||
}) as typeof setTimeout);
|
||||
|
||||
try {
|
||||
await expect(runtime.startMessageLoop()).rejects.toThrow(
|
||||
stopLoop.message,
|
||||
);
|
||||
} finally {
|
||||
setTimeoutSpy.mockRestore();
|
||||
}
|
||||
|
||||
expect(setLastTimestamp).toHaveBeenCalledWith('42');
|
||||
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
|
||||
expect(closeStdin).toHaveBeenCalledWith(
|
||||
chatJid,
|
||||
expect.objectContaining({ reason: 'paired-pending-turn-follow-up' }),
|
||||
);
|
||||
expect(enqueueMessageCheck).toHaveBeenCalledWith(
|
||||
chatJid,
|
||||
resolveGroupIpcPath(group.folder),
|
||||
);
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('codex');
|
||||
|
||||
@@ -851,10 +851,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
chatJid,
|
||||
getRecentChatMessages(chatJid, 20),
|
||||
),
|
||||
lastDeliveredMessages: labelPairedSenders(
|
||||
chatJid,
|
||||
missedMessages,
|
||||
),
|
||||
resolveChannel,
|
||||
})
|
||||
: null;
|
||||
@@ -1072,6 +1068,191 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
}
|
||||
};
|
||||
|
||||
const processQueuedGroupDispatch = async (args: {
|
||||
chatJid: string;
|
||||
group: RegisteredGroup;
|
||||
channel: Channel;
|
||||
processableGroupMessages: NewMessage[];
|
||||
}): Promise<void> => {
|
||||
const { chatJid, group, channel, processableGroupMessages } = args;
|
||||
const loopPendingTask = hasReviewerLease(chatJid)
|
||||
? getLatestOpenPairedTaskForChat(chatJid)
|
||||
: null;
|
||||
const loopCursorKey = resolveCursorKey(chatJid, loopPendingTask?.status);
|
||||
const rawPendingMessages = getMessagesSinceSeq(
|
||||
chatJid,
|
||||
deps.getLastAgentTimestamps()[loopCursorKey] || '0',
|
||||
deps.assistantName,
|
||||
);
|
||||
const pendingMessages = filterLoopingPairedBotMessages(
|
||||
chatJid,
|
||||
getProcessableMessages(chatJid, rawPendingMessages, channel),
|
||||
FAILURE_FINAL_TEXT,
|
||||
);
|
||||
const messagesToSend =
|
||||
pendingMessages.length > 0 ? pendingMessages : processableGroupMessages;
|
||||
const labeledMessagesToSend = labelPairedSenders(chatJid, messagesToSend);
|
||||
const {
|
||||
formatted,
|
||||
botOnlyFollowUpAction,
|
||||
isBotOnlyPairedFollowUp,
|
||||
loopCursorKey: dispatchCursorKey,
|
||||
endSeq,
|
||||
} = buildQueuedTurnDispatch({
|
||||
chatJid,
|
||||
timezone: deps.timezone,
|
||||
loopPendingTask,
|
||||
rawPendingMessages,
|
||||
messagesToSend,
|
||||
labeledMessagesToSend,
|
||||
formatMessages,
|
||||
});
|
||||
|
||||
if (
|
||||
await executeBotOnlyPairedFollowUpAction({
|
||||
action: botOnlyFollowUpAction,
|
||||
chatJid,
|
||||
group,
|
||||
runId: `loop-merge-ready-${Date.now().toString(36)}`,
|
||||
channel,
|
||||
log: logger,
|
||||
saveState: deps.saveState,
|
||||
lastAgentTimestamps: deps.getLastAgentTimestamps(),
|
||||
executeTurn,
|
||||
enqueueGroupMessageCheck: () =>
|
||||
enqueueScopedGroupMessageCheck(chatJid, group.folder),
|
||||
closeStdin: () =>
|
||||
deps.queue.closeStdin(chatJid, {
|
||||
reason: 'paired-pending-turn-follow-up',
|
||||
}),
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (deps.queue.sendMessage(chatJid, formatted)) {
|
||||
if (endSeq != null) {
|
||||
advanceLastAgentCursor(
|
||||
deps.getLastAgentTimestamps(),
|
||||
deps.saveState,
|
||||
chatJid,
|
||||
endSeq,
|
||||
dispatchCursorKey,
|
||||
);
|
||||
}
|
||||
logger.debug(
|
||||
{
|
||||
transition: 'typing:on',
|
||||
source: 'follow-up-queued',
|
||||
chatJid,
|
||||
group: group.name,
|
||||
groupFolder: group.folder,
|
||||
endSeq: endSeq ?? null,
|
||||
suppressed: isBotOnlyPairedFollowUp,
|
||||
},
|
||||
'Typing indicator transition',
|
||||
);
|
||||
if (!isBotOnlyPairedFollowUp) {
|
||||
await channel
|
||||
.setTyping?.(chatJid, true)
|
||||
?.catch((err) =>
|
||||
logger.warn(
|
||||
{ chatJid, err },
|
||||
'Failed to set typing indicator',
|
||||
),
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
enqueueScopedGroupMessageCheck(chatJid, group.folder);
|
||||
};
|
||||
|
||||
const processLoopGroupMessages = async (args: {
|
||||
chatJid: string;
|
||||
group: RegisteredGroup;
|
||||
groupMessages: NewMessage[];
|
||||
channel: Channel;
|
||||
}): Promise<void> => {
|
||||
const { chatJid, group, groupMessages, channel } = args;
|
||||
const isMainGroup = group.isMain === true;
|
||||
const processableGroupMessages = getProcessableMessages(
|
||||
chatJid,
|
||||
groupMessages,
|
||||
channel,
|
||||
);
|
||||
|
||||
if (processableGroupMessages.length === 0) {
|
||||
const lastIgnored = groupMessages[groupMessages.length - 1];
|
||||
if (lastIgnored?.seq != null) {
|
||||
advanceLastAgentCursor(
|
||||
deps.getLastAgentTimestamps(),
|
||||
deps.saveState,
|
||||
chatJid,
|
||||
lastIgnored.seq,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)) {
|
||||
const lastIgnored =
|
||||
processableGroupMessages[processableGroupMessages.length - 1];
|
||||
if (lastIgnored?.seq != null) {
|
||||
advanceLastAgentCursor(
|
||||
deps.getLastAgentTimestamps(),
|
||||
deps.saveState,
|
||||
chatJid,
|
||||
lastIgnored.seq,
|
||||
);
|
||||
}
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, groupFolder: group.folder },
|
||||
'Bot-collaboration timeout: no recent human message, skipping',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const loopCmdMsg = groupMessages.find(
|
||||
(msg) => extractSessionCommand(msg.content, deps.triggerPattern) !== null,
|
||||
);
|
||||
|
||||
if (loopCmdMsg) {
|
||||
if (
|
||||
isSessionCommandAllowed(
|
||||
isMainGroup,
|
||||
loopCmdMsg.is_from_me === true,
|
||||
isSessionCommandSenderAllowed(loopCmdMsg.sender),
|
||||
)
|
||||
) {
|
||||
deps.queue.closeStdin(chatJid, {
|
||||
reason: 'session-command-detected',
|
||||
});
|
||||
}
|
||||
deps.queue.enqueueMessageCheck(chatJid);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
!hasAllowedTrigger({
|
||||
chatJid,
|
||||
messages: processableGroupMessages,
|
||||
group,
|
||||
triggerPattern: deps.triggerPattern,
|
||||
hasImplicitContinuationWindow: continuationTracker.has,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
await processQueuedGroupDispatch({
|
||||
chatJid,
|
||||
group,
|
||||
channel,
|
||||
processableGroupMessages,
|
||||
});
|
||||
};
|
||||
|
||||
const startMessageLoop = async (): Promise<void> => {
|
||||
if (messageLoopRunning) {
|
||||
logger.debug('Message loop already running, skipping duplicate start');
|
||||
@@ -1120,182 +1301,12 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const isMainGroup = group.isMain === true;
|
||||
const processableGroupMessages = getProcessableMessages(
|
||||
await processLoopGroupMessages({
|
||||
chatJid,
|
||||
group,
|
||||
groupMessages,
|
||||
channel,
|
||||
);
|
||||
|
||||
if (processableGroupMessages.length === 0) {
|
||||
const lastIgnored = groupMessages[groupMessages.length - 1];
|
||||
if (lastIgnored?.seq != null) {
|
||||
advanceLastAgentCursor(
|
||||
deps.getLastAgentTimestamps(),
|
||||
deps.saveState,
|
||||
chatJid,
|
||||
lastIgnored.seq,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)
|
||||
) {
|
||||
const lastIgnored =
|
||||
processableGroupMessages[processableGroupMessages.length - 1];
|
||||
if (lastIgnored?.seq != null) {
|
||||
advanceLastAgentCursor(
|
||||
deps.getLastAgentTimestamps(),
|
||||
deps.saveState,
|
||||
chatJid,
|
||||
lastIgnored.seq,
|
||||
);
|
||||
}
|
||||
logger.info(
|
||||
{ chatJid, group: group.name, groupFolder: group.folder },
|
||||
'Bot-collaboration timeout: no recent human message, skipping',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const loopCmdMsg = groupMessages.find(
|
||||
(msg) =>
|
||||
extractSessionCommand(msg.content, deps.triggerPattern) !==
|
||||
null,
|
||||
);
|
||||
|
||||
if (loopCmdMsg) {
|
||||
if (
|
||||
isSessionCommandAllowed(
|
||||
isMainGroup,
|
||||
loopCmdMsg.is_from_me === true,
|
||||
isSessionCommandSenderAllowed(loopCmdMsg.sender),
|
||||
)
|
||||
) {
|
||||
deps.queue.closeStdin(chatJid, {
|
||||
reason: 'session-command-detected',
|
||||
});
|
||||
}
|
||||
deps.queue.enqueueMessageCheck(chatJid);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
!hasAllowedTrigger({
|
||||
chatJid,
|
||||
messages: processableGroupMessages,
|
||||
group,
|
||||
triggerPattern: deps.triggerPattern,
|
||||
hasImplicitContinuationWindow: continuationTracker.has,
|
||||
})
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use role-aware cursor for paired rooms so the reviewer
|
||||
// always sees the owner's last messages and vice versa.
|
||||
const loopPendingTask = hasReviewerLease(chatJid)
|
||||
? getLatestOpenPairedTaskForChat(chatJid)
|
||||
: null;
|
||||
const loopCursorKey = resolveCursorKey(
|
||||
chatJid,
|
||||
loopPendingTask?.status,
|
||||
);
|
||||
|
||||
const rawPendingMessages = getMessagesSinceSeq(
|
||||
chatJid,
|
||||
deps.getLastAgentTimestamps()[loopCursorKey] || '0',
|
||||
deps.assistantName,
|
||||
);
|
||||
const pendingMessages = filterLoopingPairedBotMessages(
|
||||
chatJid,
|
||||
getProcessableMessages(chatJid, rawPendingMessages, channel),
|
||||
FAILURE_FINAL_TEXT,
|
||||
);
|
||||
const messagesToSend =
|
||||
pendingMessages.length > 0
|
||||
? pendingMessages
|
||||
: processableGroupMessages;
|
||||
const labeledMessagesToSend = labelPairedSenders(
|
||||
chatJid,
|
||||
messagesToSend,
|
||||
);
|
||||
const {
|
||||
formatted,
|
||||
botOnlyFollowUpAction,
|
||||
isBotOnlyPairedFollowUp,
|
||||
loopCursorKey: dispatchCursorKey,
|
||||
endSeq,
|
||||
} = buildQueuedTurnDispatch({
|
||||
chatJid,
|
||||
timezone: deps.timezone,
|
||||
loopPendingTask,
|
||||
rawPendingMessages,
|
||||
messagesToSend,
|
||||
labeledMessagesToSend,
|
||||
formatMessages,
|
||||
});
|
||||
if (
|
||||
await executeBotOnlyPairedFollowUpAction({
|
||||
action: botOnlyFollowUpAction,
|
||||
chatJid,
|
||||
group,
|
||||
runId: `loop-merge-ready-${Date.now().toString(36)}`,
|
||||
channel,
|
||||
log: logger,
|
||||
saveState: deps.saveState,
|
||||
lastAgentTimestamps: deps.getLastAgentTimestamps(),
|
||||
executeTurn,
|
||||
enqueueGroupMessageCheck: () =>
|
||||
enqueueScopedGroupMessageCheck(chatJid, group.folder),
|
||||
closeStdin: () =>
|
||||
deps.queue.closeStdin(chatJid, {
|
||||
reason: 'paired-pending-turn-follow-up',
|
||||
}),
|
||||
})
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (deps.queue.sendMessage(chatJid, formatted)) {
|
||||
if (endSeq != null) {
|
||||
advanceLastAgentCursor(
|
||||
deps.getLastAgentTimestamps(),
|
||||
deps.saveState,
|
||||
chatJid,
|
||||
endSeq,
|
||||
dispatchCursorKey,
|
||||
);
|
||||
}
|
||||
logger.debug(
|
||||
{
|
||||
transition: 'typing:on',
|
||||
source: 'follow-up-queued',
|
||||
chatJid,
|
||||
group: group.name,
|
||||
groupFolder: group.folder,
|
||||
endSeq: endSeq ?? null,
|
||||
suppressed: isBotOnlyPairedFollowUp,
|
||||
},
|
||||
'Typing indicator transition',
|
||||
);
|
||||
if (!isBotOnlyPairedFollowUp) {
|
||||
await channel
|
||||
.setTyping?.(chatJid, true)
|
||||
?.catch((err) =>
|
||||
logger.warn(
|
||||
{ chatJid, err },
|
||||
'Failed to set typing indicator',
|
||||
),
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
enqueueScopedGroupMessageCheck(chatJid, group.folder);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
@@ -5,11 +5,11 @@ import {
|
||||
import {
|
||||
getPairedWorkspace,
|
||||
hasActiveCiWatcherForChat,
|
||||
updatePairedTask,
|
||||
} from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
||||
import {
|
||||
applyPairedTaskPatch,
|
||||
classifyVerdict,
|
||||
hasCodeChangesSinceRef,
|
||||
requestArbiterOrEscalate,
|
||||
@@ -207,10 +207,12 @@ export function handleOwnerCompletion(args: {
|
||||
|
||||
const result = markPairedTaskReviewReady(taskId);
|
||||
if (result) {
|
||||
updatePairedTask(taskId, {
|
||||
round_trip_count: task.round_trip_count + 1,
|
||||
review_requested_at: now,
|
||||
updated_at: now,
|
||||
applyPairedTaskPatch({
|
||||
taskId,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
round_trip_count: task.round_trip_count + 1,
|
||||
},
|
||||
});
|
||||
logger.info(
|
||||
{ taskId, roundTrip: task.round_trip_count + 1 },
|
||||
|
||||
@@ -155,6 +155,17 @@ export function transitionPairedTaskStatus(args: {
|
||||
});
|
||||
}
|
||||
|
||||
export function applyPairedTaskPatch(args: {
|
||||
taskId: string;
|
||||
updatedAt: string;
|
||||
patch: Omit<Parameters<typeof updatePairedTask>[1], 'updated_at'>;
|
||||
}): void {
|
||||
updatePairedTask(args.taskId, {
|
||||
...args.patch,
|
||||
updated_at: args.updatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
export function requestArbiterOrEscalate(args: {
|
||||
taskId: string;
|
||||
currentStatus: PairedTaskStatus;
|
||||
|
||||
@@ -565,7 +565,6 @@ describe('paired execution context', () => {
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
round_trip_count: 2,
|
||||
review_requested_at: expect.any(String),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
getPairedTaskById,
|
||||
getPairedWorkspace,
|
||||
hasActiveCiWatcherForChat,
|
||||
updatePairedTask,
|
||||
upsertPairedProject,
|
||||
} from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
@@ -36,6 +35,7 @@ import {
|
||||
handleReviewerCompletion,
|
||||
} from './paired-execution-context-reviewer.js';
|
||||
import {
|
||||
applyPairedTaskPatch,
|
||||
resolveCanonicalSourceRef,
|
||||
requestArbiterOrEscalate,
|
||||
transitionPairedTaskStatus,
|
||||
@@ -238,9 +238,12 @@ export function preparePairedExecutionContext(args: {
|
||||
},
|
||||
});
|
||||
} else {
|
||||
updatePairedTask(latestTask.id, {
|
||||
...(hasHuman ? { round_trip_count: 0 } : {}),
|
||||
updated_at: now,
|
||||
applyPairedTaskPatch({
|
||||
taskId: latestTask.id,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
...(hasHuman ? { round_trip_count: 0 } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -253,9 +256,12 @@ export function preparePairedExecutionContext(args: {
|
||||
if (workspace?.workspace_dir && latestTask.status === 'active') {
|
||||
const wsRef = resolveCanonicalSourceRef(workspace.workspace_dir);
|
||||
if (wsRef !== latestTask.source_ref) {
|
||||
updatePairedTask(latestTask.id, {
|
||||
source_ref: wsRef,
|
||||
updated_at: now,
|
||||
applyPairedTaskPatch({
|
||||
taskId: latestTask.id,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
source_ref: wsRef,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,6 +258,31 @@ describe('paired workspace manager', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves review_requested_at untouched when review handoff aborts before an owner workspace exists', async () => {
|
||||
const { db, manager } = await loadModules();
|
||||
db._initTestDatabase();
|
||||
|
||||
const canonicalDir = path.join(tempRoot, 'canonical');
|
||||
initCanonicalRepo(canonicalDir);
|
||||
seedPairedTask(db, canonicalDir, {
|
||||
taskId: 'paired-task-no-owner-workspace',
|
||||
groupFolder: 'no-owner-workspace-room',
|
||||
});
|
||||
|
||||
const result = manager.markPairedTaskReviewReady(
|
||||
'paired-task-no-owner-workspace',
|
||||
);
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(
|
||||
db.getPairedTaskById('paired-task-no-owner-workspace')
|
||||
?.review_requested_at,
|
||||
).toBeNull();
|
||||
expect(
|
||||
db.getPairedTaskById('paired-task-no-owner-workspace')?.status,
|
||||
).toBe('active');
|
||||
});
|
||||
|
||||
it('uses the shared DB owner workspace across service-local data dirs', async () => {
|
||||
const canonicalDir = path.join(tempRoot, 'canonical');
|
||||
fs.mkdirSync(canonicalDir, { recursive: true });
|
||||
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
getPairedProject,
|
||||
getPairedTaskById,
|
||||
getPairedWorkspace,
|
||||
updatePairedTask,
|
||||
upsertPairedWorkspace,
|
||||
} from './db.js';
|
||||
import { resolvePairedTaskWorkspacePath } from './group-folder.js';
|
||||
@@ -758,10 +757,6 @@ export function markPairedTaskReviewReady(taskId: string): {
|
||||
reviewerWorkspace: PairedWorkspace;
|
||||
} | null {
|
||||
const requestedAt = new Date().toISOString();
|
||||
updatePairedTask(taskId, {
|
||||
review_requested_at: requestedAt,
|
||||
updated_at: requestedAt,
|
||||
});
|
||||
|
||||
const ownerWorkspace = getPairedWorkspace(taskId, 'owner');
|
||||
if (!ownerWorkspace) {
|
||||
@@ -798,7 +793,6 @@ export function markPairedTaskReviewReady(taskId: string): {
|
||||
'Reviewer will mount owner workspace directly',
|
||||
);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const task = getPairedTaskById(taskId);
|
||||
if (!task) {
|
||||
return null;
|
||||
@@ -807,7 +801,10 @@ export function markPairedTaskReviewReady(taskId: string): {
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'review_ready',
|
||||
updatedAt: now,
|
||||
updatedAt: requestedAt,
|
||||
patch: {
|
||||
review_requested_at: requestedAt,
|
||||
},
|
||||
});
|
||||
|
||||
return { ownerWorkspace, reviewerWorkspace };
|
||||
|
||||
@@ -3,7 +3,11 @@ import { CronExpressionParser } from 'cron-parser';
|
||||
import fs from 'fs';
|
||||
import { getAgentOutputText } from './agent-output.js';
|
||||
import { createEvaluatedOutputHandler } from './agent-attempt.js';
|
||||
import { resolveAttemptRetryAction } from './agent-attempt-retry.js';
|
||||
import {
|
||||
executeAttemptRetryAction,
|
||||
runClaudeAttemptWithRotation,
|
||||
runCodexAttemptWithRotation,
|
||||
} from './agent-attempt-orchestration.js';
|
||||
import { getErrorMessage } from './utils.js';
|
||||
|
||||
import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js';
|
||||
@@ -29,10 +33,6 @@ import {
|
||||
} from './group-folder.js';
|
||||
import { createScopedLogger, logger } from './logger.js';
|
||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||
import {
|
||||
runClaudeRotationLoop,
|
||||
runCodexRotationLoop,
|
||||
} from './provider-retry.js';
|
||||
import {
|
||||
detectCodexRotationTrigger,
|
||||
rotateCodexToken,
|
||||
@@ -447,69 +447,53 @@ async function runTask(
|
||||
retryAfterMs?: number;
|
||||
},
|
||||
rotationMessage?: string,
|
||||
): Promise<void> => {
|
||||
const logCtx = {
|
||||
): Promise<'success' | 'error'> => {
|
||||
const logContext = {
|
||||
taskId: task.id,
|
||||
group: context.group.name,
|
||||
groupFolder: task.group_folder,
|
||||
};
|
||||
|
||||
const outcome = await runClaudeRotationLoop(
|
||||
const outcome = await runClaudeAttemptWithRotation({
|
||||
initialTrigger,
|
||||
async () => {
|
||||
const attempt = await runTaskAttempt('claude');
|
||||
runAttempt: () => runTaskAttempt('claude'),
|
||||
logContext,
|
||||
rotationMessage,
|
||||
afterAttempt: (attempt) => {
|
||||
result = attempt.attemptResult;
|
||||
error = attempt.attemptError;
|
||||
return {
|
||||
output: attempt.output,
|
||||
sawOutput: attempt.sawOutput,
|
||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||
};
|
||||
},
|
||||
logCtx,
|
||||
rotationMessage,
|
||||
);
|
||||
});
|
||||
|
||||
switch (outcome.type) {
|
||||
case 'success':
|
||||
error = null;
|
||||
return;
|
||||
case 'error':
|
||||
if (outcome.trigger) {
|
||||
error = `Claude ${outcome.trigger.reason}`;
|
||||
}
|
||||
return;
|
||||
if (outcome === 'success') {
|
||||
error = null;
|
||||
}
|
||||
return outcome;
|
||||
};
|
||||
|
||||
const retryCodexTaskWithRotation = async (
|
||||
initialTrigger: { reason: CodexRotationReason },
|
||||
rotationMessage?: string,
|
||||
): Promise<void> => {
|
||||
const outcome = await runCodexRotationLoop(
|
||||
): Promise<'success' | 'error'> => {
|
||||
const outcome = await runCodexAttemptWithRotation({
|
||||
initialTrigger,
|
||||
async () => {
|
||||
const retryAttempt = await runTaskAttempt('codex');
|
||||
result = retryAttempt.attemptResult;
|
||||
error = retryAttempt.attemptError;
|
||||
return {
|
||||
output: retryAttempt.output,
|
||||
thrownError: null,
|
||||
sawOutput: retryAttempt.sawOutput,
|
||||
streamedTriggerReason: retryAttempt.streamedTriggerReason,
|
||||
};
|
||||
},
|
||||
{
|
||||
runAttempt: () => runTaskAttempt('codex'),
|
||||
logContext: {
|
||||
taskId: task.id,
|
||||
group: context.group.name,
|
||||
groupFolder: task.group_folder,
|
||||
},
|
||||
rotationMessage,
|
||||
);
|
||||
afterAttempt: (attempt) => {
|
||||
result = attempt.attemptResult;
|
||||
error = attempt.attemptError;
|
||||
},
|
||||
});
|
||||
|
||||
if (outcome.type === 'success') {
|
||||
if (outcome === 'success') {
|
||||
error = null;
|
||||
}
|
||||
return outcome;
|
||||
};
|
||||
|
||||
const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude';
|
||||
@@ -519,25 +503,17 @@ async function runTask(
|
||||
result = attempt.attemptResult;
|
||||
error = attempt.attemptError;
|
||||
|
||||
const retryAction = resolveAttemptRetryAction({
|
||||
const retryAction = await executeAttemptRetryAction({
|
||||
provider,
|
||||
canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0,
|
||||
canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1,
|
||||
attempt,
|
||||
rotationMessage: error,
|
||||
runClaude: retryClaudeTaskWithRotation,
|
||||
runCodex: retryCodexTaskWithRotation,
|
||||
});
|
||||
|
||||
if (retryAction.kind === 'claude') {
|
||||
await retryClaudeTaskWithRotation(
|
||||
retryAction.trigger,
|
||||
retryAction.rotationMessage,
|
||||
);
|
||||
} else if (retryAction.kind === 'codex') {
|
||||
await retryCodexTaskWithRotation(
|
||||
retryAction.trigger,
|
||||
retryAction.rotationMessage,
|
||||
);
|
||||
} else if (attempt.output.status === 'error') {
|
||||
if (retryAction.kind === 'none' && attempt.output.status === 'error') {
|
||||
error = attempt.attemptError || 'Unknown error';
|
||||
}
|
||||
} // end else (non-exhausted path)
|
||||
|
||||
@@ -65,4 +65,44 @@ describe('token-rotation runtime reselection', () => {
|
||||
rateLimited: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('reloads Claude rotation state from disk written by another service', async () => {
|
||||
const mod = await import('./token-rotation.js');
|
||||
const utils = await import('./utils.js');
|
||||
const readJsonFile = vi.mocked(utils.readJsonFile);
|
||||
|
||||
readJsonFile.mockReturnValueOnce(null);
|
||||
mod.initTokenRotation();
|
||||
expect(mod.getCurrentToken()).toBe('token-1');
|
||||
|
||||
readJsonFile.mockReturnValueOnce({
|
||||
currentIndex: 1,
|
||||
rateLimits: [Date.now() + 60_000, null],
|
||||
});
|
||||
mod.reloadTokenRotationStateFromDisk();
|
||||
|
||||
expect(mod.getCurrentTokenIndex()).toBe(1);
|
||||
expect(mod.getCurrentToken()).toBe('token-2');
|
||||
});
|
||||
|
||||
it('warns when Claude rotation state cannot be persisted', async () => {
|
||||
const mod = await import('./token-rotation.js');
|
||||
const utils = await import('./utils.js');
|
||||
const { logger } = await import('./logger.js');
|
||||
|
||||
vi.mocked(utils.writeJsonFile).mockImplementation(() => {
|
||||
throw new Error('disk full');
|
||||
});
|
||||
|
||||
mod.initTokenRotation();
|
||||
expect(mod.rotateToken('rate limit')).toBe(true);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
stateFile: '/tmp/ejclaw-claude-rot-data/token-rotation-state.json',
|
||||
err: expect.any(Error),
|
||||
}),
|
||||
'Failed to persist Claude token rotation state',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -69,12 +69,15 @@ function saveState(): void {
|
||||
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
||||
};
|
||||
writeJsonFile(STATE_FILE, state);
|
||||
} catch {
|
||||
/* best effort */
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ stateFile: STATE_FILE, err },
|
||||
'Failed to persist Claude token rotation state',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function loadState(): void {
|
||||
function loadState(quiet = false): void {
|
||||
const state = readJsonFile<{
|
||||
currentIndex?: number;
|
||||
rateLimits?: (number | null)[];
|
||||
@@ -93,13 +96,27 @@ function loadState(): void {
|
||||
const until = state.rateLimits[i];
|
||||
if (typeof until === 'number' && until > now) {
|
||||
tokens[i].rateLimitedUntil = until;
|
||||
} else {
|
||||
tokens[i].rateLimitedUntil = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.info(
|
||||
{ currentIndex, tokenCount: tokens.length },
|
||||
'Token rotation state restored',
|
||||
);
|
||||
if (!quiet) {
|
||||
logger.info(
|
||||
{ currentIndex, tokenCount: tokens.length },
|
||||
'Token rotation state restored',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-read the on-disk rotation state (written by any service).
|
||||
* Call before dashboard renders so the renderer picks up rotations
|
||||
* performed by another Claude service process.
|
||||
*/
|
||||
export function reloadTokenRotationStateFromDisk(): void {
|
||||
if (tokens.length <= 1) return;
|
||||
loadState(true);
|
||||
}
|
||||
|
||||
function refreshRuntimeTokenSelection(): void {
|
||||
|
||||
Reference in New Issue
Block a user