Stabilize paired reviewer recovery and owner follow-up
This commit is contained in:
@@ -108,8 +108,7 @@ export async function runAgentProcess(
|
||||
// Reviewers always run inside a Docker container with read-only source
|
||||
// mount for kernel-level write protection. Docker is required.
|
||||
if (
|
||||
!unsafeHostPairedMode &&
|
||||
envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1' ||
|
||||
(!unsafeHostPairedMode && envOverrides?.EJCLAW_REVIEWER_RUNTIME === '1') ||
|
||||
(!unsafeHostPairedMode && envOverrides?.EJCLAW_ARBITER_RUNTIME === '1')
|
||||
) {
|
||||
const ownerWorkspaceDir =
|
||||
|
||||
@@ -120,7 +120,7 @@ export const AGENT_LANGUAGE = getEnv('AGENT_LANGUAGE') || '';
|
||||
|
||||
/** Number of consecutive owner↔reviewer round trips before arbiter is auto-requested. */
|
||||
export const ARBITER_DEADLOCK_THRESHOLD = parseInt(
|
||||
getEnv('ARBITER_DEADLOCK_THRESHOLD') || '3',
|
||||
getEnv('ARBITER_DEADLOCK_THRESHOLD') || '2',
|
||||
10,
|
||||
);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./agent-runner.js', () => ({
|
||||
runAgentProcess: vi.fn(),
|
||||
@@ -156,9 +156,14 @@ vi.mock('./session-recovery.js', () => ({
|
||||
vi.mock('./token-rotation.js', () => ({
|
||||
rotateToken: vi.fn(() => false),
|
||||
getTokenCount: vi.fn(() => 1),
|
||||
getCurrentTokenIndex: vi.fn(() => 0),
|
||||
markTokenHealthy: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./token-refresh.js', () => ({
|
||||
forceRefreshToken: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
detectCodexRotationTrigger: vi.fn((error?: string | null) => {
|
||||
const lower = (error || '').toLowerCase();
|
||||
@@ -198,6 +203,7 @@ import { runAgentForGroup } from './message-agent-executor.js';
|
||||
import * as pairedExecutionContext from './paired-execution-context.js';
|
||||
import * as sessionRecovery from './session-recovery.js';
|
||||
import * as serviceRouting from './service-routing.js';
|
||||
import * as tokenRefresh from './token-refresh.js';
|
||||
import * as tokenRotation from './token-rotation.js';
|
||||
import type { RegisteredGroup } from './types.js';
|
||||
|
||||
@@ -226,9 +232,13 @@ function makeDeps() {
|
||||
};
|
||||
}
|
||||
|
||||
const ORIGINAL_UNSAFE_HOST_PAIRED_MODE =
|
||||
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||
|
||||
describe('runAgentForGroup room memory', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
||||
async (_group, _input, _onProcess, onOutput) => {
|
||||
await onOutput?.({
|
||||
@@ -245,6 +255,15 @@ describe('runAgentForGroup room memory', () => {
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE === undefined) {
|
||||
delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||
} else {
|
||||
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE =
|
||||
ORIGINAL_UNSAFE_HOST_PAIRED_MODE;
|
||||
}
|
||||
});
|
||||
|
||||
it('injects a room memory briefing when starting a fresh session', async () => {
|
||||
const group = { ...makeGroup(), folder: 'test-group' };
|
||||
const deps = makeDeps();
|
||||
@@ -1031,6 +1050,108 @@ describe('runAgentForGroup room memory', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('starts reviewer Claude fresh in unsafe host mode instead of resuming a stored session', async () => {
|
||||
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1';
|
||||
const group = {
|
||||
...makeGroup(),
|
||||
folder: 'test-group',
|
||||
agentType: 'codex' as const,
|
||||
};
|
||||
const deps = {
|
||||
...makeDeps(),
|
||||
getSessions: () => ({ 'test-group:reviewer': 'reviewer-session' }),
|
||||
};
|
||||
|
||||
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||
chat_jid: 'group@test',
|
||||
owner_agent_type: 'codex',
|
||||
reviewer_agent_type: 'claude-code',
|
||||
arbiter_agent_type: null,
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
arbiter_service_id: null,
|
||||
activated_at: null,
|
||||
reason: null,
|
||||
explicit: false,
|
||||
});
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({
|
||||
id: 'paired-task-review',
|
||||
chat_jid: 'group@test',
|
||||
group_folder: 'test-group',
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
round_trip_count: 0,
|
||||
review_requested_at: '2026-03-31T00:00:00.000Z',
|
||||
status: 'review_ready',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-31T00:00:00.000Z',
|
||||
updated_at: '2026-03-31T00:00:00.000Z',
|
||||
});
|
||||
vi.mocked(
|
||||
pairedExecutionContext.preparePairedExecutionContext,
|
||||
).mockReturnValue({
|
||||
task: {
|
||||
id: 'paired-task-review',
|
||||
chat_jid: 'group@test',
|
||||
group_folder: 'test-group',
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
round_trip_count: 0,
|
||||
review_requested_at: '2026-03-31T00:00:00.000Z',
|
||||
status: 'review_ready',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-31T00:00:00.000Z',
|
||||
updated_at: '2026-03-31T00:00:00.000Z',
|
||||
},
|
||||
workspace: null,
|
||||
envOverrides: {
|
||||
EJCLAW_PAIRED_TASK_ID: 'paired-task-review',
|
||||
EJCLAW_PAIRED_ROLE: 'reviewer',
|
||||
EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1',
|
||||
CLAUDE_CONFIG_DIR: '/tmp/test-group-reviewer',
|
||||
},
|
||||
});
|
||||
vi.mocked(agentRunner.runAgentProcess).mockResolvedValue({
|
||||
status: 'success',
|
||||
result: 'review ok',
|
||||
newSessionId: 'review-session-new',
|
||||
});
|
||||
|
||||
await runAgentForGroup(deps, {
|
||||
group,
|
||||
prompt: 'please review',
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-review-plan-unsafe-host',
|
||||
});
|
||||
|
||||
expect(deps.clearSession).toHaveBeenCalledWith('test-group:reviewer');
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
folder: 'test-group',
|
||||
agentType: 'claude-code',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
sessionId: undefined,
|
||||
}),
|
||||
expect.any(Function),
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1',
|
||||
}),
|
||||
);
|
||||
expect(deps.persistSession).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not enqueue a second generic follow-up when reviewer approval already moved the task to merge_ready', async () => {
|
||||
const group = { ...makeGroup(), folder: 'test-group' };
|
||||
const deps = makeDeps();
|
||||
@@ -1123,7 +1244,9 @@ describe('runAgentForGroup Claude rotation', () => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(buildRoomMemoryBriefing).mockResolvedValue(undefined);
|
||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
||||
vi.mocked(tokenRotation.getCurrentTokenIndex).mockReturnValue(0);
|
||||
vi.mocked(tokenRotation.rotateToken).mockReturnValue(false);
|
||||
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null);
|
||||
});
|
||||
|
||||
it('rotates to another Claude account on usage exhaustion', async () => {
|
||||
@@ -1229,6 +1352,62 @@ describe('runAgentForGroup Claude rotation', () => {
|
||||
expect(outputs).toEqual(['새 Claude 토큰 응답입니다.']);
|
||||
});
|
||||
|
||||
it('force-refreshes the active Claude token before rotating on auth-expired', async () => {
|
||||
const outputs: string[] = [];
|
||||
|
||||
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValueOnce(
|
||||
'new-access-token',
|
||||
);
|
||||
|
||||
vi.mocked(agentRunner.runAgentProcess)
|
||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'intermediate',
|
||||
result:
|
||||
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
|
||||
});
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'final',
|
||||
result:
|
||||
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
|
||||
});
|
||||
return {
|
||||
status: 'success',
|
||||
result: null,
|
||||
};
|
||||
})
|
||||
.mockImplementationOnce(async (_group, _input, _onProcess, onOutput) => {
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'final',
|
||||
result: 'force refresh 뒤 Claude 응답입니다.',
|
||||
});
|
||||
return {
|
||||
status: 'success',
|
||||
result: null,
|
||||
};
|
||||
});
|
||||
|
||||
const result = await runAgentForGroup(makeDeps(), {
|
||||
group: makeGroup(),
|
||||
prompt: 'hello',
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-auth-expired-force-refresh',
|
||||
onOutput: async (output) => {
|
||||
if (typeof output.result === 'string') outputs.push(output.result);
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBe('success');
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||
expect(tokenRefresh.forceRefreshToken).toHaveBeenCalledWith(0);
|
||||
expect(tokenRotation.rotateToken).not.toHaveBeenCalled();
|
||||
expect(tokenRotation.markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||
expect(outputs).toEqual(['force refresh 뒤 Claude 응답입니다.']);
|
||||
});
|
||||
|
||||
it('suppresses Claude 502 HTML and returns error when no rotation is available', async () => {
|
||||
const outputs: string[] = [];
|
||||
|
||||
@@ -1505,6 +1684,48 @@ describe('runAgentForGroup Claude rotation', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('drops a stale Claude session id before retrying a fresh session', async () => {
|
||||
const deps = {
|
||||
...makeDeps(),
|
||||
getSessions: () => ({ 'test-claude': 'stale-session-id' }),
|
||||
};
|
||||
|
||||
vi.mocked(sessionRecovery.shouldRetryFreshSessionOnAgentFailure)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(false);
|
||||
|
||||
vi.mocked(agentRunner.runAgentProcess)
|
||||
.mockImplementationOnce(async (_group, input) => {
|
||||
expect(input.sessionId).toBe('stale-session-id');
|
||||
throw new Error('No conversation found with session ID: stale-session-id');
|
||||
})
|
||||
.mockImplementationOnce(async (_group, input, _onProcess, onOutput) => {
|
||||
expect(input.sessionId).toBeUndefined();
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'final',
|
||||
result: 'fresh retry success',
|
||||
});
|
||||
return {
|
||||
status: 'success',
|
||||
result: 'fresh retry success',
|
||||
newSessionId: 'fresh-session-id',
|
||||
};
|
||||
});
|
||||
|
||||
const result = await runAgentForGroup(deps, {
|
||||
group: makeGroup(),
|
||||
prompt: 'hello',
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-stale-session-id-retry',
|
||||
onOutput: async () => {},
|
||||
});
|
||||
|
||||
expect(result).toBe('success');
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(2);
|
||||
expect(deps.clearSession).toHaveBeenCalledWith('test-claude');
|
||||
});
|
||||
|
||||
it('suppresses a usage-exhausted banner even when Claude already emitted progress text', async () => {
|
||||
const outputs: string[] = [];
|
||||
|
||||
|
||||
@@ -150,12 +150,22 @@ export async function runAgentForGroup(
|
||||
activeRole,
|
||||
group.agentType,
|
||||
);
|
||||
const unsafeHostPairedMode =
|
||||
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE === '1';
|
||||
const forceFreshClaudeReviewerSession =
|
||||
reviewerMode && isClaudeCodeAgent && unsafeHostPairedMode;
|
||||
const storedSessionId = sessions[sessionFolder];
|
||||
if (forceFreshClaudeReviewerSession && storedSessionId) {
|
||||
deps.clearSession(sessionFolder);
|
||||
}
|
||||
// Arbiter always starts fresh — never resume a previous session
|
||||
const sessionId =
|
||||
activeRole === 'arbiter' || args.forcedAgentType
|
||||
let currentSessionId =
|
||||
activeRole === 'arbiter' ||
|
||||
args.forcedAgentType ||
|
||||
forceFreshClaudeReviewerSession
|
||||
? undefined
|
||||
: sessions[sessionFolder];
|
||||
const memoryBriefing = sessionId
|
||||
: storedSessionId;
|
||||
const memoryBriefing = currentSessionId
|
||||
? undefined
|
||||
: await buildRoomMemoryBriefing({
|
||||
groupFolder: group.folder,
|
||||
@@ -185,7 +195,9 @@ export async function runAgentForGroup(
|
||||
|
||||
let resetSessionRequested = false;
|
||||
|
||||
const canRotateToken = isClaudeCodeAgent && getTokenCount() > 1;
|
||||
const claudeTokenCount = isClaudeCodeAgent ? getTokenCount() : 0;
|
||||
const canRetryClaudeCredentials = isClaudeCodeAgent && claudeTokenCount > 0;
|
||||
const canRotateToken = isClaudeCodeAgent && claudeTokenCount > 1;
|
||||
const roomRoleContext = buildRoomRoleContext(
|
||||
currentLease,
|
||||
effectiveServiceId,
|
||||
@@ -212,6 +224,7 @@ export async function runAgentForGroup(
|
||||
pairedExecutionContext.envOverrides[effortKey] = roleConfig.effort;
|
||||
}
|
||||
}
|
||||
|
||||
const log = createScopedLogger({
|
||||
chatJid,
|
||||
groupName: group.name,
|
||||
@@ -243,11 +256,27 @@ export async function runAgentForGroup(
|
||||
reviewerMode,
|
||||
arbiterMode,
|
||||
sessionFolder,
|
||||
resumedSession: sessionId ?? null,
|
||||
resumedSession: currentSessionId ?? null,
|
||||
},
|
||||
'Resolved execution target for agent turn',
|
||||
);
|
||||
|
||||
const clearRoleSdkSessions = (): void => {
|
||||
const configDir = pairedExecutionContext?.envOverrides?.CLAUDE_CONFIG_DIR;
|
||||
if (!configDir) return;
|
||||
for (const sdkDir of ['.claude', '.codex']) {
|
||||
const sessionsDir = path.join(configDir, sdkDir, 'sessions');
|
||||
if (fs.existsSync(sessionsDir)) {
|
||||
fs.rmSync(sessionsDir, { recursive: true, force: true });
|
||||
log.info({ sessionsDir }, 'Cleared SDK sessions for fresh Claude turn');
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (forceFreshClaudeReviewerSession) {
|
||||
clearRoleSdkSessions();
|
||||
}
|
||||
|
||||
// ── MoA prompt enrichment ─────────────────────────────────────
|
||||
// When MoA is enabled and we're in arbiter mode, query external API
|
||||
// models (Kimi, GLM, etc.) in parallel for their opinions, then inject
|
||||
@@ -394,7 +423,7 @@ export async function runAgentForGroup(
|
||||
|
||||
const agentInput = {
|
||||
prompt: effectivePrompt,
|
||||
sessionId,
|
||||
sessionId: currentSessionId,
|
||||
memoryBriefing,
|
||||
groupFolder: group.folder,
|
||||
chatJid,
|
||||
@@ -451,6 +480,7 @@ export async function runAgentForGroup(
|
||||
sawVisibleOutput: false,
|
||||
sawSuccessNullResultWithoutOutput: false,
|
||||
};
|
||||
const attemptSessionId = provider === 'claude' ? currentSessionId : undefined;
|
||||
|
||||
const wrappedOnOutput = onOutput
|
||||
? async (output: AgentOutput) => {
|
||||
@@ -476,7 +506,7 @@ export async function runAgentForGroup(
|
||||
effectiveServiceId,
|
||||
effectiveAgentType,
|
||||
sessionFolder,
|
||||
resumedSession: sessionId ?? null,
|
||||
resumedSession: attemptSessionId ?? null,
|
||||
streamedSessionId: output.newSessionId ?? null,
|
||||
roomRoleServiceId: roomRoleContext?.serviceId ?? null,
|
||||
roomRole: roomRoleContext?.role ?? null,
|
||||
@@ -499,9 +529,11 @@ export async function runAgentForGroup(
|
||||
if (
|
||||
provider === 'claude' &&
|
||||
output.newSessionId &&
|
||||
!resetSessionRequested
|
||||
!resetSessionRequested &&
|
||||
!forceFreshClaudeReviewerSession
|
||||
) {
|
||||
deps.persistSession(sessionFolder, output.newSessionId);
|
||||
currentSessionId = output.newSessionId;
|
||||
}
|
||||
const evaluation = evaluateStreamedOutput(output, streamedState, {
|
||||
agentType: isClaudeCodeAgent ? 'claude-code' : 'codex',
|
||||
@@ -510,7 +542,7 @@ export async function runAgentForGroup(
|
||||
trackSuccessNullResult: true,
|
||||
shortCircuitTriggeredErrors:
|
||||
provider === 'claude'
|
||||
? canRotateToken
|
||||
? canRetryClaudeCredentials
|
||||
: getCodexAccountCount() > 1,
|
||||
});
|
||||
streamedState = evaluation.state;
|
||||
@@ -601,7 +633,7 @@ export async function runAgentForGroup(
|
||||
effectiveGroup,
|
||||
{
|
||||
...agentInput,
|
||||
sessionId: provider === 'claude' ? sessionId : undefined,
|
||||
sessionId: attemptSessionId,
|
||||
},
|
||||
(proc, processName, ipcDir) =>
|
||||
deps.queue.registerProcess(chatJid, proc, processName, ipcDir),
|
||||
@@ -609,8 +641,13 @@ export async function runAgentForGroup(
|
||||
pairedExecutionContext?.envOverrides,
|
||||
);
|
||||
|
||||
if (provider === 'claude' && output.newSessionId) {
|
||||
if (
|
||||
provider === 'claude' &&
|
||||
output.newSessionId &&
|
||||
!forceFreshClaudeReviewerSession
|
||||
) {
|
||||
deps.persistSession(sessionFolder, output.newSessionId);
|
||||
currentSessionId = output.newSessionId;
|
||||
}
|
||||
|
||||
providerLog.info(
|
||||
@@ -804,23 +841,9 @@ export async function runAgentForGroup(
|
||||
})));
|
||||
|
||||
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
|
||||
currentSessionId = undefined;
|
||||
deps.clearSession(sessionFolder);
|
||||
// Also clear SDK session state inside the reviewer/arbiter config dir
|
||||
// so the persistent container doesn't resume the same stale session.
|
||||
const configDir =
|
||||
pairedExecutionContext?.envOverrides?.CLAUDE_CONFIG_DIR;
|
||||
if (configDir) {
|
||||
for (const sdkDir of ['.claude', '.codex']) {
|
||||
const sessionsDir = path.join(configDir, sdkDir, 'sessions');
|
||||
if (fs.existsSync(sessionsDir)) {
|
||||
fs.rmSync(sessionsDir, { recursive: true, force: true });
|
||||
log.info(
|
||||
{ sessionsDir },
|
||||
'Cleared container SDK sessions for stale session recovery',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
clearRoleSdkSessions();
|
||||
log.warn(
|
||||
'Cleared poisoned Claude session before visible output, retrying fresh session',
|
||||
);
|
||||
@@ -828,6 +851,7 @@ export async function runAgentForGroup(
|
||||
primaryAttempt = await runAttempt('claude');
|
||||
|
||||
if (isRetryableClaudeSessionFailure(primaryAttempt)) {
|
||||
currentSessionId = undefined;
|
||||
deps.clearSession(sessionFolder);
|
||||
log.warn('Fresh Claude retry also hit a retryable session failure');
|
||||
|
||||
@@ -840,7 +864,7 @@ export async function runAgentForGroup(
|
||||
|
||||
if (primaryAttempt.error) {
|
||||
if (
|
||||
canRotateToken &&
|
||||
canRetryClaudeCredentials &&
|
||||
provider === 'claude' &&
|
||||
!primaryAttempt.sawOutput
|
||||
) {
|
||||
@@ -913,7 +937,7 @@ export async function runAgentForGroup(
|
||||
}
|
||||
|
||||
if (
|
||||
canRotateToken &&
|
||||
canRetryClaudeCredentials &&
|
||||
provider === 'claude' &&
|
||||
!primaryAttempt.sawOutput &&
|
||||
primaryAttempt.streamedTriggerReason &&
|
||||
@@ -945,7 +969,7 @@ export async function runAgentForGroup(
|
||||
|
||||
if (output.status === 'error') {
|
||||
if (
|
||||
canRotateToken &&
|
||||
canRetryClaudeCredentials &&
|
||||
provider === 'claude' &&
|
||||
!primaryAttempt.sawOutput
|
||||
) {
|
||||
|
||||
@@ -1704,6 +1704,230 @@ describe('createMessageRuntime', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('codex');
|
||||
const channel = makeChannel(chatJid);
|
||||
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({
|
||||
id: 'task-active-owner-follow-up',
|
||||
chat_jid: chatJid,
|
||||
group_folder: group.folder,
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: '2026-03-30T00:00:00.000Z',
|
||||
round_trip_count: 1,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-30T00:00:00.000Z',
|
||||
updated_at: '2026-03-30T00:00:00.000Z',
|
||||
});
|
||||
vi.mocked(db.getMessagesSince).mockReturnValue([]);
|
||||
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
||||
{
|
||||
id: 1,
|
||||
task_id: 'task-active-owner-follow-up',
|
||||
turn_number: 1,
|
||||
role: 'owner',
|
||||
output_text: 'owner 초안',
|
||||
created_at: '2026-03-30T00:00:01.000Z',
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
task_id: 'task-active-owner-follow-up',
|
||||
turn_number: 2,
|
||||
role: 'reviewer',
|
||||
output_text: '리뷰어가 수정 요청을 남김',
|
||||
created_at: '2026-03-30T00:00:02.000Z',
|
||||
},
|
||||
]);
|
||||
vi.mocked(db.getRecentChatMessages).mockReturnValue([
|
||||
{
|
||||
id: 'human-1',
|
||||
chat_jid: chatJid,
|
||||
sender: 'user@test',
|
||||
sender_name: '눈쟁이',
|
||||
content: '이 기능 마무리해줘',
|
||||
timestamp: '2026-03-30T00:00:00.500Z',
|
||||
seq: 1,
|
||||
is_bot_message: false,
|
||||
} as any,
|
||||
]);
|
||||
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
||||
async (_group, input, _onProcess, onOutput) => {
|
||||
expect(input.prompt).toContain('이 기능 마무리해줘');
|
||||
expect(input.prompt).toContain('리뷰어가 수정 요청을 남김');
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'final',
|
||||
result: 'owner가 reviewer 피드백을 반영했습니다.',
|
||||
newSessionId: 'session-owner-follow-up',
|
||||
});
|
||||
return {
|
||||
status: 'success',
|
||||
result: 'owner가 reviewer 피드백을 반영했습니다.',
|
||||
newSessionId: 'session-owner-follow-up',
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const runtime = createMessageRuntime({
|
||||
assistantName: 'Andy',
|
||||
idleTimeout: 1_000,
|
||||
pollInterval: 1_000,
|
||||
timezone: 'UTC',
|
||||
triggerPattern: /^@Andy\b/i,
|
||||
channels: [channel],
|
||||
queue: {
|
||||
registerProcess: vi.fn(),
|
||||
closeStdin: vi.fn(),
|
||||
notifyIdle: vi.fn(),
|
||||
} as any,
|
||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||
getSessions: () => ({}),
|
||||
getLastTimestamp: () => '',
|
||||
setLastTimestamp: vi.fn(),
|
||||
getLastAgentTimestamps: () => ({}),
|
||||
saveState: vi.fn(),
|
||||
persistSession: vi.fn(),
|
||||
clearSession: vi.fn(),
|
||||
});
|
||||
|
||||
const result = await runtime.processGroupMessages(chatJid, {
|
||||
runId: 'run-active-owner-follow-up',
|
||||
reason: 'messages',
|
||||
});
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
||||
expect(channel.sendMessage).toHaveBeenCalledWith(
|
||||
chatJid,
|
||||
'owner가 reviewer 피드백을 반영했습니다.',
|
||||
);
|
||||
});
|
||||
|
||||
it('builds owner follow-up prompts from paired turn outputs instead of raw reviewer bot delivery text', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('codex');
|
||||
const channel = makeChannel(chatJid);
|
||||
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({
|
||||
id: 'task-active-bot-follow-up',
|
||||
chat_jid: chatJid,
|
||||
group_folder: group.folder,
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: '2026-03-30T00:00:00.000Z',
|
||||
round_trip_count: 1,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-30T00:00:00.000Z',
|
||||
updated_at: '2026-03-30T00:00:00.000Z',
|
||||
});
|
||||
vi.mocked(db.getMessagesSince).mockReturnValue([
|
||||
{
|
||||
id: 'reviewer-bot-message-active',
|
||||
chat_jid: chatJid,
|
||||
sender: 'reviewer-bot@test',
|
||||
sender_name: '리뷰어',
|
||||
content: 'DONE_WITH_CONCERNS\n\n리뷰어 디스코드 출력 원문',
|
||||
timestamp: '2026-03-30T00:00:04.000Z',
|
||||
seq: 42,
|
||||
is_bot_message: true,
|
||||
} as any,
|
||||
]);
|
||||
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
||||
{
|
||||
id: 1,
|
||||
task_id: 'task-active-bot-follow-up',
|
||||
turn_number: 1,
|
||||
role: 'owner',
|
||||
output_text: 'owner 초안',
|
||||
created_at: '2026-03-30T00:00:01.000Z',
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
task_id: 'task-active-bot-follow-up',
|
||||
turn_number: 2,
|
||||
role: 'reviewer',
|
||||
output_text: 'paired_turn_outputs 에 저장된 reviewer 요약',
|
||||
created_at: '2026-03-30T00:00:02.000Z',
|
||||
},
|
||||
]);
|
||||
vi.mocked(db.getRecentChatMessages).mockReturnValue([
|
||||
{
|
||||
id: 'human-1',
|
||||
chat_jid: chatJid,
|
||||
sender: 'user@test',
|
||||
sender_name: '눈쟁이',
|
||||
content: '이 기능 마무리해줘',
|
||||
timestamp: '2026-03-30T00:00:00.500Z',
|
||||
seq: 1,
|
||||
is_bot_message: false,
|
||||
} as any,
|
||||
]);
|
||||
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
||||
async (_group, input, _onProcess, onOutput) => {
|
||||
expect(input.prompt).toContain('paired_turn_outputs 에 저장된 reviewer 요약');
|
||||
expect(input.prompt).not.toContain('리뷰어 디스코드 출력 원문');
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'final',
|
||||
result: 'owner follow-up ok',
|
||||
newSessionId: 'session-owner-bot-follow-up',
|
||||
});
|
||||
return {
|
||||
status: 'success',
|
||||
result: 'owner follow-up ok',
|
||||
newSessionId: 'session-owner-bot-follow-up',
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const runtime = createMessageRuntime({
|
||||
assistantName: 'Andy',
|
||||
idleTimeout: 1_000,
|
||||
pollInterval: 1_000,
|
||||
timezone: 'UTC',
|
||||
triggerPattern: /^@Andy\b/i,
|
||||
channels: [channel],
|
||||
queue: {
|
||||
registerProcess: vi.fn(),
|
||||
closeStdin: vi.fn(),
|
||||
notifyIdle: vi.fn(),
|
||||
} as any,
|
||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
||||
getSessions: () => ({}),
|
||||
getLastTimestamp: () => '',
|
||||
setLastTimestamp: vi.fn(),
|
||||
getLastAgentTimestamps: () => ({}),
|
||||
saveState: vi.fn(),
|
||||
persistSession: vi.fn(),
|
||||
clearSession: vi.fn(),
|
||||
});
|
||||
|
||||
const result = await runtime.processGroupMessages(chatJid, {
|
||||
runId: 'run-active-bot-follow-up',
|
||||
reason: 'messages',
|
||||
});
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(agentRunner.runAgentProcess).toHaveBeenCalledTimes(1);
|
||||
expect(channel.sendMessage).toHaveBeenCalledWith(chatJid, 'owner follow-up ok');
|
||||
});
|
||||
|
||||
it('reuses the shared arbiter prompt builder for pending arbiter turns', async () => {
|
||||
const chatJid = 'group@test';
|
||||
const group = makeGroup('codex');
|
||||
|
||||
@@ -270,6 +270,30 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
return `User request:\n---\n${userMessage}\n---\n\nReview the latest owner changes in the workspace.`;
|
||||
};
|
||||
|
||||
const buildOwnerPendingPrompt = (
|
||||
task: PairedTask,
|
||||
chatJid: string,
|
||||
timezone: string,
|
||||
): string => {
|
||||
const turnOutputs = getPairedTurnOutputs(task.id);
|
||||
if (turnOutputs.length > 0) {
|
||||
const humanMessages = getRecentChatMessages(chatJid, 20).filter(
|
||||
(message) => !message.is_bot_message,
|
||||
);
|
||||
return formatMessages(
|
||||
mergeHumanAndTurnOutputMessages(chatJid, humanMessages, turnOutputs),
|
||||
timezone,
|
||||
);
|
||||
}
|
||||
|
||||
const userMessage = getLastHumanMessageContent(chatJid);
|
||||
if (!userMessage) {
|
||||
return 'Continue the owner turn using the latest reviewer or arbiter feedback.';
|
||||
}
|
||||
|
||||
return `User request:\n---\n${userMessage}\n---\n\nContinue the owner turn using the latest reviewer or arbiter feedback.`;
|
||||
};
|
||||
|
||||
const buildArbiterPromptForTask = (
|
||||
task: PairedTask,
|
||||
chatJid: string,
|
||||
@@ -825,6 +849,22 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
||||
};
|
||||
}
|
||||
|
||||
if (taskStatus === 'active') {
|
||||
const turnOutputs = getPairedTurnOutputs(task.id);
|
||||
const lastTurnOutput = turnOutputs[turnOutputs.length - 1];
|
||||
if (
|
||||
lastTurnOutput &&
|
||||
(lastTurnOutput.role === 'reviewer' ||
|
||||
lastTurnOutput.role === 'arbiter')
|
||||
) {
|
||||
return {
|
||||
prompt: buildOwnerPendingPrompt(task, chatJid, deps.timezone),
|
||||
channel: resolveChannel(taskStatus),
|
||||
cursor,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./db.js', () => ({
|
||||
createPairedTask: vi.fn(),
|
||||
@@ -78,6 +78,12 @@ const failoverOwnerContext: RoomRoleContext = {
|
||||
failoverOwner: true,
|
||||
};
|
||||
|
||||
const ORIGINAL_UNSAFE_HOST_PAIRED_MODE =
|
||||
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||
const ORIGINAL_REVIEWER_RUNTIME = process.env.EJCLAW_REVIEWER_RUNTIME;
|
||||
const ORIGINAL_CLAUDE_REVIEWER_READONLY =
|
||||
process.env.EJCLAW_CLAUDE_REVIEWER_READONLY;
|
||||
|
||||
function createCanonicalRepoWithCommit(commitMessage: string): string {
|
||||
const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-finalize-'));
|
||||
execFileSync('git', ['init'], { cwd: repoDir, stdio: 'ignore' });
|
||||
@@ -150,6 +156,9 @@ function buildWorkspace(
|
||||
|
||||
describe('paired execution context', () => {
|
||||
beforeEach(() => {
|
||||
delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||
delete process.env.EJCLAW_REVIEWER_RUNTIME;
|
||||
delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY;
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue(undefined);
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(undefined);
|
||||
@@ -176,6 +185,28 @@ describe('paired execution context', () => {
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) {
|
||||
delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||
} else {
|
||||
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE =
|
||||
ORIGINAL_UNSAFE_HOST_PAIRED_MODE;
|
||||
}
|
||||
|
||||
if (ORIGINAL_REVIEWER_RUNTIME == null) {
|
||||
delete process.env.EJCLAW_REVIEWER_RUNTIME;
|
||||
} else {
|
||||
process.env.EJCLAW_REVIEWER_RUNTIME = ORIGINAL_REVIEWER_RUNTIME;
|
||||
}
|
||||
|
||||
if (ORIGINAL_CLAUDE_REVIEWER_READONLY == null) {
|
||||
delete process.env.EJCLAW_CLAUDE_REVIEWER_READONLY;
|
||||
} else {
|
||||
process.env.EJCLAW_CLAUDE_REVIEWER_READONLY =
|
||||
ORIGINAL_CLAUDE_REVIEWER_READONLY;
|
||||
}
|
||||
});
|
||||
|
||||
it('creates an owner execution with a worktree override', () => {
|
||||
const result = preparePairedExecutionContext({
|
||||
group,
|
||||
@@ -413,6 +444,7 @@ describe('paired execution context', () => {
|
||||
EJCLAW_PAIRED_ROLE: 'reviewer',
|
||||
EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1',
|
||||
});
|
||||
expect(result?.envOverrides.EJCLAW_CLAUDE_REVIEWER_READONLY).toBe('1');
|
||||
expect(result?.envOverrides.EJCLAW_REVIEWER_RUNTIME).toBeUndefined();
|
||||
expect(result?.envOverrides.CLAUDE_CONFIG_DIR).toContain(
|
||||
'/data/sessions/paired-room-reviewer',
|
||||
|
||||
@@ -387,6 +387,9 @@ export function preparePairedExecutionContext(args: {
|
||||
envOverrides.CLAUDE_CONFIG_DIR = reviewerSessionDir;
|
||||
if (unsafeHostPairedMode) {
|
||||
envOverrides.EJCLAW_UNSAFE_HOST_PAIRED_MODE = '1';
|
||||
if (REVIEWER_AGENT_TYPE === 'claude-code') {
|
||||
envOverrides.EJCLAW_CLAUDE_REVIEWER_READONLY = '1';
|
||||
}
|
||||
} else {
|
||||
envOverrides.EJCLAW_REVIEWER_RUNTIME = '1';
|
||||
}
|
||||
|
||||
@@ -219,10 +219,9 @@ describe('paired workspace manager', () => {
|
||||
packageManager: 'pnpm' as const,
|
||||
}));
|
||||
vi.doMock('./workspace-package-manager.js', async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import('./workspace-package-manager.js')>(
|
||||
'./workspace-package-manager.js',
|
||||
);
|
||||
const actual = await vi.importActual<
|
||||
typeof import('./workspace-package-manager.js')
|
||||
>('./workspace-package-manager.js');
|
||||
return {
|
||||
...actual,
|
||||
ensureWorkspaceDependenciesInstalled:
|
||||
|
||||
@@ -12,21 +12,30 @@ vi.mock('./logger.js', () => ({
|
||||
vi.mock('./token-rotation.js', () => ({
|
||||
rotateToken: vi.fn(() => false),
|
||||
getTokenCount: vi.fn(() => 1),
|
||||
getCurrentTokenIndex: vi.fn(() => 0),
|
||||
markTokenHealthy: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./token-refresh.js', () => ({
|
||||
forceRefreshToken: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
import { runClaudeRotationLoop } from './provider-retry.js';
|
||||
import {
|
||||
getCurrentTokenIndex,
|
||||
getTokenCount,
|
||||
markTokenHealthy,
|
||||
rotateToken,
|
||||
} from './token-rotation.js';
|
||||
import { forceRefreshToken } from './token-refresh.js';
|
||||
|
||||
describe('runClaudeRotationLoop', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(getTokenCount).mockReturnValue(1);
|
||||
vi.mocked(getCurrentTokenIndex).mockReturnValue(0);
|
||||
vi.mocked(rotateToken).mockReturnValue(false);
|
||||
vi.mocked(forceRefreshToken).mockResolvedValue(null);
|
||||
});
|
||||
|
||||
it('rotates and succeeds after an org-access-denied trigger', async () => {
|
||||
@@ -85,6 +94,60 @@ describe('runClaudeRotationLoop', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('force-refreshes the active token before rotating on auth-expired', async () => {
|
||||
vi.mocked(forceRefreshToken).mockResolvedValueOnce('new-access-token');
|
||||
|
||||
const outcome = await runClaudeRotationLoop(
|
||||
{ reason: 'auth-expired' },
|
||||
async () => ({
|
||||
output: { status: 'success', result: 'ok' },
|
||||
sawOutput: true,
|
||||
}),
|
||||
{ runId: 'force-refresh-auth-expired' },
|
||||
);
|
||||
|
||||
expect(outcome).toEqual({ type: 'success', sawOutput: true });
|
||||
expect(forceRefreshToken).toHaveBeenCalledWith(0);
|
||||
expect(rotateToken).not.toHaveBeenCalled();
|
||||
expect(markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('falls back to rotation when force refresh does not recover auth-expired', async () => {
|
||||
vi.mocked(getTokenCount).mockReturnValue(2);
|
||||
vi.mocked(forceRefreshToken).mockResolvedValueOnce('new-access-token');
|
||||
vi.mocked(rotateToken).mockReturnValueOnce(true);
|
||||
|
||||
let attempts = 0;
|
||||
const outcome = await runClaudeRotationLoop(
|
||||
{ reason: 'auth-expired' },
|
||||
async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) {
|
||||
return {
|
||||
output: {
|
||||
status: 'error',
|
||||
result: null,
|
||||
error:
|
||||
'Failed to authenticate. API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
|
||||
},
|
||||
sawOutput: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
output: { status: 'success', result: 'ok' },
|
||||
sawOutput: true,
|
||||
};
|
||||
},
|
||||
{ runId: 'force-refresh-then-rotate' },
|
||||
);
|
||||
|
||||
expect(outcome).toEqual({ type: 'success', sawOutput: true });
|
||||
expect(forceRefreshToken).toHaveBeenCalledWith(0);
|
||||
expect(rotateToken).toHaveBeenCalledTimes(1);
|
||||
expect(markTokenHealthy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('uses structured public output text as the next rotation message', async () => {
|
||||
vi.mocked(getTokenCount).mockReturnValue(2);
|
||||
vi.mocked(rotateToken).mockReturnValueOnce(true).mockReturnValueOnce(false);
|
||||
|
||||
@@ -15,10 +15,12 @@ import {
|
||||
import { logger } from './logger.js';
|
||||
import { getErrorMessage } from './utils.js';
|
||||
import {
|
||||
getCurrentTokenIndex,
|
||||
rotateToken,
|
||||
getTokenCount,
|
||||
markTokenHealthy,
|
||||
} from './token-rotation.js';
|
||||
import { forceRefreshToken } from './token-refresh.js';
|
||||
import { clearGlobalFailover } from './service-routing.js';
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────
|
||||
@@ -40,6 +42,113 @@ export type RotationOutcome =
|
||||
| { type: 'success'; sawOutput: boolean }
|
||||
| { type: 'error'; trigger?: TriggerInfo };
|
||||
|
||||
type AttemptOutcome =
|
||||
| { type: 'success'; sawOutput: boolean }
|
||||
| { type: 'error'; trigger?: TriggerInfo }
|
||||
| {
|
||||
type: 'continue';
|
||||
trigger: TriggerInfo;
|
||||
rotationMessage?: string;
|
||||
};
|
||||
|
||||
function evaluateClaudeAttempt(
|
||||
attempt: RotationAttemptResult,
|
||||
logContext: Record<string, unknown>,
|
||||
): AttemptOutcome {
|
||||
if (attempt.thrownError) {
|
||||
if (!attempt.sawOutput) {
|
||||
const errMsg = getErrorMessage(attempt.thrownError);
|
||||
const retryTrigger = attempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRetry: true,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||
}
|
||||
: classifyRotationTrigger(errMsg);
|
||||
if (retryTrigger.shouldRetry) {
|
||||
return {
|
||||
type: 'continue',
|
||||
trigger: {
|
||||
reason: retryTrigger.reason,
|
||||
retryAfterMs: retryTrigger.retryAfterMs,
|
||||
},
|
||||
rotationMessage: errMsg,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'claude', err: attempt.thrownError },
|
||||
'Rotated Claude account also threw',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
const output = attempt.output;
|
||||
if (!output) {
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'claude' },
|
||||
'Rotated Claude account produced no output object',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
if (
|
||||
!attempt.sawOutput &&
|
||||
attempt.streamedTriggerReason &&
|
||||
output.status !== 'error'
|
||||
) {
|
||||
return {
|
||||
type: 'continue',
|
||||
trigger: {
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||
},
|
||||
rotationMessage: getAgentOutputText(output) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (!attempt.sawOutput && attempt.sawSuccessNullResult) {
|
||||
logger.warn(
|
||||
{ ...logContext, provider: 'claude' },
|
||||
'All rotated tokens returned success with null result',
|
||||
);
|
||||
return { type: 'error', trigger: { reason: 'success-null-result' } };
|
||||
}
|
||||
|
||||
if (output.status === 'error') {
|
||||
if (!attempt.sawOutput) {
|
||||
const retryTrigger = attempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRetry: true,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||
}
|
||||
: classifyRotationTrigger(output.error);
|
||||
if (retryTrigger.shouldRetry) {
|
||||
return {
|
||||
type: 'continue',
|
||||
trigger: {
|
||||
reason: retryTrigger.reason,
|
||||
retryAfterMs: retryTrigger.retryAfterMs,
|
||||
},
|
||||
rotationMessage: output.error ?? undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'claude', error: output.error },
|
||||
'Rotated Claude account failed',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
markTokenHealthy();
|
||||
clearGlobalFailover();
|
||||
return { type: 'success', sawOutput: attempt.sawOutput };
|
||||
}
|
||||
|
||||
// ── Shared rotation loop ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
@@ -56,112 +165,70 @@ export async function runClaudeRotationLoop(
|
||||
): Promise<RotationOutcome> {
|
||||
let trigger = initialTrigger;
|
||||
let lastRotationMessage = rotationMessage;
|
||||
const attemptedForcedRefreshIndexes = new Set<number>();
|
||||
|
||||
while (shouldRotateClaudeToken(trigger.reason)) {
|
||||
if (trigger.reason === 'auth-expired') {
|
||||
const tokenIndex = getCurrentTokenIndex();
|
||||
if (
|
||||
tokenIndex != null &&
|
||||
!attemptedForcedRefreshIndexes.has(tokenIndex)
|
||||
) {
|
||||
attemptedForcedRefreshIndexes.add(tokenIndex);
|
||||
const refreshedToken = await forceRefreshToken(tokenIndex);
|
||||
if (refreshedToken) {
|
||||
logger.info(
|
||||
{ ...logContext, tokenIndex },
|
||||
'Claude auth-expired recovered by force-refreshing current token',
|
||||
);
|
||||
|
||||
const refreshAttemptOutcome = evaluateClaudeAttempt(
|
||||
await runAttempt(),
|
||||
logContext,
|
||||
);
|
||||
if (refreshAttemptOutcome.type === 'success') {
|
||||
return refreshAttemptOutcome;
|
||||
}
|
||||
if (refreshAttemptOutcome.type === 'error') {
|
||||
return refreshAttemptOutcome;
|
||||
}
|
||||
|
||||
trigger = refreshAttemptOutcome.trigger;
|
||||
lastRotationMessage = refreshAttemptOutcome.rotationMessage;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
!(
|
||||
getTokenCount() > 1 &&
|
||||
// Respect per-token cooldowns so exhausted auth/quota failures can
|
||||
// terminate instead of cycling forever.
|
||||
rotateToken(lastRotationMessage)
|
||||
)
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
while (
|
||||
shouldRotateClaudeToken(trigger.reason) &&
|
||||
getTokenCount() > 1 &&
|
||||
// Respect per-token cooldowns so exhausted auth/quota failures can
|
||||
// terminate instead of cycling forever.
|
||||
rotateToken(lastRotationMessage)
|
||||
) {
|
||||
logger.info(
|
||||
{ ...logContext, reason: trigger.reason },
|
||||
'Claude account unavailable, retrying with rotated account',
|
||||
);
|
||||
|
||||
const attempt = await runAttempt();
|
||||
|
||||
// ── Thrown error (exception from spawn/process) ──
|
||||
if (attempt.thrownError) {
|
||||
if (!attempt.sawOutput) {
|
||||
const errMsg = getErrorMessage(attempt.thrownError);
|
||||
const retryTrigger = attempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRetry: true,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||
}
|
||||
: classifyRotationTrigger(errMsg);
|
||||
if (retryTrigger.shouldRetry) {
|
||||
trigger = {
|
||||
reason: retryTrigger.reason,
|
||||
retryAfterMs: retryTrigger.retryAfterMs,
|
||||
};
|
||||
lastRotationMessage = errMsg;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'claude', err: attempt.thrownError },
|
||||
'Rotated Claude account also threw',
|
||||
);
|
||||
return { type: 'error' };
|
||||
const rotatedAttemptOutcome = evaluateClaudeAttempt(
|
||||
await runAttempt(),
|
||||
logContext,
|
||||
);
|
||||
if (rotatedAttemptOutcome.type === 'success') {
|
||||
return rotatedAttemptOutcome;
|
||||
}
|
||||
if (rotatedAttemptOutcome.type === 'error') {
|
||||
return rotatedAttemptOutcome;
|
||||
}
|
||||
|
||||
const output = attempt.output;
|
||||
if (!output) {
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'claude' },
|
||||
'Rotated Claude account produced no output object',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
// ── Streamed trigger in non-error success ──
|
||||
if (
|
||||
!attempt.sawOutput &&
|
||||
attempt.streamedTriggerReason &&
|
||||
output.status !== 'error'
|
||||
) {
|
||||
trigger = {
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||
};
|
||||
lastRotationMessage = getAgentOutputText(output) ?? undefined;
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Success with null result ──
|
||||
if (!attempt.sawOutput && attempt.sawSuccessNullResult) {
|
||||
logger.warn(
|
||||
{ ...logContext, provider: 'claude' },
|
||||
'All rotated tokens returned success with null result',
|
||||
);
|
||||
return { type: 'error', trigger: { reason: 'success-null-result' } };
|
||||
}
|
||||
|
||||
// ── Error status ──
|
||||
if (output.status === 'error') {
|
||||
if (!attempt.sawOutput) {
|
||||
const retryTrigger = attempt.streamedTriggerReason
|
||||
? {
|
||||
shouldRetry: true,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
||||
}
|
||||
: classifyRotationTrigger(output.error);
|
||||
if (retryTrigger.shouldRetry) {
|
||||
trigger = {
|
||||
reason: retryTrigger.reason,
|
||||
retryAfterMs: retryTrigger.retryAfterMs,
|
||||
};
|
||||
lastRotationMessage = output.error ?? undefined;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'claude', error: output.error },
|
||||
'Rotated Claude account failed',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
// ── Success ──
|
||||
markTokenHealthy();
|
||||
clearGlobalFailover();
|
||||
return { type: 'success', sawOutput: attempt.sawOutput };
|
||||
trigger = rotatedAttemptOutcome.trigger;
|
||||
lastRotationMessage = rotatedAttemptOutcome.rotationMessage;
|
||||
}
|
||||
|
||||
// ── All tokens exhausted ──
|
||||
|
||||
@@ -54,9 +54,14 @@ vi.mock('./agent-error-detection.js', async (importOriginal) => {
|
||||
vi.mock('./token-rotation.js', () => ({
|
||||
rotateToken: vi.fn(() => false),
|
||||
getTokenCount: vi.fn(() => 1),
|
||||
getCurrentTokenIndex: vi.fn(() => 0),
|
||||
markTokenHealthy: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./token-refresh.js', () => ({
|
||||
forceRefreshToken: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
detectCodexRotationTrigger: vi.fn((error?: string | null) => {
|
||||
const lower = (error || '').toLowerCase();
|
||||
@@ -142,6 +147,7 @@ import { _initTestDatabase, createTask, getTaskById } from './db.js';
|
||||
import * as codexTokenRotation from './codex-token-rotation.js';
|
||||
import { TIMEZONE } from './config.js';
|
||||
import * as serviceRouting from './service-routing.js';
|
||||
import * as tokenRefresh from './token-refresh.js';
|
||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||
import { TASK_STATUS_MESSAGE_PREFIX } from './task-watch-status.js';
|
||||
import * as tokenRotation from './token-rotation.js';
|
||||
@@ -183,9 +189,12 @@ describe('task scheduler', () => {
|
||||
});
|
||||
// No fallback provider setup needed
|
||||
vi.mocked(tokenRotation.getTokenCount).mockReturnValue(1);
|
||||
vi.mocked(tokenRotation.getCurrentTokenIndex).mockReturnValue(0);
|
||||
vi.mocked(tokenRotation.markTokenHealthy).mockClear();
|
||||
vi.mocked(tokenRotation.rotateToken).mockClear();
|
||||
vi.mocked(tokenRotation.rotateToken).mockReturnValue(false);
|
||||
vi.mocked(tokenRefresh.forceRefreshToken).mockReset();
|
||||
vi.mocked(tokenRefresh.forceRefreshToken).mockResolvedValue(null);
|
||||
vi.mocked(codexTokenRotation.rotateCodexToken).mockClear();
|
||||
vi.mocked(codexTokenRotation.rotateCodexToken).mockReturnValue(false);
|
||||
vi.mocked(codexTokenRotation.getCodexAccountCount).mockReturnValue(1);
|
||||
|
||||
@@ -237,6 +237,7 @@ async function refreshToken(
|
||||
*/
|
||||
async function checkAndRefreshAccount(
|
||||
accountIndex: number,
|
||||
opts?: { force?: boolean },
|
||||
): Promise<string | null> {
|
||||
const creds = readCredentials(accountIndex);
|
||||
if (!creds?.claudeAiOauth) return null;
|
||||
@@ -250,7 +251,7 @@ async function checkAndRefreshAccount(
|
||||
const now = Date.now();
|
||||
const remaining = expiresAt - now;
|
||||
|
||||
if (remaining > REFRESH_BEFORE_EXPIRY_MS) {
|
||||
if (!opts?.force && remaining > REFRESH_BEFORE_EXPIRY_MS) {
|
||||
logger.debug(
|
||||
{ accountIndex, remainingMin: Math.round(remaining / 60000) },
|
||||
'Token still valid, no refresh needed',
|
||||
@@ -260,7 +261,12 @@ async function checkAndRefreshAccount(
|
||||
|
||||
const isExpired = remaining <= 0;
|
||||
logger.info(
|
||||
{ accountIndex, remainingMin: Math.round(remaining / 60000), isExpired },
|
||||
{
|
||||
accountIndex,
|
||||
remainingMin: Math.round(remaining / 60000),
|
||||
isExpired,
|
||||
force: opts?.force === true,
|
||||
},
|
||||
'Refreshing Claude OAuth token',
|
||||
);
|
||||
|
||||
@@ -407,7 +413,9 @@ export async function forceRefreshToken(
|
||||
): Promise<string | null> {
|
||||
logger.info({ tokenIndex }, 'Force-refreshing token after 401');
|
||||
|
||||
const newAccessToken = await checkAndRefreshAccount(tokenIndex);
|
||||
const newAccessToken = await checkAndRefreshAccount(tokenIndex, {
|
||||
force: true,
|
||||
});
|
||||
if (newAccessToken) {
|
||||
const allTokens = getAllTokens();
|
||||
if (tokenIndex < allTokens.length) {
|
||||
|
||||
@@ -155,6 +155,13 @@ export function getCurrentToken(): string | undefined {
|
||||
return tokens[currentIndex % tokens.length]?.token;
|
||||
}
|
||||
|
||||
/** Get the current active token index. */
|
||||
export function getCurrentTokenIndex(): number | null {
|
||||
if (tokens.length === 0) return null;
|
||||
refreshRuntimeTokenSelection();
|
||||
return currentIndex % tokens.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to rotate to the next available (non-rate-limited) token.
|
||||
* Returns true if a fresh token was found, false if all are exhausted.
|
||||
|
||||
@@ -20,7 +20,9 @@ describe('verification helpers', () => {
|
||||
});
|
||||
|
||||
it('builds deterministic commands for each profile', () => {
|
||||
const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ejclaw-verification-'));
|
||||
const repoDir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ejclaw-verification-'),
|
||||
);
|
||||
fs.writeFileSync(path.join(repoDir, 'package.json'), JSON.stringify({}));
|
||||
|
||||
expect(buildVerificationCommand('test', repoDir)).toMatchObject({
|
||||
|
||||
@@ -77,7 +77,11 @@ export function buildVerificationCommand(
|
||||
repoDir: string = process.cwd(),
|
||||
): VerificationCommandSpec {
|
||||
const scriptName =
|
||||
profile === 'test' ? 'test' : profile === 'typecheck' ? 'typecheck' : 'build';
|
||||
profile === 'test'
|
||||
? 'test'
|
||||
: profile === 'typecheck'
|
||||
? 'typecheck'
|
||||
: 'build';
|
||||
const command = buildWorkspaceScriptCommand(repoDir, scriptName);
|
||||
return {
|
||||
file: command.file,
|
||||
|
||||
@@ -57,7 +57,10 @@ describe('workspace package manager helpers', () => {
|
||||
JSON.stringify({ packageManager: 'bun@1.3.11' }),
|
||||
);
|
||||
fs.writeFileSync(path.join(mixedRepo, 'bun.lock'), '');
|
||||
fs.writeFileSync(path.join(mixedRepo, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n');
|
||||
fs.writeFileSync(
|
||||
path.join(mixedRepo, 'pnpm-lock.yaml'),
|
||||
'lockfileVersion: 9.0\n',
|
||||
);
|
||||
|
||||
expect(detectWorkspacePackageManager(pnpmRepo)).toBe('pnpm');
|
||||
expect(detectWorkspacePackageManager(bunRepo)).toBe('bun');
|
||||
@@ -118,7 +121,10 @@ describe('workspace package manager helpers', () => {
|
||||
scripts: { typecheck: 'tsc --noEmit' },
|
||||
}),
|
||||
);
|
||||
fs.writeFileSync(path.join(repoDir, 'pnpm-lock.yaml'), 'lockfileVersion: 9.0\n');
|
||||
fs.writeFileSync(
|
||||
path.join(repoDir, 'pnpm-lock.yaml'),
|
||||
'lockfileVersion: 9.0\n',
|
||||
);
|
||||
|
||||
execFileSyncMock.mockImplementation((_file, _args, options) => {
|
||||
const cwd = (options as { cwd: string }).cwd;
|
||||
|
||||
@@ -44,7 +44,9 @@ function readPackageJsonMetadata(repoDir: string): PackageJsonMetadata | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
return JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) as PackageJsonMetadata;
|
||||
return JSON.parse(
|
||||
fs.readFileSync(packageJsonPath, 'utf-8'),
|
||||
) as PackageJsonMetadata;
|
||||
}
|
||||
|
||||
function detectPackageManagerFromField(
|
||||
@@ -60,10 +62,7 @@ function detectPackageManagerFromField(
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasLockfile(
|
||||
repoDir: string,
|
||||
...names: readonly string[]
|
||||
): boolean {
|
||||
function hasLockfile(repoDir: string, ...names: readonly string[]): boolean {
|
||||
return names.some((name) => fs.existsSync(path.join(repoDir, name)));
|
||||
}
|
||||
|
||||
@@ -91,7 +90,9 @@ function detectYarnBerry(
|
||||
packageManagerField: string | undefined,
|
||||
): boolean {
|
||||
if (packageManagerField?.startsWith('yarn@')) {
|
||||
const version = packageManagerField.slice('yarn@'.length).split(/[+-]/, 1)[0];
|
||||
const version = packageManagerField
|
||||
.slice('yarn@'.length)
|
||||
.split(/[+-]/, 1)[0];
|
||||
const major = Number.parseInt(version.split('.', 1)[0] ?? '', 10);
|
||||
if (Number.isFinite(major) && major >= 2) {
|
||||
return true;
|
||||
@@ -142,11 +143,7 @@ function computeInstallFingerprint(repoDir: string): string | null {
|
||||
}
|
||||
|
||||
function readInstallFingerprint(repoDir: string): string | null {
|
||||
const statePath = path.join(
|
||||
repoDir,
|
||||
'node_modules',
|
||||
INSTALL_STATE_FILENAME,
|
||||
);
|
||||
const statePath = path.join(repoDir, 'node_modules', INSTALL_STATE_FILENAME);
|
||||
if (!fs.existsSync(statePath)) {
|
||||
return null;
|
||||
}
|
||||
@@ -356,7 +353,9 @@ export function ensureWorkspaceDependenciesInstalled(
|
||||
? (error as { stdout: string }).stdout.trim()
|
||||
: '';
|
||||
const detail =
|
||||
stderr || stdout || (error instanceof Error ? error.message : String(error));
|
||||
stderr ||
|
||||
stdout ||
|
||||
(error instanceof Error ? error.message : String(error));
|
||||
throw new Error(
|
||||
`Failed to install workspace dependencies with "${command.commandText}" in ${repoDir}: ${detail}`,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user