Merge branch 'codex/owner/ejclaw'
# Conflicts: # src/message-agent-executor-rules.test.ts # src/message-agent-executor-rules.ts # src/message-agent-executor.ts # src/message-runtime-flow.ts # src/message-runtime.test.ts # src/message-runtime.ts # src/paired-execution-context-arbiter.ts # src/paired-execution-context-owner.ts # src/paired-execution-context-reviewer.ts # src/paired-execution-context-shared.ts # src/paired-execution-context.ts # src/paired-workspace-manager.test.ts # src/task-scheduler.ts
This commit is contained in:
@@ -37,7 +37,8 @@ Run commands directly—don't tell the user to run them.
|
|||||||
```bash
|
```bash
|
||||||
bun run build # Build main project
|
bun run build # Build main project
|
||||||
bun run build:runners # Install + build both runners
|
bun run build:runners # Install + build both runners
|
||||||
bun run build:container # Rebuild reviewer Docker image
|
bun run build:runtime # Build host runtime only
|
||||||
|
bun run build:container # Optional: rebuild reviewer Docker image
|
||||||
bun run dev # Dev mode with hot reload
|
bun run dev # Dev mode with hot reload
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -53,6 +54,9 @@ Deploy:
|
|||||||
bun run deploy
|
bun run deploy
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`deploy` rebuilds only the host runtime. Rebuild the reviewer Docker image
|
||||||
|
separately with `bun run build:container` only when you need the container path.
|
||||||
|
|
||||||
## Service Stack Architecture
|
## Service Stack Architecture
|
||||||
|
|
||||||
Single unified service manages all three Discord bots in one process:
|
Single unified service manages all three Discord bots in one process:
|
||||||
|
|||||||
@@ -8,9 +8,10 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"build:runners": "bun install --cwd runners/agent-runner && bun run --cwd runners/agent-runner build && bun install --cwd runners/codex-runner && bun run --cwd runners/codex-runner build",
|
"build:runners": "bun install --cwd runners/agent-runner && bun run --cwd runners/agent-runner build && bun install --cwd runners/codex-runner && bun run --cwd runners/codex-runner build",
|
||||||
|
"build:runtime": "bun run build && bun run build:runners",
|
||||||
"build:container": "docker build -f container/Dockerfile -t ejclaw-reviewer:latest .",
|
"build:container": "docker build -f container/Dockerfile -t ejclaw-reviewer:latest .",
|
||||||
"build:all": "bun run build && bun run build:runners && bun run build:container",
|
"build:all": "bun run build:runtime && bun run build:container",
|
||||||
"deploy": "git pull --ff-only && bun run build:all && systemctl --user restart ejclaw",
|
"deploy": "git pull --ff-only && bun run build:runtime && systemctl --user restart ejclaw",
|
||||||
"start": "bun dist/index.js",
|
"start": "bun dist/index.js",
|
||||||
"dev": "bun --watch src/index.ts",
|
"dev": "bun --watch src/index.ts",
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ You may receive reference opinions from external models appended to your prompt.
|
|||||||
## Rules
|
## Rules
|
||||||
|
|
||||||
- Base your verdict on evidence (code, test output, logs), not on who said what first
|
- Base your verdict on evidence (code, test output, logs), not on who said what first
|
||||||
|
- Distinguish reviewer snapshot limits from real product bugs. Reviewer workspaces may intentionally omit heavy artifacts like `node_modules`, `dist`, and `build`; inability to run direct local test/typecheck/build there is not, by itself, a blocker if dedicated verification evidence exists
|
||||||
|
- When verification evidence exists from the dedicated verification path, judge that evidence on its merits instead of requiring the reviewer to reproduce the same result from the lightweight reviewer snapshot
|
||||||
- Your verdict is final for this deadlock cycle — after it, work resumes normally
|
- Your verdict is final for this deadlock cycle — after it, work resumes normally
|
||||||
- You do NOT implement or review code — you only judge the disagreement
|
- You do NOT implement or review code — you only judge the disagreement
|
||||||
- Keep your verdict concise — state the decision, the evidence, and the required action
|
- Keep your verdict concise — state the decision, the evidence, and the required action
|
||||||
|
|||||||
@@ -30,6 +30,8 @@ Push back with evidence when the owner is wrong. Hold your ground when you are r
|
|||||||
## Rules
|
## Rules
|
||||||
|
|
||||||
- Judge completion only by verification output. "It should work now" means run it. "I'm confident" means nothing — confidence is not evidence. "I tested earlier" means test again if code changed since. "It's a trivial change" means verify anyway
|
- Judge completion only by verification output. "It should work now" means run it. "I'm confident" means nothing — confidence is not evidence. "I tested earlier" means test again if code changed since. "It's a trivial change" means verify anyway
|
||||||
|
- Reviewer workspaces may intentionally exclude heavy runtime artifacts such as `node_modules`, `dist`, and `build`. Do not treat the inability to run direct local test/typecheck/build from the reviewer snapshot as a product bug by itself
|
||||||
|
- When test/typecheck/build evidence is needed, prefer the dedicated verification path (`run_verification`) over assuming the reviewer snapshot can execute the full project locally
|
||||||
- Stagnation: **Spinning** (same error 3+), **Oscillation** (alternating approaches), **Diminishing returns** (shrinking improvement), **No progress** (discussion without change) — name the pattern and report: **Status**, **Attempted**, **Recommendation**
|
- Stagnation: **Spinning** (same error 3+), **Oscillation** (alternating approaches), **Diminishing returns** (shrinking improvement), **No progress** (discussion without change) — name the pattern and report: **Status**, **Attempted**, **Recommendation**
|
||||||
- Implementation, commits, and pushes require agreement from both sides. Either can veto
|
- Implementation, commits, and pushes require agreement from both sides. Either can veto
|
||||||
- Keep reviews concise — approve quickly when there is nothing to critique
|
- Keep reviews concise — approve quickly when there is nothing to critique
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import fs from 'fs';
|
|||||||
import os from 'os';
|
import os from 'os';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { describe, expect, it } from 'vitest';
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
assertReadonlyWorkspaceRepoConnectivity,
|
assertReadonlyWorkspaceRepoConnectivity,
|
||||||
@@ -11,6 +11,22 @@ import {
|
|||||||
isReviewerRuntime,
|
isReviewerRuntime,
|
||||||
} from '../src/reviewer-runtime.js';
|
} from '../src/reviewer-runtime.js';
|
||||||
|
|
||||||
|
const ORIGINAL_UNSAFE_HOST_PAIRED_MODE =
|
||||||
|
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (ORIGINAL_UNSAFE_HOST_PAIRED_MODE == null) {
|
||||||
|
delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||||
|
} else {
|
||||||
|
process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE =
|
||||||
|
ORIGINAL_UNSAFE_HOST_PAIRED_MODE;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
delete process.env.EJCLAW_UNSAFE_HOST_PAIRED_MODE;
|
||||||
|
});
|
||||||
|
|
||||||
function createTempRepo(prefix: string): string {
|
function createTempRepo(prefix: string): string {
|
||||||
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||||
execFileSync('git', ['init'], {
|
execFileSync('git', ['init'], {
|
||||||
|
|||||||
182
src/agent-attempt-orchestration.ts
Normal file
182
src/agent-attempt-orchestration.ts
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
import type { AgentOutput } from './agent-runner.js';
|
||||||
|
import {
|
||||||
|
resolveAttemptRetryAction,
|
||||||
|
type AttemptRetryAction,
|
||||||
|
type AttemptRetryState,
|
||||||
|
} from './agent-attempt-retry.js';
|
||||||
|
import type {
|
||||||
|
AgentTriggerReason,
|
||||||
|
CodexRotationReason,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
|
import {
|
||||||
|
runClaudeRotationLoop,
|
||||||
|
runCodexRotationLoop,
|
||||||
|
} from './provider-retry.js';
|
||||||
|
|
||||||
|
type RotationLogContext = Record<string, unknown>;
|
||||||
|
|
||||||
|
type ClaudeRetryAttemptState = Pick<
|
||||||
|
AttemptRetryState,
|
||||||
|
'sawOutput' | 'streamedTriggerReason'
|
||||||
|
> & {
|
||||||
|
output?: Pick<AgentOutput, 'status' | 'result' | 'output' | 'error'>;
|
||||||
|
error?: unknown;
|
||||||
|
sawSuccessNullResultWithoutOutput?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type CodexRetryAttemptState = Pick<
|
||||||
|
AttemptRetryState,
|
||||||
|
'sawOutput' | 'streamedTriggerReason'
|
||||||
|
> & {
|
||||||
|
output?: Pick<AgentOutput, 'status' | 'result' | 'output' | 'error'>;
|
||||||
|
error?: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ExecutedAttemptRetryAction =
|
||||||
|
| { kind: 'none' }
|
||||||
|
| {
|
||||||
|
kind: 'claude';
|
||||||
|
trigger: { reason: AgentTriggerReason; retryAfterMs?: number };
|
||||||
|
rotationMessage?: string;
|
||||||
|
result: 'success' | 'error';
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
kind: 'codex';
|
||||||
|
trigger: { reason: CodexRotationReason };
|
||||||
|
rotationMessage?: string;
|
||||||
|
result: 'success' | 'error';
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function runClaudeAttemptWithRotation<
|
||||||
|
TAttempt extends ClaudeRetryAttemptState,
|
||||||
|
>(args: {
|
||||||
|
initialTrigger: {
|
||||||
|
reason: AgentTriggerReason;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
};
|
||||||
|
runAttempt: () => Promise<TAttempt>;
|
||||||
|
logContext: RotationLogContext;
|
||||||
|
rotationMessage?: string;
|
||||||
|
afterAttempt?: (attempt: TAttempt) => Promise<void> | void;
|
||||||
|
onSuccess?: (outcome: { sawOutput: boolean }) => Promise<void> | void;
|
||||||
|
}): Promise<'success' | 'error'> {
|
||||||
|
const outcome = await runClaudeRotationLoop(
|
||||||
|
args.initialTrigger,
|
||||||
|
async () => {
|
||||||
|
const attempt = await args.runAttempt();
|
||||||
|
await args.afterAttempt?.(attempt);
|
||||||
|
return {
|
||||||
|
output: attempt.output,
|
||||||
|
thrownError: attempt.error,
|
||||||
|
sawOutput: attempt.sawOutput,
|
||||||
|
sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput,
|
||||||
|
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
args.logContext,
|
||||||
|
args.rotationMessage,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (outcome.type === 'success') {
|
||||||
|
await args.onSuccess?.({ sawOutput: outcome.sawOutput });
|
||||||
|
return 'success';
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runCodexAttemptWithRotation<
|
||||||
|
TAttempt extends CodexRetryAttemptState,
|
||||||
|
>(args: {
|
||||||
|
initialTrigger: { reason: CodexRotationReason };
|
||||||
|
runAttempt: () => Promise<TAttempt>;
|
||||||
|
logContext: RotationLogContext;
|
||||||
|
rotationMessage?: string;
|
||||||
|
afterAttempt?: (attempt: TAttempt) => Promise<void> | void;
|
||||||
|
}): Promise<'success' | 'error'> {
|
||||||
|
const outcome = await runCodexRotationLoop(
|
||||||
|
args.initialTrigger,
|
||||||
|
async () => {
|
||||||
|
const attempt = await args.runAttempt();
|
||||||
|
await args.afterAttempt?.(attempt);
|
||||||
|
return {
|
||||||
|
output: attempt.output,
|
||||||
|
thrownError: attempt.error,
|
||||||
|
sawOutput: attempt.sawOutput,
|
||||||
|
streamedTriggerReason: attempt.streamedTriggerReason,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
args.logContext,
|
||||||
|
args.rotationMessage,
|
||||||
|
);
|
||||||
|
|
||||||
|
return outcome.type === 'success' ? 'success' : 'error';
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function executeAttemptRetryAction(args: {
|
||||||
|
provider: 'claude' | 'codex';
|
||||||
|
canRetryClaudeCredentials: boolean;
|
||||||
|
canRetryCodex: boolean;
|
||||||
|
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||||
|
rotationMessage?: string | null;
|
||||||
|
runClaude: (
|
||||||
|
trigger: { reason: AgentTriggerReason; retryAfterMs?: number },
|
||||||
|
rotationMessage?: string,
|
||||||
|
) => Promise<'success' | 'error'>;
|
||||||
|
runCodex: (
|
||||||
|
trigger: { reason: CodexRotationReason },
|
||||||
|
rotationMessage?: string,
|
||||||
|
) => Promise<'success' | 'error'>;
|
||||||
|
}): Promise<ExecutedAttemptRetryAction> {
|
||||||
|
const retryAction = resolveAttemptRetryAction({
|
||||||
|
provider: args.provider,
|
||||||
|
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
||||||
|
canRetryCodex: args.canRetryCodex,
|
||||||
|
attempt: args.attempt,
|
||||||
|
rotationMessage: args.rotationMessage,
|
||||||
|
});
|
||||||
|
|
||||||
|
return executeResolvedAttemptRetryAction({
|
||||||
|
retryAction,
|
||||||
|
runClaude: args.runClaude,
|
||||||
|
runCodex: args.runCodex,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function executeResolvedAttemptRetryAction(args: {
|
||||||
|
retryAction: AttemptRetryAction;
|
||||||
|
runClaude: (
|
||||||
|
trigger: { reason: AgentTriggerReason; retryAfterMs?: number },
|
||||||
|
rotationMessage?: string,
|
||||||
|
) => Promise<'success' | 'error'>;
|
||||||
|
runCodex: (
|
||||||
|
trigger: { reason: CodexRotationReason },
|
||||||
|
rotationMessage?: string,
|
||||||
|
) => Promise<'success' | 'error'>;
|
||||||
|
}): Promise<ExecutedAttemptRetryAction> {
|
||||||
|
if (args.retryAction.kind === 'claude') {
|
||||||
|
return {
|
||||||
|
kind: 'claude',
|
||||||
|
trigger: args.retryAction.trigger,
|
||||||
|
rotationMessage: args.retryAction.rotationMessage,
|
||||||
|
result: await args.runClaude(
|
||||||
|
args.retryAction.trigger,
|
||||||
|
args.retryAction.rotationMessage,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.retryAction.kind === 'codex') {
|
||||||
|
return {
|
||||||
|
kind: 'codex',
|
||||||
|
trigger: args.retryAction.trigger,
|
||||||
|
rotationMessage: args.retryAction.rotationMessage,
|
||||||
|
result: await args.runCodex(
|
||||||
|
args.retryAction.trigger,
|
||||||
|
args.retryAction.rotationMessage,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { kind: 'none' };
|
||||||
|
}
|
||||||
164
src/agent-attempt-retry.ts
Normal file
164
src/agent-attempt-retry.ts
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
import {
|
||||||
|
classifyRotationTrigger,
|
||||||
|
type AgentTriggerReason,
|
||||||
|
type CodexRotationReason,
|
||||||
|
isCodexRotationReason,
|
||||||
|
} from './agent-error-detection.js';
|
||||||
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
|
export interface AttemptStreamedTrigger {
|
||||||
|
reason: AgentTriggerReason;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AttemptRetryState {
|
||||||
|
sawOutput: boolean;
|
||||||
|
retryableSessionFailureDetected?: boolean;
|
||||||
|
streamedTriggerReason?: AttemptStreamedTrigger;
|
||||||
|
error?: unknown;
|
||||||
|
outputError?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRetryableClaudeSessionFailureAttempt(args: {
|
||||||
|
attempt: AttemptRetryState;
|
||||||
|
isClaudeCodeAgent: boolean;
|
||||||
|
provider: 'claude' | 'codex';
|
||||||
|
shouldRetryFreshSessionOnAgentFailure: (args: {
|
||||||
|
result: null;
|
||||||
|
error: string;
|
||||||
|
}) => boolean;
|
||||||
|
}): boolean {
|
||||||
|
if (
|
||||||
|
!args.isClaudeCodeAgent ||
|
||||||
|
args.provider !== 'claude' ||
|
||||||
|
args.attempt.sawOutput
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.retryableSessionFailureDetected === true) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.error == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return args.shouldRetryFreshSessionOnAgentFailure({
|
||||||
|
result: null,
|
||||||
|
error: getErrorMessage(args.attempt.error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveClaudeRetryTrigger(args: {
|
||||||
|
canRetryClaudeCredentials: boolean;
|
||||||
|
provider: 'claude' | 'codex';
|
||||||
|
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||||
|
fallbackMessage?: string | null;
|
||||||
|
}): { reason: AgentTriggerReason; retryAfterMs?: number } | null {
|
||||||
|
if (
|
||||||
|
!(
|
||||||
|
args.canRetryClaudeCredentials &&
|
||||||
|
args.provider === 'claude' &&
|
||||||
|
!args.attempt.sawOutput
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.streamedTriggerReason) {
|
||||||
|
return {
|
||||||
|
reason: args.attempt.streamedTriggerReason.reason,
|
||||||
|
retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const trigger = classifyRotationTrigger(args.fallbackMessage);
|
||||||
|
if (!trigger.shouldRetry) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
reason: trigger.reason,
|
||||||
|
retryAfterMs: trigger.retryAfterMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveCodexRetryTrigger(args: {
|
||||||
|
canRetryCodex: boolean;
|
||||||
|
attempt: Pick<AttemptRetryState, 'streamedTriggerReason'>;
|
||||||
|
rotationMessage?: string | null;
|
||||||
|
}): { reason: CodexRotationReason } | null {
|
||||||
|
if (!args.canRetryCodex) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.attempt.streamedTriggerReason) {
|
||||||
|
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
reason: args.attempt.streamedTriggerReason.reason,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const trigger = detectCodexRotationTrigger(args.rotationMessage);
|
||||||
|
if (!trigger.shouldRotate) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { reason: trigger.reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AttemptRetryAction =
|
||||||
|
| {
|
||||||
|
kind: 'claude';
|
||||||
|
trigger: { reason: AgentTriggerReason; retryAfterMs?: number };
|
||||||
|
rotationMessage?: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
kind: 'codex';
|
||||||
|
trigger: { reason: CodexRotationReason };
|
||||||
|
rotationMessage?: string;
|
||||||
|
}
|
||||||
|
| { kind: 'none' };
|
||||||
|
|
||||||
|
export function resolveAttemptRetryAction(args: {
|
||||||
|
provider: 'claude' | 'codex';
|
||||||
|
canRetryClaudeCredentials: boolean;
|
||||||
|
canRetryCodex: boolean;
|
||||||
|
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||||
|
rotationMessage?: string | null;
|
||||||
|
}): AttemptRetryAction {
|
||||||
|
const normalizedRotationMessage = args.rotationMessage ?? undefined;
|
||||||
|
|
||||||
|
const claudeTrigger = resolveClaudeRetryTrigger({
|
||||||
|
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
||||||
|
provider: args.provider,
|
||||||
|
attempt: args.attempt,
|
||||||
|
fallbackMessage: normalizedRotationMessage,
|
||||||
|
});
|
||||||
|
if (claudeTrigger) {
|
||||||
|
return {
|
||||||
|
kind: 'claude',
|
||||||
|
trigger: claudeTrigger,
|
||||||
|
rotationMessage: normalizedRotationMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const codexTrigger = resolveCodexRetryTrigger({
|
||||||
|
canRetryCodex: args.canRetryCodex,
|
||||||
|
attempt: args.attempt,
|
||||||
|
rotationMessage: normalizedRotationMessage,
|
||||||
|
});
|
||||||
|
if (codexTrigger) {
|
||||||
|
return {
|
||||||
|
kind: 'codex',
|
||||||
|
trigger: codexTrigger,
|
||||||
|
rotationMessage: normalizedRotationMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { kind: 'none' };
|
||||||
|
}
|
||||||
@@ -121,4 +121,25 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
|||||||
expect(active).toBeDefined();
|
expect(active).toBeDefined();
|
||||||
expect(active!.index).toBe(1); // fallback picks next non-rate-limited
|
expect(active!.index).toBe(1); // fallback picks next non-rate-limited
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('warns when Codex rotation state cannot be persisted', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
const utils = await import('./utils.js');
|
||||||
|
const { logger } = await import('./logger.js');
|
||||||
|
|
||||||
|
vi.mocked(utils.writeJsonFile).mockImplementation(() => {
|
||||||
|
throw new Error('disk full');
|
||||||
|
});
|
||||||
|
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
expect(mod.rotateCodexToken('rate limit')).toBe(true);
|
||||||
|
|
||||||
|
expect(logger.warn).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
stateFile: '/tmp/ejclaw-codex-rot-data/codex-rotation-state.json',
|
||||||
|
err: expect.any(Error),
|
||||||
|
}),
|
||||||
|
'Failed to persist Codex rotation state',
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -138,8 +138,11 @@ function saveCodexState(): void {
|
|||||||
resetD7Ats: accounts.map((a) => a.resetD7At ?? null),
|
resetD7Ats: accounts.map((a) => a.resetD7At ?? null),
|
||||||
};
|
};
|
||||||
writeJsonFile(STATE_FILE, state);
|
writeJsonFile(STATE_FILE, state);
|
||||||
} catch {
|
} catch (err) {
|
||||||
/* best effort */
|
logger.warn(
|
||||||
|
{ stateFile: STATE_FILE, err },
|
||||||
|
'Failed to persist Codex rotation state',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
33
src/logger.test.ts
Normal file
33
src/logger.test.ts
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
describe('logger singleton', () => {
|
||||||
|
it('reuses the root logger and does not add duplicate process listeners across module reloads', async () => {
|
||||||
|
vi.resetModules();
|
||||||
|
|
||||||
|
const beforeUncaught = process.listeners('uncaughtException').length;
|
||||||
|
const beforeUnhandled = process.listeners('unhandledRejection').length;
|
||||||
|
const beforeExit = process.listeners('exit').length;
|
||||||
|
|
||||||
|
const first = await import('./logger.js');
|
||||||
|
|
||||||
|
const afterFirstUncaught = process.listeners('uncaughtException').length;
|
||||||
|
const afterFirstUnhandled = process.listeners('unhandledRejection').length;
|
||||||
|
const afterFirstExit = process.listeners('exit').length;
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
|
||||||
|
const second = await import('./logger.js');
|
||||||
|
|
||||||
|
const afterSecondUncaught = process.listeners('uncaughtException').length;
|
||||||
|
const afterSecondUnhandled = process.listeners('unhandledRejection').length;
|
||||||
|
const afterSecondExit = process.listeners('exit').length;
|
||||||
|
|
||||||
|
expect(second.logger).toBe(first.logger);
|
||||||
|
expect(afterFirstUncaught).toBeGreaterThanOrEqual(beforeUncaught);
|
||||||
|
expect(afterFirstUnhandled).toBeGreaterThanOrEqual(beforeUnhandled);
|
||||||
|
expect(afterSecondUncaught).toBe(afterFirstUncaught);
|
||||||
|
expect(afterSecondUnhandled).toBe(afterFirstUnhandled);
|
||||||
|
expect(afterFirstExit).toBeGreaterThanOrEqual(beforeExit);
|
||||||
|
expect(afterSecondExit).toBe(afterFirstExit);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,12 +1,34 @@
|
|||||||
import pino, { type Logger } from 'pino';
|
import pino, { type Logger } from 'pino';
|
||||||
|
|
||||||
const serviceName = (process.env.ASSISTANT_NAME || 'claude').toLowerCase();
|
const serviceName = (process.env.ASSISTANT_NAME || 'claude').toLowerCase();
|
||||||
|
const isTestEnv =
|
||||||
|
process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
|
||||||
|
|
||||||
export const logger = pino({
|
type LoggerGlobalState = typeof globalThis & {
|
||||||
level: process.env.LOG_LEVEL || 'info',
|
__ejclawLogger?: Logger;
|
||||||
name: serviceName,
|
__ejclawProcessHandlersInstalled?: boolean;
|
||||||
transport: { target: 'pino-pretty', options: { colorize: true } },
|
};
|
||||||
});
|
|
||||||
|
const globalState = globalThis as LoggerGlobalState;
|
||||||
|
|
||||||
|
function createRootLogger(): Logger {
|
||||||
|
const baseOptions = {
|
||||||
|
level: process.env.LOG_LEVEL || 'info',
|
||||||
|
name: serviceName,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (isTestEnv) {
|
||||||
|
return pino(baseOptions);
|
||||||
|
}
|
||||||
|
|
||||||
|
return pino({
|
||||||
|
...baseOptions,
|
||||||
|
transport: { target: 'pino-pretty', options: { colorize: true } },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export const logger =
|
||||||
|
globalState.__ejclawLogger ?? (globalState.__ejclawLogger = createRootLogger());
|
||||||
|
|
||||||
type LogBindings = Record<string, unknown>;
|
type LogBindings = Record<string, unknown>;
|
||||||
|
|
||||||
@@ -30,11 +52,17 @@ export function createScopedLogger(bindings: LogBindings): Logger {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Route uncaught errors through pino so they get timestamps in stderr
|
// Route uncaught errors through pino so they get timestamps in stderr
|
||||||
process.on('uncaughtException', (err) => {
|
function handleUncaughtException(err: unknown): void {
|
||||||
logger.fatal({ err }, 'Uncaught exception');
|
logger.fatal({ err }, 'Uncaught exception');
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
});
|
}
|
||||||
|
|
||||||
process.on('unhandledRejection', (reason) => {
|
function handleUnhandledRejection(reason: unknown): void {
|
||||||
logger.error({ err: reason }, 'Unhandled rejection');
|
logger.error({ err: reason }, 'Unhandled rejection');
|
||||||
});
|
}
|
||||||
|
|
||||||
|
if (!globalState.__ejclawProcessHandlersInstalled) {
|
||||||
|
process.on('uncaughtException', handleUncaughtException);
|
||||||
|
process.on('unhandledRejection', handleUnhandledRejection);
|
||||||
|
globalState.__ejclawProcessHandlersInstalled = true;
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import { classifyRotationTrigger } from './agent-error-detection.js';
|
|||||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
||||||
import {
|
import {
|
||||||
isRetryableClaudeSessionFailureAttempt,
|
isRetryableClaudeSessionFailureAttempt,
|
||||||
|
resolveAttemptRetryAction,
|
||||||
resolveClaudeRetryTrigger,
|
resolveClaudeRetryTrigger,
|
||||||
resolveCodexRetryTrigger,
|
resolveCodexRetryTrigger,
|
||||||
resolvePairedFollowUpQueueAction,
|
resolvePairedFollowUpQueueAction,
|
||||||
@@ -88,6 +89,47 @@ describe('message-agent-executor-rules', () => {
|
|||||||
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
|
expect(detectCodexRotationTrigger).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('resolves a shared Claude retry action from fallback output text', () => {
|
||||||
|
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||||
|
shouldRetry: true,
|
||||||
|
reason: '429',
|
||||||
|
retryAfterMs: 30000,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(
|
||||||
|
resolveAttemptRetryAction({
|
||||||
|
provider: 'claude',
|
||||||
|
canRetryClaudeCredentials: true,
|
||||||
|
canRetryCodex: false,
|
||||||
|
attempt: { sawOutput: false },
|
||||||
|
rotationMessage: '429 rate limit',
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
kind: 'claude',
|
||||||
|
trigger: { reason: '429', retryAfterMs: 30000 },
|
||||||
|
rotationMessage: '429 rate limit',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves a shared Codex retry action from streamed state', () => {
|
||||||
|
expect(
|
||||||
|
resolveAttemptRetryAction({
|
||||||
|
provider: 'codex',
|
||||||
|
canRetryClaudeCredentials: false,
|
||||||
|
canRetryCodex: true,
|
||||||
|
attempt: {
|
||||||
|
sawOutput: false,
|
||||||
|
streamedTriggerReason: { reason: 'auth-expired' },
|
||||||
|
},
|
||||||
|
rotationMessage: 'oauth token expired',
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
kind: 'codex',
|
||||||
|
trigger: { reason: 'auth-expired' },
|
||||||
|
rotationMessage: 'oauth token expired',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('detects retryable Claude session failures from either flag or classifier', () => {
|
it('detects retryable Claude session failures from either flag or classifier', () => {
|
||||||
expect(
|
expect(
|
||||||
isRetryableClaudeSessionFailureAttempt({
|
isRetryableClaudeSessionFailureAttempt({
|
||||||
|
|||||||
@@ -1,116 +1,14 @@
|
|||||||
import {
|
|
||||||
classifyRotationTrigger,
|
|
||||||
type AgentTriggerReason,
|
|
||||||
type CodexRotationReason,
|
|
||||||
isCodexRotationReason,
|
|
||||||
} from './agent-error-detection.js';
|
|
||||||
import { detectCodexRotationTrigger } from './codex-token-rotation.js';
|
|
||||||
import { resolveNextTurnAction } from './message-runtime-rules.js';
|
import { resolveNextTurnAction } from './message-runtime-rules.js';
|
||||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||||
import { getErrorMessage } from './utils.js';
|
export {
|
||||||
|
isRetryableClaudeSessionFailureAttempt,
|
||||||
export interface ExecutorStreamedTrigger {
|
resolveAttemptRetryAction,
|
||||||
reason: AgentTriggerReason;
|
resolveClaudeRetryTrigger,
|
||||||
retryAfterMs?: number;
|
resolveCodexRetryTrigger,
|
||||||
}
|
type AttemptRetryAction,
|
||||||
|
type AttemptRetryState as ExecutorAttemptState,
|
||||||
export interface ExecutorAttemptState {
|
type AttemptStreamedTrigger as ExecutorStreamedTrigger,
|
||||||
sawOutput: boolean;
|
} from './agent-attempt-retry.js';
|
||||||
retryableSessionFailureDetected?: boolean;
|
|
||||||
streamedTriggerReason?: ExecutorStreamedTrigger;
|
|
||||||
error?: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isRetryableClaudeSessionFailureAttempt(args: {
|
|
||||||
attempt: ExecutorAttemptState;
|
|
||||||
isClaudeCodeAgent: boolean;
|
|
||||||
provider: 'claude' | 'codex';
|
|
||||||
shouldRetryFreshSessionOnAgentFailure: (args: {
|
|
||||||
result: null;
|
|
||||||
error: string;
|
|
||||||
}) => boolean;
|
|
||||||
}): boolean {
|
|
||||||
if (
|
|
||||||
!args.isClaudeCodeAgent ||
|
|
||||||
args.provider !== 'claude' ||
|
|
||||||
args.attempt.sawOutput
|
|
||||||
) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (args.attempt.retryableSessionFailureDetected === true) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (args.attempt.error == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return args.shouldRetryFreshSessionOnAgentFailure({
|
|
||||||
result: null,
|
|
||||||
error: getErrorMessage(args.attempt.error),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveClaudeRetryTrigger(args: {
|
|
||||||
canRetryClaudeCredentials: boolean;
|
|
||||||
provider: 'claude' | 'codex';
|
|
||||||
attempt: Pick<ExecutorAttemptState, 'sawOutput' | 'streamedTriggerReason'>;
|
|
||||||
fallbackMessage?: string | null;
|
|
||||||
}): { reason: AgentTriggerReason; retryAfterMs?: number } | null {
|
|
||||||
if (
|
|
||||||
!(
|
|
||||||
args.canRetryClaudeCredentials &&
|
|
||||||
args.provider === 'claude' &&
|
|
||||||
!args.attempt.sawOutput
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (args.attempt.streamedTriggerReason) {
|
|
||||||
return {
|
|
||||||
reason: args.attempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: args.attempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const trigger = classifyRotationTrigger(args.fallbackMessage);
|
|
||||||
if (!trigger.shouldRetry) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
reason: trigger.reason,
|
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resolveCodexRetryTrigger(args: {
|
|
||||||
canRetryCodex: boolean;
|
|
||||||
attempt: Pick<ExecutorAttemptState, 'streamedTriggerReason'>;
|
|
||||||
rotationMessage?: string | null;
|
|
||||||
}): { reason: CodexRotationReason } | null {
|
|
||||||
if (!args.canRetryCodex) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (args.attempt.streamedTriggerReason) {
|
|
||||||
if (!isCodexRotationReason(args.attempt.streamedTriggerReason.reason)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
reason: args.attempt.streamedTriggerReason.reason,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const trigger = detectCodexRotationTrigger(args.rotationMessage);
|
|
||||||
if (!trigger.shouldRotate) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return { reason: trigger.reason };
|
|
||||||
}
|
|
||||||
|
|
||||||
export type PairedFollowUpQueueAction =
|
export type PairedFollowUpQueueAction =
|
||||||
| 'generic'
|
| 'generic'
|
||||||
|
|||||||
@@ -5,6 +5,12 @@ import { getErrorMessage } from './utils.js';
|
|||||||
|
|
||||||
import { getAgentOutputText } from './agent-output.js';
|
import { getAgentOutputText } from './agent-output.js';
|
||||||
import { createEvaluatedOutputHandler } from './agent-attempt.js';
|
import { createEvaluatedOutputHandler } from './agent-attempt.js';
|
||||||
|
import {
|
||||||
|
executeAttemptRetryAction,
|
||||||
|
runClaudeAttemptWithRotation,
|
||||||
|
runCodexAttemptWithRotation,
|
||||||
|
} from './agent-attempt-orchestration.js';
|
||||||
|
import { isRetryableClaudeSessionFailureAttempt } from './agent-attempt-retry.js';
|
||||||
import {
|
import {
|
||||||
AgentOutput,
|
AgentOutput,
|
||||||
runAgentProcess,
|
runAgentProcess,
|
||||||
@@ -30,18 +36,9 @@ import {
|
|||||||
} from './paired-execution-context.js';
|
} from './paired-execution-context.js';
|
||||||
import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js';
|
import { resolveCodexFallbackHandoff } from './paired-turn-fallback.js';
|
||||||
import { resolveExecutionTarget } from './message-runtime-rules.js';
|
import { resolveExecutionTarget } from './message-runtime-rules.js';
|
||||||
import {
|
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
||||||
isRetryableClaudeSessionFailureAttempt,
|
|
||||||
resolveClaudeRetryTrigger,
|
|
||||||
resolveCodexRetryTrigger,
|
|
||||||
resolvePairedFollowUpQueueAction,
|
|
||||||
} from './message-agent-executor-rules.js';
|
|
||||||
import { buildRoomRoleContext } from './room-role-context.js';
|
import { buildRoomRoleContext } from './room-role-context.js';
|
||||||
import { type AgentTriggerReason } from './agent-error-detection.js';
|
import { type AgentTriggerReason } from './agent-error-detection.js';
|
||||||
import {
|
|
||||||
runClaudeRotationLoop,
|
|
||||||
runCodexRotationLoop,
|
|
||||||
} from './provider-retry.js';
|
|
||||||
import {
|
import {
|
||||||
shouldResetSessionOnAgentFailure,
|
shouldResetSessionOnAgentFailure,
|
||||||
shouldRetryFreshSessionOnAgentFailure,
|
shouldRetryFreshSessionOnAgentFailure,
|
||||||
@@ -650,27 +647,17 @@ export async function runAgentForGroup(
|
|||||||
initialTrigger: { reason: CodexRotationReason },
|
initialTrigger: { reason: CodexRotationReason },
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<'success' | 'error'> => {
|
): Promise<'success' | 'error'> => {
|
||||||
const outcome = await runCodexRotationLoop(
|
return runCodexAttemptWithRotation({
|
||||||
initialTrigger,
|
initialTrigger,
|
||||||
async () => {
|
runAttempt: () => runAttempt('codex'),
|
||||||
const attempt = await runAttempt('codex');
|
logContext: {
|
||||||
return {
|
|
||||||
output: attempt.output,
|
|
||||||
thrownError: attempt.error,
|
|
||||||
sawOutput: attempt.sawOutput,
|
|
||||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
{
|
|
||||||
chatJid,
|
chatJid,
|
||||||
group: group.name,
|
group: group.name,
|
||||||
groupFolder: group.folder,
|
groupFolder: group.folder,
|
||||||
runId,
|
runId,
|
||||||
},
|
},
|
||||||
rotationMessage,
|
rotationMessage,
|
||||||
);
|
});
|
||||||
|
|
||||||
return outcome.type === 'success' ? 'success' : 'error';
|
|
||||||
};
|
};
|
||||||
|
|
||||||
type AgentAttempt = Awaited<ReturnType<typeof runAttempt>>;
|
type AgentAttempt = Awaited<ReturnType<typeof runAttempt>>;
|
||||||
@@ -689,78 +676,63 @@ export async function runAgentForGroup(
|
|||||||
runId,
|
runId,
|
||||||
};
|
};
|
||||||
|
|
||||||
const outcome = await runClaudeRotationLoop(
|
return runClaudeAttemptWithRotation({
|
||||||
initialTrigger,
|
initialTrigger,
|
||||||
async () => {
|
runAttempt: () => runAttempt('claude'),
|
||||||
const attempt = await runAttempt('claude');
|
logContext: logCtx,
|
||||||
return {
|
|
||||||
output: attempt.output,
|
|
||||||
thrownError: attempt.error,
|
|
||||||
sawOutput: attempt.sawOutput,
|
|
||||||
sawSuccessNullResult: attempt.sawSuccessNullResultWithoutOutput,
|
|
||||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
logCtx,
|
|
||||||
rotationMessage,
|
rotationMessage,
|
||||||
);
|
onSuccess: ({ sawOutput }) => {
|
||||||
|
pairedSawOutput = sawOutput;
|
||||||
switch (outcome.type) {
|
},
|
||||||
case 'success':
|
});
|
||||||
pairedSawOutput = outcome.sawOutput;
|
|
||||||
return 'success';
|
|
||||||
case 'error':
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const retryClaudeAttemptIfNeeded = async (
|
const retryClaudeAttemptIfNeeded = async (
|
||||||
attempt: AgentAttempt,
|
attempt: AgentAttempt,
|
||||||
rotationMessage?: string | null,
|
rotationMessage?: string | null,
|
||||||
): Promise<'success' | 'error' | null> => {
|
): Promise<'success' | 'error' | null> => {
|
||||||
const trigger = resolveClaudeRetryTrigger({
|
const retryAction = await executeAttemptRetryAction({
|
||||||
canRetryClaudeCredentials,
|
|
||||||
provider,
|
provider,
|
||||||
|
canRetryClaudeCredentials,
|
||||||
|
canRetryCodex: false,
|
||||||
attempt,
|
attempt,
|
||||||
fallbackMessage: rotationMessage,
|
rotationMessage,
|
||||||
|
runClaude: retryClaudeWithRotation,
|
||||||
|
runCodex: retryCodexWithRotation,
|
||||||
});
|
});
|
||||||
if (!trigger) {
|
if (retryAction.kind !== 'claude') {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await retryClaudeWithRotation(
|
if (retryAction.result === 'error') {
|
||||||
trigger,
|
return maybeHandoffAfterError(retryAction.trigger.reason, attempt);
|
||||||
rotationMessage ?? undefined,
|
|
||||||
);
|
|
||||||
if (result === 'error') {
|
|
||||||
return maybeHandoffAfterError(trigger.reason, attempt);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
pairedExecutionStatus = 'succeeded';
|
||||||
return result;
|
return retryAction.result;
|
||||||
};
|
};
|
||||||
|
|
||||||
const retryCodexAttemptIfNeeded = async (
|
const retryCodexAttemptIfNeeded = async (
|
||||||
attempt: AgentAttempt,
|
attempt: AgentAttempt,
|
||||||
rotationMessage?: string | null,
|
rotationMessage?: string | null,
|
||||||
): Promise<'success' | 'error' | null> => {
|
): Promise<'success' | 'error' | null> => {
|
||||||
const trigger = resolveCodexRetryTrigger({
|
const retryAction = await executeAttemptRetryAction({
|
||||||
|
provider,
|
||||||
|
canRetryClaudeCredentials: false,
|
||||||
canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1,
|
canRetryCodex: !isClaudeCodeAgent && getCodexAccountCount() > 1,
|
||||||
attempt,
|
attempt,
|
||||||
rotationMessage,
|
rotationMessage,
|
||||||
|
runClaude: retryClaudeWithRotation,
|
||||||
|
runCodex: retryCodexWithRotation,
|
||||||
});
|
});
|
||||||
if (!trigger) {
|
if (retryAction.kind !== 'codex') {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await retryCodexWithRotation(
|
if (retryAction.result === 'success') {
|
||||||
{ reason: trigger.reason },
|
|
||||||
rotationMessage ?? undefined,
|
|
||||||
);
|
|
||||||
if (result === 'success') {
|
|
||||||
pairedExecutionStatus = 'succeeded';
|
pairedExecutionStatus = 'succeeded';
|
||||||
}
|
}
|
||||||
return result;
|
return retryAction.result;
|
||||||
};
|
};
|
||||||
|
|
||||||
const maybeHandoffAfterError = (
|
const maybeHandoffAfterError = (
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
buildArbiterPromptForTask,
|
buildArbiterPromptForTask,
|
||||||
buildFinalizePendingPrompt,
|
buildFinalizePendingPrompt,
|
||||||
buildOwnerPendingPrompt,
|
buildOwnerPendingPrompt,
|
||||||
|
buildPairedTurnPrompt,
|
||||||
buildReviewerPendingPrompt,
|
buildReviewerPendingPrompt,
|
||||||
} from './message-runtime-prompts.js';
|
} from './message-runtime-prompts.js';
|
||||||
import {
|
import {
|
||||||
@@ -47,23 +48,24 @@ export type BotOnlyPairedFollowUpAction =
|
|||||||
nextRole: 'owner' | 'reviewer' | 'arbiter';
|
nextRole: 'owner' | 'reviewer' | 'arbiter';
|
||||||
};
|
};
|
||||||
|
|
||||||
export function resolveLastDeliveredBotRole(
|
export type QueuedTurnDispatch = {
|
||||||
|
formatted: string;
|
||||||
|
botOnlyFollowUpAction: BotOnlyPairedFollowUpAction;
|
||||||
|
isBotOnlyPairedFollowUp: boolean;
|
||||||
|
loopCursorKey: string;
|
||||||
|
endSeq: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isBotOnlyPairedRoomTurn(
|
||||||
|
chatJid: string,
|
||||||
messages: NewMessage[],
|
messages: NewMessage[],
|
||||||
): PairedRoomRole | null {
|
): boolean {
|
||||||
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
return (
|
||||||
const message = messages[index];
|
hasReviewerLease(chatJid) &&
|
||||||
if (!message?.is_bot_message) {
|
messages.every(
|
||||||
continue;
|
(message) => message.is_from_me === true || !!message.is_bot_message,
|
||||||
}
|
)
|
||||||
if (
|
);
|
||||||
message.sender_name === 'owner' ||
|
|
||||||
message.sender_name === 'reviewer' ||
|
|
||||||
message.sender_name === 'arbiter'
|
|
||||||
) {
|
|
||||||
return message.sender_name;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildPendingPairedTurn(args: {
|
export function buildPendingPairedTurn(args: {
|
||||||
@@ -77,7 +79,6 @@ export function buildPendingPairedTurn(args: {
|
|||||||
labeledRecentMessages: Parameters<
|
labeledRecentMessages: Parameters<
|
||||||
typeof buildArbiterPromptForTask
|
typeof buildArbiterPromptForTask
|
||||||
>[0]['labeledRecentMessages'];
|
>[0]['labeledRecentMessages'];
|
||||||
lastDeliveredMessages?: NewMessage[];
|
|
||||||
resolveChannel: (taskStatus?: string | null) => Channel | null;
|
resolveChannel: (taskStatus?: string | null) => Channel | null;
|
||||||
}): PendingPairedTurn {
|
}): PendingPairedTurn {
|
||||||
const {
|
const {
|
||||||
@@ -96,10 +97,7 @@ export function buildPendingPairedTurn(args: {
|
|||||||
const lastTurnOutput = turnOutputs[turnOutputs.length - 1];
|
const lastTurnOutput = turnOutputs[turnOutputs.length - 1];
|
||||||
const nextTurnAction = resolveNextTurnAction({
|
const nextTurnAction = resolveNextTurnAction({
|
||||||
taskStatus,
|
taskStatus,
|
||||||
lastTurnOutputRole:
|
lastTurnOutputRole: lastTurnOutput?.role ?? null,
|
||||||
resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ??
|
|
||||||
lastTurnOutput?.role ??
|
|
||||||
null,
|
|
||||||
});
|
});
|
||||||
const recentMessages = getRecentChatMessages(chatJid, 20);
|
const recentMessages = getRecentChatMessages(chatJid, 20);
|
||||||
const lastHumanMessage = getLastHumanMessageContent(chatJid);
|
const lastHumanMessage = getLastHumanMessageContent(chatJid);
|
||||||
@@ -234,7 +232,6 @@ export function resolveBotOnlyPairedFollowUpAction(args: {
|
|||||||
chatJid: string;
|
chatJid: string;
|
||||||
task: PairedTask | null | undefined;
|
task: PairedTask | null | undefined;
|
||||||
isBotOnlyPairedFollowUp: boolean;
|
isBotOnlyPairedFollowUp: boolean;
|
||||||
lastDeliveredMessages?: NewMessage[];
|
|
||||||
pendingCursorSource:
|
pendingCursorSource:
|
||||||
| { seq?: number | null; timestamp?: string | null }
|
| { seq?: number | null; timestamp?: string | null }
|
||||||
| undefined;
|
| undefined;
|
||||||
@@ -249,10 +246,7 @@ export function resolveBotOnlyPairedFollowUpAction(args: {
|
|||||||
const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1);
|
const lastTurnOutput = getPairedTurnOutputs(task.id).at(-1);
|
||||||
const nextTurnAction = resolveNextTurnAction({
|
const nextTurnAction = resolveNextTurnAction({
|
||||||
taskStatus: task.status,
|
taskStatus: task.status,
|
||||||
lastTurnOutputRole:
|
lastTurnOutputRole: lastTurnOutput?.role ?? null,
|
||||||
resolveLastDeliveredBotRole(args.lastDeliveredMessages ?? []) ??
|
|
||||||
lastTurnOutput?.role ??
|
|
||||||
null,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (nextTurnAction.kind === 'finalize-owner-turn') {
|
if (nextTurnAction.kind === 'finalize-owner-turn') {
|
||||||
@@ -384,6 +378,53 @@ export async function executeBotOnlyPairedFollowUpAction(args: {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function buildQueuedTurnDispatch(args: {
|
||||||
|
chatJid: string;
|
||||||
|
timezone: string;
|
||||||
|
loopPendingTask: PairedTask | null | undefined;
|
||||||
|
rawPendingMessages: NewMessage[];
|
||||||
|
messagesToSend: NewMessage[];
|
||||||
|
labeledMessagesToSend: NewMessage[];
|
||||||
|
formatMessages: (messages: NewMessage[], timezone: string) => string;
|
||||||
|
}): QueuedTurnDispatch {
|
||||||
|
const loopCursorKey = resolveCursorKey(
|
||||||
|
args.chatJid,
|
||||||
|
args.loopPendingTask?.status,
|
||||||
|
);
|
||||||
|
const formatted = args.loopPendingTask
|
||||||
|
? buildPairedTurnPrompt({
|
||||||
|
taskId: args.loopPendingTask.id,
|
||||||
|
chatJid: args.chatJid,
|
||||||
|
timezone: args.timezone,
|
||||||
|
missedMessages: args.messagesToSend,
|
||||||
|
labeledFallbackMessages: args.labeledMessagesToSend,
|
||||||
|
turnOutputs: getPairedTurnOutputs(args.loopPendingTask.id),
|
||||||
|
})
|
||||||
|
: args.formatMessages(args.labeledMessagesToSend, args.timezone);
|
||||||
|
const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn(
|
||||||
|
args.chatJid,
|
||||||
|
args.messagesToSend,
|
||||||
|
);
|
||||||
|
const pendingCursorSource =
|
||||||
|
args.rawPendingMessages.length > 0
|
||||||
|
? args.rawPendingMessages[args.rawPendingMessages.length - 1]
|
||||||
|
: args.messagesToSend[args.messagesToSend.length - 1];
|
||||||
|
const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({
|
||||||
|
chatJid: args.chatJid,
|
||||||
|
task: args.loopPendingTask,
|
||||||
|
isBotOnlyPairedFollowUp,
|
||||||
|
pendingCursorSource,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
formatted,
|
||||||
|
botOnlyFollowUpAction,
|
||||||
|
isBotOnlyPairedFollowUp,
|
||||||
|
loopCursorKey,
|
||||||
|
endSeq: args.messagesToSend[args.messagesToSend.length - 1]?.seq ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function shouldSkipGenericFollowUpAfterDeliveryRetry(args: {
|
export function shouldSkipGenericFollowUpAfterDeliveryRetry(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
deliveryRole: PairedRoomRole;
|
deliveryRole: PairedRoomRole;
|
||||||
|
|||||||
@@ -1827,124 +1827,6 @@ describe('createMessageRuntime', () => {
|
|||||||
expect(sendMessage).not.toHaveBeenCalled();
|
expect(sendMessage).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('requeues owner follow-ups from reviewer bot-only messages even when paired turn output storage is stale', async () => {
|
|
||||||
const chatJid = 'group@test';
|
|
||||||
const group = makeGroup('codex');
|
|
||||||
const ownerChannel = makeChannel(chatJid);
|
|
||||||
const reviewerChannel = {
|
|
||||||
...makeChannel(chatJid, 'discord-review', false),
|
|
||||||
isOwnMessage: vi.fn((message) => message.sender === 'reviewer-bot@test'),
|
|
||||||
};
|
|
||||||
const enqueueMessageCheck = vi.fn();
|
|
||||||
const closeStdin = vi.fn();
|
|
||||||
const sendMessage = vi.fn(() => false);
|
|
||||||
const setLastTimestamp = vi.fn();
|
|
||||||
const saveState = vi.fn();
|
|
||||||
const lastAgentTimestamps: Record<string, string> = {};
|
|
||||||
const stopLoop = new Error('stop-message-loop');
|
|
||||||
|
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
|
||||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue({
|
|
||||||
id: 'task-active-owner-follow-up-stale-output',
|
|
||||||
chat_jid: chatJid,
|
|
||||||
group_folder: group.folder,
|
|
||||||
owner_service_id: 'codex-main',
|
|
||||||
reviewer_service_id: 'claude',
|
|
||||||
title: null,
|
|
||||||
source_ref: 'HEAD',
|
|
||||||
plan_notes: null,
|
|
||||||
review_requested_at: '2026-03-30T00:00:00.000Z',
|
|
||||||
round_trip_count: 1,
|
|
||||||
status: 'active',
|
|
||||||
arbiter_verdict: null,
|
|
||||||
arbiter_requested_at: null,
|
|
||||||
completion_reason: null,
|
|
||||||
created_at: '2026-03-30T00:00:00.000Z',
|
|
||||||
updated_at: '2026-03-30T00:00:00.000Z',
|
|
||||||
});
|
|
||||||
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
|
||||||
{
|
|
||||||
id: 1,
|
|
||||||
task_id: 'task-active-owner-follow-up-stale-output',
|
|
||||||
turn_number: 1,
|
|
||||||
role: 'owner',
|
|
||||||
output_text: 'owner 초안',
|
|
||||||
created_at: '2026-03-30T00:00:01.000Z',
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
vi.mocked(db.getNewMessages).mockReturnValue({
|
|
||||||
messages: [
|
|
||||||
{
|
|
||||||
id: 'reviewer-bot-message-owner-follow-up-stale-output',
|
|
||||||
chat_jid: chatJid,
|
|
||||||
sender: 'reviewer-bot@test',
|
|
||||||
sender_name: '리뷰어',
|
|
||||||
content: 'DONE_WITH_CONCERNS\n\nreviewer direct message',
|
|
||||||
timestamp: '2026-03-30T00:00:04.000Z',
|
|
||||||
seq: 42,
|
|
||||||
is_bot_message: true,
|
|
||||||
} as any,
|
|
||||||
],
|
|
||||||
newTimestamp: '42',
|
|
||||||
});
|
|
||||||
|
|
||||||
const runtime = createMessageRuntime({
|
|
||||||
assistantName: 'Andy',
|
|
||||||
idleTimeout: 60_000,
|
|
||||||
pollInterval: 123,
|
|
||||||
timezone: 'UTC',
|
|
||||||
triggerPattern: /^@Andy\b/i,
|
|
||||||
channels: [ownerChannel, reviewerChannel],
|
|
||||||
queue: {
|
|
||||||
registerProcess: vi.fn(),
|
|
||||||
closeStdin,
|
|
||||||
enqueueMessageCheck,
|
|
||||||
notifyIdle: vi.fn(),
|
|
||||||
sendMessage,
|
|
||||||
} as any,
|
|
||||||
getRegisteredGroups: () => ({ [chatJid]: group }),
|
|
||||||
getSessions: () => ({}),
|
|
||||||
getLastTimestamp: () => '',
|
|
||||||
setLastTimestamp,
|
|
||||||
getLastAgentTimestamps: () => lastAgentTimestamps,
|
|
||||||
saveState,
|
|
||||||
persistSession: vi.fn(),
|
|
||||||
clearSession: vi.fn(),
|
|
||||||
});
|
|
||||||
|
|
||||||
const originalSetTimeout = global.setTimeout;
|
|
||||||
const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation(((
|
|
||||||
handler: any,
|
|
||||||
timeout?: number,
|
|
||||||
...args: any[]
|
|
||||||
) => {
|
|
||||||
if (timeout === 123) {
|
|
||||||
throw stopLoop;
|
|
||||||
}
|
|
||||||
return (originalSetTimeout as any)(handler, timeout, ...args);
|
|
||||||
}) as typeof setTimeout);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await expect(runtime.startMessageLoop()).rejects.toThrow(
|
|
||||||
stopLoop.message,
|
|
||||||
);
|
|
||||||
} finally {
|
|
||||||
setTimeoutSpy.mockRestore();
|
|
||||||
}
|
|
||||||
|
|
||||||
expect(setLastTimestamp).toHaveBeenCalledWith('42');
|
|
||||||
expect(agentRunner.runAgentProcess).not.toHaveBeenCalled();
|
|
||||||
expect(closeStdin).toHaveBeenCalledWith(
|
|
||||||
chatJid,
|
|
||||||
expect.objectContaining({ reason: 'paired-pending-turn-follow-up' }),
|
|
||||||
);
|
|
||||||
expect(enqueueMessageCheck).toHaveBeenCalledWith(
|
|
||||||
chatJid,
|
|
||||||
resolveGroupIpcPath(group.folder),
|
|
||||||
);
|
|
||||||
expect(sendMessage).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => {
|
it('auto-runs an owner follow-up when a task returns to active after reviewer feedback', async () => {
|
||||||
const chatJid = 'group@test';
|
const chatJid = 'group@test';
|
||||||
const group = makeGroup('codex');
|
const group = makeGroup('codex');
|
||||||
|
|||||||
@@ -43,11 +43,11 @@ import {
|
|||||||
} from './message-runtime-rules.js';
|
} from './message-runtime-rules.js';
|
||||||
import { runAgentForGroup } from './message-agent-executor.js';
|
import { runAgentForGroup } from './message-agent-executor.js';
|
||||||
import {
|
import {
|
||||||
|
buildQueuedTurnDispatch,
|
||||||
buildPendingPairedTurn,
|
buildPendingPairedTurn,
|
||||||
executeBotOnlyPairedFollowUpAction,
|
executeBotOnlyPairedFollowUpAction,
|
||||||
executePendingPairedTurn,
|
executePendingPairedTurn,
|
||||||
resolveBotOnlyPairedFollowUpAction,
|
isBotOnlyPairedRoomTurn,
|
||||||
resolveLastDeliveredBotRole,
|
|
||||||
shouldSkipGenericFollowUpAfterDeliveryRetry,
|
shouldSkipGenericFollowUpAfterDeliveryRetry,
|
||||||
} from './message-runtime-flow.js';
|
} from './message-runtime-flow.js';
|
||||||
import { MessageTurnController } from './message-turn-controller.js';
|
import { MessageTurnController } from './message-turn-controller.js';
|
||||||
@@ -58,6 +58,7 @@ import {
|
|||||||
buildPairedTurnPrompt,
|
buildPairedTurnPrompt,
|
||||||
buildReviewerPendingPrompt,
|
buildReviewerPendingPrompt,
|
||||||
} from './message-runtime-prompts.js';
|
} from './message-runtime-prompts.js';
|
||||||
|
import { transitionPairedTaskStatus } from './paired-execution-context-shared.js';
|
||||||
import {
|
import {
|
||||||
extractSessionCommand,
|
extractSessionCommand,
|
||||||
handleSessionCommand,
|
handleSessionCommand,
|
||||||
@@ -207,15 +208,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const isBotOnlyPairedRoomTurn = (
|
|
||||||
chatJid: string,
|
|
||||||
messages: NewMessage[],
|
|
||||||
): boolean =>
|
|
||||||
hasReviewerLease(chatJid) &&
|
|
||||||
messages.every(
|
|
||||||
(message) => message.is_from_me === true || !!message.is_bot_message,
|
|
||||||
);
|
|
||||||
|
|
||||||
const enqueueScopedGroupMessageCheck = (
|
const enqueueScopedGroupMessageCheck = (
|
||||||
chatJid: string,
|
chatJid: string,
|
||||||
groupFolder: string,
|
groupFolder: string,
|
||||||
@@ -859,7 +851,6 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
chatJid,
|
chatJid,
|
||||||
getRecentChatMessages(chatJid, 20),
|
getRecentChatMessages(chatJid, 20),
|
||||||
),
|
),
|
||||||
lastDeliveredMessages: labelPairedSenders(chatJid, missedMessages),
|
|
||||||
resolveChannel,
|
resolveChannel,
|
||||||
})
|
})
|
||||||
: null;
|
: null;
|
||||||
@@ -937,10 +928,15 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
if (hasReviewerLease(chatJid)) {
|
if (hasReviewerLease(chatJid)) {
|
||||||
const task = getLatestOpenPairedTaskForChat(chatJid);
|
const task = getLatestOpenPairedTaskForChat(chatJid);
|
||||||
if (task) {
|
if (task) {
|
||||||
updatePairedTask(task.id, {
|
const now = new Date().toISOString();
|
||||||
status: 'completed',
|
transitionPairedTaskStatus({
|
||||||
completion_reason: 'stopped',
|
taskId: task.id,
|
||||||
updated_at: new Date().toISOString(),
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'completed',
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
completion_reason: 'stopped',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1072,6 +1068,191 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const processQueuedGroupDispatch = async (args: {
|
||||||
|
chatJid: string;
|
||||||
|
group: RegisteredGroup;
|
||||||
|
channel: Channel;
|
||||||
|
processableGroupMessages: NewMessage[];
|
||||||
|
}): Promise<void> => {
|
||||||
|
const { chatJid, group, channel, processableGroupMessages } = args;
|
||||||
|
const loopPendingTask = hasReviewerLease(chatJid)
|
||||||
|
? getLatestOpenPairedTaskForChat(chatJid)
|
||||||
|
: null;
|
||||||
|
const loopCursorKey = resolveCursorKey(chatJid, loopPendingTask?.status);
|
||||||
|
const rawPendingMessages = getMessagesSinceSeq(
|
||||||
|
chatJid,
|
||||||
|
deps.getLastAgentTimestamps()[loopCursorKey] || '0',
|
||||||
|
deps.assistantName,
|
||||||
|
);
|
||||||
|
const pendingMessages = filterLoopingPairedBotMessages(
|
||||||
|
chatJid,
|
||||||
|
getProcessableMessages(chatJid, rawPendingMessages, channel),
|
||||||
|
FAILURE_FINAL_TEXT,
|
||||||
|
);
|
||||||
|
const messagesToSend =
|
||||||
|
pendingMessages.length > 0 ? pendingMessages : processableGroupMessages;
|
||||||
|
const labeledMessagesToSend = labelPairedSenders(chatJid, messagesToSend);
|
||||||
|
const {
|
||||||
|
formatted,
|
||||||
|
botOnlyFollowUpAction,
|
||||||
|
isBotOnlyPairedFollowUp,
|
||||||
|
loopCursorKey: dispatchCursorKey,
|
||||||
|
endSeq,
|
||||||
|
} = buildQueuedTurnDispatch({
|
||||||
|
chatJid,
|
||||||
|
timezone: deps.timezone,
|
||||||
|
loopPendingTask,
|
||||||
|
rawPendingMessages,
|
||||||
|
messagesToSend,
|
||||||
|
labeledMessagesToSend,
|
||||||
|
formatMessages,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (
|
||||||
|
await executeBotOnlyPairedFollowUpAction({
|
||||||
|
action: botOnlyFollowUpAction,
|
||||||
|
chatJid,
|
||||||
|
group,
|
||||||
|
runId: `loop-merge-ready-${Date.now().toString(36)}`,
|
||||||
|
channel,
|
||||||
|
log: logger,
|
||||||
|
saveState: deps.saveState,
|
||||||
|
lastAgentTimestamps: deps.getLastAgentTimestamps(),
|
||||||
|
executeTurn,
|
||||||
|
enqueueGroupMessageCheck: () =>
|
||||||
|
enqueueScopedGroupMessageCheck(chatJid, group.folder),
|
||||||
|
closeStdin: () =>
|
||||||
|
deps.queue.closeStdin(chatJid, {
|
||||||
|
reason: 'paired-pending-turn-follow-up',
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (deps.queue.sendMessage(chatJid, formatted)) {
|
||||||
|
if (endSeq != null) {
|
||||||
|
advanceLastAgentCursor(
|
||||||
|
deps.getLastAgentTimestamps(),
|
||||||
|
deps.saveState,
|
||||||
|
chatJid,
|
||||||
|
endSeq,
|
||||||
|
dispatchCursorKey,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
transition: 'typing:on',
|
||||||
|
source: 'follow-up-queued',
|
||||||
|
chatJid,
|
||||||
|
group: group.name,
|
||||||
|
groupFolder: group.folder,
|
||||||
|
endSeq: endSeq ?? null,
|
||||||
|
suppressed: isBotOnlyPairedFollowUp,
|
||||||
|
},
|
||||||
|
'Typing indicator transition',
|
||||||
|
);
|
||||||
|
if (!isBotOnlyPairedFollowUp) {
|
||||||
|
await channel
|
||||||
|
.setTyping?.(chatJid, true)
|
||||||
|
?.catch((err) =>
|
||||||
|
logger.warn(
|
||||||
|
{ chatJid, err },
|
||||||
|
'Failed to set typing indicator',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
enqueueScopedGroupMessageCheck(chatJid, group.folder);
|
||||||
|
};
|
||||||
|
|
||||||
|
const processLoopGroupMessages = async (args: {
|
||||||
|
chatJid: string;
|
||||||
|
group: RegisteredGroup;
|
||||||
|
groupMessages: NewMessage[];
|
||||||
|
channel: Channel;
|
||||||
|
}): Promise<void> => {
|
||||||
|
const { chatJid, group, groupMessages, channel } = args;
|
||||||
|
const isMainGroup = group.isMain === true;
|
||||||
|
const processableGroupMessages = getProcessableMessages(
|
||||||
|
chatJid,
|
||||||
|
groupMessages,
|
||||||
|
channel,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (processableGroupMessages.length === 0) {
|
||||||
|
const lastIgnored = groupMessages[groupMessages.length - 1];
|
||||||
|
if (lastIgnored?.seq != null) {
|
||||||
|
advanceLastAgentCursor(
|
||||||
|
deps.getLastAgentTimestamps(),
|
||||||
|
deps.saveState,
|
||||||
|
chatJid,
|
||||||
|
lastIgnored.seq,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)) {
|
||||||
|
const lastIgnored =
|
||||||
|
processableGroupMessages[processableGroupMessages.length - 1];
|
||||||
|
if (lastIgnored?.seq != null) {
|
||||||
|
advanceLastAgentCursor(
|
||||||
|
deps.getLastAgentTimestamps(),
|
||||||
|
deps.saveState,
|
||||||
|
chatJid,
|
||||||
|
lastIgnored.seq,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
logger.info(
|
||||||
|
{ chatJid, group: group.name, groupFolder: group.folder },
|
||||||
|
'Bot-collaboration timeout: no recent human message, skipping',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const loopCmdMsg = groupMessages.find(
|
||||||
|
(msg) => extractSessionCommand(msg.content, deps.triggerPattern) !== null,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (loopCmdMsg) {
|
||||||
|
if (
|
||||||
|
isSessionCommandAllowed(
|
||||||
|
isMainGroup,
|
||||||
|
loopCmdMsg.is_from_me === true,
|
||||||
|
isSessionCommandSenderAllowed(loopCmdMsg.sender),
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
deps.queue.closeStdin(chatJid, {
|
||||||
|
reason: 'session-command-detected',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
deps.queue.enqueueMessageCheck(chatJid);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!hasAllowedTrigger({
|
||||||
|
chatJid,
|
||||||
|
messages: processableGroupMessages,
|
||||||
|
group,
|
||||||
|
triggerPattern: deps.triggerPattern,
|
||||||
|
hasImplicitContinuationWindow: continuationTracker.has,
|
||||||
|
})
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await processQueuedGroupDispatch({
|
||||||
|
chatJid,
|
||||||
|
group,
|
||||||
|
channel,
|
||||||
|
processableGroupMessages,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const startMessageLoop = async (): Promise<void> => {
|
const startMessageLoop = async (): Promise<void> => {
|
||||||
if (messageLoopRunning) {
|
if (messageLoopRunning) {
|
||||||
logger.debug('Message loop already running, skipping duplicate start');
|
logger.debug('Message loop already running, skipping duplicate start');
|
||||||
@@ -1120,193 +1301,12 @@ export function createMessageRuntime(deps: MessageRuntimeDeps): {
|
|||||||
);
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
await processLoopGroupMessages({
|
||||||
const isMainGroup = group.isMain === true;
|
|
||||||
const processableGroupMessages = getProcessableMessages(
|
|
||||||
chatJid,
|
chatJid,
|
||||||
|
group,
|
||||||
groupMessages,
|
groupMessages,
|
||||||
channel,
|
channel,
|
||||||
);
|
|
||||||
|
|
||||||
if (processableGroupMessages.length === 0) {
|
|
||||||
const lastIgnored = groupMessages[groupMessages.length - 1];
|
|
||||||
if (lastIgnored?.seq != null) {
|
|
||||||
advanceLastAgentCursor(
|
|
||||||
deps.getLastAgentTimestamps(),
|
|
||||||
deps.saveState,
|
|
||||||
chatJid,
|
|
||||||
lastIgnored.seq,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
shouldSkipBotOnlyCollaboration(chatJid, processableGroupMessages)
|
|
||||||
) {
|
|
||||||
const lastIgnored =
|
|
||||||
processableGroupMessages[processableGroupMessages.length - 1];
|
|
||||||
if (lastIgnored?.seq != null) {
|
|
||||||
advanceLastAgentCursor(
|
|
||||||
deps.getLastAgentTimestamps(),
|
|
||||||
deps.saveState,
|
|
||||||
chatJid,
|
|
||||||
lastIgnored.seq,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
logger.info(
|
|
||||||
{ chatJid, group: group.name, groupFolder: group.folder },
|
|
||||||
'Bot-collaboration timeout: no recent human message, skipping',
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const loopCmdMsg = groupMessages.find(
|
|
||||||
(msg) =>
|
|
||||||
extractSessionCommand(msg.content, deps.triggerPattern) !==
|
|
||||||
null,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (loopCmdMsg) {
|
|
||||||
if (
|
|
||||||
isSessionCommandAllowed(
|
|
||||||
isMainGroup,
|
|
||||||
loopCmdMsg.is_from_me === true,
|
|
||||||
isSessionCommandSenderAllowed(loopCmdMsg.sender),
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
deps.queue.closeStdin(chatJid, {
|
|
||||||
reason: 'session-command-detected',
|
|
||||||
});
|
|
||||||
}
|
|
||||||
deps.queue.enqueueMessageCheck(chatJid);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
!hasAllowedTrigger({
|
|
||||||
chatJid,
|
|
||||||
messages: processableGroupMessages,
|
|
||||||
group,
|
|
||||||
triggerPattern: deps.triggerPattern,
|
|
||||||
hasImplicitContinuationWindow: continuationTracker.has,
|
|
||||||
})
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use role-aware cursor for paired rooms so the reviewer
|
|
||||||
// always sees the owner's last messages and vice versa.
|
|
||||||
const loopPendingTask = hasReviewerLease(chatJid)
|
|
||||||
? getLatestOpenPairedTaskForChat(chatJid)
|
|
||||||
: null;
|
|
||||||
const loopCursorKey = resolveCursorKey(
|
|
||||||
chatJid,
|
|
||||||
loopPendingTask?.status,
|
|
||||||
);
|
|
||||||
|
|
||||||
const rawPendingMessages = getMessagesSinceSeq(
|
|
||||||
chatJid,
|
|
||||||
deps.getLastAgentTimestamps()[loopCursorKey] || '0',
|
|
||||||
deps.assistantName,
|
|
||||||
);
|
|
||||||
const pendingMessages = filterLoopingPairedBotMessages(
|
|
||||||
chatJid,
|
|
||||||
getProcessableMessages(chatJid, rawPendingMessages, channel),
|
|
||||||
FAILURE_FINAL_TEXT,
|
|
||||||
);
|
|
||||||
const messagesToSend =
|
|
||||||
pendingMessages.length > 0
|
|
||||||
? pendingMessages
|
|
||||||
: processableGroupMessages;
|
|
||||||
const labeledMessagesToSend = labelPairedSenders(
|
|
||||||
chatJid,
|
|
||||||
messagesToSend,
|
|
||||||
);
|
|
||||||
const formatted = loopPendingTask
|
|
||||||
? buildPairedTurnPrompt({
|
|
||||||
taskId: loopPendingTask.id,
|
|
||||||
chatJid,
|
|
||||||
timezone: deps.timezone,
|
|
||||||
missedMessages: messagesToSend,
|
|
||||||
labeledFallbackMessages: labeledMessagesToSend,
|
|
||||||
turnOutputs: getPairedTurnOutputs(loopPendingTask.id),
|
|
||||||
})
|
|
||||||
: formatMessages(labeledMessagesToSend, deps.timezone);
|
|
||||||
const isBotOnlyPairedFollowUp = isBotOnlyPairedRoomTurn(
|
|
||||||
chatJid,
|
|
||||||
messagesToSend,
|
|
||||||
);
|
|
||||||
const pendingCursorSource =
|
|
||||||
rawPendingMessages.length > 0
|
|
||||||
? rawPendingMessages[rawPendingMessages.length - 1]
|
|
||||||
: messagesToSend[messagesToSend.length - 1];
|
|
||||||
const botOnlyFollowUpAction = resolveBotOnlyPairedFollowUpAction({
|
|
||||||
chatJid,
|
|
||||||
task: loopPendingTask,
|
|
||||||
isBotOnlyPairedFollowUp,
|
|
||||||
lastDeliveredMessages: labeledMessagesToSend,
|
|
||||||
pendingCursorSource,
|
|
||||||
});
|
});
|
||||||
if (
|
|
||||||
await executeBotOnlyPairedFollowUpAction({
|
|
||||||
action: botOnlyFollowUpAction,
|
|
||||||
chatJid,
|
|
||||||
group,
|
|
||||||
runId: `loop-merge-ready-${Date.now().toString(36)}`,
|
|
||||||
channel,
|
|
||||||
log: logger,
|
|
||||||
saveState: deps.saveState,
|
|
||||||
lastAgentTimestamps: deps.getLastAgentTimestamps(),
|
|
||||||
executeTurn,
|
|
||||||
enqueueGroupMessageCheck: () =>
|
|
||||||
enqueueScopedGroupMessageCheck(chatJid, group.folder),
|
|
||||||
closeStdin: () =>
|
|
||||||
deps.queue.closeStdin(chatJid, {
|
|
||||||
reason: 'paired-pending-turn-follow-up',
|
|
||||||
}),
|
|
||||||
})
|
|
||||||
) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (deps.queue.sendMessage(chatJid, formatted)) {
|
|
||||||
const endSeq = messagesToSend[messagesToSend.length - 1]?.seq;
|
|
||||||
if (endSeq != null) {
|
|
||||||
advanceLastAgentCursor(
|
|
||||||
deps.getLastAgentTimestamps(),
|
|
||||||
deps.saveState,
|
|
||||||
chatJid,
|
|
||||||
endSeq,
|
|
||||||
loopCursorKey,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
transition: 'typing:on',
|
|
||||||
source: 'follow-up-queued',
|
|
||||||
chatJid,
|
|
||||||
group: group.name,
|
|
||||||
groupFolder: group.folder,
|
|
||||||
endSeq: endSeq ?? null,
|
|
||||||
suppressed: isBotOnlyPairedFollowUp,
|
|
||||||
},
|
|
||||||
'Typing indicator transition',
|
|
||||||
);
|
|
||||||
if (!isBotOnlyPairedFollowUp) {
|
|
||||||
await channel
|
|
||||||
.setTyping?.(chatJid, true)
|
|
||||||
?.catch((err) =>
|
|
||||||
logger.warn(
|
|
||||||
{ chatJid, err },
|
|
||||||
'Failed to set typing indicator',
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
enqueueScopedGroupMessageCheck(chatJid, group.folder);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
|
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
|
||||||
import { updatePairedTask } from './db.js';
|
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { classifyArbiterVerdict } from './paired-execution-context-shared.js';
|
import {
|
||||||
|
classifyArbiterVerdict,
|
||||||
|
transitionPairedTaskStatus,
|
||||||
|
} from './paired-execution-context-shared.js';
|
||||||
import type { PairedTask } from './types.js';
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
export function handleFailedArbiterExecution(args: {
|
export function handleFailedArbiterExecution(args: {
|
||||||
@@ -15,7 +17,12 @@ export function handleFailedArbiterExecution(args: {
|
|||||||
? 'arbiter_requested'
|
? 'arbiter_requested'
|
||||||
: task.status;
|
: task.status;
|
||||||
if (fallbackStatus !== task.status) {
|
if (fallbackStatus !== task.status) {
|
||||||
updatePairedTask(taskId, { status: fallbackStatus, updated_at: now });
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: fallbackStatus,
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
taskId,
|
taskId,
|
||||||
@@ -45,11 +52,15 @@ export function handleArbiterCompletion(args: {
|
|||||||
case 'proceed':
|
case 'proceed':
|
||||||
case 'revise':
|
case 'revise':
|
||||||
case 'reset':
|
case 'reset':
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'active',
|
taskId,
|
||||||
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1),
|
currentStatus: 'in_arbitration',
|
||||||
arbiter_verdict: arbiterVerdict,
|
nextStatus: 'active',
|
||||||
updated_at: now,
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1),
|
||||||
|
arbiter_verdict: arbiterVerdict,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, arbiterVerdict },
|
{ taskId, arbiterVerdict },
|
||||||
@@ -57,20 +68,28 @@ export function handleArbiterCompletion(args: {
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
case 'escalate':
|
case 'escalate':
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'completed',
|
taskId,
|
||||||
arbiter_verdict: 'escalate',
|
currentStatus: 'in_arbitration',
|
||||||
completion_reason: 'arbiter_escalated',
|
nextStatus: 'completed',
|
||||||
updated_at: now,
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
completion_reason: 'arbiter_escalated',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info({ taskId }, 'Arbiter escalated to user — task completed');
|
logger.info({ taskId }, 'Arbiter escalated to user — task completed');
|
||||||
return;
|
return;
|
||||||
default:
|
default:
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'active',
|
taskId,
|
||||||
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1),
|
currentStatus: 'in_arbitration',
|
||||||
arbiter_verdict: 'unknown',
|
nextStatus: 'active',
|
||||||
updated_at: now,
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
round_trip_count: Math.max(0, ARBITER_DEADLOCK_THRESHOLD - 1),
|
||||||
|
arbiter_verdict: 'unknown',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ taskId, summary: summary?.slice(0, 200) },
|
{ taskId, summary: summary?.slice(0, 200) },
|
||||||
|
|||||||
@@ -5,15 +5,16 @@ import {
|
|||||||
import {
|
import {
|
||||||
getPairedWorkspace,
|
getPairedWorkspace,
|
||||||
hasActiveCiWatcherForChat,
|
hasActiveCiWatcherForChat,
|
||||||
updatePairedTask,
|
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
||||||
import {
|
import {
|
||||||
|
applyPairedTaskPatch,
|
||||||
classifyVerdict,
|
classifyVerdict,
|
||||||
hasCodeChangesSinceRef,
|
hasCodeChangesSinceRef,
|
||||||
requestArbiterOrEscalate,
|
requestArbiterOrEscalate,
|
||||||
resolveCanonicalSourceRef,
|
resolveCanonicalSourceRef,
|
||||||
|
transitionPairedTaskStatus,
|
||||||
} from './paired-execution-context-shared.js';
|
} from './paired-execution-context-shared.js';
|
||||||
import type { PairedTask } from './types.js';
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
@@ -24,7 +25,12 @@ export function handleFailedOwnerExecution(args: {
|
|||||||
const { task, taskId } = args;
|
const { task, taskId } = args;
|
||||||
if (task.status !== 'active') {
|
if (task.status !== 'active') {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
updatePairedTask(taskId, { status: 'active', updated_at: now });
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'active',
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, role: 'owner', previousStatus: task.status },
|
{ taskId, role: 'owner', previousStatus: task.status },
|
||||||
'Reset task to active after failed execution',
|
'Reset task to active after failed execution',
|
||||||
@@ -44,6 +50,7 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') {
|
if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
now,
|
now,
|
||||||
arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter',
|
arbiterLogMessage: 'Owner blocked during finalize — requesting arbiter',
|
||||||
escalateLogMessage: 'Owner blocked during finalize — escalating to user',
|
escalateLogMessage: 'Owner blocked during finalize — escalating to user',
|
||||||
@@ -60,6 +67,7 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
|
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
now,
|
now,
|
||||||
arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter',
|
arbiterLogMessage: 'Owner finalize loop detected — requesting arbiter',
|
||||||
escalateLogMessage: 'Owner finalize loop detected — escalating to user',
|
escalateLogMessage: 'Owner finalize loop detected — escalating to user',
|
||||||
@@ -71,7 +79,12 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
updatePairedTask(taskId, { status: 'active', updated_at: now });
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'active',
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
taskId,
|
taskId,
|
||||||
@@ -92,6 +105,7 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
|
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
now,
|
now,
|
||||||
arbiterLogMessage:
|
arbiterLogMessage:
|
||||||
'Owner finalize DONE loop detected — requesting arbiter',
|
'Owner finalize DONE loop detected — requesting arbiter',
|
||||||
@@ -116,10 +130,14 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'completed',
|
taskId,
|
||||||
completion_reason: 'done',
|
currentStatus: task.status,
|
||||||
updated_at: now,
|
nextStatus: 'completed',
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
completion_reason: 'done',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, hasNewChanges, summary: summary?.slice(0, 100) },
|
{ taskId, hasNewChanges, summary: summary?.slice(0, 100) },
|
||||||
@@ -161,6 +179,7 @@ export function handleOwnerCompletion(args: {
|
|||||||
if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') {
|
if (ownerVerdict === 'blocked' || ownerVerdict === 'needs_context') {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
now,
|
now,
|
||||||
arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter',
|
arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter',
|
||||||
escalateLogMessage: 'Owner blocked/needs_context — escalating to user',
|
escalateLogMessage: 'Owner blocked/needs_context — escalating to user',
|
||||||
@@ -188,10 +207,12 @@ export function handleOwnerCompletion(args: {
|
|||||||
|
|
||||||
const result = markPairedTaskReviewReady(taskId);
|
const result = markPairedTaskReviewReady(taskId);
|
||||||
if (result) {
|
if (result) {
|
||||||
updatePairedTask(taskId, {
|
applyPairedTaskPatch({
|
||||||
round_trip_count: task.round_trip_count + 1,
|
taskId,
|
||||||
review_requested_at: now,
|
updatedAt: now,
|
||||||
updated_at: now,
|
patch: {
|
||||||
|
round_trip_count: task.round_trip_count + 1,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, roundTrip: task.round_trip_count + 1 },
|
{ taskId, roundTrip: task.round_trip_count + 1 },
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
classifyVerdict,
|
classifyVerdict,
|
||||||
requestArbiterOrEscalate,
|
requestArbiterOrEscalate,
|
||||||
resolveCanonicalSourceRef,
|
resolveCanonicalSourceRef,
|
||||||
|
transitionPairedTaskStatus,
|
||||||
} from './paired-execution-context-shared.js';
|
} from './paired-execution-context-shared.js';
|
||||||
import type { PairedTask } from './types.js';
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
@@ -29,11 +30,15 @@ export function handleFailedReviewerExecution(args: {
|
|||||||
verdict === 'done' && ownerWs?.workspace_dir
|
verdict === 'done' && ownerWs?.workspace_dir
|
||||||
? resolveCanonicalSourceRef(ownerWs.workspace_dir)
|
? resolveCanonicalSourceRef(ownerWs.workspace_dir)
|
||||||
: task.source_ref;
|
: task.source_ref;
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: verdict === 'done' ? 'merge_ready' : 'completed',
|
taskId,
|
||||||
...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}),
|
currentStatus: task.status,
|
||||||
...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}),
|
nextStatus: verdict === 'done' ? 'merge_ready' : 'completed',
|
||||||
updated_at: now,
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
...(verdict === 'done' ? { source_ref: approvedSourceRef } : {}),
|
||||||
|
...(verdict !== 'done' ? { completion_reason: 'escalated' } : {}),
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
@@ -53,7 +58,12 @@ export function handleFailedReviewerExecution(args: {
|
|||||||
? 'review_ready'
|
? 'review_ready'
|
||||||
: task.status;
|
: task.status;
|
||||||
if (fallbackStatus !== task.status) {
|
if (fallbackStatus !== task.status) {
|
||||||
updatePairedTask(taskId, { status: fallbackStatus, updated_at: now });
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: fallbackStatus,
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
taskId,
|
taskId,
|
||||||
@@ -81,10 +91,14 @@ export function handleReviewerCompletion(args: {
|
|||||||
const approvedSourceRef = ownerWs?.workspace_dir
|
const approvedSourceRef = ownerWs?.workspace_dir
|
||||||
? resolveCanonicalSourceRef(ownerWs.workspace_dir)
|
? resolveCanonicalSourceRef(ownerWs.workspace_dir)
|
||||||
: task.source_ref;
|
: task.source_ref;
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'merge_ready',
|
taskId,
|
||||||
source_ref: approvedSourceRef,
|
currentStatus: task.status,
|
||||||
updated_at: now,
|
nextStatus: 'merge_ready',
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
source_ref: approvedSourceRef,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
@@ -102,6 +116,7 @@ export function handleReviewerCompletion(args: {
|
|||||||
case 'needs_context':
|
case 'needs_context':
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
now,
|
now,
|
||||||
arbiterLogMessage:
|
arbiterLogMessage:
|
||||||
'Reviewer blocked/needs_context — requesting arbiter before escalating',
|
'Reviewer blocked/needs_context — requesting arbiter before escalating',
|
||||||
@@ -116,6 +131,7 @@ export function handleReviewerCompletion(args: {
|
|||||||
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
|
if (task.round_trip_count >= ARBITER_DEADLOCK_THRESHOLD) {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
taskId,
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
now,
|
now,
|
||||||
arbiterLogMessage:
|
arbiterLogMessage:
|
||||||
'Deadlock detected — requesting arbiter intervention',
|
'Deadlock detected — requesting arbiter intervention',
|
||||||
@@ -129,7 +145,12 @@ export function handleReviewerCompletion(args: {
|
|||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
updatePairedTask(taskId, { status: 'active', updated_at: now });
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'active',
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, verdict },
|
{ taskId, verdict },
|
||||||
'Reviewer has feedback, task set back to active for owner',
|
'Reviewer has feedback, task set back to active for owner',
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { execFileSync } from 'child_process';
|
|||||||
import { isArbiterEnabled } from './config.js';
|
import { isArbiterEnabled } from './config.js';
|
||||||
import { updatePairedTask } from './db.js';
|
import { updatePairedTask } from './db.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import type { PairedTaskStatus } from './types.js';
|
||||||
|
|
||||||
export type Verdict =
|
export type Verdict =
|
||||||
| 'done'
|
| 'done'
|
||||||
@@ -93,29 +94,116 @@ export function hasCodeChangesSinceRef(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ALLOWED_PAIRED_STATUS_TRANSITIONS: Record<
|
||||||
|
PairedTaskStatus,
|
||||||
|
ReadonlySet<PairedTaskStatus>
|
||||||
|
> = {
|
||||||
|
active: new Set(['review_ready', 'arbiter_requested', 'completed']),
|
||||||
|
review_ready: new Set([
|
||||||
|
'active',
|
||||||
|
'in_review',
|
||||||
|
'arbiter_requested',
|
||||||
|
'completed',
|
||||||
|
]),
|
||||||
|
in_review: new Set([
|
||||||
|
'active',
|
||||||
|
'review_ready',
|
||||||
|
'merge_ready',
|
||||||
|
'arbiter_requested',
|
||||||
|
'completed',
|
||||||
|
]),
|
||||||
|
merge_ready: new Set(['active', 'arbiter_requested', 'completed']),
|
||||||
|
completed: new Set(),
|
||||||
|
arbiter_requested: new Set(['in_arbitration', 'completed']),
|
||||||
|
in_arbitration: new Set(['active', 'arbiter_requested', 'completed']),
|
||||||
|
};
|
||||||
|
|
||||||
|
export function assertPairedTaskStatusTransition(args: {
|
||||||
|
currentStatus: PairedTaskStatus;
|
||||||
|
nextStatus: PairedTaskStatus;
|
||||||
|
}): void {
|
||||||
|
const { currentStatus, nextStatus } = args;
|
||||||
|
if (currentStatus === nextStatus) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ALLOWED_PAIRED_STATUS_TRANSITIONS[currentStatus].has(nextStatus)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Invalid paired task status transition: ${currentStatus} -> ${nextStatus}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function transitionPairedTaskStatus(args: {
|
||||||
|
taskId: string;
|
||||||
|
currentStatus: PairedTaskStatus;
|
||||||
|
nextStatus: PairedTaskStatus;
|
||||||
|
updatedAt: string;
|
||||||
|
patch?: Omit<Parameters<typeof updatePairedTask>[1], 'status' | 'updated_at'>;
|
||||||
|
}): void {
|
||||||
|
assertPairedTaskStatusTransition({
|
||||||
|
currentStatus: args.currentStatus,
|
||||||
|
nextStatus: args.nextStatus,
|
||||||
|
});
|
||||||
|
|
||||||
|
updatePairedTask(args.taskId, {
|
||||||
|
...args.patch,
|
||||||
|
status: args.nextStatus,
|
||||||
|
updated_at: args.updatedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function applyPairedTaskPatch(args: {
|
||||||
|
taskId: string;
|
||||||
|
updatedAt: string;
|
||||||
|
patch: Omit<Parameters<typeof updatePairedTask>[1], 'updated_at'>;
|
||||||
|
}): void {
|
||||||
|
updatePairedTask(args.taskId, {
|
||||||
|
...args.patch,
|
||||||
|
updated_at: args.updatedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function requestArbiterOrEscalate(args: {
|
export function requestArbiterOrEscalate(args: {
|
||||||
taskId: string;
|
taskId: string;
|
||||||
|
currentStatus: PairedTaskStatus;
|
||||||
now: string;
|
now: string;
|
||||||
arbiterLogMessage: string;
|
arbiterLogMessage: string;
|
||||||
escalateLogMessage: string;
|
escalateLogMessage: string;
|
||||||
logContext?: Record<string, unknown>;
|
logContext?: Record<string, unknown>;
|
||||||
}): void {
|
}): void {
|
||||||
const { taskId, now, arbiterLogMessage, escalateLogMessage, logContext } =
|
const {
|
||||||
args;
|
taskId,
|
||||||
|
currentStatus,
|
||||||
|
now,
|
||||||
|
arbiterLogMessage,
|
||||||
|
escalateLogMessage,
|
||||||
|
logContext,
|
||||||
|
} = args;
|
||||||
if (isArbiterEnabled()) {
|
if (isArbiterEnabled()) {
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'arbiter_requested',
|
taskId,
|
||||||
arbiter_requested_at: now,
|
currentStatus,
|
||||||
updated_at: now,
|
nextStatus: 'arbiter_requested',
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
arbiter_requested_at: now,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(logContext ?? { taskId }, arbiterLogMessage);
|
logger.info(logContext ?? { taskId }, arbiterLogMessage);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
updatePairedTask(taskId, {
|
transitionPairedTaskStatus({
|
||||||
status: 'completed',
|
taskId,
|
||||||
completion_reason: 'escalated',
|
currentStatus,
|
||||||
updated_at: now,
|
nextStatus: 'completed',
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
completion_reason: 'escalated',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
logger.info(logContext ?? { taskId }, escalateLogMessage);
|
logger.info(logContext ?? { taskId }, escalateLogMessage);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -565,7 +565,6 @@ describe('paired execution context', () => {
|
|||||||
'task-1',
|
'task-1',
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
round_trip_count: 2,
|
round_trip_count: 2,
|
||||||
review_requested_at: expect.any(String),
|
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ import {
|
|||||||
getPairedTaskById,
|
getPairedTaskById,
|
||||||
getPairedWorkspace,
|
getPairedWorkspace,
|
||||||
hasActiveCiWatcherForChat,
|
hasActiveCiWatcherForChat,
|
||||||
updatePairedTask,
|
|
||||||
upsertPairedProject,
|
upsertPairedProject,
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
@@ -36,8 +35,10 @@ import {
|
|||||||
handleReviewerCompletion,
|
handleReviewerCompletion,
|
||||||
} from './paired-execution-context-reviewer.js';
|
} from './paired-execution-context-reviewer.js';
|
||||||
import {
|
import {
|
||||||
|
applyPairedTaskPatch,
|
||||||
resolveCanonicalSourceRef,
|
resolveCanonicalSourceRef,
|
||||||
requestArbiterOrEscalate,
|
requestArbiterOrEscalate,
|
||||||
|
transitionPairedTaskStatus,
|
||||||
} from './paired-execution-context-shared.js';
|
} from './paired-execution-context-shared.js';
|
||||||
import {
|
import {
|
||||||
markPairedTaskReviewReady,
|
markPairedTaskReviewReady,
|
||||||
@@ -226,11 +227,25 @@ export function preparePairedExecutionContext(args: {
|
|||||||
latestTask.status === 'review_ready' ||
|
latestTask.status === 'review_ready' ||
|
||||||
latestTask.status === 'in_review';
|
latestTask.status === 'in_review';
|
||||||
if (hasHuman || needsStatusReset) {
|
if (hasHuman || needsStatusReset) {
|
||||||
updatePairedTask(latestTask.id, {
|
if (needsStatusReset) {
|
||||||
...(hasHuman ? { round_trip_count: 0 } : {}),
|
transitionPairedTaskStatus({
|
||||||
...(needsStatusReset ? { status: 'active' as const } : {}),
|
taskId: latestTask.id,
|
||||||
updated_at: now,
|
currentStatus: latestTask.status,
|
||||||
});
|
nextStatus: 'active',
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
...(hasHuman ? { round_trip_count: 0 } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
applyPairedTaskPatch({
|
||||||
|
taskId: latestTask.id,
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
...(hasHuman ? { round_trip_count: 0 } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Use a stable per-channel worktree (not per-task) so the Claude SDK
|
// Use a stable per-channel worktree (not per-task) so the Claude SDK
|
||||||
// session persists across tasks. Different channels still get isolation.
|
// session persists across tasks. Different channels still get isolation.
|
||||||
@@ -241,9 +256,12 @@ export function preparePairedExecutionContext(args: {
|
|||||||
if (workspace?.workspace_dir && latestTask.status === 'active') {
|
if (workspace?.workspace_dir && latestTask.status === 'active') {
|
||||||
const wsRef = resolveCanonicalSourceRef(workspace.workspace_dir);
|
const wsRef = resolveCanonicalSourceRef(workspace.workspace_dir);
|
||||||
if (wsRef !== latestTask.source_ref) {
|
if (wsRef !== latestTask.source_ref) {
|
||||||
updatePairedTask(latestTask.id, {
|
applyPairedTaskPatch({
|
||||||
source_ref: wsRef,
|
taskId: latestTask.id,
|
||||||
updated_at: now,
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
source_ref: wsRef,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -253,9 +271,11 @@ export function preparePairedExecutionContext(args: {
|
|||||||
blockMessage = reviewerWorkspace.blockMessage;
|
blockMessage = reviewerWorkspace.blockMessage;
|
||||||
const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask;
|
const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask;
|
||||||
if (workspace && refreshedTask.status === 'review_ready') {
|
if (workspace && refreshedTask.status === 'review_ready') {
|
||||||
updatePairedTask(latestTask.id, {
|
transitionPairedTaskStatus({
|
||||||
status: 'in_review',
|
taskId: latestTask.id,
|
||||||
updated_at: now,
|
currentStatus: refreshedTask.status,
|
||||||
|
nextStatus: 'in_review',
|
||||||
|
updatedAt: now,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if (roomRoleContext.role === 'arbiter') {
|
} else if (roomRoleContext.role === 'arbiter') {
|
||||||
@@ -265,9 +285,11 @@ export function preparePairedExecutionContext(args: {
|
|||||||
blockMessage = reviewerWorkspace.blockMessage;
|
blockMessage = reviewerWorkspace.blockMessage;
|
||||||
const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask;
|
const refreshedTask = getPairedTaskById(latestTask.id) ?? latestTask;
|
||||||
if (workspace && refreshedTask.status === 'arbiter_requested') {
|
if (workspace && refreshedTask.status === 'arbiter_requested') {
|
||||||
updatePairedTask(latestTask.id, {
|
transitionPairedTaskStatus({
|
||||||
status: 'in_arbitration',
|
taskId: latestTask.id,
|
||||||
updated_at: now,
|
currentStatus: refreshedTask.status,
|
||||||
|
nextStatus: 'in_arbitration',
|
||||||
|
updatedAt: now,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -258,6 +258,31 @@ describe('paired workspace manager', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('leaves review_requested_at untouched when review handoff aborts before an owner workspace exists', async () => {
|
||||||
|
const { db, manager } = await loadModules();
|
||||||
|
db._initTestDatabase();
|
||||||
|
|
||||||
|
const canonicalDir = path.join(tempRoot, 'canonical');
|
||||||
|
initCanonicalRepo(canonicalDir);
|
||||||
|
seedPairedTask(db, canonicalDir, {
|
||||||
|
taskId: 'paired-task-no-owner-workspace',
|
||||||
|
groupFolder: 'no-owner-workspace-room',
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = manager.markPairedTaskReviewReady(
|
||||||
|
'paired-task-no-owner-workspace',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toBeNull();
|
||||||
|
expect(
|
||||||
|
db.getPairedTaskById('paired-task-no-owner-workspace')
|
||||||
|
?.review_requested_at,
|
||||||
|
).toBeNull();
|
||||||
|
expect(
|
||||||
|
db.getPairedTaskById('paired-task-no-owner-workspace')?.status,
|
||||||
|
).toBe('active');
|
||||||
|
});
|
||||||
|
|
||||||
it('uses the shared DB owner workspace across service-local data dirs', async () => {
|
it('uses the shared DB owner workspace across service-local data dirs', async () => {
|
||||||
const canonicalDir = path.join(tempRoot, 'canonical');
|
const canonicalDir = path.join(tempRoot, 'canonical');
|
||||||
fs.mkdirSync(canonicalDir, { recursive: true });
|
fs.mkdirSync(canonicalDir, { recursive: true });
|
||||||
|
|||||||
@@ -8,11 +8,11 @@ import {
|
|||||||
getPairedProject,
|
getPairedProject,
|
||||||
getPairedTaskById,
|
getPairedTaskById,
|
||||||
getPairedWorkspace,
|
getPairedWorkspace,
|
||||||
updatePairedTask,
|
|
||||||
upsertPairedWorkspace,
|
upsertPairedWorkspace,
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
import { resolvePairedTaskWorkspacePath } from './group-folder.js';
|
import { resolvePairedTaskWorkspacePath } from './group-folder.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import { transitionPairedTaskStatus } from './paired-execution-context-shared.js';
|
||||||
import type { PairedTask, PairedWorkspace } from './types.js';
|
import type { PairedTask, PairedWorkspace } from './types.js';
|
||||||
import { ensureWorkspaceDependenciesInstalled } from './workspace-package-manager.js';
|
import { ensureWorkspaceDependenciesInstalled } from './workspace-package-manager.js';
|
||||||
|
|
||||||
@@ -757,10 +757,6 @@ export function markPairedTaskReviewReady(taskId: string): {
|
|||||||
reviewerWorkspace: PairedWorkspace;
|
reviewerWorkspace: PairedWorkspace;
|
||||||
} | null {
|
} | null {
|
||||||
const requestedAt = new Date().toISOString();
|
const requestedAt = new Date().toISOString();
|
||||||
updatePairedTask(taskId, {
|
|
||||||
review_requested_at: requestedAt,
|
|
||||||
updated_at: requestedAt,
|
|
||||||
});
|
|
||||||
|
|
||||||
const ownerWorkspace = getPairedWorkspace(taskId, 'owner');
|
const ownerWorkspace = getPairedWorkspace(taskId, 'owner');
|
||||||
if (!ownerWorkspace) {
|
if (!ownerWorkspace) {
|
||||||
@@ -797,10 +793,18 @@ export function markPairedTaskReviewReady(taskId: string): {
|
|||||||
'Reviewer will mount owner workspace directly',
|
'Reviewer will mount owner workspace directly',
|
||||||
);
|
);
|
||||||
|
|
||||||
const now = new Date().toISOString();
|
const task = getPairedTaskById(taskId);
|
||||||
updatePairedTask(taskId, {
|
if (!task) {
|
||||||
status: 'review_ready',
|
return null;
|
||||||
updated_at: now,
|
}
|
||||||
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'review_ready',
|
||||||
|
updatedAt: requestedAt,
|
||||||
|
patch: {
|
||||||
|
review_requested_at: requestedAt,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return { ownerWorkspace, reviewerWorkspace };
|
return { ownerWorkspace, reviewerWorkspace };
|
||||||
|
|||||||
@@ -3,6 +3,11 @@ import { CronExpressionParser } from 'cron-parser';
|
|||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { getAgentOutputText } from './agent-output.js';
|
import { getAgentOutputText } from './agent-output.js';
|
||||||
import { createEvaluatedOutputHandler } from './agent-attempt.js';
|
import { createEvaluatedOutputHandler } from './agent-attempt.js';
|
||||||
|
import {
|
||||||
|
executeAttemptRetryAction,
|
||||||
|
runClaudeAttemptWithRotation,
|
||||||
|
runCodexAttemptWithRotation,
|
||||||
|
} from './agent-attempt-orchestration.js';
|
||||||
import { getErrorMessage } from './utils.js';
|
import { getErrorMessage } from './utils.js';
|
||||||
|
|
||||||
import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js';
|
import { ASSISTANT_NAME, SCHEDULER_POLL_INTERVAL, TIMEZONE } from './config.js';
|
||||||
@@ -28,10 +33,6 @@ import {
|
|||||||
} from './group-folder.js';
|
} from './group-folder.js';
|
||||||
import { createScopedLogger, logger } from './logger.js';
|
import { createScopedLogger, logger } from './logger.js';
|
||||||
import { createTaskStatusTracker } from './task-status-tracker.js';
|
import { createTaskStatusTracker } from './task-status-tracker.js';
|
||||||
import {
|
|
||||||
runClaudeRotationLoop,
|
|
||||||
runCodexRotationLoop,
|
|
||||||
} from './provider-retry.js';
|
|
||||||
import {
|
import {
|
||||||
detectCodexRotationTrigger,
|
detectCodexRotationTrigger,
|
||||||
rotateCodexToken,
|
rotateCodexToken,
|
||||||
@@ -42,7 +43,6 @@ import {
|
|||||||
classifyRotationTrigger,
|
classifyRotationTrigger,
|
||||||
type AgentTriggerReason,
|
type AgentTriggerReason,
|
||||||
type CodexRotationReason,
|
type CodexRotationReason,
|
||||||
isCodexRotationReason,
|
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import {
|
import {
|
||||||
getTokenCount,
|
getTokenCount,
|
||||||
@@ -447,69 +447,53 @@ async function runTask(
|
|||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
},
|
},
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<void> => {
|
): Promise<'success' | 'error'> => {
|
||||||
const logCtx = {
|
const logContext = {
|
||||||
taskId: task.id,
|
taskId: task.id,
|
||||||
group: context.group.name,
|
group: context.group.name,
|
||||||
groupFolder: task.group_folder,
|
groupFolder: task.group_folder,
|
||||||
};
|
};
|
||||||
|
|
||||||
const outcome = await runClaudeRotationLoop(
|
const outcome = await runClaudeAttemptWithRotation({
|
||||||
initialTrigger,
|
initialTrigger,
|
||||||
async () => {
|
runAttempt: () => runTaskAttempt('claude'),
|
||||||
const attempt = await runTaskAttempt('claude');
|
logContext,
|
||||||
|
rotationMessage,
|
||||||
|
afterAttempt: (attempt) => {
|
||||||
result = attempt.attemptResult;
|
result = attempt.attemptResult;
|
||||||
error = attempt.attemptError;
|
error = attempt.attemptError;
|
||||||
return {
|
|
||||||
output: attempt.output,
|
|
||||||
sawOutput: attempt.sawOutput,
|
|
||||||
streamedTriggerReason: attempt.streamedTriggerReason,
|
|
||||||
};
|
|
||||||
},
|
},
|
||||||
logCtx,
|
});
|
||||||
rotationMessage,
|
|
||||||
);
|
|
||||||
|
|
||||||
switch (outcome.type) {
|
if (outcome === 'success') {
|
||||||
case 'success':
|
error = null;
|
||||||
error = null;
|
|
||||||
return;
|
|
||||||
case 'error':
|
|
||||||
if (outcome.trigger) {
|
|
||||||
error = `Claude ${outcome.trigger.reason}`;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
return outcome;
|
||||||
};
|
};
|
||||||
|
|
||||||
const retryCodexTaskWithRotation = async (
|
const retryCodexTaskWithRotation = async (
|
||||||
initialTrigger: { reason: CodexRotationReason },
|
initialTrigger: { reason: CodexRotationReason },
|
||||||
rotationMessage?: string,
|
rotationMessage?: string,
|
||||||
): Promise<void> => {
|
): Promise<'success' | 'error'> => {
|
||||||
const outcome = await runCodexRotationLoop(
|
const outcome = await runCodexAttemptWithRotation({
|
||||||
initialTrigger,
|
initialTrigger,
|
||||||
async () => {
|
runAttempt: () => runTaskAttempt('codex'),
|
||||||
const retryAttempt = await runTaskAttempt('codex');
|
logContext: {
|
||||||
result = retryAttempt.attemptResult;
|
|
||||||
error = retryAttempt.attemptError;
|
|
||||||
return {
|
|
||||||
output: retryAttempt.output,
|
|
||||||
thrownError: null,
|
|
||||||
sawOutput: retryAttempt.sawOutput,
|
|
||||||
streamedTriggerReason: retryAttempt.streamedTriggerReason,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
{
|
|
||||||
taskId: task.id,
|
taskId: task.id,
|
||||||
group: context.group.name,
|
group: context.group.name,
|
||||||
groupFolder: task.group_folder,
|
groupFolder: task.group_folder,
|
||||||
},
|
},
|
||||||
rotationMessage,
|
rotationMessage,
|
||||||
);
|
afterAttempt: (attempt) => {
|
||||||
|
result = attempt.attemptResult;
|
||||||
|
error = attempt.attemptError;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
if (outcome.type === 'success') {
|
if (outcome === 'success') {
|
||||||
error = null;
|
error = null;
|
||||||
}
|
}
|
||||||
|
return outcome;
|
||||||
};
|
};
|
||||||
|
|
||||||
const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude';
|
const provider = context.taskAgentType === 'codex' ? 'codex' : 'claude';
|
||||||
@@ -519,56 +503,17 @@ async function runTask(
|
|||||||
result = attempt.attemptResult;
|
result = attempt.attemptResult;
|
||||||
error = attempt.attemptError;
|
error = attempt.attemptError;
|
||||||
|
|
||||||
if (
|
const retryAction = await executeAttemptRetryAction({
|
||||||
provider === 'claude' &&
|
provider,
|
||||||
attempt.streamedTriggerReason &&
|
canRetryClaudeCredentials: provider === 'claude' && getTokenCount() > 0,
|
||||||
!attempt.sawOutput
|
canRetryCodex: provider === 'codex' && getCodexAccountCount() > 1,
|
||||||
) {
|
attempt,
|
||||||
await retryClaudeTaskWithRotation(attempt.streamedTriggerReason);
|
rotationMessage: error,
|
||||||
} else if (
|
runClaude: retryClaudeTaskWithRotation,
|
||||||
provider === 'codex' &&
|
runCodex: retryCodexTaskWithRotation,
|
||||||
attempt.streamedTriggerReason &&
|
});
|
||||||
!attempt.sawOutput &&
|
|
||||||
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
if (retryAction.kind === 'none' && attempt.output.status === 'error') {
|
||||||
) {
|
|
||||||
await retryCodexTaskWithRotation(
|
|
||||||
{
|
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
|
||||||
},
|
|
||||||
typeof attempt.output.error === 'string'
|
|
||||||
? attempt.output.error
|
|
||||||
: undefined,
|
|
||||||
);
|
|
||||||
} else if (attempt.output.status === 'error' && provider === 'claude') {
|
|
||||||
const trigger = attempt.streamedTriggerReason
|
|
||||||
? {
|
|
||||||
shouldRetry: true,
|
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
|
||||||
retryAfterMs: attempt.streamedTriggerReason.retryAfterMs,
|
|
||||||
}
|
|
||||||
: classifyRotationTrigger(error);
|
|
||||||
if (trigger.shouldRetry) {
|
|
||||||
await retryClaudeTaskWithRotation({
|
|
||||||
reason: trigger.reason,
|
|
||||||
retryAfterMs: trigger.retryAfterMs,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else if (attempt.output.status === 'error' && provider === 'codex') {
|
|
||||||
const trigger =
|
|
||||||
attempt.streamedTriggerReason &&
|
|
||||||
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
|
||||||
? {
|
|
||||||
shouldRotate: true as const,
|
|
||||||
reason: attempt.streamedTriggerReason.reason,
|
|
||||||
}
|
|
||||||
: detectCodexRotationTrigger(error);
|
|
||||||
if (trigger.shouldRotate) {
|
|
||||||
await retryCodexTaskWithRotation(
|
|
||||||
{ reason: trigger.reason },
|
|
||||||
error || undefined,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else if (attempt.output.status === 'error') {
|
|
||||||
error = attempt.attemptError || 'Unknown error';
|
error = attempt.attemptError || 'Unknown error';
|
||||||
}
|
}
|
||||||
} // end else (non-exhausted path)
|
} // end else (non-exhausted path)
|
||||||
|
|||||||
@@ -65,4 +65,44 @@ describe('token-rotation runtime reselection', () => {
|
|||||||
rateLimited: 0,
|
rateLimited: 0,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reloads Claude rotation state from disk written by another service', async () => {
|
||||||
|
const mod = await import('./token-rotation.js');
|
||||||
|
const utils = await import('./utils.js');
|
||||||
|
const readJsonFile = vi.mocked(utils.readJsonFile);
|
||||||
|
|
||||||
|
readJsonFile.mockReturnValueOnce(null);
|
||||||
|
mod.initTokenRotation();
|
||||||
|
expect(mod.getCurrentToken()).toBe('token-1');
|
||||||
|
|
||||||
|
readJsonFile.mockReturnValueOnce({
|
||||||
|
currentIndex: 1,
|
||||||
|
rateLimits: [Date.now() + 60_000, null],
|
||||||
|
});
|
||||||
|
mod.reloadTokenRotationStateFromDisk();
|
||||||
|
|
||||||
|
expect(mod.getCurrentTokenIndex()).toBe(1);
|
||||||
|
expect(mod.getCurrentToken()).toBe('token-2');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('warns when Claude rotation state cannot be persisted', async () => {
|
||||||
|
const mod = await import('./token-rotation.js');
|
||||||
|
const utils = await import('./utils.js');
|
||||||
|
const { logger } = await import('./logger.js');
|
||||||
|
|
||||||
|
vi.mocked(utils.writeJsonFile).mockImplementation(() => {
|
||||||
|
throw new Error('disk full');
|
||||||
|
});
|
||||||
|
|
||||||
|
mod.initTokenRotation();
|
||||||
|
expect(mod.rotateToken('rate limit')).toBe(true);
|
||||||
|
|
||||||
|
expect(logger.warn).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
stateFile: '/tmp/ejclaw-claude-rot-data/token-rotation-state.json',
|
||||||
|
err: expect.any(Error),
|
||||||
|
}),
|
||||||
|
'Failed to persist Claude token rotation state',
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -69,12 +69,15 @@ function saveState(): void {
|
|||||||
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
rateLimits: tokens.map((t) => t.rateLimitedUntil),
|
||||||
};
|
};
|
||||||
writeJsonFile(STATE_FILE, state);
|
writeJsonFile(STATE_FILE, state);
|
||||||
} catch {
|
} catch (err) {
|
||||||
/* best effort */
|
logger.warn(
|
||||||
|
{ stateFile: STATE_FILE, err },
|
||||||
|
'Failed to persist Claude token rotation state',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadState(): void {
|
function loadState(quiet = false): void {
|
||||||
const state = readJsonFile<{
|
const state = readJsonFile<{
|
||||||
currentIndex?: number;
|
currentIndex?: number;
|
||||||
rateLimits?: (number | null)[];
|
rateLimits?: (number | null)[];
|
||||||
@@ -93,13 +96,27 @@ function loadState(): void {
|
|||||||
const until = state.rateLimits[i];
|
const until = state.rateLimits[i];
|
||||||
if (typeof until === 'number' && until > now) {
|
if (typeof until === 'number' && until > now) {
|
||||||
tokens[i].rateLimitedUntil = until;
|
tokens[i].rateLimitedUntil = until;
|
||||||
|
} else {
|
||||||
|
tokens[i].rateLimitedUntil = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.info(
|
if (!quiet) {
|
||||||
{ currentIndex, tokenCount: tokens.length },
|
logger.info(
|
||||||
'Token rotation state restored',
|
{ currentIndex, tokenCount: tokens.length },
|
||||||
);
|
'Token rotation state restored',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Re-read the on-disk rotation state (written by any service).
|
||||||
|
* Call before dashboard renders so the renderer picks up rotations
|
||||||
|
* performed by another Claude service process.
|
||||||
|
*/
|
||||||
|
export function reloadTokenRotationStateFromDisk(): void {
|
||||||
|
if (tokens.length <= 1) return;
|
||||||
|
loadState(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
function refreshRuntimeTokenSelection(): void {
|
function refreshRuntimeTokenSelection(): void {
|
||||||
|
|||||||
Reference in New Issue
Block a user