Compare commits
76 Commits
2d4b136c00
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9095d95a8 | ||
|
|
4d3ab20378 | ||
|
|
5d60df8122 | ||
|
|
2612e8a6ca | ||
|
|
2be6c8db8d | ||
|
|
f9b1e74838 | ||
|
|
822ac34c0e | ||
|
|
e14ac3dfca | ||
|
|
1d8358d1e1 | ||
|
|
be9f2379c0 | ||
|
|
a621e85432 | ||
|
|
80ddb1aa0d | ||
|
|
ee4c5559b1 | ||
|
|
73fd71b39a | ||
|
|
9c46cf6761 | ||
|
|
aec70bc388 | ||
|
|
1a85ebf7eb | ||
|
|
728a8c2ec9 | ||
|
|
f6ce3122bf | ||
|
|
4ddaa88f8a | ||
|
|
b3a927b241 | ||
|
|
ef2b26e5da | ||
|
|
0ca24debfd | ||
|
|
edeeed9476 | ||
|
|
e324a1baa1 | ||
|
|
efc8c00380 | ||
|
|
2313d6cf3e | ||
|
|
65510e0fc3 | ||
|
|
de54f13469 | ||
|
|
5f06673ac6 | ||
|
|
dca56d9493 | ||
|
|
1dd948a7ac | ||
|
|
7eb97dd921 | ||
|
|
8883cd166d | ||
|
|
5c47024e44 | ||
|
|
337d46461c | ||
|
|
29ca23a558 | ||
|
|
80bc078a24 | ||
|
|
184a8eff6a | ||
|
|
ac0cd3e41d | ||
|
|
1af074786b | ||
|
|
8cf351580b | ||
|
|
d7b5166862 | ||
|
|
c001905678 | ||
|
|
f1679ac686 | ||
|
|
807828e0ce | ||
|
|
879d16235f | ||
|
|
0cbb7e3b9c | ||
|
|
32468e0214 | ||
|
|
2dab27f443 | ||
|
|
22602da1ec | ||
|
|
63a4515241 | ||
|
|
b4197875fa | ||
|
|
3d36e15f67 | ||
|
|
3894d4f406 | ||
|
|
7fc558acb6 | ||
|
|
5648b61075 | ||
|
|
03d4c81192 | ||
|
|
453157f6c3 | ||
|
|
f28399021a | ||
|
|
f4de795b1e | ||
|
|
db715c6329 | ||
|
|
90c17c6890 | ||
|
|
e3c8de61f1 | ||
|
|
0d87778b78 | ||
|
|
047f0d3b70 | ||
|
|
65059719ce | ||
|
|
32fbb00dd2 | ||
|
|
cdb59ce9fe | ||
|
|
85085e2782 | ||
|
|
cf9bf08197 | ||
|
|
890991394d | ||
|
|
4e90e1d7ec | ||
|
|
7ed0af98a2 | ||
|
|
51512c9aeb | ||
|
|
1509108e04 |
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"env": {
|
"env": {
|
||||||
"CLAUDE_AUTOCOMPACT_PCT_OVERRIDE": "50"
|
"CLAUDE_AUTOCOMPACT_PCT_OVERRIDE": "70"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -39,6 +39,7 @@ tmp/
|
|||||||
backups/
|
backups/
|
||||||
dist-backups/
|
dist-backups/
|
||||||
.deploy-backups/
|
.deploy-backups/
|
||||||
|
recovery/
|
||||||
cache/
|
cache/
|
||||||
runners/*/node_modules/
|
runners/*/node_modules/
|
||||||
runners/codex-runner-backups/
|
runners/codex-runner-backups/
|
||||||
|
|||||||
@@ -204,7 +204,7 @@ function buildRoomOptions(snapshots: StatusSnapshot[]): RoomOption[] {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return [...rooms.values()].sort((a, b) =>
|
return Array.from(rooms.values()).sort((a, b) =>
|
||||||
`${a.name} ${a.folder}`.localeCompare(`${b.name} ${b.folder}`),
|
`${a.name} ${a.folder}`.localeCompare(`${b.name} ${b.folder}`),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -247,7 +247,7 @@ function DashboardErrorCard({
|
|||||||
<strong>{t.error.api}</strong>
|
<strong>{t.error.api}</strong>
|
||||||
<small>{humanizeError(error, t)}</small>
|
<small>{humanizeError(error, t)}</small>
|
||||||
</span>
|
</span>
|
||||||
<button disabled={refreshing} onClick={onRetry}>
|
<button disabled={refreshing} onClick={onRetry} type="button">
|
||||||
{t.actions.retry}
|
{t.actions.retry}
|
||||||
</button>
|
</button>
|
||||||
</section>
|
</section>
|
||||||
|
|||||||
207
apps/dashboard/src/GlassesApp.tsx
Normal file
207
apps/dashboard/src/GlassesApp.tsx
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
import {
|
||||||
|
type DashboardInboxAction,
|
||||||
|
type DashboardOverview,
|
||||||
|
type StatusSnapshot,
|
||||||
|
fetchDashboardData,
|
||||||
|
runInboxAction,
|
||||||
|
sendRoomMessage,
|
||||||
|
} from './api';
|
||||||
|
import { isLocale, matchLocale, type Locale } from './i18n';
|
||||||
|
import { GlassesPanel } from './GlassesPanel';
|
||||||
|
import './styles.css';
|
||||||
|
import './glasses.css';
|
||||||
|
|
||||||
|
interface GlassesState {
|
||||||
|
overview: DashboardOverview;
|
||||||
|
snapshots: StatusSnapshot[];
|
||||||
|
}
|
||||||
|
|
||||||
|
type InboxItem = DashboardOverview['inbox'][number];
|
||||||
|
type InboxActionKey = `${string}:${DashboardInboxAction}`;
|
||||||
|
|
||||||
|
const REFRESH_INTERVAL_MS = 15_000;
|
||||||
|
const DASHBOARD_STALE_MS = 75_000;
|
||||||
|
const LOCALE_STORAGE_KEY = 'ejclaw.dashboard.locale.v2';
|
||||||
|
|
||||||
|
function makeClientRequestId(): string {
|
||||||
|
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readInitialLocale(): Locale {
|
||||||
|
const stored =
|
||||||
|
typeof window === 'undefined'
|
||||||
|
? null
|
||||||
|
: window.localStorage.getItem(LOCALE_STORAGE_KEY);
|
||||||
|
if (isLocale(stored)) return stored;
|
||||||
|
|
||||||
|
const languages =
|
||||||
|
typeof navigator === 'undefined'
|
||||||
|
? []
|
||||||
|
: [...(navigator.languages || []), navigator.language];
|
||||||
|
for (const language of languages) {
|
||||||
|
const matched = matchLocale(language);
|
||||||
|
if (matched) return matched;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'en';
|
||||||
|
}
|
||||||
|
|
||||||
|
function dashboardAgeMs(value: string | null | undefined): number | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const date = new Date(value);
|
||||||
|
if (Number.isNaN(date.getTime())) return null;
|
||||||
|
return Math.max(0, Date.now() - date.getTime());
|
||||||
|
}
|
||||||
|
|
||||||
|
function freshnessLabel(
|
||||||
|
online: boolean,
|
||||||
|
generatedAt: string | null | undefined,
|
||||||
|
): string {
|
||||||
|
if (!online) return 'offline';
|
||||||
|
const age = dashboardAgeMs(generatedAt);
|
||||||
|
if (age !== null && age > DASHBOARD_STALE_MS) return 'stale';
|
||||||
|
return 'fresh';
|
||||||
|
}
|
||||||
|
|
||||||
|
function readNickname(): string {
|
||||||
|
if (typeof window === 'undefined') return '';
|
||||||
|
return window.localStorage.getItem('ejclaw-nickname') ?? '';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function GlassesApp() {
|
||||||
|
const [data, setData] = useState<GlassesState | null>(null);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [refreshing, setRefreshing] = useState(false);
|
||||||
|
const [online, setOnline] = useState(() =>
|
||||||
|
typeof navigator === 'undefined' ? true : navigator.onLine,
|
||||||
|
);
|
||||||
|
const [locale] = useState<Locale>(readInitialLocale);
|
||||||
|
const [inboxActionKey, setInboxActionKey] = useState<InboxActionKey | null>(
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
const [roomMessageKey, setRoomMessageKey] = useState<string | null>(null);
|
||||||
|
|
||||||
|
async function refresh(showSpinner = false) {
|
||||||
|
if (showSpinner) setRefreshing(true);
|
||||||
|
try {
|
||||||
|
const nextData = await fetchDashboardData();
|
||||||
|
setData({
|
||||||
|
overview: nextData.overview,
|
||||||
|
snapshots: nextData.snapshots,
|
||||||
|
});
|
||||||
|
setError(null);
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : String(err));
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
setRefreshing(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleInboxAction(
|
||||||
|
item: InboxItem,
|
||||||
|
action: DashboardInboxAction,
|
||||||
|
) {
|
||||||
|
const actionKey: InboxActionKey = `${item.id}:${action}`;
|
||||||
|
setInboxActionKey(actionKey);
|
||||||
|
try {
|
||||||
|
await runInboxAction(item.id, action, {
|
||||||
|
lastOccurredAt: item.lastOccurredAt,
|
||||||
|
requestId: makeClientRequestId(),
|
||||||
|
});
|
||||||
|
await refresh(false);
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : String(err));
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setInboxActionKey(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleRoomMessage(
|
||||||
|
roomJid: string,
|
||||||
|
text: string,
|
||||||
|
requestId: string,
|
||||||
|
) {
|
||||||
|
setRoomMessageKey(roomJid);
|
||||||
|
try {
|
||||||
|
await sendRoomMessage(roomJid, text, requestId, readNickname() || null);
|
||||||
|
void refresh(false);
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : String(err));
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setRoomMessageKey(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
document.documentElement.lang = locale;
|
||||||
|
}, [locale]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
function handleOnline() {
|
||||||
|
setOnline(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleOffline() {
|
||||||
|
setOnline(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
window.addEventListener('online', handleOnline);
|
||||||
|
window.addEventListener('offline', handleOffline);
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener('online', handleOnline);
|
||||||
|
window.removeEventListener('offline', handleOffline);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void refresh();
|
||||||
|
const id = window.setInterval(() => {
|
||||||
|
void refresh();
|
||||||
|
}, REFRESH_INTERVAL_MS);
|
||||||
|
return () => window.clearInterval(id);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (loading && !data) {
|
||||||
|
return (
|
||||||
|
<main className="glasses-shell" aria-busy="true">
|
||||||
|
<div className="glasses-empty">
|
||||||
|
<strong>EJClaw</strong>
|
||||||
|
<span>Loading</span>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!data) {
|
||||||
|
return (
|
||||||
|
<main className="glasses-shell">
|
||||||
|
<p className="glasses-error">{error ?? 'Dashboard unavailable'}</p>
|
||||||
|
</main>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<GlassesPanel
|
||||||
|
createRequestId={makeClientRequestId}
|
||||||
|
error={error}
|
||||||
|
freshnessText={freshnessLabel(online, data.overview.generatedAt)}
|
||||||
|
inboxActionKey={inboxActionKey}
|
||||||
|
locale={locale}
|
||||||
|
onInboxAction={handleInboxAction}
|
||||||
|
onRefresh={() => void refresh(true)}
|
||||||
|
onSendRoomMessage={handleRoomMessage}
|
||||||
|
overview={data.overview}
|
||||||
|
refreshing={refreshing}
|
||||||
|
roomMessageKey={roomMessageKey}
|
||||||
|
snapshots={data.snapshots}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
120
apps/dashboard/src/GlassesPanel.test.ts
Normal file
120
apps/dashboard/src/GlassesPanel.test.ts
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
import { createElement } from 'react';
|
||||||
|
import { renderToStaticMarkup } from 'react-dom/server';
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { GlassesPanel } from './GlassesPanel';
|
||||||
|
import type { DashboardOverview, StatusSnapshot } from './api';
|
||||||
|
|
||||||
|
const overview: DashboardOverview = {
|
||||||
|
generatedAt: '2026-06-07T13:00:00.000Z',
|
||||||
|
services: [],
|
||||||
|
rooms: {
|
||||||
|
total: 1,
|
||||||
|
active: 1,
|
||||||
|
waiting: 0,
|
||||||
|
inactive: 0,
|
||||||
|
},
|
||||||
|
tasks: {
|
||||||
|
total: 1,
|
||||||
|
active: 1,
|
||||||
|
paused: 0,
|
||||||
|
completed: 0,
|
||||||
|
watchers: {
|
||||||
|
active: 0,
|
||||||
|
paused: 0,
|
||||||
|
completed: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
usage: {
|
||||||
|
rows: [],
|
||||||
|
fetchedAt: null,
|
||||||
|
},
|
||||||
|
inbox: [
|
||||||
|
{
|
||||||
|
createdAt: '2026-06-07T12:59:30.000Z',
|
||||||
|
groupKey: 'task:review',
|
||||||
|
id: 'inbox-1',
|
||||||
|
kind: 'reviewer-request',
|
||||||
|
lastOccurredAt: '2026-06-07T12:59:30.000Z',
|
||||||
|
occurrences: 1,
|
||||||
|
occurredAt: '2026-06-07T12:59:30.000Z',
|
||||||
|
roomJid: 'dc:ops',
|
||||||
|
roomName: 'Ops',
|
||||||
|
severity: 'warn',
|
||||||
|
source: 'paired-task',
|
||||||
|
summary: 'Reviewer requested owner changes',
|
||||||
|
taskId: 'task-1',
|
||||||
|
taskStatus: 'active',
|
||||||
|
title: 'Review follow-up',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const snapshots: StatusSnapshot[] = [
|
||||||
|
{
|
||||||
|
agentType: 'codex',
|
||||||
|
assistantName: 'Codex',
|
||||||
|
serviceId: 'codex-main',
|
||||||
|
updatedAt: '2026-06-07T13:00:00.000Z',
|
||||||
|
entries: [
|
||||||
|
{
|
||||||
|
agentType: 'codex',
|
||||||
|
elapsedMs: 12000,
|
||||||
|
folder: 'ejclaw',
|
||||||
|
jid: 'dc:ops',
|
||||||
|
name: 'Ops',
|
||||||
|
pendingMessages: false,
|
||||||
|
pendingTasks: 1,
|
||||||
|
status: 'processing',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
describe('GlassesPanel', () => {
|
||||||
|
it('renders a compact display queue and voice input surface', () => {
|
||||||
|
const html = renderToStaticMarkup(
|
||||||
|
createElement(GlassesPanel, {
|
||||||
|
createRequestId: () => 'req-1',
|
||||||
|
error: null,
|
||||||
|
freshnessText: 'fresh',
|
||||||
|
inboxActionKey: null,
|
||||||
|
locale: 'ko',
|
||||||
|
onInboxAction: async () => true,
|
||||||
|
onRefresh: () => {},
|
||||||
|
onSendRoomMessage: async () => true,
|
||||||
|
overview,
|
||||||
|
refreshing: false,
|
||||||
|
roomMessageKey: null,
|
||||||
|
snapshots,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(html).toContain('glasses-shell');
|
||||||
|
expect(html).toContain('Review follow-up');
|
||||||
|
expect(html).toContain('리뷰');
|
||||||
|
expect(html).toContain('Voice');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('tolerates partial status snapshot payloads while data is refreshing', () => {
|
||||||
|
const html = renderToStaticMarkup(
|
||||||
|
createElement(GlassesPanel, {
|
||||||
|
createRequestId: () => 'req-1',
|
||||||
|
error: null,
|
||||||
|
freshnessText: 'fresh',
|
||||||
|
inboxActionKey: null,
|
||||||
|
locale: 'ko',
|
||||||
|
onInboxAction: async () => true,
|
||||||
|
onRefresh: () => {},
|
||||||
|
onSendRoomMessage: async () => true,
|
||||||
|
overview,
|
||||||
|
refreshing: false,
|
||||||
|
roomMessageKey: null,
|
||||||
|
snapshots: [{} as StatusSnapshot],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(html).toContain('glasses-shell');
|
||||||
|
expect(html).toContain('Review follow-up');
|
||||||
|
});
|
||||||
|
});
|
||||||
607
apps/dashboard/src/GlassesPanel.tsx
Normal file
607
apps/dashboard/src/GlassesPanel.tsx
Normal file
@@ -0,0 +1,607 @@
|
|||||||
|
import {
|
||||||
|
type KeyboardEvent,
|
||||||
|
useEffect,
|
||||||
|
useMemo,
|
||||||
|
useRef,
|
||||||
|
useState,
|
||||||
|
} from 'react';
|
||||||
|
import { Check, Inbox, Mic, RefreshCw, Send, X } from 'lucide-react';
|
||||||
|
|
||||||
|
import {
|
||||||
|
type DashboardInboxAction,
|
||||||
|
type DashboardOverview,
|
||||||
|
type StatusSnapshot,
|
||||||
|
} from './api';
|
||||||
|
import { formatDate } from './dashboardHelpers';
|
||||||
|
import type { Locale } from './i18n';
|
||||||
|
|
||||||
|
type InboxItem = DashboardOverview['inbox'][number];
|
||||||
|
type GlassesMode = 'queue' | 'voice';
|
||||||
|
|
||||||
|
interface RoomChoice {
|
||||||
|
jid: string;
|
||||||
|
label: string;
|
||||||
|
status: StatusSnapshot['entries'][number]['status'];
|
||||||
|
pendingTasks: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SpeechRecognitionResultLike {
|
||||||
|
readonly 0?: { transcript?: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SpeechRecognitionEventLike {
|
||||||
|
readonly results: ArrayLike<SpeechRecognitionResultLike>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SpeechRecognitionLike {
|
||||||
|
continuous: boolean;
|
||||||
|
interimResults: boolean;
|
||||||
|
lang: string;
|
||||||
|
maxAlternatives: number;
|
||||||
|
onend: (() => void) | null;
|
||||||
|
onerror: (() => void) | null;
|
||||||
|
onresult: ((event: SpeechRecognitionEventLike) => void) | null;
|
||||||
|
start: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
|
||||||
|
|
||||||
|
interface SpeechWindow extends Window {
|
||||||
|
SpeechRecognition?: SpeechRecognitionConstructor;
|
||||||
|
webkitSpeechRecognition?: SpeechRecognitionConstructor;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GlassesPanelProps {
|
||||||
|
createRequestId: () => string;
|
||||||
|
error: string | null;
|
||||||
|
freshnessText: string;
|
||||||
|
inboxActionKey: `${string}:${DashboardInboxAction}` | null;
|
||||||
|
locale: Locale;
|
||||||
|
onInboxAction: (
|
||||||
|
item: InboxItem,
|
||||||
|
action: DashboardInboxAction,
|
||||||
|
) => Promise<boolean>;
|
||||||
|
onRefresh: () => void;
|
||||||
|
onSendRoomMessage: (
|
||||||
|
roomJid: string,
|
||||||
|
text: string,
|
||||||
|
requestId: string,
|
||||||
|
) => Promise<boolean>;
|
||||||
|
overview: DashboardOverview;
|
||||||
|
refreshing: boolean;
|
||||||
|
roomMessageKey: string | null;
|
||||||
|
snapshots: StatusSnapshot[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const QUEUE_KIND_ORDER: Record<InboxItem['kind'], number> = {
|
||||||
|
approval: 0,
|
||||||
|
'reviewer-request': 1,
|
||||||
|
'arbiter-request': 2,
|
||||||
|
'ci-failure': 3,
|
||||||
|
mention: 4,
|
||||||
|
'pending-room': 5,
|
||||||
|
};
|
||||||
|
|
||||||
|
const SEVERITY_ORDER: Record<InboxItem['severity'], number> = {
|
||||||
|
error: 0,
|
||||||
|
warn: 1,
|
||||||
|
info: 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
const SPEECH_LOCALES: Record<Locale, string> = {
|
||||||
|
en: 'en-US',
|
||||||
|
ja: 'ja-JP',
|
||||||
|
ko: 'ko-KR',
|
||||||
|
zh: 'zh-CN',
|
||||||
|
};
|
||||||
|
|
||||||
|
function getSpeechRecognition(): SpeechRecognitionConstructor | null {
|
||||||
|
if (typeof window === 'undefined') return null;
|
||||||
|
const speechWindow = window as SpeechWindow;
|
||||||
|
return (
|
||||||
|
speechWindow.SpeechRecognition ??
|
||||||
|
speechWindow.webkitSpeechRecognition ??
|
||||||
|
null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function inboxActionsFor(item: InboxItem): DashboardInboxAction[] {
|
||||||
|
if (
|
||||||
|
item.source === 'paired-task' &&
|
||||||
|
(item.kind === 'reviewer-request' ||
|
||||||
|
item.kind === 'approval' ||
|
||||||
|
item.kind === 'arbiter-request')
|
||||||
|
) {
|
||||||
|
return ['run', 'decline', 'dismiss'];
|
||||||
|
}
|
||||||
|
return ['dismiss'];
|
||||||
|
}
|
||||||
|
|
||||||
|
function actionLabel(item: InboxItem, action: DashboardInboxAction): string {
|
||||||
|
if (action === 'dismiss') return '닫기';
|
||||||
|
if (action === 'decline') return '거절';
|
||||||
|
if (item.kind === 'approval') return '최종화';
|
||||||
|
if (item.kind === 'reviewer-request') return '리뷰';
|
||||||
|
if (item.kind === 'arbiter-request') return '중재';
|
||||||
|
return '실행';
|
||||||
|
}
|
||||||
|
|
||||||
|
function actionIcon(action: DashboardInboxAction) {
|
||||||
|
if (action === 'decline') return <X size={16} aria-hidden />;
|
||||||
|
if (action === 'dismiss') return <Check size={16} aria-hidden />;
|
||||||
|
return <Send size={16} aria-hidden />;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sortInboxItems(items: InboxItem[]): InboxItem[] {
|
||||||
|
return [...items].sort((a, b) => {
|
||||||
|
const severity = SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity];
|
||||||
|
if (severity !== 0) return severity;
|
||||||
|
const kind = QUEUE_KIND_ORDER[a.kind] - QUEUE_KIND_ORDER[b.kind];
|
||||||
|
if (kind !== 0) return kind;
|
||||||
|
return b.lastOccurredAt.localeCompare(a.lastOccurredAt);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildRoomChoices(snapshots: StatusSnapshot[]): RoomChoice[] {
|
||||||
|
const rooms = new Map<string, RoomChoice>();
|
||||||
|
for (const snapshot of snapshots) {
|
||||||
|
const entries = Array.isArray(snapshot.entries) ? snapshot.entries : [];
|
||||||
|
for (const entry of entries) {
|
||||||
|
const existing = rooms.get(entry.jid);
|
||||||
|
const label = entry.name || entry.folder || entry.jid;
|
||||||
|
if (!existing) {
|
||||||
|
rooms.set(entry.jid, {
|
||||||
|
jid: entry.jid,
|
||||||
|
label,
|
||||||
|
pendingTasks: entry.pendingTasks,
|
||||||
|
status: entry.status,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
entry.status === 'processing' ||
|
||||||
|
entry.pendingTasks > existing.pendingTasks
|
||||||
|
) {
|
||||||
|
rooms.set(entry.jid, {
|
||||||
|
jid: entry.jid,
|
||||||
|
label,
|
||||||
|
pendingTasks: entry.pendingTasks,
|
||||||
|
status: entry.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [...rooms.values()].sort((a, b) => {
|
||||||
|
if (a.status === 'processing' && b.status !== 'processing') return -1;
|
||||||
|
if (b.status === 'processing' && a.status !== 'processing') return 1;
|
||||||
|
return b.pendingTasks - a.pendingTasks || a.label.localeCompare(b.label);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampIndex(index: number, length: number): number {
|
||||||
|
if (length <= 0) return 0;
|
||||||
|
return Math.max(0, Math.min(index, length - 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
function nextIndex(index: number, length: number, delta: number): number {
|
||||||
|
if (length <= 0) return 0;
|
||||||
|
return (index + delta + length) % length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readTranscript(event: SpeechRecognitionEventLike): string {
|
||||||
|
return Array.from(event.results)
|
||||||
|
.map((result) => result[0]?.transcript?.trim() ?? '')
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(' ')
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
interface QueueCardProps {
|
||||||
|
inboxActionKey: `${string}:${DashboardInboxAction}` | null;
|
||||||
|
inboxActions: DashboardInboxAction[];
|
||||||
|
inboxItems: InboxItem[];
|
||||||
|
locale: Locale;
|
||||||
|
onInboxAction: (
|
||||||
|
item: InboxItem,
|
||||||
|
action: DashboardInboxAction,
|
||||||
|
) => Promise<boolean>;
|
||||||
|
selectedActionIndex: number;
|
||||||
|
selectedInbox: InboxItem | undefined;
|
||||||
|
selectedInboxIndex: number;
|
||||||
|
setSelectedActionIndex: (index: number) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function QueueCard({
|
||||||
|
inboxActionKey,
|
||||||
|
inboxActions,
|
||||||
|
inboxItems,
|
||||||
|
locale,
|
||||||
|
onInboxAction,
|
||||||
|
selectedActionIndex,
|
||||||
|
selectedInbox,
|
||||||
|
selectedInboxIndex,
|
||||||
|
setSelectedActionIndex,
|
||||||
|
}: QueueCardProps) {
|
||||||
|
if (!selectedInbox) {
|
||||||
|
return (
|
||||||
|
<section className="glasses-card glasses-queue-card" aria-live="polite">
|
||||||
|
<div className="glasses-empty">
|
||||||
|
<strong>Queue clear</strong>
|
||||||
|
<span>새 요청 없음</span>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="glasses-card glasses-queue-card" aria-live="polite">
|
||||||
|
<div className="glasses-card-head">
|
||||||
|
<span className={`glasses-pill sev-${selectedInbox.severity}`}>
|
||||||
|
{selectedInbox.kind}
|
||||||
|
</span>
|
||||||
|
<small>
|
||||||
|
{selectedInboxIndex + 1}/{inboxItems.length}
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
<h2>{selectedInbox.title}</h2>
|
||||||
|
<p>{selectedInbox.summary}</p>
|
||||||
|
<dl className="glasses-meta">
|
||||||
|
<div>
|
||||||
|
<dt>Target</dt>
|
||||||
|
<dd>
|
||||||
|
{selectedInbox.roomName ??
|
||||||
|
selectedInbox.groupFolder ??
|
||||||
|
selectedInbox.roomJid ??
|
||||||
|
selectedInbox.taskId ??
|
||||||
|
'-'}
|
||||||
|
</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Time</dt>
|
||||||
|
<dd>{formatDate(selectedInbox.lastOccurredAt, locale)}</dd>
|
||||||
|
</div>
|
||||||
|
</dl>
|
||||||
|
<div className="glasses-actions">
|
||||||
|
{inboxActions.map((action, index) => {
|
||||||
|
const actionKey = `${selectedInbox.id}:${action}`;
|
||||||
|
const busy = inboxActionKey === actionKey;
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
aria-busy={busy || undefined}
|
||||||
|
aria-pressed={index === selectedActionIndex}
|
||||||
|
className={
|
||||||
|
index === selectedActionIndex ? 'is-active' : undefined
|
||||||
|
}
|
||||||
|
disabled={busy || Boolean(inboxActionKey)}
|
||||||
|
key={action}
|
||||||
|
onClick={() => {
|
||||||
|
setSelectedActionIndex(index);
|
||||||
|
void onInboxAction(selectedInbox, action);
|
||||||
|
}}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
{actionIcon(action)}
|
||||||
|
{busy ? '처리 중' : actionLabel(selectedInbox, action)}
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface VoiceCardProps {
|
||||||
|
canListen: boolean;
|
||||||
|
listening: boolean;
|
||||||
|
onSendVoiceText: () => void;
|
||||||
|
onStartListening: () => void;
|
||||||
|
selectedRoom: RoomChoice | undefined;
|
||||||
|
sendingVoice: boolean;
|
||||||
|
setSelectedRoomIndex: (index: number) => void;
|
||||||
|
setVoiceText: (value: string) => void;
|
||||||
|
rooms: RoomChoice[];
|
||||||
|
voiceText: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function VoiceCard({
|
||||||
|
canListen,
|
||||||
|
listening,
|
||||||
|
onSendVoiceText,
|
||||||
|
onStartListening,
|
||||||
|
selectedRoom,
|
||||||
|
sendingVoice,
|
||||||
|
setSelectedRoomIndex,
|
||||||
|
setVoiceText,
|
||||||
|
rooms,
|
||||||
|
voiceText,
|
||||||
|
}: VoiceCardProps) {
|
||||||
|
return (
|
||||||
|
<section className="glasses-card glasses-voice-card">
|
||||||
|
<div className="glasses-card-head">
|
||||||
|
<span className="glasses-pill">voice</span>
|
||||||
|
<small>{selectedRoom?.label ?? 'No room'}</small>
|
||||||
|
</div>
|
||||||
|
<textarea
|
||||||
|
aria-label="음성 또는 키보드 입력"
|
||||||
|
onChange={(event) => setVoiceText(event.target.value)}
|
||||||
|
placeholder="말하거나 입력..."
|
||||||
|
value={voiceText}
|
||||||
|
/>
|
||||||
|
<div className="glasses-room-strip" aria-label="room target">
|
||||||
|
{rooms.slice(0, 4).map((room, index) => (
|
||||||
|
<button
|
||||||
|
aria-pressed={room.jid === selectedRoom?.jid}
|
||||||
|
className={room.jid === selectedRoom?.jid ? 'is-active' : undefined}
|
||||||
|
key={room.jid}
|
||||||
|
onClick={() => setSelectedRoomIndex(index)}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
{room.label}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<div className="glasses-actions">
|
||||||
|
<button
|
||||||
|
disabled={!canListen || listening}
|
||||||
|
onClick={onStartListening}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<Mic size={16} aria-hidden />
|
||||||
|
{listening ? '듣는 중' : '말하기'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="is-active"
|
||||||
|
disabled={!selectedRoom || !voiceText.trim() || sendingVoice}
|
||||||
|
onClick={onSendVoiceText}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<Send size={16} aria-hidden />
|
||||||
|
{sendingVoice ? '전송 중' : '전송'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GlassesHeaderProps {
|
||||||
|
onRefresh: () => void;
|
||||||
|
refreshing: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function GlassesHeader({ onRefresh, refreshing }: GlassesHeaderProps) {
|
||||||
|
return (
|
||||||
|
<header className="glasses-header">
|
||||||
|
<div>
|
||||||
|
<span className="glasses-kicker">EJClaw</span>
|
||||||
|
<h1>Display</h1>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
aria-busy={refreshing || undefined}
|
||||||
|
aria-label="새로고침"
|
||||||
|
className="glasses-icon-button"
|
||||||
|
disabled={refreshing}
|
||||||
|
onClick={onRefresh}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<RefreshCw size={18} aria-hidden />
|
||||||
|
</button>
|
||||||
|
</header>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ModeTabsProps {
|
||||||
|
mode: GlassesMode;
|
||||||
|
setMode: (mode: GlassesMode) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ModeTabs({ mode, setMode }: ModeTabsProps) {
|
||||||
|
return (
|
||||||
|
<nav className="glasses-tabs" aria-label="display modes">
|
||||||
|
<button
|
||||||
|
aria-pressed={mode === 'queue'}
|
||||||
|
className={mode === 'queue' ? 'is-active' : undefined}
|
||||||
|
onClick={() => setMode('queue')}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<Inbox size={16} aria-hidden />
|
||||||
|
Queue
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
aria-pressed={mode === 'voice'}
|
||||||
|
className={mode === 'voice' ? 'is-active' : undefined}
|
||||||
|
onClick={() => setMode('voice')}
|
||||||
|
type="button"
|
||||||
|
>
|
||||||
|
<Mic size={16} aria-hidden />
|
||||||
|
Voice
|
||||||
|
</button>
|
||||||
|
</nav>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function footerStatus(
|
||||||
|
mode: GlassesMode,
|
||||||
|
busyActionKey: string | null,
|
||||||
|
selectedRoom: RoomChoice | undefined,
|
||||||
|
): string {
|
||||||
|
if (mode === 'queue') return busyActionKey ?? 'ready';
|
||||||
|
return selectedRoom?.status ?? 'ready';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function GlassesPanel({
|
||||||
|
createRequestId,
|
||||||
|
error,
|
||||||
|
freshnessText,
|
||||||
|
inboxActionKey,
|
||||||
|
locale,
|
||||||
|
onInboxAction,
|
||||||
|
onRefresh,
|
||||||
|
onSendRoomMessage,
|
||||||
|
overview,
|
||||||
|
refreshing,
|
||||||
|
roomMessageKey,
|
||||||
|
snapshots,
|
||||||
|
}: GlassesPanelProps) {
|
||||||
|
const shellRef = useRef<HTMLElement | null>(null);
|
||||||
|
const [mode, setMode] = useState<GlassesMode>('queue');
|
||||||
|
const [selectedInboxIndex, setSelectedInboxIndex] = useState(0);
|
||||||
|
const [selectedActionIndex, setSelectedActionIndex] = useState(0);
|
||||||
|
const [selectedRoomIndex, setSelectedRoomIndex] = useState(0);
|
||||||
|
const [voiceText, setVoiceText] = useState('');
|
||||||
|
const [listening, setListening] = useState(false);
|
||||||
|
const inboxItems = useMemo(
|
||||||
|
() => sortInboxItems(Array.isArray(overview.inbox) ? overview.inbox : []),
|
||||||
|
[overview.inbox],
|
||||||
|
);
|
||||||
|
const rooms = useMemo(
|
||||||
|
() => buildRoomChoices(Array.isArray(snapshots) ? snapshots : []),
|
||||||
|
[snapshots],
|
||||||
|
);
|
||||||
|
const selectedInbox =
|
||||||
|
inboxItems[clampIndex(selectedInboxIndex, inboxItems.length)];
|
||||||
|
const inboxActions = selectedInbox ? inboxActionsFor(selectedInbox) : [];
|
||||||
|
const selectedAction =
|
||||||
|
inboxActions[clampIndex(selectedActionIndex, inboxActions.length)];
|
||||||
|
const selectedRoom = rooms[clampIndex(selectedRoomIndex, rooms.length)];
|
||||||
|
const canListen = getSpeechRecognition() !== null;
|
||||||
|
const busyActionKey =
|
||||||
|
selectedInbox && selectedAction
|
||||||
|
? `${selectedInbox.id}:${selectedAction}`
|
||||||
|
: null;
|
||||||
|
const sendingVoice = selectedRoom
|
||||||
|
? roomMessageKey === selectedRoom.jid
|
||||||
|
: false;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
shellRef.current?.focus();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setSelectedInboxIndex((value) => clampIndex(value, inboxItems.length));
|
||||||
|
}, [inboxItems.length]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setSelectedActionIndex((value) => clampIndex(value, inboxActions.length));
|
||||||
|
}, [inboxActions.length]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setSelectedRoomIndex((value) => clampIndex(value, rooms.length));
|
||||||
|
}, [rooms.length]);
|
||||||
|
|
||||||
|
function startListening() {
|
||||||
|
const SpeechRecognition = getSpeechRecognition();
|
||||||
|
if (!SpeechRecognition || listening) return;
|
||||||
|
const recognition = new SpeechRecognition();
|
||||||
|
recognition.lang = SPEECH_LOCALES[locale];
|
||||||
|
recognition.continuous = false;
|
||||||
|
recognition.interimResults = false;
|
||||||
|
recognition.maxAlternatives = 1;
|
||||||
|
recognition.onresult = (event) => {
|
||||||
|
const nextText = readTranscript(event);
|
||||||
|
if (nextText) setVoiceText(nextText);
|
||||||
|
};
|
||||||
|
recognition.onerror = () => setListening(false);
|
||||||
|
recognition.onend = () => setListening(false);
|
||||||
|
setListening(true);
|
||||||
|
recognition.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSelectedAction() {
|
||||||
|
if (!selectedInbox || !selectedAction || inboxActionKey) return;
|
||||||
|
await onInboxAction(selectedInbox, selectedAction);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendVoiceText() {
|
||||||
|
const text = voiceText.trim();
|
||||||
|
if (!selectedRoom || !text || sendingVoice) return;
|
||||||
|
const ok = await onSendRoomMessage(
|
||||||
|
selectedRoom.jid,
|
||||||
|
text,
|
||||||
|
createRequestId(),
|
||||||
|
);
|
||||||
|
if (ok) setVoiceText('');
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleKeyDown(event: KeyboardEvent<HTMLElement>) {
|
||||||
|
if (
|
||||||
|
event.target instanceof HTMLInputElement ||
|
||||||
|
event.target instanceof HTMLTextAreaElement
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') {
|
||||||
|
event.preventDefault();
|
||||||
|
setMode((value) => (value === 'queue' ? 'voice' : 'queue'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.key === 'ArrowUp' || event.key === 'ArrowDown') {
|
||||||
|
event.preventDefault();
|
||||||
|
const delta = event.key === 'ArrowDown' ? 1 : -1;
|
||||||
|
if (mode === 'queue') {
|
||||||
|
setSelectedInboxIndex((value) =>
|
||||||
|
nextIndex(value, inboxItems.length, delta),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
setSelectedRoomIndex((value) => nextIndex(value, rooms.length, delta));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.key === 'Enter') {
|
||||||
|
event.preventDefault();
|
||||||
|
if (mode === 'queue') void runSelectedAction();
|
||||||
|
else void sendVoiceText();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<main
|
||||||
|
aria-label="EJClaw Ray-Ban Display"
|
||||||
|
className="glasses-shell"
|
||||||
|
onKeyDown={handleKeyDown}
|
||||||
|
ref={shellRef}
|
||||||
|
tabIndex={0}
|
||||||
|
>
|
||||||
|
<GlassesHeader onRefresh={onRefresh} refreshing={refreshing} />
|
||||||
|
|
||||||
|
<section className="glasses-status-row" aria-label="상태">
|
||||||
|
<span>{freshnessText}</span>
|
||||||
|
<strong>{inboxItems.length} items</strong>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{error ? <p className="glasses-error">{error}</p> : null}
|
||||||
|
|
||||||
|
<ModeTabs mode={mode} setMode={setMode} />
|
||||||
|
|
||||||
|
{mode === 'queue' ? (
|
||||||
|
<QueueCard
|
||||||
|
inboxActionKey={inboxActionKey}
|
||||||
|
inboxActions={inboxActions}
|
||||||
|
inboxItems={inboxItems}
|
||||||
|
locale={locale}
|
||||||
|
onInboxAction={onInboxAction}
|
||||||
|
selectedActionIndex={selectedActionIndex}
|
||||||
|
selectedInbox={selectedInbox}
|
||||||
|
selectedInboxIndex={selectedInboxIndex}
|
||||||
|
setSelectedActionIndex={setSelectedActionIndex}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<VoiceCard
|
||||||
|
canListen={canListen}
|
||||||
|
listening={listening}
|
||||||
|
onSendVoiceText={() => void sendVoiceText()}
|
||||||
|
onStartListening={startListening}
|
||||||
|
rooms={rooms}
|
||||||
|
selectedRoom={selectedRoom}
|
||||||
|
sendingVoice={sendingVoice}
|
||||||
|
setSelectedRoomIndex={setSelectedRoomIndex}
|
||||||
|
setVoiceText={setVoiceText}
|
||||||
|
voiceText={voiceText}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<footer className="glasses-footer">
|
||||||
|
<span>{footerStatus(mode, busyActionKey, selectedRoom)}</span>
|
||||||
|
</footer>
|
||||||
|
</main>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -300,7 +300,7 @@ export function RoomBoardV2({
|
|||||||
const filtered = allEntries.filter(
|
const filtered = allEntries.filter(
|
||||||
(entry) => filter === 'all' || entry.status === filter,
|
(entry) => filter === 'all' || entry.status === filter,
|
||||||
);
|
);
|
||||||
const sorted = [...filtered].sort((a, b) => {
|
const sorted = Array.from(filtered).sort((a, b) => {
|
||||||
if (sort === 'name') return a.name.localeCompare(b.name);
|
if (sort === 'name') return a.name.localeCompare(b.name);
|
||||||
if (sort === 'queue') {
|
if (sort === 'queue') {
|
||||||
const aQ = a.pendingTasks + (a.pendingMessages ? 1 : 0);
|
const aQ = a.pendingTasks + (a.pendingMessages ? 1 : 0);
|
||||||
@@ -321,10 +321,11 @@ export function RoomBoardV2({
|
|||||||
const nextJid = selectedEntry?.jid ?? null;
|
const nextJid = selectedEntry?.jid ?? null;
|
||||||
if (nextJid !== selectedJid) {
|
if (nextJid !== selectedJid) {
|
||||||
onSelectedJidChange(nextJid);
|
onSelectedJidChange(nextJid);
|
||||||
setMobileDetailOpen(false);
|
|
||||||
}
|
}
|
||||||
}, [onSelectedJidChange, selectedEntry?.jid, selectedJid]);
|
}, [onSelectedJidChange, selectedEntry?.jid, selectedJid]);
|
||||||
|
|
||||||
|
const detailOpen = mobileDetailOpen && selectedEntry?.jid === selectedJid;
|
||||||
|
|
||||||
if (allEntries.length === 0) {
|
if (allEntries.length === 0) {
|
||||||
return <EmptyState>{t.rooms.empty}</EmptyState>;
|
return <EmptyState>{t.rooms.empty}</EmptyState>;
|
||||||
}
|
}
|
||||||
@@ -359,9 +360,7 @@ export function RoomBoardV2({
|
|||||||
{sorted.length === 0 ? (
|
{sorted.length === 0 ? (
|
||||||
<EmptyState>{t.rooms.empty}</EmptyState>
|
<EmptyState>{t.rooms.empty}</EmptyState>
|
||||||
) : (
|
) : (
|
||||||
<div
|
<div className={`rooms-twopane${detailOpen ? ' is-detail-open' : ''}`}>
|
||||||
className={`rooms-twopane${mobileDetailOpen ? ' is-detail-open' : ''}`}
|
|
||||||
>
|
|
||||||
<RoomsList
|
<RoomsList
|
||||||
entries={sorted}
|
entries={sorted}
|
||||||
inbox={inbox}
|
inbox={inbox}
|
||||||
|
|||||||
@@ -91,6 +91,56 @@ interface TaskEditFormProps {
|
|||||||
t: Messages;
|
t: Messages;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const TASK_TIME_FORMATTERS: Record<Locale, Intl.DateTimeFormat> = {
|
||||||
|
en: new Intl.DateTimeFormat(localeTags.en, {
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
}),
|
||||||
|
ja: new Intl.DateTimeFormat(localeTags.ja, {
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
}),
|
||||||
|
ko: new Intl.DateTimeFormat(localeTags.ko, {
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
}),
|
||||||
|
zh: new Intl.DateTimeFormat(localeTags.zh, {
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const EN_TASK_DATE_TIME_FORMATTER = new Intl.DateTimeFormat(localeTags.en, {
|
||||||
|
month: 'short',
|
||||||
|
day: 'numeric',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const RELATIVE_TIME_FORMATTERS: Record<Locale, Intl.RelativeTimeFormat> = {
|
||||||
|
en: new Intl.RelativeTimeFormat(localeTags.en, {
|
||||||
|
numeric: 'auto',
|
||||||
|
style: 'short',
|
||||||
|
}),
|
||||||
|
ja: new Intl.RelativeTimeFormat(localeTags.ja, {
|
||||||
|
numeric: 'auto',
|
||||||
|
style: 'short',
|
||||||
|
}),
|
||||||
|
ko: new Intl.RelativeTimeFormat(localeTags.ko, {
|
||||||
|
numeric: 'auto',
|
||||||
|
style: 'short',
|
||||||
|
}),
|
||||||
|
zh: new Intl.RelativeTimeFormat(localeTags.zh, {
|
||||||
|
numeric: 'auto',
|
||||||
|
style: 'short',
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
function formatTaskDate(
|
function formatTaskDate(
|
||||||
value: string | null | undefined,
|
value: string | null | undefined,
|
||||||
locale: Locale,
|
locale: Locale,
|
||||||
@@ -98,22 +148,12 @@ function formatTaskDate(
|
|||||||
if (!value) return '-';
|
if (!value) return '-';
|
||||||
const date = new Date(value);
|
const date = new Date(value);
|
||||||
if (Number.isNaN(date.getTime())) return value;
|
if (Number.isNaN(date.getTime())) return value;
|
||||||
const time = new Intl.DateTimeFormat(localeTags[locale], {
|
const time = TASK_TIME_FORMATTERS[locale].format(date);
|
||||||
hour: '2-digit',
|
|
||||||
minute: '2-digit',
|
|
||||||
hour12: false,
|
|
||||||
}).format(date);
|
|
||||||
if (locale === 'ko')
|
if (locale === 'ko')
|
||||||
return `${date.getMonth() + 1}월 ${date.getDate()}일 ${time}`;
|
return `${date.getMonth() + 1}월 ${date.getDate()}일 ${time}`;
|
||||||
if (locale === 'ja' || locale === 'zh')
|
if (locale === 'ja' || locale === 'zh')
|
||||||
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
||||||
return new Intl.DateTimeFormat(localeTags[locale], {
|
return EN_TASK_DATE_TIME_FORMATTER.format(date);
|
||||||
month: 'short',
|
|
||||||
day: 'numeric',
|
|
||||||
hour: '2-digit',
|
|
||||||
minute: '2-digit',
|
|
||||||
hour12: false,
|
|
||||||
}).format(date);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatRelativeDate(
|
function formatRelativeDate(
|
||||||
@@ -135,10 +175,10 @@ function formatRelativeDate(
|
|||||||
];
|
];
|
||||||
const [unit, unitMs] =
|
const [unit, unitMs] =
|
||||||
units.find(([, threshold]) => absMs >= threshold) ?? units.at(-1)!;
|
units.find(([, threshold]) => absMs >= threshold) ?? units.at(-1)!;
|
||||||
return new Intl.RelativeTimeFormat(localeTags[locale], {
|
return RELATIVE_TIME_FORMATTERS[locale].format(
|
||||||
numeric: 'auto',
|
Math.round(diffMs / unitMs),
|
||||||
style: 'short',
|
unit,
|
||||||
}).format(Math.round(diffMs / unitMs), unit);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function safePreview(
|
function safePreview(
|
||||||
|
|||||||
@@ -141,7 +141,7 @@ function UsageSpeed({ row, t }: { row: UsageRow; t: Messages }) {
|
|||||||
export function UsagePanel({ overview, t }: UsagePanelProps) {
|
export function UsagePanel({ overview, t }: UsagePanelProps) {
|
||||||
const rows = useMemo(
|
const rows = useMemo(
|
||||||
() =>
|
() =>
|
||||||
[...overview.usage.rows].sort((a, b) => {
|
Array.from(overview.usage.rows).sort((a, b) => {
|
||||||
if (usageActive(a) !== usageActive(b)) return usageActive(a) ? -1 : 1;
|
if (usageActive(a) !== usageActive(b)) return usageActive(a) ? -1 : 1;
|
||||||
return usagePeak(b) - usagePeak(a);
|
return usagePeak(b) - usagePeak(a);
|
||||||
}),
|
}),
|
||||||
@@ -194,24 +194,28 @@ export function UsagePanel({ overview, t }: UsagePanelProps) {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="usage-matrix" role="table" aria-label={t.panels.usage}>
|
<table className="usage-matrix" aria-label={t.panels.usage}>
|
||||||
<div className="usage-matrix-head" role="row">
|
<thead>
|
||||||
<span>{t.usage.usage}</span>
|
<tr className="usage-matrix-head">
|
||||||
<span>{t.usage.quota.h5}</span>
|
<th scope="col">{t.usage.usage}</th>
|
||||||
<span>{t.usage.quota.d7}</span>
|
<th scope="col">{t.usage.quota.h5}</th>
|
||||||
<span>{t.usage.speed}</span>
|
<th scope="col">{t.usage.quota.d7}</th>
|
||||||
</div>
|
<th scope="col">{t.usage.speed}</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
{groups.map((group) => (
|
{groups.map((group) => (
|
||||||
<div className="usage-group" key={group.key} role="rowgroup">
|
<tbody className="usage-group" key={group.key}>
|
||||||
<div className="usage-group-label" role="row">
|
<tr className="usage-group-label">
|
||||||
<span>{group.label}</span>
|
<th colSpan={4} scope="colgroup">
|
||||||
</div>
|
{group.label}
|
||||||
|
</th>
|
||||||
|
</tr>
|
||||||
{group.rows.map((row) => {
|
{group.rows.map((row) => {
|
||||||
const risk = usageRiskLevel(row);
|
const risk = usageRiskLevel(row);
|
||||||
const { account, plan } = usageNameParts(row);
|
const { account, plan } = usageNameParts(row);
|
||||||
return (
|
return (
|
||||||
<section className={`usage-row usage-${risk}`} key={row.name}>
|
<tr className={`usage-row usage-${risk}`} key={row.name}>
|
||||||
<div className="usage-account">
|
<th className="usage-account" scope="row">
|
||||||
<strong>{account}</strong>
|
<strong>{account}</strong>
|
||||||
<div>
|
<div>
|
||||||
{usageActive(row) ? (
|
{usageActive(row) ? (
|
||||||
@@ -224,26 +228,32 @@ export function UsagePanel({ overview, t }: UsagePanelProps) {
|
|||||||
</span>
|
</span>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</th>
|
||||||
<UsageQuotaMeter
|
<td>
|
||||||
row={row}
|
<UsageQuotaMeter
|
||||||
rowName={account}
|
row={row}
|
||||||
window="h5"
|
rowName={account}
|
||||||
t={t}
|
window="h5"
|
||||||
/>
|
t={t}
|
||||||
<UsageQuotaMeter
|
/>
|
||||||
row={row}
|
</td>
|
||||||
rowName={account}
|
<td>
|
||||||
window="d7"
|
<UsageQuotaMeter
|
||||||
t={t}
|
row={row}
|
||||||
/>
|
rowName={account}
|
||||||
<UsageSpeed row={row} t={t} />
|
window="d7"
|
||||||
</section>
|
t={t}
|
||||||
|
/>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<UsageSpeed row={row} t={t} />
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
</div>
|
</tbody>
|
||||||
))}
|
))}
|
||||||
</div>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,34 @@
|
|||||||
import type { DashboardTask, DashboardTaskAction } from './api';
|
import type { DashboardTask, DashboardTaskAction } from './api';
|
||||||
import { localeTags, type Locale, type Messages } from './i18n';
|
import { localeTags, type Locale, type Messages } from './i18n';
|
||||||
|
|
||||||
|
const SHORT_TIME_FORMAT_OPTIONS = {
|
||||||
|
hour: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
minute: '2-digit',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const MONTH_DAY_TIME_FORMAT_OPTIONS = {
|
||||||
|
day: 'numeric',
|
||||||
|
hour: '2-digit',
|
||||||
|
hour12: false,
|
||||||
|
minute: '2-digit',
|
||||||
|
month: 'short',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const SHORT_TIME_FORMATTERS: Record<Locale, Intl.DateTimeFormat> = {
|
||||||
|
en: new Intl.DateTimeFormat(localeTags.en, SHORT_TIME_FORMAT_OPTIONS),
|
||||||
|
ja: new Intl.DateTimeFormat(localeTags.ja, SHORT_TIME_FORMAT_OPTIONS),
|
||||||
|
ko: new Intl.DateTimeFormat(localeTags.ko, SHORT_TIME_FORMAT_OPTIONS),
|
||||||
|
zh: new Intl.DateTimeFormat(localeTags.zh, SHORT_TIME_FORMAT_OPTIONS),
|
||||||
|
};
|
||||||
|
|
||||||
|
const MONTH_DAY_TIME_FORMATTERS: Record<Locale, Intl.DateTimeFormat> = {
|
||||||
|
en: new Intl.DateTimeFormat(localeTags.en, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||||
|
ja: new Intl.DateTimeFormat(localeTags.ja, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||||
|
ko: new Intl.DateTimeFormat(localeTags.ko, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||||
|
zh: new Intl.DateTimeFormat(localeTags.zh, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||||
|
};
|
||||||
|
|
||||||
export function formatDate(
|
export function formatDate(
|
||||||
value: string | null | undefined,
|
value: string | null | undefined,
|
||||||
locale: Locale,
|
locale: Locale,
|
||||||
@@ -31,30 +59,16 @@ export function formatDate(
|
|||||||
}
|
}
|
||||||
const sameDay = new Date().toDateString() === date.toDateString();
|
const sameDay = new Date().toDateString() === date.toDateString();
|
||||||
if (sameDay) {
|
if (sameDay) {
|
||||||
return new Intl.DateTimeFormat(localeTags[locale], {
|
return SHORT_TIME_FORMATTERS[locale].format(date);
|
||||||
hour: '2-digit',
|
|
||||||
hour12: false,
|
|
||||||
minute: '2-digit',
|
|
||||||
}).format(date);
|
|
||||||
}
|
}
|
||||||
const time = new Intl.DateTimeFormat(localeTags[locale], {
|
const time = SHORT_TIME_FORMATTERS[locale].format(date);
|
||||||
hour: '2-digit',
|
|
||||||
hour12: false,
|
|
||||||
minute: '2-digit',
|
|
||||||
}).format(date);
|
|
||||||
if (locale === 'ko')
|
if (locale === 'ko')
|
||||||
return `${date.getMonth() + 1}월 ${date.getDate()}일 ${time}`;
|
return `${date.getMonth() + 1}월 ${date.getDate()}일 ${time}`;
|
||||||
if (locale === 'ja')
|
if (locale === 'ja')
|
||||||
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
||||||
if (locale === 'zh')
|
if (locale === 'zh')
|
||||||
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
||||||
return new Intl.DateTimeFormat(localeTags[locale], {
|
return MONTH_DAY_TIME_FORMATTERS[locale].format(date);
|
||||||
day: 'numeric',
|
|
||||||
hour: '2-digit',
|
|
||||||
hour12: false,
|
|
||||||
minute: '2-digit',
|
|
||||||
month: 'short',
|
|
||||||
}).format(date);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function taskActionsFor(task: DashboardTask): DashboardTaskAction[] {
|
export function taskActionsFor(task: DashboardTask): DashboardTaskAction[] {
|
||||||
|
|||||||
293
apps/dashboard/src/glasses.css
Normal file
293
apps/dashboard/src/glasses.css
Normal file
@@ -0,0 +1,293 @@
|
|||||||
|
body:has(.glasses-shell) {
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
background: #000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-shell {
|
||||||
|
display: grid;
|
||||||
|
grid-template-rows: auto auto auto minmax(0, 1fr) auto;
|
||||||
|
gap: 8px;
|
||||||
|
width: min(100vw, 600px);
|
||||||
|
height: min(100vh, 600px);
|
||||||
|
min-height: 0;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 14px;
|
||||||
|
overflow: hidden;
|
||||||
|
background: #050706;
|
||||||
|
color: #f4f1e9;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-header,
|
||||||
|
.glasses-status-row,
|
||||||
|
.glasses-card-head,
|
||||||
|
.glasses-actions,
|
||||||
|
.glasses-footer,
|
||||||
|
.glasses-tabs,
|
||||||
|
.glasses-room-strip {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-header {
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-kicker,
|
||||||
|
.glasses-status-row,
|
||||||
|
.glasses-card-head small,
|
||||||
|
.glasses-meta dt,
|
||||||
|
.glasses-footer {
|
||||||
|
color: #a8afa6;
|
||||||
|
font-size: 11px;
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-header h1 {
|
||||||
|
margin: 0;
|
||||||
|
font-size: 26px;
|
||||||
|
line-height: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-icon-button {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
width: 42px;
|
||||||
|
height: 42px;
|
||||||
|
padding: 0;
|
||||||
|
border-radius: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-status-row {
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-status-row strong {
|
||||||
|
color: #6cb087;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-error {
|
||||||
|
margin: 0;
|
||||||
|
padding: 8px 10px;
|
||||||
|
border: 1px solid rgba(224, 100, 75, 0.45);
|
||||||
|
border-radius: 8px;
|
||||||
|
background: rgba(224, 100, 75, 0.12);
|
||||||
|
color: #ffb4a4;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-tabs {
|
||||||
|
gap: 6px;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-tabs button,
|
||||||
|
.glasses-actions button,
|
||||||
|
.glasses-room-strip button {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 6px;
|
||||||
|
min-height: 42px;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 8px;
|
||||||
|
background: rgba(255, 255, 255, 0.06);
|
||||||
|
color: #f4f1e9;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-tabs button:hover:not(:disabled),
|
||||||
|
.glasses-actions button:hover:not(:disabled),
|
||||||
|
.glasses-room-strip button:hover:not(:disabled) {
|
||||||
|
background: rgba(255, 255, 255, 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-tabs button {
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-tabs button.is-active,
|
||||||
|
.glasses-tabs button[aria-pressed='true'],
|
||||||
|
.glasses-actions button.is-active,
|
||||||
|
.glasses-actions button[aria-pressed='true'],
|
||||||
|
.glasses-room-strip button.is-active,
|
||||||
|
.glasses-room-strip button[aria-pressed='true'] {
|
||||||
|
border-color: rgba(232, 152, 112, 0.72);
|
||||||
|
background: rgba(214, 130, 88, 0.2);
|
||||||
|
color: #fff4ec;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-tabs button.is-active:hover:not(:disabled),
|
||||||
|
.glasses-tabs button[aria-pressed='true']:hover:not(:disabled),
|
||||||
|
.glasses-actions button.is-active:hover:not(:disabled),
|
||||||
|
.glasses-actions button[aria-pressed='true']:hover:not(:disabled),
|
||||||
|
.glasses-room-strip button.is-active:hover:not(:disabled),
|
||||||
|
.glasses-room-strip button[aria-pressed='true']:hover:not(:disabled) {
|
||||||
|
background: rgba(214, 130, 88, 0.26);
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-card {
|
||||||
|
display: grid;
|
||||||
|
align-content: start;
|
||||||
|
gap: 10px;
|
||||||
|
min-height: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 8px;
|
||||||
|
background: #111512;
|
||||||
|
padding: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-card-head {
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-pill {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
max-width: 100%;
|
||||||
|
min-height: 24px;
|
||||||
|
padding: 0 8px;
|
||||||
|
overflow: hidden;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.12);
|
||||||
|
border-radius: 999px;
|
||||||
|
background: rgba(255, 255, 255, 0.05);
|
||||||
|
color: #d7ded2;
|
||||||
|
font-size: 11px;
|
||||||
|
font-weight: 700;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
text-transform: uppercase;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-pill.sev-error {
|
||||||
|
border-color: rgba(224, 100, 75, 0.5);
|
||||||
|
color: #ffb4a4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-pill.sev-warn {
|
||||||
|
border-color: rgba(212, 160, 74, 0.5);
|
||||||
|
color: #ffd38b;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-queue-card h2 {
|
||||||
|
margin: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
font-size: 22px;
|
||||||
|
line-height: 1.08;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-queue-card p {
|
||||||
|
display: -webkit-box;
|
||||||
|
min-height: 0;
|
||||||
|
margin: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
color: #d2d5ce;
|
||||||
|
font-size: 14px;
|
||||||
|
line-height: 1.35;
|
||||||
|
-webkit-box-orient: vertical;
|
||||||
|
-webkit-line-clamp: 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-meta {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 1fr;
|
||||||
|
gap: 8px;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-meta div {
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-meta dt,
|
||||||
|
.glasses-meta dd {
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-meta dd {
|
||||||
|
margin: 2px 0 0;
|
||||||
|
color: #f4f1e9;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-actions {
|
||||||
|
gap: 6px;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-actions button {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
padding: 0 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-empty {
|
||||||
|
display: grid;
|
||||||
|
place-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
min-height: 220px;
|
||||||
|
color: #a8afa6;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-empty strong {
|
||||||
|
color: #f4f1e9;
|
||||||
|
font-size: 24px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-voice-card textarea {
|
||||||
|
width: 100%;
|
||||||
|
min-height: 126px;
|
||||||
|
resize: none;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||||
|
border-radius: 8px;
|
||||||
|
background: rgba(255, 255, 255, 0.04);
|
||||||
|
color: #f4f1e9;
|
||||||
|
font: inherit;
|
||||||
|
font-size: 18px;
|
||||||
|
line-height: 1.35;
|
||||||
|
padding: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-voice-card textarea:focus-visible {
|
||||||
|
border-color: #d68258;
|
||||||
|
outline: 2px solid rgba(214, 130, 88, 0.2);
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-room-strip {
|
||||||
|
gap: 6px;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-room-strip button {
|
||||||
|
min-width: 0;
|
||||||
|
max-width: 50%;
|
||||||
|
overflow: hidden;
|
||||||
|
padding: 0 10px;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
|
||||||
|
.glasses-footer {
|
||||||
|
min-height: 18px;
|
||||||
|
justify-content: center;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 640px) {
|
||||||
|
.glasses-shell {
|
||||||
|
width: 100vw;
|
||||||
|
height: 100vh;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,13 @@ import { StrictMode } from 'react';
|
|||||||
import { createRoot } from 'react-dom/client';
|
import { createRoot } from 'react-dom/client';
|
||||||
|
|
||||||
import App from './App';
|
import App from './App';
|
||||||
|
import { GlassesApp } from './GlassesApp';
|
||||||
|
|
||||||
|
function isGlassesRoute(): boolean {
|
||||||
|
const pathname = window.location.pathname.replace(/\/+$/, '') || '/';
|
||||||
|
const search = new URLSearchParams(window.location.search);
|
||||||
|
return pathname === '/glasses' || search.get('display') === 'rayban';
|
||||||
|
}
|
||||||
|
|
||||||
const root = document.getElementById('root');
|
const root = document.getElementById('root');
|
||||||
|
|
||||||
@@ -10,7 +17,5 @@ if (!root) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
createRoot(root).render(
|
createRoot(root).render(
|
||||||
<StrictMode>
|
<StrictMode>{isGlassesRoute() ? <GlassesApp /> : <App />}</StrictMode>,
|
||||||
<App />
|
|
||||||
</StrictMode>,
|
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -169,6 +169,41 @@ function mergeAdjacentBotChunks(entries: RoomThreadEntry[]): RoomThreadEntry[] {
|
|||||||
return merged;
|
return merged;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function collectOutputEntries(outputs: RoomOutput[]): RoomThreadEntry[] {
|
||||||
|
const entries: RoomThreadEntry[] = [];
|
||||||
|
for (const output of outputs) {
|
||||||
|
const entry = toOutputEntry(output);
|
||||||
|
if (entry) entries.push(entry);
|
||||||
|
}
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectChatEntries(
|
||||||
|
messages: RoomMessage[],
|
||||||
|
outputEntries: RoomThreadEntry[],
|
||||||
|
): RoomThreadEntry[] {
|
||||||
|
const entries: RoomThreadEntry[] = [];
|
||||||
|
for (const message of messages) {
|
||||||
|
if (!isThreadChatMessage(message)) continue;
|
||||||
|
const entry = toMessageEntry(message);
|
||||||
|
if (!isDuplicateOutputMessage(entry, outputEntries)) entries.push(entry);
|
||||||
|
}
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectOptimisticPending(
|
||||||
|
pendingMessages: RoomMessage[],
|
||||||
|
confirmedSet: Set<string>,
|
||||||
|
): RoomThreadEntry[] {
|
||||||
|
const entries: RoomThreadEntry[] = [];
|
||||||
|
for (const message of pendingMessages) {
|
||||||
|
if (confirmedSet.has(messageKey(message))) continue;
|
||||||
|
if (isInternalProtocolPayload(message.content)) continue;
|
||||||
|
entries.push(toMessageEntry(message));
|
||||||
|
}
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
export function buildRoomThreadEntries({
|
export function buildRoomThreadEntries({
|
||||||
messages,
|
messages,
|
||||||
outputs,
|
outputs,
|
||||||
@@ -179,20 +214,14 @@ export function buildRoomThreadEntries({
|
|||||||
pendingMessages?: RoomMessage[];
|
pendingMessages?: RoomMessage[];
|
||||||
}): RoomThreadEntry[] {
|
}): RoomThreadEntry[] {
|
||||||
const confirmedSet = new Set(messages.map(messageKey));
|
const confirmedSet = new Set(messages.map(messageKey));
|
||||||
const outputEntries = outputs
|
const outputEntries = collectOutputEntries(outputs);
|
||||||
.map(toOutputEntry)
|
const chatEntries = collectChatEntries(messages, outputEntries);
|
||||||
.filter((entry): entry is RoomThreadEntry => Boolean(entry));
|
const optimisticPending = collectOptimisticPending(
|
||||||
const chatEntries = messages
|
pendingMessages,
|
||||||
.filter(isThreadChatMessage)
|
confirmedSet,
|
||||||
.map(toMessageEntry)
|
|
||||||
.filter((entry) => !isDuplicateOutputMessage(entry, outputEntries));
|
|
||||||
const optimisticPending = pendingMessages
|
|
||||||
.filter((message) => !confirmedSet.has(messageKey(message)))
|
|
||||||
.filter((message) => !isInternalProtocolPayload(message.content))
|
|
||||||
.map(toMessageEntry);
|
|
||||||
|
|
||||||
const entries = [...chatEntries, ...optimisticPending, ...outputEntries].sort(
|
|
||||||
(a, b) => a.timestamp.localeCompare(b.timestamp),
|
|
||||||
);
|
);
|
||||||
|
const entries = chatEntries
|
||||||
|
.concat(optimisticPending, outputEntries)
|
||||||
|
.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
||||||
return mergeAdjacentBotChunks(entries);
|
return mergeAdjacentBotChunks(entries);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2227,6 +2227,19 @@ dd,
|
|||||||
.usage-matrix {
|
.usage-matrix {
|
||||||
display: grid;
|
display: grid;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
border-spacing: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.usage-matrix thead,
|
||||||
|
.usage-matrix tbody {
|
||||||
|
display: grid;
|
||||||
|
}
|
||||||
|
|
||||||
|
.usage-matrix th,
|
||||||
|
.usage-matrix td {
|
||||||
|
padding: 0;
|
||||||
|
border: 0;
|
||||||
|
text-align: inherit;
|
||||||
}
|
}
|
||||||
|
|
||||||
.usage-matrix-head,
|
.usage-matrix-head,
|
||||||
@@ -2258,7 +2271,6 @@ dd,
|
|||||||
}
|
}
|
||||||
|
|
||||||
.usage-group-label {
|
.usage-group-label {
|
||||||
padding: 7px 10px;
|
|
||||||
color: var(--accent-strong);
|
color: var(--accent-strong);
|
||||||
font-size: 11px;
|
font-size: 11px;
|
||||||
font-weight: 800;
|
font-weight: 800;
|
||||||
@@ -2267,6 +2279,14 @@ dd,
|
|||||||
background: rgba(191, 95, 44, 0.07);
|
background: rgba(191, 95, 44, 0.07);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.usage-group-label th {
|
||||||
|
padding: 7px 10px;
|
||||||
|
color: inherit;
|
||||||
|
font-weight: inherit;
|
||||||
|
letter-spacing: inherit;
|
||||||
|
text-transform: inherit;
|
||||||
|
}
|
||||||
|
|
||||||
.usage-row {
|
.usage-row {
|
||||||
--meter: var(--green);
|
--meter: var(--green);
|
||||||
padding: 8px 10px;
|
padding: 8px 10px;
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ export function useSelectedRoomActivity({
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!active || !selectedRoomJid) {
|
if (!active || !selectedRoomJid) {
|
||||||
setRoomActivityLoading(false);
|
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,5 +63,10 @@ export function useSelectedRoomActivity({
|
|||||||
};
|
};
|
||||||
}, [active, pollMs, refreshRoom, selectedRoomJid]);
|
}, [active, pollMs, refreshRoom, selectedRoomJid]);
|
||||||
|
|
||||||
return { refreshRoom, roomActivity, roomActivityLoading };
|
return {
|
||||||
|
refreshRoom,
|
||||||
|
roomActivity,
|
||||||
|
roomActivityLoading:
|
||||||
|
active && selectedRoomJid !== null ? roomActivityLoading : false,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,3 +5,99 @@ This file is for mutable memory shared across Claude groups.
|
|||||||
Use it for durable facts, preferences, and shared context that may change over time.
|
Use it for durable facts, preferences, and shared context that may change over time.
|
||||||
|
|
||||||
Do not store platform-wide operating rules here. Those now live in `prompts/claude-platform.md`.
|
Do not store platform-wide operating rules here. Those now live in `prompts/claude-platform.md`.
|
||||||
|
|
||||||
|
## Stored credentials
|
||||||
|
|
||||||
|
Shared credentials live at `/home/claude/.config/ejclaw/secrets.json` (chmod 600, owner-only). Read with the Read tool when a session in any channel asks to "저장해둔 계정토큰으로 로그인" or otherwise needs a registered token.
|
||||||
|
|
||||||
|
Schema: `credentials.<host>.{type, host, token, note, added_at}`.
|
||||||
|
|
||||||
|
Currently stored:
|
||||||
|
- `git.tkrmagid.kr` — Gitea personal access token. Use via `Authorization: token <value>` header, or embed in HTTPS URL as `https://<user>:<token>@git.tkrmagid.kr/...`. For `git clone/push`, prefer the URL form or `git -c http.extraHeader="Authorization: token <value>" clone ...`. Do not paste the raw token into chat replies.
|
||||||
|
- `sudo` — local sudo password for the `claude` user on this host. Use via `echo "$PW" | sudo -S <cmd>` (read the password from `credentials.sudo.password` with the Read tool, then pipe). Do not paste the raw password into chat replies.
|
||||||
|
|
||||||
|
To add or rotate a credential, edit `secrets.json` and append a new entry under `credentials`; update this list with the host and intended use.
|
||||||
|
|
||||||
|
## Room mode policy
|
||||||
|
|
||||||
|
All rooms default to `tribunal` (paired) mode. Owner runs the work, reviewer/arbiter (claude-code) verifies. New rooms registered via `bun setup/index.ts --step register` are also `tribunal` by default.
|
||||||
|
|
||||||
|
If the user in a channel says any of these — "클로드 사용하지 말자", "paired 모드 끄자", "리뷰어 끄자", "이 방은 single 로 바꿔줘" — switch that channel back to `single` mode. Use:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun -e "import { initDatabase, setExplicitRoomMode } from './src/db.js'; initDatabase(); setExplicitRoomMode('<chatJid>', 'single');"
|
||||||
|
```
|
||||||
|
|
||||||
|
The reverse phrase ("paired 켜자", "리뷰어 다시 켜자") flips it back to `'tribunal'`. Acknowledge the change in chat and confirm the new mode.
|
||||||
|
|
||||||
|
## Git backups
|
||||||
|
|
||||||
|
- 2026-05-27 10:53 KST: EJClaw 정상 동작 상태를 git commit `1509108` (`backup current stable ejclaw state`)로 백업했다. "이번 요청만 리뷰어 사용" 기능 작업은 이 백업 커밋 이후에 시작한 변경이다.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# CLAUDE.md
|
||||||
|
|
||||||
|
Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
|
||||||
|
|
||||||
|
**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
|
||||||
|
|
||||||
|
## 1. Think Before Coding
|
||||||
|
|
||||||
|
**Don't assume. Don't hide confusion. Surface tradeoffs.**
|
||||||
|
|
||||||
|
Before implementing:
|
||||||
|
- State your assumptions explicitly. If uncertain, ask.
|
||||||
|
- If multiple interpretations exist, present them - don't pick silently.
|
||||||
|
- If a simpler approach exists, say so. Push back when warranted.
|
||||||
|
- If something is unclear, stop. Name what's confusing. Ask.
|
||||||
|
|
||||||
|
## 2. Simplicity First
|
||||||
|
|
||||||
|
**Minimum code that solves the problem. Nothing speculative.**
|
||||||
|
|
||||||
|
- No features beyond what was asked.
|
||||||
|
- No abstractions for single-use code.
|
||||||
|
- No "flexibility" or "configurability" that wasn't requested.
|
||||||
|
- No error handling for impossible scenarios.
|
||||||
|
- If you write 200 lines and it could be 50, rewrite it.
|
||||||
|
|
||||||
|
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
|
||||||
|
|
||||||
|
## 3. Surgical Changes
|
||||||
|
|
||||||
|
**Touch only what you must. Clean up only your own mess.**
|
||||||
|
|
||||||
|
When editing existing code:
|
||||||
|
- Don't "improve" adjacent code, comments, or formatting.
|
||||||
|
- Don't refactor things that aren't broken.
|
||||||
|
- Match existing style, even if you'd do it differently.
|
||||||
|
- If you notice unrelated dead code, mention it - don't delete it.
|
||||||
|
|
||||||
|
When your changes create orphans:
|
||||||
|
- Remove imports/variables/functions that YOUR changes made unused.
|
||||||
|
- Don't remove pre-existing dead code unless asked.
|
||||||
|
|
||||||
|
The test: Every changed line should trace directly to the user's request.
|
||||||
|
|
||||||
|
## 4. Goal-Driven Execution
|
||||||
|
|
||||||
|
**Define success criteria. Loop until verified.**
|
||||||
|
|
||||||
|
Transform tasks into verifiable goals:
|
||||||
|
- "Add validation" → "Write tests for invalid inputs, then make them pass"
|
||||||
|
- "Fix the bug" → "Write a test that reproduces it, then make it pass"
|
||||||
|
- "Refactor X" → "Ensure tests pass before and after"
|
||||||
|
|
||||||
|
For multi-step tasks, state a brief plan:
|
||||||
|
```
|
||||||
|
1. [Step] → verify: [check]
|
||||||
|
2. [Step] → verify: [check]
|
||||||
|
3. [Step] → verify: [check]
|
||||||
|
```
|
||||||
|
|
||||||
|
Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.
|
||||||
|
|||||||
3859
package-lock.json
generated
Normal file
3859
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -51,6 +51,8 @@ If the first visible line is not one of these statuses, the output is invalid; d
|
|||||||
- **BLOCKED** — Cannot proceed without user decision
|
- **BLOCKED** — Cannot proceed without user decision
|
||||||
- **NEEDS_CONTEXT** — Missing information from user
|
- **NEEDS_CONTEXT** — Missing information from user
|
||||||
|
|
||||||
|
Do not start with non-status review labels like **APPROVE** or **REVISE**. Use **TASK_DONE** for approval and **DONE_WITH_CONCERNS** for change requests.
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
|
|
||||||
- Judge completion only by verification output. "It should work now" means run it. "I'm confident" means nothing — confidence is not evidence. "I tested earlier" means test again if code changed since. "It's a trivial change" means verify anyway
|
- Judge completion only by verification output. "It should work now" means run it. "I'm confident" means nothing — confidence is not evidence. "I tested earlier" means test again if code changed since. "It's a trivial change" means verify anyway
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ You are the **owner** (implementer) in this paired room.
|
|||||||
## Critical review
|
## Critical review
|
||||||
|
|
||||||
Before accepting any proposal from the reviewer, run it through:
|
Before accepting any proposal from the reviewer, run it through:
|
||||||
|
|
||||||
1. **Essence** — Is the stated problem the actual problem?
|
1. **Essence** — Is the stated problem the actual problem?
|
||||||
2. **Root cause** — Are we fixing the root cause or treating a symptom?
|
2. **Root cause** — Are we fixing the root cause or treating a symptom?
|
||||||
3. **Prerequisites** — What must be true before this approach can work?
|
3. **Prerequisites** — What must be true before this approach can work?
|
||||||
|
|||||||
1
restart.sh
Normal file
1
restart.sh
Normal file
@@ -0,0 +1 @@
|
|||||||
|
systemctl --user restart ejclaw
|
||||||
@@ -12,7 +12,6 @@ import path from 'path';
|
|||||||
import { CronExpressionParser } from 'cron-parser';
|
import { CronExpressionParser } from 'cron-parser';
|
||||||
import {
|
import {
|
||||||
DEFAULT_SCHEDULE_TASK_CONTEXT_MODE,
|
DEFAULT_SCHEDULE_TASK_CONTEXT_MODE,
|
||||||
DEFAULT_TASK_CONTEXT_MODE,
|
|
||||||
DEFAULT_WATCH_CI_CONTEXT_MODE,
|
DEFAULT_WATCH_CI_CONTEXT_MODE,
|
||||||
EJCLAW_ENV,
|
EJCLAW_ENV,
|
||||||
TASK_CONTEXT_MODES,
|
TASK_CONTEXT_MODES,
|
||||||
@@ -766,8 +765,10 @@ If folder is omitted, the host generates one automatically.`,
|
|||||||
name: z.string().describe('Display name for the room'),
|
name: z.string().describe('Display name for the room'),
|
||||||
room_mode: z
|
room_mode: z
|
||||||
.enum(['single', 'tribunal'])
|
.enum(['single', 'tribunal'])
|
||||||
.default('single')
|
.default('tribunal')
|
||||||
.describe('single=owner only, tribunal=owner/reviewer roles enabled'),
|
.describe(
|
||||||
|
'single=owner only, tribunal=owner/reviewer roles enabled (default)',
|
||||||
|
),
|
||||||
owner_agent_type: z
|
owner_agent_type: z
|
||||||
.enum(['claude-code', 'codex'])
|
.enum(['claude-code', 'codex'])
|
||||||
.optional()
|
.optional()
|
||||||
@@ -832,7 +833,7 @@ If folder is omitted, the host generates one automatically.`,
|
|||||||
type: 'assign_room',
|
type: 'assign_room',
|
||||||
jid: args.jid,
|
jid: args.jid,
|
||||||
name: args.name,
|
name: args.name,
|
||||||
room_mode: args.room_mode || 'single',
|
room_mode: args.room_mode || 'tribunal',
|
||||||
owner_agent_type: args.owner_agent_type,
|
owner_agent_type: args.owner_agent_type,
|
||||||
reviewer_agent_type: args.reviewer_agent_type,
|
reviewer_agent_type: args.reviewer_agent_type,
|
||||||
arbiter_agent_type: args.arbiter_agent_type,
|
arbiter_agent_type: args.arbiter_agent_type,
|
||||||
|
|||||||
@@ -153,6 +153,7 @@ export async function runVerificationRequestDirect(
|
|||||||
`Failed to parse verification response from ${pathToFileURL(helperPath).href}: ${
|
`Failed to parse verification response from ${pathToFileURL(helperPath).href}: ${
|
||||||
error instanceof Error ? error.message : String(error)
|
error instanceof Error ? error.message : String(error)
|
||||||
}${detail ? `\n${detail}` : ''}`,
|
}${detail ? `\n${detail}` : ''}`,
|
||||||
|
{ cause: error },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,4 @@
|
|||||||
import {
|
import { WATCH_CI_PROMPT_PREFIX } from 'ejclaw-runners-shared';
|
||||||
DEFAULT_WATCH_CI_CONTEXT_MODE,
|
|
||||||
WATCH_CI_PROMPT_PREFIX,
|
|
||||||
} from 'ejclaw-runners-shared';
|
|
||||||
|
|
||||||
export const DEFAULT_WATCH_CI_INTERVAL_SECONDS = 60;
|
export const DEFAULT_WATCH_CI_INTERVAL_SECONDS = 60;
|
||||||
export const MIN_WATCH_CI_INTERVAL_SECONDS = 30;
|
export const MIN_WATCH_CI_INTERVAL_SECONDS = 30;
|
||||||
|
|||||||
@@ -80,9 +80,7 @@ async function measure(page: Page, sel: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function evalExpr(page: Page, expr: string) {
|
async function evalExpr(page: Page, expr: string) {
|
||||||
const result = await page.evaluate((e) => {
|
const result = await page.evaluate(expr);
|
||||||
return eval(e);
|
|
||||||
}, expr);
|
|
||||||
console.log(JSON.stringify(result, null, 2));
|
console.log(JSON.stringify(result, null, 2));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ const STEPS: Record<
|
|||||||
'restart-stack': () => import('./restart-stack.js'),
|
'restart-stack': () => import('./restart-stack.js'),
|
||||||
service: () => import('./service.js'),
|
service: () => import('./service.js'),
|
||||||
verify: () => import('./verify.js'),
|
verify: () => import('./verify.js'),
|
||||||
|
login: () => import('./login.js'),
|
||||||
};
|
};
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
async function main(): Promise<void> {
|
||||||
|
|||||||
228
setup/login.ts
Normal file
228
setup/login.ts
Normal file
@@ -0,0 +1,228 @@
|
|||||||
|
/**
|
||||||
|
* Step: login — Sign in to Claude (Pro/Max) via OAuth PKCE manual flow.
|
||||||
|
*
|
||||||
|
* Mirrors the URL Claude Code's `claude auth login --claudeai` actually
|
||||||
|
* builds (captured from the official CLI). This is the manual / paste-code
|
||||||
|
* variant: the user visits the authorize URL, completes login in their
|
||||||
|
* browser, and pastes the code back. We exchange it for tokens and write
|
||||||
|
* `~/.claude/.credentials.json`.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* bun setup/index.ts --step login # phase 1: print authorize URL
|
||||||
|
* bun setup/index.ts --step login --code <c> # phase 2: exchange the code
|
||||||
|
*
|
||||||
|
* Phase 1 stashes the PKCE verifier + state in /tmp/ejclaw-claude-login.json
|
||||||
|
* (mode 0600). Phase 2 reads it back, exchanges the code, writes credentials.
|
||||||
|
*
|
||||||
|
* This step is run by hand for re-auth. Once `.credentials.json` exists, the
|
||||||
|
* existing token-refresh loop (src/token-refresh.ts) keeps it fresh.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import crypto from 'crypto';
|
||||||
|
import fs from 'fs';
|
||||||
|
import os from 'os';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { logger } from '../src/logger.js';
|
||||||
|
import { emitStatus } from './status.js';
|
||||||
|
|
||||||
|
const CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
|
||||||
|
const AUTHORIZE_URL = 'https://claude.com/cai/oauth/authorize';
|
||||||
|
const TOKEN_URL = 'https://platform.claude.com/v1/oauth/token';
|
||||||
|
const REDIRECT_URI = 'https://platform.claude.com/oauth/code/callback';
|
||||||
|
const SCOPES = [
|
||||||
|
'org:create_api_key',
|
||||||
|
'user:profile',
|
||||||
|
'user:inference',
|
||||||
|
'user:sessions:claude_code',
|
||||||
|
'user:mcp_servers',
|
||||||
|
'user:file_upload',
|
||||||
|
];
|
||||||
|
|
||||||
|
const STATE_FILE = path.join(os.tmpdir(), 'ejclaw-claude-login.json');
|
||||||
|
const CREDS_PATH = path.join(os.homedir(), '.claude', '.credentials.json');
|
||||||
|
|
||||||
|
interface PendingState {
|
||||||
|
verifier: string;
|
||||||
|
state: string;
|
||||||
|
createdAt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ExchangeResponse {
|
||||||
|
access_token: string;
|
||||||
|
refresh_token?: string;
|
||||||
|
expires_in: number;
|
||||||
|
scope?: string;
|
||||||
|
account?: {
|
||||||
|
email_address?: string;
|
||||||
|
has_claude_max?: boolean;
|
||||||
|
has_claude_pro?: boolean;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function base64url(buf: Buffer): string {
|
||||||
|
return buf
|
||||||
|
.toString('base64')
|
||||||
|
.replace(/=+$/g, '')
|
||||||
|
.replace(/\+/g, '-')
|
||||||
|
.replace(/\//g, '_');
|
||||||
|
}
|
||||||
|
|
||||||
|
function generatePkce(): { verifier: string; challenge: string } {
|
||||||
|
const verifier = base64url(crypto.randomBytes(32));
|
||||||
|
const challenge = base64url(
|
||||||
|
crypto.createHash('sha256').update(verifier).digest(),
|
||||||
|
);
|
||||||
|
return { verifier, challenge };
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAuthorizeUrl(state: string, challenge: string): string {
|
||||||
|
const url = new URL(AUTHORIZE_URL);
|
||||||
|
url.searchParams.append('code', 'true');
|
||||||
|
url.searchParams.append('client_id', CLIENT_ID);
|
||||||
|
url.searchParams.append('response_type', 'code');
|
||||||
|
url.searchParams.append('redirect_uri', REDIRECT_URI);
|
||||||
|
url.searchParams.append('scope', SCOPES.join(' '));
|
||||||
|
url.searchParams.append('code_challenge', challenge);
|
||||||
|
url.searchParams.append('code_challenge_method', 'S256');
|
||||||
|
url.searchParams.append('state', state);
|
||||||
|
return url.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function writePendingState(p: PendingState): void {
|
||||||
|
fs.writeFileSync(STATE_FILE, JSON.stringify(p), { mode: 0o600 });
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPendingState(): PendingState | null {
|
||||||
|
if (!fs.existsSync(STATE_FILE)) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as PendingState;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function exchangeCode(
|
||||||
|
code: string,
|
||||||
|
verifier: string,
|
||||||
|
state: string,
|
||||||
|
): Promise<ExchangeResponse> {
|
||||||
|
// The success page can hand the user a value of `code#state` or just
|
||||||
|
// `code` — accept both.
|
||||||
|
const [bareCode, embeddedState] = code.split('#');
|
||||||
|
const stateForExchange = embeddedState || state;
|
||||||
|
const body = JSON.stringify({
|
||||||
|
grant_type: 'authorization_code',
|
||||||
|
code: bareCode,
|
||||||
|
redirect_uri: REDIRECT_URI,
|
||||||
|
client_id: CLIENT_ID,
|
||||||
|
code_verifier: verifier,
|
||||||
|
state: stateForExchange,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await fetch(TOKEN_URL, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const text = await res.text().catch(() => '');
|
||||||
|
throw new Error(
|
||||||
|
`Token exchange failed (${res.status}): ${text.slice(0, 400)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return (await res.json()) as ExchangeResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeCredentials(resp: ExchangeResponse): void {
|
||||||
|
const expiresAt = Date.now() + resp.expires_in * 1000;
|
||||||
|
const creds = {
|
||||||
|
claudeAiOauth: {
|
||||||
|
accessToken: resp.access_token,
|
||||||
|
refreshToken: resp.refresh_token || '',
|
||||||
|
expiresAt,
|
||||||
|
scopes: resp.scope ? resp.scope.split(' ') : SCOPES,
|
||||||
|
subscriptionType: resp.account?.has_claude_max
|
||||||
|
? 'max'
|
||||||
|
: resp.account?.has_claude_pro
|
||||||
|
? 'pro'
|
||||||
|
: '',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const dir = path.dirname(CREDS_PATH);
|
||||||
|
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||||
|
const tmp = `${CREDS_PATH}.tmp`;
|
||||||
|
fs.writeFileSync(tmp, JSON.stringify(creds, null, 2), { mode: 0o600 });
|
||||||
|
fs.renameSync(tmp, CREDS_PATH);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Args {
|
||||||
|
code: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArgs(args: string[]): Args {
|
||||||
|
let code: string | null = null;
|
||||||
|
for (let i = 0; i < args.length; i++) {
|
||||||
|
if (args[i] === '--code' && args[i + 1]) {
|
||||||
|
code = args[++i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { code };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function run(args: string[]): Promise<void> {
|
||||||
|
const { code } = parseArgs(args);
|
||||||
|
|
||||||
|
if (!code) {
|
||||||
|
// Phase 1: produce the authorize URL.
|
||||||
|
const { verifier, challenge } = generatePkce();
|
||||||
|
const state = base64url(crypto.randomBytes(32));
|
||||||
|
writePendingState({ verifier, state, createdAt: Date.now() });
|
||||||
|
const url = buildAuthorizeUrl(state, challenge);
|
||||||
|
|
||||||
|
console.log('AUTHORIZE_URL=' + url);
|
||||||
|
console.log(
|
||||||
|
'NEXT: open the URL above in your browser, complete the login,' +
|
||||||
|
' and re-run this command with --code <pasted_code>.',
|
||||||
|
);
|
||||||
|
emitStatus('LOGIN_URL', { STATUS: 'awaiting_code', URL: url });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 2: exchange the code.
|
||||||
|
const pending = readPendingState();
|
||||||
|
if (!pending) {
|
||||||
|
emitStatus('LOGIN', {
|
||||||
|
STATUS: 'failed',
|
||||||
|
ERROR: 'no_pending_state — run phase 1 first',
|
||||||
|
});
|
||||||
|
process.exit(4);
|
||||||
|
}
|
||||||
|
if (Date.now() - pending.createdAt > 30 * 60 * 1000) {
|
||||||
|
logger.warn(
|
||||||
|
'Pending login state is older than 30 minutes — proceeding anyway',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const resp = await exchangeCode(code, pending.verifier, pending.state);
|
||||||
|
writeCredentials(resp);
|
||||||
|
fs.unlinkSync(STATE_FILE);
|
||||||
|
const newScopes = (resp.scope || SCOPES.join(' ')).split(' ').sort();
|
||||||
|
logger.info(
|
||||||
|
{ scopes: newScopes, expiresInMin: Math.round(resp.expires_in / 60) },
|
||||||
|
'Wrote ~/.claude/.credentials.json',
|
||||||
|
);
|
||||||
|
emitStatus('LOGIN', {
|
||||||
|
STATUS: 'success',
|
||||||
|
CREDENTIALS_PATH: CREDS_PATH,
|
||||||
|
SCOPES: newScopes.join(','),
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
logger.error({ err: message }, 'Token exchange failed');
|
||||||
|
emitStatus('LOGIN', { STATUS: 'failed', ERROR: message });
|
||||||
|
process.exit(4);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -99,7 +99,9 @@ export function restartStackServices(
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
if (managedServiceCaller) {
|
if (managedServiceCaller) {
|
||||||
throw new Error(MANAGED_SERVICE_CALLER_FALLBACK_MESSAGE);
|
throw new Error(MANAGED_SERVICE_CALLER_FALLBACK_MESSAGE, {
|
||||||
|
cause: error,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return restartStackServicesDirect(projectRoot, deps);
|
return restartStackServicesDirect(projectRoot, deps);
|
||||||
|
|||||||
@@ -113,6 +113,15 @@ WorkingDirectory=${projectRoot}
|
|||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
RestartPreventExitStatus=${STARTUP_PRECONDITION_EXIT_CODE}
|
RestartPreventExitStatus=${STARTUP_PRECONDITION_EXIT_CODE}
|
||||||
|
# Per-turn agent runners do heavy file I/O (git clone, builds, video/ffmpeg
|
||||||
|
# scratch) inside this service cgroup. The kernel keeps those reads as
|
||||||
|
# reclaimable page cache, which makes systemd's MemoryCurrent climb to several
|
||||||
|
# GB even though real (anon) usage stays a few hundred MB. MemoryHigh applies
|
||||||
|
# gentle reclaim pressure so that cache is dropped before the number balloons,
|
||||||
|
# while MemoryMax stays unset (infinity) so a legitimate burst is never
|
||||||
|
# OOM-killed — it is throttled at worst.
|
||||||
|
MemoryAccounting=yes
|
||||||
|
MemoryHigh=3G
|
||||||
${envLines.join('\n')}
|
${envLines.join('\n')}
|
||||||
StandardOutput=append:${projectRoot}/logs/${def.logName}.log
|
StandardOutput=append:${projectRoot}/logs/${def.logName}.log
|
||||||
StandardError=append:${projectRoot}/logs/${def.logName}.error.log
|
StandardError=append:${projectRoot}/logs/${def.logName}.error.log
|
||||||
|
|||||||
@@ -111,6 +111,20 @@ describe('systemd unit generation', () => {
|
|||||||
expect(unit).toContain('RestartPreventExitStatus=78');
|
expect(unit).toContain('RestartPreventExitStatus=78');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('bounds reclaimable page cache with a soft memory limit', () => {
|
||||||
|
const unit = buildSystemdUnit(
|
||||||
|
baseServiceDef,
|
||||||
|
'/home/user/ejclaw',
|
||||||
|
'/usr/bin/node',
|
||||||
|
'/home/user',
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
expect(unit).toContain('MemoryAccounting=yes');
|
||||||
|
expect(unit).toContain('MemoryHigh=3G');
|
||||||
|
// MemoryMax must stay unset so a real burst is throttled, never OOM-killed.
|
||||||
|
expect(unit).not.toContain('MemoryMax=');
|
||||||
|
});
|
||||||
|
|
||||||
it('sets correct ExecStart', () => {
|
it('sets correct ExecStart', () => {
|
||||||
const unit = buildSystemdUnit(
|
const unit = buildSystemdUnit(
|
||||||
baseServiceDef,
|
baseServiceDef,
|
||||||
|
|||||||
29
src/agent-attempt-retry.test.ts
Normal file
29
src/agent-attempt-retry.test.ts
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { resolveAttemptRetryAction } from './agent-attempt-retry.js';
|
||||||
|
|
||||||
|
describe('resolveAttemptRetryAction', () => {
|
||||||
|
it('uses the streamed Codex trigger message as the rotation message', () => {
|
||||||
|
const errorMessage =
|
||||||
|
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
|
||||||
|
|
||||||
|
const action = resolveAttemptRetryAction({
|
||||||
|
provider: 'codex',
|
||||||
|
canRetryClaudeCredentials: false,
|
||||||
|
canRetryCodex: true,
|
||||||
|
attempt: {
|
||||||
|
sawOutput: false,
|
||||||
|
streamedTriggerReason: {
|
||||||
|
reason: 'auth-expired',
|
||||||
|
message: errorMessage,
|
||||||
|
} as any,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(action).toEqual({
|
||||||
|
kind: 'codex',
|
||||||
|
trigger: { reason: 'auth-expired' },
|
||||||
|
rotationMessage: errorMessage,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -10,6 +10,7 @@ import { getErrorMessage } from './utils.js';
|
|||||||
export interface AttemptStreamedTrigger {
|
export interface AttemptStreamedTrigger {
|
||||||
reason: AgentTriggerReason;
|
reason: AgentTriggerReason;
|
||||||
retryAfterMs?: number;
|
retryAfterMs?: number;
|
||||||
|
message?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AttemptRetryState {
|
export interface AttemptRetryState {
|
||||||
@@ -131,7 +132,10 @@ export function resolveAttemptRetryAction(args: {
|
|||||||
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||||
rotationMessage?: string | null;
|
rotationMessage?: string | null;
|
||||||
}): AttemptRetryAction {
|
}): AttemptRetryAction {
|
||||||
const normalizedRotationMessage = args.rotationMessage ?? undefined;
|
const normalizedRotationMessage =
|
||||||
|
args.rotationMessage ??
|
||||||
|
args.attempt.streamedTriggerReason?.message ??
|
||||||
|
undefined;
|
||||||
|
|
||||||
const claudeTrigger = resolveClaudeRetryTrigger({
|
const claudeTrigger = resolveClaudeRetryTrigger({
|
||||||
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
|
|||||||
import {
|
import {
|
||||||
classifyAgentError,
|
classifyAgentError,
|
||||||
classifyClaudeAuthError,
|
classifyClaudeAuthError,
|
||||||
|
classifyCodexAuthError,
|
||||||
detectClaudeProviderFailureMessage,
|
detectClaudeProviderFailureMessage,
|
||||||
isClaudeOrgAccessDeniedMessage,
|
isClaudeOrgAccessDeniedMessage,
|
||||||
shouldRotateClaudeToken,
|
shouldRotateClaudeToken,
|
||||||
@@ -66,6 +67,45 @@ describe('agent-error-detection', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('classifies Codex reused refresh-token errors as auth-expired', () => {
|
||||||
|
expect(
|
||||||
|
classifyCodexAuthError(
|
||||||
|
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.',
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('classifies a Codex auth-expired trigger reason as auth-expired', () => {
|
||||||
|
expect(classifyCodexAuthError('auth-expired')).toEqual({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not classify the internal Codex pool-unavailable sentinel as an auth failure', () => {
|
||||||
|
expect(
|
||||||
|
classifyCodexAuthError(
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
),
|
||||||
|
).toEqual({ category: 'none', reason: '' });
|
||||||
|
expect(
|
||||||
|
classifyCodexAuthError(
|
||||||
|
'Codex rotation pool unavailable: all rotation accounts are currently dead, rate-limited, or locked',
|
||||||
|
),
|
||||||
|
).toEqual({ category: 'none', reason: '' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('classifies Codex workspace credit exhaustion as rate-limit', () => {
|
||||||
|
expect(classifyAgentError('Workspace out of credits')).toEqual({
|
||||||
|
category: 'rate-limit',
|
||||||
|
reason: '429',
|
||||||
|
retryAfterMs: undefined,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
||||||
expect(shouldRotateClaudeToken('429')).toBe(true);
|
expect(shouldRotateClaudeToken('429')).toBe(true);
|
||||||
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
||||||
|
|||||||
@@ -231,6 +231,35 @@ const NONE: AgentErrorClassification = {
|
|||||||
reason: '',
|
reason: '',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export function isCodexPoolUnavailableError(
|
||||||
|
error: string | null | undefined,
|
||||||
|
): boolean {
|
||||||
|
if (!error) return false;
|
||||||
|
return (
|
||||||
|
/all\s+codex(?:\s+rotation)?\s+accounts(?:\s+are)?\s+unavailable/i.test(
|
||||||
|
error,
|
||||||
|
) || /codex\s+rotation\s+pool\s+unavailable/i.test(error)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isTerminalCodexAccountFailure(
|
||||||
|
error: string | null | undefined,
|
||||||
|
): boolean {
|
||||||
|
if (!error) return false;
|
||||||
|
if (isCodexPoolUnavailableError(error)) return true;
|
||||||
|
if (classifyCodexAuthError(error).category !== 'none') return true;
|
||||||
|
const lower = error.toLowerCase();
|
||||||
|
if (
|
||||||
|
classifyAgentError(error).category === 'rate-limit' &&
|
||||||
|
(lower.includes('workspace out of credits') ||
|
||||||
|
lower.includes('out of credits') ||
|
||||||
|
lower.includes('codex'))
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Classify an agent error string into a category.
|
* Classify an agent error string into a category.
|
||||||
* Handles patterns common to both Claude and Codex: 429, 503, network.
|
* Handles patterns common to both Claude and Codex: 429, 503, network.
|
||||||
@@ -249,6 +278,8 @@ export function classifyAgentError(
|
|||||||
lower.includes('429') ||
|
lower.includes('429') ||
|
||||||
lower.includes('rate limit') ||
|
lower.includes('rate limit') ||
|
||||||
lower.includes('usage limit') ||
|
lower.includes('usage limit') ||
|
||||||
|
lower.includes('out of credits') ||
|
||||||
|
lower.includes('workspace out of credits') ||
|
||||||
lower.includes('hit your limit') ||
|
lower.includes('hit your limit') ||
|
||||||
lower.includes('too many requests') ||
|
lower.includes('too many requests') ||
|
||||||
lower.includes('rate_limit')
|
lower.includes('rate_limit')
|
||||||
@@ -260,8 +291,11 @@ export function classifyAgentError(
|
|||||||
return { category: 'rate-limit', reason: '429', retryAfterMs };
|
return { category: 'rate-limit', reason: '429', retryAfterMs };
|
||||||
}
|
}
|
||||||
|
|
||||||
// 503 / Overloaded
|
const hasApi5xx = /\bapi error:\s*5\d\d\b/i.test(error);
|
||||||
|
|
||||||
|
// 5xx / Overloaded
|
||||||
if (
|
if (
|
||||||
|
hasApi5xx ||
|
||||||
lower.includes('503') ||
|
lower.includes('503') ||
|
||||||
lower.includes('overloaded') ||
|
lower.includes('overloaded') ||
|
||||||
lower.includes('selected model is at capacity') ||
|
lower.includes('selected model is at capacity') ||
|
||||||
@@ -332,14 +366,21 @@ export function classifyCodexAuthError(
|
|||||||
error: string | null | undefined,
|
error: string | null | undefined,
|
||||||
): AgentErrorClassification {
|
): AgentErrorClassification {
|
||||||
if (!error) return NONE;
|
if (!error) return NONE;
|
||||||
|
if (isCodexPoolUnavailableError(error)) return NONE;
|
||||||
const lower = error.toLowerCase();
|
const lower = error.toLowerCase();
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
lower.includes('auth-expired') ||
|
||||||
|
lower.includes('auth expired') ||
|
||||||
lower.includes('401') ||
|
lower.includes('401') ||
|
||||||
lower.includes('authentication_error') ||
|
lower.includes('authentication_error') ||
|
||||||
lower.includes('failed to authenticate') ||
|
lower.includes('failed to authenticate') ||
|
||||||
|
lower.includes('access token could not be refreshed') ||
|
||||||
lower.includes('oauth token has expired') ||
|
lower.includes('oauth token has expired') ||
|
||||||
|
lower.includes('refresh token was already used') ||
|
||||||
lower.includes('refresh your existing token') ||
|
lower.includes('refresh your existing token') ||
|
||||||
|
lower.includes('log out and sign in again') ||
|
||||||
|
lower.includes('app_session_terminated') ||
|
||||||
lower.includes('unauthorized')
|
lower.includes('unauthorized')
|
||||||
) {
|
) {
|
||||||
return { category: 'auth-expired', reason: 'auth-expired' };
|
return { category: 'auth-expired', reason: 'auth-expired' };
|
||||||
|
|||||||
@@ -5,9 +5,24 @@ import path from 'path';
|
|||||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
import { EJCLAW_ENV } from 'ejclaw-runners-shared';
|
import { EJCLAW_ENV } from 'ejclaw-runners-shared';
|
||||||
|
|
||||||
const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({
|
const {
|
||||||
|
mockReadEnvFile,
|
||||||
|
mockGetActiveCodexAuthPath,
|
||||||
|
mockGetCodexAccountCount,
|
||||||
|
mockClaimCodexAuthLease,
|
||||||
|
mockFindCodexAccountIndexByAuthPath,
|
||||||
|
} = vi.hoisted(() => ({
|
||||||
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
||||||
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
||||||
|
mockGetCodexAccountCount: vi.fn<() => number>(),
|
||||||
|
mockClaimCodexAuthLease: vi.fn<
|
||||||
|
() => {
|
||||||
|
authPath: string;
|
||||||
|
accountIndex: number;
|
||||||
|
release: () => void;
|
||||||
|
} | null
|
||||||
|
>(),
|
||||||
|
mockFindCodexAccountIndexByAuthPath: vi.fn<() => number | null>(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
@@ -29,6 +44,9 @@ vi.mock('./env.js', () => ({
|
|||||||
|
|
||||||
vi.mock('./codex-token-rotation.js', () => ({
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
getActiveCodexAuthPath: mockGetActiveCodexAuthPath,
|
getActiveCodexAuthPath: mockGetActiveCodexAuthPath,
|
||||||
|
getCodexAccountCount: mockGetCodexAccountCount,
|
||||||
|
claimCodexAuthLease: mockClaimCodexAuthLease,
|
||||||
|
findCodexAccountIndexByAuthPath: mockFindCodexAccountIndexByAuthPath,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./token-rotation.js', () => ({
|
vi.mock('./token-rotation.js', () => ({
|
||||||
@@ -159,6 +177,12 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -240,6 +264,84 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
expect(auth).toEqual(rotatedAuth);
|
expect(auth).toEqual(rotatedAuth);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('fails fast instead of launching Codex without OAuth when rotation accounts exist but no lease is available', () => {
|
||||||
|
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||||
|
fs.writeFileSync(
|
||||||
|
rotatedAuthPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: { access_token: 'locked-token' },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(1);
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
|
||||||
|
mockReadEnvFile.mockReturnValue({});
|
||||||
|
|
||||||
|
expect(() => prepareGroupEnvironment(group, false, 'dc:test')).toThrow(
|
||||||
|
/Codex rotation pool unavailable/,
|
||||||
|
);
|
||||||
|
|
||||||
|
const authPath = path.join(
|
||||||
|
tempRoot,
|
||||||
|
'sessions',
|
||||||
|
group.folder,
|
||||||
|
'services',
|
||||||
|
'codex-main',
|
||||||
|
'.codex',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
expect(fs.existsSync(authPath)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('prepareGroupEnvironment prompt stacks', () => {
|
||||||
|
let tempRoot: string;
|
||||||
|
let previousCwd: string;
|
||||||
|
let previousOpenAiKey: string | undefined;
|
||||||
|
let previousCodexOpenAiKey: string | undefined;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
tempRoot = fs.mkdtempSync(path.join('/tmp', 'ejclaw-agent-env-'));
|
||||||
|
previousCwd = process.cwd();
|
||||||
|
process.chdir(tempRoot);
|
||||||
|
|
||||||
|
process.env.EJ_TEST_ROOT = tempRoot;
|
||||||
|
process.env.EJ_TEST_HOME = path.join(tempRoot, 'home');
|
||||||
|
previousOpenAiKey = process.env.OPENAI_API_KEY;
|
||||||
|
previousCodexOpenAiKey = process.env.CODEX_OPENAI_API_KEY;
|
||||||
|
delete process.env.OPENAI_API_KEY;
|
||||||
|
delete process.env.CODEX_OPENAI_API_KEY;
|
||||||
|
|
||||||
|
fs.mkdirSync(process.env.EJ_TEST_HOME, { recursive: true });
|
||||||
|
fs.mkdirSync(path.join(process.env.EJ_TEST_HOME, '.codex'), {
|
||||||
|
recursive: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
mockReadEnvFile.mockReset();
|
||||||
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.chdir(previousCwd);
|
||||||
|
delete process.env.EJ_TEST_ROOT;
|
||||||
|
delete process.env.EJ_TEST_HOME;
|
||||||
|
if (previousOpenAiKey) process.env.OPENAI_API_KEY = previousOpenAiKey;
|
||||||
|
else delete process.env.OPENAI_API_KEY;
|
||||||
|
if (previousCodexOpenAiKey) {
|
||||||
|
process.env.CODEX_OPENAI_API_KEY = previousCodexOpenAiKey;
|
||||||
|
} else {
|
||||||
|
delete process.env.CODEX_OPENAI_API_KEY;
|
||||||
|
}
|
||||||
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => {
|
it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => {
|
||||||
vi.mocked(config.isReviewService).mockReturnValue(true);
|
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||||
@@ -411,6 +513,12 @@ describe('prepareGroupEnvironment Codex MCP room role env', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
resetRoutingMocks();
|
resetRoutingMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -484,6 +592,12 @@ describe('prepareGroupEnvironment room skill overrides', () => {
|
|||||||
});
|
});
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -616,6 +730,12 @@ describe('prepareGroupEnvironment Codex goals handling', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
vi.mocked(config.isReviewService).mockReturnValue(false);
|
vi.mocked(config.isReviewService).mockReturnValue(false);
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false);
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false);
|
||||||
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||||
@@ -706,6 +826,12 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
|||||||
|
|
||||||
mockReadEnvFile.mockReset();
|
mockReadEnvFile.mockReset();
|
||||||
mockGetActiveCodexAuthPath.mockReset();
|
mockGetActiveCodexAuthPath.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReset();
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(0);
|
||||||
|
mockClaimCodexAuthLease.mockReset();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||||
|
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -715,6 +841,37 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
|||||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not claim a Codex auth lease for Claude read-only sessions', () => {
|
||||||
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||||
|
mockGetCodexAccountCount.mockReturnValue(1);
|
||||||
|
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||||
|
fs.writeFileSync(rotatedAuthPath, '{"auth_mode":"chatgpt"}\n');
|
||||||
|
const release = vi.fn();
|
||||||
|
mockClaimCodexAuthLease.mockReturnValue({
|
||||||
|
authPath: rotatedAuthPath,
|
||||||
|
accountIndex: 0,
|
||||||
|
release,
|
||||||
|
});
|
||||||
|
|
||||||
|
const sessionDir = path.join(tempRoot, 'readonly-claude-session');
|
||||||
|
const prepared = prepareReadonlySessionEnvironment({
|
||||||
|
sessionDir,
|
||||||
|
chatJid: 'dc:test',
|
||||||
|
isMain: false,
|
||||||
|
groupFolder: 'codex-test-group',
|
||||||
|
agentType: 'claude-code',
|
||||||
|
memoryBriefing: 'memory briefing',
|
||||||
|
role: 'reviewer',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(prepared.codexSessionAuth).toBeNull();
|
||||||
|
expect(mockClaimCodexAuthLease).not.toHaveBeenCalled();
|
||||||
|
expect(release).not.toHaveBeenCalled();
|
||||||
|
expect(fs.existsSync(path.join(sessionDir, '.codex', 'auth.json'))).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('writes matching AGENTS.md and copies host codex auth/config into the role-scoped session', () => {
|
it('writes matching AGENTS.md and copies host codex auth/config into the role-scoped session', () => {
|
||||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,13 @@ import {
|
|||||||
} from './config.js';
|
} from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { readEnvFile } from './env.js';
|
import { readEnvFile } from './env.js';
|
||||||
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
import {
|
||||||
|
claimCodexAuthLease,
|
||||||
|
findCodexAccountIndexByAuthPath,
|
||||||
|
getActiveCodexAuthPath,
|
||||||
|
getCodexAccountCount,
|
||||||
|
type CodexAuthLease,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
import { readCodexFeatureFromFile } from './codex-config-features.js';
|
import { readCodexFeatureFromFile } from './codex-config-features.js';
|
||||||
import { ensureClaudeSessionSettings } from './claude-session-settings.js';
|
import { ensureClaudeSessionSettings } from './claude-session-settings.js';
|
||||||
import {
|
import {
|
||||||
@@ -140,17 +146,32 @@ function ensureClaudeGlobalSettingsFile(sessionDir: string): void {
|
|||||||
fs.writeFileSync(settingsFile, '{}\n');
|
fs.writeFileSync(settingsFile, '{}\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
function syncHostCodexSessionFiles(sessionCodexDir: string): void {
|
export interface PreparedCodexSessionAuth {
|
||||||
|
canonicalAuthPath: string;
|
||||||
|
sessionAuthPath: string;
|
||||||
|
accountIndex: number | null;
|
||||||
|
lease?: CodexAuthLease;
|
||||||
|
}
|
||||||
|
|
||||||
|
function syncHostCodexSessionFiles(
|
||||||
|
sessionCodexDir: string,
|
||||||
|
): PreparedCodexSessionAuth | null {
|
||||||
const hostCodexDir = path.join(os.homedir(), '.codex');
|
const hostCodexDir = path.join(os.homedir(), '.codex');
|
||||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||||
|
|
||||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||||
const rotatedAuthSrc = getActiveCodexAuthPath();
|
const hasRotationAccounts = getCodexAccountCount() > 0;
|
||||||
|
const lease = claimCodexAuthLease();
|
||||||
|
const rotatedAuthSrc =
|
||||||
|
lease?.authPath ?? (!hasRotationAccounts ? getActiveCodexAuthPath() : null);
|
||||||
|
const fallbackAuthSrc = path.join(hostCodexDir, 'auth.json');
|
||||||
const authSrc =
|
const authSrc =
|
||||||
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||||
? rotatedAuthSrc
|
? rotatedAuthSrc
|
||||||
: path.join(hostCodexDir, 'auth.json');
|
: !hasRotationAccounts && fs.existsSync(fallbackAuthSrc)
|
||||||
if (fs.existsSync(authSrc)) {
|
? fallbackAuthSrc
|
||||||
|
: null;
|
||||||
|
if (authSrc) {
|
||||||
fs.copyFileSync(authSrc, authDst);
|
fs.copyFileSync(authSrc, authDst);
|
||||||
} else if (fs.existsSync(authDst)) {
|
} else if (fs.existsSync(authDst)) {
|
||||||
fs.unlinkSync(authDst);
|
fs.unlinkSync(authDst);
|
||||||
@@ -163,6 +184,28 @@ function syncHostCodexSessionFiles(sessionCodexDir: string): void {
|
|||||||
fs.copyFileSync(src, dst);
|
fs.copyFileSync(src, dst);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!rotatedAuthSrc ||
|
||||||
|
authSrc !== rotatedAuthSrc ||
|
||||||
|
!fs.existsSync(authDst)
|
||||||
|
) {
|
||||||
|
lease?.release();
|
||||||
|
if (hasRotationAccounts) {
|
||||||
|
throw new Error(
|
||||||
|
'Codex rotation pool unavailable: all rotation accounts are currently dead, rate-limited, or locked; re-auth or clear stale state before launching Codex',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
canonicalAuthPath: rotatedAuthSrc,
|
||||||
|
sessionAuthPath: authDst,
|
||||||
|
accountIndex:
|
||||||
|
lease?.accountIndex ?? findCodexAccountIndexByAuthPath(rotatedAuthSrc),
|
||||||
|
...(lease ? { lease } : {}),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function upsertEjclawMcpServerSection(args: {
|
function upsertEjclawMcpServerSection(args: {
|
||||||
@@ -357,7 +400,7 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
useFailoverPromptPack: boolean;
|
useFailoverPromptPack: boolean;
|
||||||
memoryBriefing?: string;
|
memoryBriefing?: string;
|
||||||
skillOverrides?: StoredRoomSkillOverride[];
|
skillOverrides?: StoredRoomSkillOverride[];
|
||||||
}): void {
|
}): PreparedCodexSessionAuth | null {
|
||||||
// API key auth intentionally removed — Codex uses OAuth only.
|
// API key auth intentionally removed — Codex uses OAuth only.
|
||||||
// Never pass any API key to Codex child process to prevent API billing.
|
// Never pass any API key to Codex child process to prevent API billing.
|
||||||
delete args.env.OPENAI_API_KEY;
|
delete args.env.OPENAI_API_KEY;
|
||||||
@@ -376,7 +419,7 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
if (codexEffort) args.env.CODEX_EFFORT = codexEffort;
|
if (codexEffort) args.env.CODEX_EFFORT = codexEffort;
|
||||||
|
|
||||||
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
||||||
syncHostCodexSessionFiles(sessionCodexDir);
|
const codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||||
|
|
||||||
const overlayPath = path.join(args.groupDir, '.codex', 'config.toml');
|
const overlayPath = path.join(args.groupDir, '.codex', 'config.toml');
|
||||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||||
@@ -506,16 +549,19 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
delete args.env.ANTHROPIC_BASE_URL;
|
delete args.env.ANTHROPIC_BASE_URL;
|
||||||
delete args.env.CLAUDE_CODE_OAUTH_TOKEN;
|
delete args.env.CLAUDE_CODE_OAUTH_TOKEN;
|
||||||
args.env.CODEX_HOME = sessionCodexDir;
|
args.env.CODEX_HOME = sessionCodexDir;
|
||||||
|
return codexSessionAuth;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PreparedGroupEnvironment {
|
export interface PreparedGroupEnvironment {
|
||||||
env: Record<string, string>;
|
env: Record<string, string>;
|
||||||
groupDir: string;
|
groupDir: string;
|
||||||
runnerDir: string;
|
runnerDir: string;
|
||||||
|
codexSessionAuth?: PreparedCodexSessionAuth | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PreparedReadonlySessionEnvironment {
|
export interface PreparedReadonlySessionEnvironment {
|
||||||
codexHomeDir?: string;
|
codexHomeDir?: string;
|
||||||
|
codexSessionAuth?: PreparedCodexSessionAuth | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function prepareGroupEnvironment(
|
export function prepareGroupEnvironment(
|
||||||
@@ -657,8 +703,9 @@ export function prepareGroupEnvironment(
|
|||||||
runtimeTaskId,
|
runtimeTaskId,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
|
||||||
if (agentType === 'codex') {
|
if (agentType === 'codex') {
|
||||||
prepareCodexSessionEnvironment({
|
codexSessionAuth = prepareCodexSessionEnvironment({
|
||||||
env,
|
env,
|
||||||
envVars,
|
envVars,
|
||||||
projectRoot,
|
projectRoot,
|
||||||
@@ -677,7 +724,7 @@ export function prepareGroupEnvironment(
|
|||||||
prepareClaudeEnvironment({ env, envVars, group });
|
prepareClaudeEnvironment({ env, envVars, group });
|
||||||
}
|
}
|
||||||
|
|
||||||
return { env, groupDir, runnerDir };
|
return { env, groupDir, runnerDir, codexSessionAuth };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -716,6 +763,7 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
skillOverrides,
|
skillOverrides,
|
||||||
} = args;
|
} = args;
|
||||||
const projectRoot = process.cwd();
|
const projectRoot = process.cwd();
|
||||||
|
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
|
||||||
|
|
||||||
fs.mkdirSync(sessionDir, { recursive: true });
|
fs.mkdirSync(sessionDir, { recursive: true });
|
||||||
ensureClaudeSessionSettings(sessionDir);
|
ensureClaudeSessionSettings(sessionDir);
|
||||||
@@ -784,32 +832,39 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
const sessionClaudeMdPath = path.join(sessionDir, 'CLAUDE.md');
|
const sessionClaudeMdPath = path.join(sessionDir, 'CLAUDE.md');
|
||||||
if (sessionClaudeMd) {
|
if (sessionClaudeMd) {
|
||||||
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
||||||
const sessionCodexDir = path.join(sessionDir, '.codex');
|
if (agentType === 'codex') {
|
||||||
syncHostCodexSessionFiles(sessionCodexDir);
|
const sessionCodexDir = path.join(sessionDir, '.codex');
|
||||||
fs.writeFileSync(
|
codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||||
path.join(sessionCodexDir, 'AGENTS.md'),
|
fs.writeFileSync(
|
||||||
sessionClaudeMd + '\n',
|
path.join(sessionCodexDir, 'AGENTS.md'),
|
||||||
);
|
sessionClaudeMd + '\n',
|
||||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
);
|
||||||
const mcpServerPath = path.join(
|
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||||
projectRoot,
|
const mcpServerPath = path.join(
|
||||||
'runners',
|
projectRoot,
|
||||||
'agent-runner',
|
'runners',
|
||||||
'dist',
|
'agent-runner',
|
||||||
'ipc-mcp-stdio.js',
|
'dist',
|
||||||
);
|
'ipc-mcp-stdio.js',
|
||||||
if (fs.existsSync(mcpServerPath)) {
|
);
|
||||||
upsertEjclawMcpServerSection({
|
if (fs.existsSync(mcpServerPath)) {
|
||||||
sessionConfigPath,
|
upsertEjclawMcpServerSection({
|
||||||
mcpServerPath,
|
sessionConfigPath,
|
||||||
ipcDir,
|
mcpServerPath,
|
||||||
hostIpcDir,
|
ipcDir,
|
||||||
chatJid,
|
hostIpcDir,
|
||||||
groupFolder,
|
chatJid,
|
||||||
isMain,
|
groupFolder,
|
||||||
agentType,
|
isMain,
|
||||||
roomRole: role,
|
agentType,
|
||||||
workDir,
|
roomRole: role,
|
||||||
|
workDir,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fs.rmSync(path.join(sessionDir, '.codex'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -826,5 +881,5 @@ export function prepareReadonlySessionEnvironment(args: {
|
|||||||
} else if (fs.existsSync(sessionClaudeMdPath)) {
|
} else if (fs.existsSync(sessionClaudeMdPath)) {
|
||||||
fs.unlinkSync(sessionClaudeMdPath);
|
fs.unlinkSync(sessionClaudeMdPath);
|
||||||
}
|
}
|
||||||
return { codexHomeDir };
|
return { codexHomeDir, codexSessionAuth };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,11 @@ interface RunSpawnedAgentProcessArgs {
|
|||||||
logsDir: string;
|
logsDir: string;
|
||||||
startTime: number;
|
startTime: number;
|
||||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||||
|
onTerminalStreamedOutputFlushed?: (output: AgentOutput) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isTerminalStreamedOutput(output: AgentOutput): boolean {
|
||||||
|
return (output.phase ?? 'final') !== 'progress';
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseLegacyAgentOutput(stdout: string): AgentOutput {
|
function parseLegacyAgentOutput(stdout: string): AgentOutput {
|
||||||
@@ -55,6 +60,44 @@ function logStreamedOutputDeliveryError(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function logStreamedAgentErrorOutput(
|
||||||
|
output: AgentOutput,
|
||||||
|
group: RegisteredGroup,
|
||||||
|
input: AgentInput,
|
||||||
|
): void {
|
||||||
|
if (output.status !== 'error') return;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
group: group.name,
|
||||||
|
chatJid: input.chatJid,
|
||||||
|
runId: input.runId,
|
||||||
|
error: output.error,
|
||||||
|
newSessionId: output.newSessionId,
|
||||||
|
},
|
||||||
|
'Streamed agent error output',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function chainStreamedOutputDelivery(args: {
|
||||||
|
outputChain: Promise<void>;
|
||||||
|
parsed: AgentOutput;
|
||||||
|
onOutput: (output: AgentOutput) => Promise<void>;
|
||||||
|
onTerminalStreamedOutputFlushed?: (output: AgentOutput) => void;
|
||||||
|
group: RegisteredGroup;
|
||||||
|
input: AgentInput;
|
||||||
|
}): Promise<void> {
|
||||||
|
return args.outputChain.then(async () => {
|
||||||
|
try {
|
||||||
|
await args.onOutput(args.parsed);
|
||||||
|
if (isTerminalStreamedOutput(args.parsed)) {
|
||||||
|
args.onTerminalStreamedOutputFlushed?.(args.parsed);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logStreamedOutputDeliveryError(err, args.group, args.input);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function writeTimeoutLog(args: {
|
function writeTimeoutLog(args: {
|
||||||
logsDir: string;
|
logsDir: string;
|
||||||
input: AgentInput;
|
input: AgentInput;
|
||||||
@@ -105,6 +148,11 @@ export function runSpawnedAgentProcess(
|
|||||||
let stdoutTruncated = false;
|
let stdoutTruncated = false;
|
||||||
let stderrTruncated = false;
|
let stderrTruncated = false;
|
||||||
|
|
||||||
|
// Use UTF-8 string decoding on streams to avoid splitting multi-byte
|
||||||
|
// characters (e.g. Korean) at chunk boundaries, which produces U+FFFD.
|
||||||
|
stdoutStream.setEncoding('utf8');
|
||||||
|
stderrStream.setEncoding('utf8');
|
||||||
|
|
||||||
// Streaming output: parse OUTPUT_START/END marker pairs as they arrive.
|
// Streaming output: parse OUTPUT_START/END marker pairs as they arrive.
|
||||||
let parseBuffer = '';
|
let parseBuffer = '';
|
||||||
let newSessionId: string | undefined;
|
let newSessionId: string | undefined;
|
||||||
@@ -183,21 +231,16 @@ export function runSpawnedAgentProcess(
|
|||||||
}
|
}
|
||||||
hadStreamingOutput = true;
|
hadStreamingOutput = true;
|
||||||
resetTimeout();
|
resetTimeout();
|
||||||
if (parsed.status === 'error') {
|
logStreamedAgentErrorOutput(parsed, group, input);
|
||||||
logger.warn(
|
outputChain = chainStreamedOutputDelivery({
|
||||||
{
|
outputChain,
|
||||||
group: group.name,
|
parsed,
|
||||||
chatJid: input.chatJid,
|
onOutput,
|
||||||
runId: input.runId,
|
onTerminalStreamedOutputFlushed:
|
||||||
error: parsed.error,
|
args.onTerminalStreamedOutputFlushed,
|
||||||
newSessionId: parsed.newSessionId,
|
group,
|
||||||
},
|
input,
|
||||||
'Streamed agent error output',
|
});
|
||||||
);
|
|
||||||
}
|
|
||||||
outputChain = outputChain
|
|
||||||
.then(() => onOutput(parsed))
|
|
||||||
.catch((err) => logStreamedOutputDeliveryError(err, group, input));
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -137,6 +137,72 @@ function emitOutputMarker(
|
|||||||
proc.stdout.push(`${OUTPUT_START_MARKER}\n${json}\n${OUTPUT_END_MARKER}\n`);
|
proc.stdout.push(`${OUTPUT_START_MARKER}\n${json}\n${OUTPUT_END_MARKER}\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function mockCodexLeaseEnvironment(releaseLease: () => void) {
|
||||||
|
return vi
|
||||||
|
.spyOn(agentRunnerEnvironment, 'prepareGroupEnvironment')
|
||||||
|
.mockReturnValueOnce({
|
||||||
|
env: {
|
||||||
|
EJCLAW_IPC_DIR: '/tmp/ejclaw-test-data/ipc/test-group',
|
||||||
|
},
|
||||||
|
groupDir: '/tmp/ejclaw-test-groups/test-group',
|
||||||
|
runnerDir: '/tmp/ejclaw-test-runners/codex-runner',
|
||||||
|
codexSessionAuth: {
|
||||||
|
canonicalAuthPath: '/tmp/codex-account/auth.json',
|
||||||
|
sessionAuthPath: '/tmp/codex-session/auth.json',
|
||||||
|
accountIndex: 0,
|
||||||
|
lease: {
|
||||||
|
accountIndex: 0,
|
||||||
|
authPath: '/tmp/codex-account/auth.json',
|
||||||
|
release: releaseLease,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function expectCodexLeaseReleasedAfterFinalOutputFlush() {
|
||||||
|
const releaseLease = vi.fn();
|
||||||
|
const prepareGroupEnvironmentSpy = mockCodexLeaseEnvironment(releaseLease);
|
||||||
|
const onOutput = vi.fn(async () => {});
|
||||||
|
const codexGroup: RegisteredGroup = {
|
||||||
|
...testGroup,
|
||||||
|
agentType: 'codex',
|
||||||
|
};
|
||||||
|
let resultPromise: Promise<AgentOutput> | undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
resultPromise = runAgentProcess(
|
||||||
|
codexGroup,
|
||||||
|
{
|
||||||
|
...testInput,
|
||||||
|
runId: 'run-codex-lease-final',
|
||||||
|
},
|
||||||
|
() => {},
|
||||||
|
onOutput,
|
||||||
|
);
|
||||||
|
|
||||||
|
emitOutputMarker(fakeProc, {
|
||||||
|
status: 'success',
|
||||||
|
result: 'TASK_DONE\n작업 완료',
|
||||||
|
phase: 'final',
|
||||||
|
newSessionId: 'codex-session-lease-final',
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
|
expect(onOutput).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
result: 'TASK_DONE\n작업 완료',
|
||||||
|
phase: 'final',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(releaseLease).toHaveBeenCalledTimes(1);
|
||||||
|
} finally {
|
||||||
|
fakeProc.emit('close', 0);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
await resultPromise?.catch(() => undefined);
|
||||||
|
prepareGroupEnvironmentSpy.mockRestore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
describe('agent-runner timeout behavior', () => {
|
describe('agent-runner timeout behavior', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
@@ -157,23 +223,18 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
onOutput,
|
onOutput,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Emit output with a result
|
|
||||||
emitOutputMarker(fakeProc, {
|
emitOutputMarker(fakeProc, {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
result: 'Here is my response',
|
result: 'Here is my response',
|
||||||
newSessionId: 'session-123',
|
newSessionId: 'session-123',
|
||||||
});
|
});
|
||||||
|
|
||||||
// Let output processing settle
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
// Fire the hard timeout (IDLE_TIMEOUT + 30s = 1830000ms)
|
|
||||||
await vi.advanceTimersByTimeAsync(1830000);
|
await vi.advanceTimersByTimeAsync(1830000);
|
||||||
|
|
||||||
// Emit close event (as if agent was stopped by the timeout)
|
|
||||||
fakeProc.emit('close', 137);
|
fakeProc.emit('close', 137);
|
||||||
|
|
||||||
// Let the promise resolve
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
const result = await resultPromise;
|
const result = await resultPromise;
|
||||||
@@ -193,10 +254,8 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
onOutput,
|
onOutput,
|
||||||
);
|
);
|
||||||
|
|
||||||
// No output emitted — fire the hard timeout
|
|
||||||
await vi.advanceTimersByTimeAsync(1830000);
|
await vi.advanceTimersByTimeAsync(1830000);
|
||||||
|
|
||||||
// Emit close event
|
|
||||||
fakeProc.emit('close', 137);
|
fakeProc.emit('close', 137);
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
@@ -216,7 +275,6 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
onOutput,
|
onOutput,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Emit output
|
|
||||||
emitOutputMarker(fakeProc, {
|
emitOutputMarker(fakeProc, {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
result: 'Done',
|
result: 'Done',
|
||||||
@@ -225,7 +283,6 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
// Normal exit (no timeout)
|
|
||||||
fakeProc.emit('close', 0);
|
fakeProc.emit('close', 0);
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
@@ -235,6 +292,10 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
expect(result.newSessionId).toBe('session-456');
|
expect(result.newSessionId).toBe('session-456');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('releases Codex auth lease after final streamed output flushes even before process close', async () => {
|
||||||
|
await expectCodexLeaseReleasedAfterFinalOutputFlush();
|
||||||
|
});
|
||||||
|
|
||||||
it('preserves streamed progress phase metadata', async () => {
|
it('preserves streamed progress phase metadata', async () => {
|
||||||
const onOutput = vi.fn(async () => {});
|
const onOutput = vi.fn(async () => {});
|
||||||
const resultPromise = runAgentProcess(
|
const resultPromise = runAgentProcess(
|
||||||
@@ -276,6 +337,18 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('agent-runner environment wiring', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
fakeProc = createFakeProcess();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
it('passes the actual chat JID into codex runner MCP env', async () => {
|
it('passes the actual chat JID into codex runner MCP env', async () => {
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
@@ -413,6 +486,55 @@ describe('agent-runner timeout behavior', () => {
|
|||||||
expect(spawnEnv?.CODEX_HOME).toBe('/tmp/host-reviewer-session/.codex');
|
expect(spawnEnv?.CODEX_HOME).toBe('/tmp/host-reviewer-session/.codex');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('releases the initial Codex auth lease when read-only session preparation fails', async () => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
fakeProc = createFakeProcess();
|
||||||
|
|
||||||
|
const releaseLease = vi.fn();
|
||||||
|
const prepareGroupEnvironmentSpy = mockCodexLeaseEnvironment(releaseLease);
|
||||||
|
const readonlyError = new Error('Codex rotation pool unavailable');
|
||||||
|
const prepareReadonlySessionEnvironmentSpy = vi
|
||||||
|
.spyOn(agentRunnerEnvironment, 'prepareReadonlySessionEnvironment')
|
||||||
|
.mockImplementationOnce(() => {
|
||||||
|
throw readonlyError;
|
||||||
|
});
|
||||||
|
const codexGroup: RegisteredGroup = {
|
||||||
|
...testGroup,
|
||||||
|
agentType: 'codex',
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await expect(
|
||||||
|
runAgentProcess(
|
||||||
|
codexGroup,
|
||||||
|
{
|
||||||
|
...testInput,
|
||||||
|
roomRoleContext: {
|
||||||
|
serviceId: 'codex-review',
|
||||||
|
role: 'reviewer',
|
||||||
|
ownerServiceId: 'codex-main',
|
||||||
|
reviewerServiceId: 'codex-review',
|
||||||
|
failoverOwner: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
() => {},
|
||||||
|
async () => {},
|
||||||
|
{
|
||||||
|
EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1',
|
||||||
|
CLAUDE_CONFIG_DIR: '/tmp/host-reviewer-session',
|
||||||
|
EJCLAW_WORK_DIR: '/tmp/paired/task-1/reviewer',
|
||||||
|
},
|
||||||
|
),
|
||||||
|
).rejects.toThrow(readonlyError);
|
||||||
|
|
||||||
|
expect(prepareReadonlySessionEnvironmentSpy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(releaseLease).toHaveBeenCalledTimes(1);
|
||||||
|
} finally {
|
||||||
|
prepareReadonlySessionEnvironmentSpy.mockRestore();
|
||||||
|
prepareGroupEnvironmentSpy.mockRestore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('serializes roomRoleContext into the runner stdin payload', async () => {
|
it('serializes roomRoleContext into the runner stdin payload', async () => {
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
fakeProc = createFakeProcess();
|
fakeProc = createFakeProcess();
|
||||||
@@ -634,6 +756,18 @@ OUROBOROS_LLM_BACKEND = "codex"
|
|||||||
expect(toml).toContain('OUROBOROS_AGENT_RUNTIME = "codex"');
|
expect(toml).toContain('OUROBOROS_AGENT_RUNTIME = "codex"');
|
||||||
expect(toml).toContain('[mcp_servers.ejclaw]');
|
expect(toml).toContain('[mcp_servers.ejclaw]');
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('agent-runner output flushing', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
fakeProc = createFakeProcess();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
it('waits for queued streamed output before resolving an error exit', async () => {
|
it('waits for queued streamed output before resolving an error exit', async () => {
|
||||||
let releaseOutputs: (() => void) | undefined;
|
let releaseOutputs: (() => void) | undefined;
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ import {
|
|||||||
import {
|
import {
|
||||||
prepareReadonlySessionEnvironment,
|
prepareReadonlySessionEnvironment,
|
||||||
prepareGroupEnvironment,
|
prepareGroupEnvironment,
|
||||||
|
type PreparedCodexSessionAuth,
|
||||||
} from './agent-runner-environment.js';
|
} from './agent-runner-environment.js';
|
||||||
|
import { syncCodexSessionAuthBack } from './codex-token-rotation.js';
|
||||||
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
||||||
import { getStoredRoomSkillOverrides } from './db.js';
|
import { getStoredRoomSkillOverrides } from './db.js';
|
||||||
export {
|
export {
|
||||||
@@ -76,6 +78,47 @@ function readRoomSkillOverridesForRunner(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function releaseCodexAuthSession(
|
||||||
|
auth: PreparedCodexSessionAuth | null | undefined,
|
||||||
|
): void {
|
||||||
|
auth?.lease?.release();
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizeCodexAuthSession(
|
||||||
|
auth: PreparedCodexSessionAuth | null | undefined,
|
||||||
|
): void {
|
||||||
|
if (!auth) return;
|
||||||
|
try {
|
||||||
|
syncCodexSessionAuthBack({
|
||||||
|
canonicalAuthPath: auth.canonicalAuthPath,
|
||||||
|
sessionAuthPath: auth.sessionAuthPath,
|
||||||
|
accountIndex: auth.accountIndex,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
err,
|
||||||
|
canonicalAuthPath: auth.canonicalAuthPath,
|
||||||
|
accountIndex: auth.accountIndex,
|
||||||
|
},
|
||||||
|
'Failed to sync Codex session auth back to canonical slot',
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
auth.lease?.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createCodexAuthSessionFinalizer(
|
||||||
|
getAuth: () => PreparedCodexSessionAuth | null | undefined,
|
||||||
|
): () => void {
|
||||||
|
let finalized = false;
|
||||||
|
return () => {
|
||||||
|
if (finalized) return;
|
||||||
|
finalized = true;
|
||||||
|
finalizeCodexAuthSession(getAuth());
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function runAgentProcess(
|
export async function runAgentProcess(
|
||||||
group: RegisteredGroup,
|
group: RegisteredGroup,
|
||||||
input: AgentInput,
|
input: AgentInput,
|
||||||
@@ -92,18 +135,15 @@ export async function runAgentProcess(
|
|||||||
// ── Host process mode (owner) ───────────────────────────────────
|
// ── Host process mode (owner) ───────────────────────────────────
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
||||||
const { env, groupDir, runnerDir } = prepareGroupEnvironment(
|
const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, {
|
||||||
group,
|
memoryBriefing: input.memoryBriefing,
|
||||||
input.isMain,
|
runtimeTaskId: input.runtimeTaskId,
|
||||||
input.chatJid,
|
useTaskScopedSession: input.useTaskScopedSession,
|
||||||
{
|
skillOverrides,
|
||||||
memoryBriefing: input.memoryBriefing,
|
roomRole: input.roomRoleContext?.role,
|
||||||
runtimeTaskId: input.runtimeTaskId,
|
});
|
||||||
useTaskScopedSession: input.useTaskScopedSession,
|
const { env, groupDir, runnerDir } = prepared;
|
||||||
skillOverrides,
|
let codexSessionAuth = prepared.codexSessionAuth;
|
||||||
roomRole: input.roomRoleContext?.role,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// Apply env overrides (caller-provided)
|
// Apply env overrides (caller-provided)
|
||||||
if (envOverrides) {
|
if (envOverrides) {
|
||||||
@@ -117,20 +157,32 @@ export async function runAgentProcess(
|
|||||||
(input.roomRoleContext?.role === 'reviewer' ||
|
(input.roomRoleContext?.role === 'reviewer' ||
|
||||||
input.roomRoleContext?.role === 'arbiter')
|
input.roomRoleContext?.role === 'arbiter')
|
||||||
) {
|
) {
|
||||||
const readonlySession = prepareReadonlySessionEnvironment({
|
const isCodexGroup = (group.agentType || 'claude-code') === 'codex';
|
||||||
sessionDir: envOverrides.CLAUDE_CONFIG_DIR,
|
const previousCodexSessionAuth = codexSessionAuth;
|
||||||
chatJid: input.chatJid,
|
let readonlySession: ReturnType<typeof prepareReadonlySessionEnvironment>;
|
||||||
isMain: input.isMain,
|
try {
|
||||||
groupFolder: group.folder,
|
readonlySession = prepareReadonlySessionEnvironment({
|
||||||
agentType: group.agentType || 'claude-code',
|
sessionDir: envOverrides.CLAUDE_CONFIG_DIR,
|
||||||
memoryBriefing: input.memoryBriefing,
|
chatJid: input.chatJid,
|
||||||
role: input.roomRoleContext.role,
|
isMain: input.isMain,
|
||||||
ipcDir: env[EJCLAW_ENV.ipcDir],
|
groupFolder: group.folder,
|
||||||
hostIpcDir: env[EJCLAW_ENV.hostIpcDir],
|
agentType: group.agentType || 'claude-code',
|
||||||
workDir: envOverrides[EJCLAW_ENV.workDir] || env[EJCLAW_ENV.workDir],
|
memoryBriefing: input.memoryBriefing,
|
||||||
skillOverrides,
|
role: input.roomRoleContext.role,
|
||||||
});
|
ipcDir: env[EJCLAW_ENV.ipcDir],
|
||||||
if ((group.agentType || 'claude-code') === 'codex') {
|
hostIpcDir: env[EJCLAW_ENV.hostIpcDir],
|
||||||
|
workDir: envOverrides[EJCLAW_ENV.workDir] || env[EJCLAW_ENV.workDir],
|
||||||
|
skillOverrides,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
if (isCodexGroup) {
|
||||||
|
releaseCodexAuthSession(previousCodexSessionAuth);
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (isCodexGroup) {
|
||||||
|
releaseCodexAuthSession(previousCodexSessionAuth);
|
||||||
|
codexSessionAuth = readonlySession.codexSessionAuth;
|
||||||
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
|
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
|
||||||
if (readonlySession.codexHomeDir) {
|
if (readonlySession.codexHomeDir) {
|
||||||
env.HOME = readonlySession.codexHomeDir;
|
env.HOME = readonlySession.codexHomeDir;
|
||||||
@@ -144,6 +196,9 @@ export async function runAgentProcess(
|
|||||||
const safeName = group.folder.replace(/[^a-zA-Z0-9-]/g, '-');
|
const safeName = group.folder.replace(/[^a-zA-Z0-9-]/g, '-');
|
||||||
const processSuffix = input.runId || `${Date.now()}`;
|
const processSuffix = input.runId || `${Date.now()}`;
|
||||||
const processName = `ejclaw-${safeName}-${processSuffix}`;
|
const processName = `ejclaw-${safeName}-${processSuffix}`;
|
||||||
|
const finalizeCodexAuthSessionOnce = createCodexAuthSessionFinalizer(
|
||||||
|
() => codexSessionAuth,
|
||||||
|
);
|
||||||
|
|
||||||
// Check if runner is built
|
// Check if runner is built
|
||||||
const distEntry = path.join(runnerDir, 'dist', 'index.js');
|
const distEntry = path.join(runnerDir, 'dist', 'index.js');
|
||||||
@@ -152,6 +207,7 @@ export async function runAgentProcess(
|
|||||||
{ runnerDir, chatJid: input.chatJid, runId: input.runId },
|
{ runnerDir, chatJid: input.chatJid, runId: input.runId },
|
||||||
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
|
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
|
||||||
);
|
);
|
||||||
|
releaseCodexAuthSession(codexSessionAuth);
|
||||||
return {
|
return {
|
||||||
status: 'error',
|
status: 'error',
|
||||||
result: null,
|
result: null,
|
||||||
@@ -199,6 +255,23 @@ export async function runAgentProcess(
|
|||||||
logsDir,
|
logsDir,
|
||||||
startTime,
|
startTime,
|
||||||
onOutput,
|
onOutput,
|
||||||
}).then(resolve);
|
onTerminalStreamedOutputFlushed: finalizeCodexAuthSessionOnce,
|
||||||
|
})
|
||||||
|
.then((output) => {
|
||||||
|
finalizeCodexAuthSessionOnce();
|
||||||
|
resolve(output);
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
finalizeCodexAuthSessionOnce();
|
||||||
|
logger.error(
|
||||||
|
{ err, processName, chatJid: input.chatJid, runId: input.runId },
|
||||||
|
'Spawned agent process runner failed',
|
||||||
|
);
|
||||||
|
resolve({
|
||||||
|
status: 'error',
|
||||||
|
result: null,
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
179
src/auto-paired-mode.ts
Normal file
179
src/auto-paired-mode.ts
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
/**
|
||||||
|
* Auto paired-mode degrade/restore.
|
||||||
|
*
|
||||||
|
* When all configured Claude accounts are at/over the 95% exhaustion threshold,
|
||||||
|
* paired (tribunal) rooms cannot meaningfully run their reviewer/arbiter
|
||||||
|
* because the gate blocks Claude turns. To keep the user-facing rooms
|
||||||
|
* responsive, we temporarily flip selected rooms to `single` mode and notify
|
||||||
|
* each affected channel. When usage recovers we flip them back to `tribunal`.
|
||||||
|
*
|
||||||
|
* Selection (Option B agreed with the user):
|
||||||
|
* - The most-recently-active room (per chats.last_message_time)
|
||||||
|
* - All rooms that are currently in-flight (queue status === 'processing')
|
||||||
|
* - Deduplicated; rooms whose owner is `claude-code` are skipped because
|
||||||
|
* degrading them does not help — the 95% gate blocks them either way.
|
||||||
|
*
|
||||||
|
* Restore behavior:
|
||||||
|
* - Only restore a room if it is still in 'single' (don't override a manual
|
||||||
|
* user change made during the degrade window).
|
||||||
|
*/
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { getClaudeUsageExhaustion } from './claude-usage.js';
|
||||||
|
import { DATA_DIR } from './config.js';
|
||||||
|
import {
|
||||||
|
getAllChats,
|
||||||
|
getAllRoomBindings,
|
||||||
|
getEffectiveRoomMode,
|
||||||
|
getStoredRoomSettings,
|
||||||
|
setExplicitRoomMode,
|
||||||
|
} from './db.js';
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
|
|
||||||
|
const STATE_PATH = path.join(DATA_DIR, 'auto-paired-mode-state.json');
|
||||||
|
|
||||||
|
const DEGRADE_NOTICE =
|
||||||
|
'클로드 사용량이 95%를 넘어 잠시 single 모드로 전환했어요. 사용량이 회복되면 paired 모드로 다시 켤게요.';
|
||||||
|
const RESTORE_NOTICE = '클로드 사용량이 회복돼서 paired 모드로 다시 켰어요.';
|
||||||
|
|
||||||
|
interface DowngradeRecord {
|
||||||
|
chatJid: string;
|
||||||
|
atIso: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AutoPairedModeState {
|
||||||
|
exhausted: boolean;
|
||||||
|
downgraded: DowngradeRecord[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AutoPairedModeDeps {
|
||||||
|
queue?: {
|
||||||
|
getStatuses(jids: string[]): Array<{ jid: string; status: string }>;
|
||||||
|
};
|
||||||
|
sendNotice: (chatJid: string, text: string) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadState(): AutoPairedModeState {
|
||||||
|
const raw = readJsonFile<AutoPairedModeState>(STATE_PATH);
|
||||||
|
if (!raw || typeof raw !== 'object') {
|
||||||
|
return { exhausted: false, downgraded: [] };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
exhausted: Boolean(raw.exhausted),
|
||||||
|
downgraded: Array.isArray(raw.downgraded) ? raw.downgraded : [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveState(state: AutoPairedModeState): void {
|
||||||
|
try {
|
||||||
|
fs.mkdirSync(path.dirname(STATE_PATH), { recursive: true });
|
||||||
|
writeJsonFile(STATE_PATH, state);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err }, 'auto-paired-mode: failed to persist state');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function pickCandidates(deps: AutoPairedModeDeps): string[] {
|
||||||
|
const bindings = getAllRoomBindings();
|
||||||
|
const allJids = Object.keys(bindings);
|
||||||
|
if (allJids.length === 0) return [];
|
||||||
|
|
||||||
|
const candidates = new Set<string>();
|
||||||
|
|
||||||
|
// Most-recently-active room first (chats ordered DESC by last_message_time).
|
||||||
|
const chats = getAllChats();
|
||||||
|
const recent = chats.find((c) => allJids.includes(c.jid));
|
||||||
|
if (recent) candidates.add(recent.jid);
|
||||||
|
|
||||||
|
// Currently in-flight rooms.
|
||||||
|
if (deps.queue) {
|
||||||
|
for (const status of deps.queue.getStatuses(allJids)) {
|
||||||
|
if (status.status === 'processing') candidates.add(status.jid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip claude-owned rooms (Option B).
|
||||||
|
const result: string[] = [];
|
||||||
|
for (const jid of candidates) {
|
||||||
|
const settings = getStoredRoomSettings(jid);
|
||||||
|
if (settings?.ownerAgentType === 'claude-code') continue;
|
||||||
|
if (getEffectiveRoomMode(jid) !== 'tribunal') continue;
|
||||||
|
result.push(jid);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function evaluateAndApplyAutoPairedMode(
|
||||||
|
deps: AutoPairedModeDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
const exhaustion = getClaudeUsageExhaustion();
|
||||||
|
const state = loadState();
|
||||||
|
|
||||||
|
if (exhaustion.exhausted && !state.exhausted) {
|
||||||
|
// Transition: healthy → exhausted. Downgrade selected rooms.
|
||||||
|
const targets = pickCandidates(deps);
|
||||||
|
const downgraded: DowngradeRecord[] = [];
|
||||||
|
for (const jid of targets) {
|
||||||
|
try {
|
||||||
|
setExplicitRoomMode(jid, 'single');
|
||||||
|
downgraded.push({ chatJid: jid, atIso: new Date().toISOString() });
|
||||||
|
logger.info(
|
||||||
|
{ jid },
|
||||||
|
'auto-paired-mode: downgraded room to single (claude exhausted)',
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await deps.sendNotice(jid, DEGRADE_NOTICE);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, jid },
|
||||||
|
'auto-paired-mode: failed to send degrade notice',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err, jid }, 'auto-paired-mode: failed to downgrade room');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saveState({ exhausted: true, downgraded });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!exhaustion.exhausted && state.exhausted) {
|
||||||
|
// Transition: exhausted → healthy. Restore previously-downgraded rooms.
|
||||||
|
for (const record of state.downgraded) {
|
||||||
|
try {
|
||||||
|
if (getEffectiveRoomMode(record.chatJid) !== 'single') {
|
||||||
|
// Respect manual user changes during the degrade window.
|
||||||
|
logger.info(
|
||||||
|
{ jid: record.chatJid },
|
||||||
|
'auto-paired-mode: skip restore (room is no longer single)',
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
setExplicitRoomMode(record.chatJid, 'tribunal');
|
||||||
|
logger.info(
|
||||||
|
{ jid: record.chatJid },
|
||||||
|
'auto-paired-mode: restored room to tribunal',
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await deps.sendNotice(record.chatJid, RESTORE_NOTICE);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, jid: record.chatJid },
|
||||||
|
'auto-paired-mode: failed to send restore notice',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, jid: record.chatJid },
|
||||||
|
'auto-paired-mode: failed to restore room',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saveState({ exhausted: false, downgraded: [] });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Steady state — nothing to do.
|
||||||
|
}
|
||||||
@@ -129,11 +129,7 @@ vi.mock('discord.js', () => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
import {
|
import { DiscordChannel, DiscordChannelOpts } from './discord.js';
|
||||||
DiscordChannel,
|
|
||||||
DiscordChannelOpts,
|
|
||||||
chunkForDiscord,
|
|
||||||
} from './discord.js';
|
|
||||||
import { logger } from '../logger.js';
|
import { logger } from '../logger.js';
|
||||||
|
|
||||||
// --- Test helpers ---
|
// --- Test helpers ---
|
||||||
@@ -981,6 +977,28 @@ describe('sendMessage', () => {
|
|||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('surfaces rejected attachments in the sent message instead of dropping them silently', async () => {
|
||||||
|
const opts = createTestOpts();
|
||||||
|
const channel = new DiscordChannel('test-token', opts);
|
||||||
|
await channel.connect();
|
||||||
|
const mockChannel = {
|
||||||
|
send: vi.fn().mockResolvedValue({ id: 'discord-message-1' }),
|
||||||
|
sendTyping: vi.fn(),
|
||||||
|
};
|
||||||
|
currentClient().channels.fetch.mockResolvedValue(mockChannel);
|
||||||
|
|
||||||
|
await channel.sendMessage('dc:1234567890123456', '샘플을 첨부합니다.', {
|
||||||
|
attachments: [{ path: '/home/claude/missing-sample.wav' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockChannel.send).toHaveBeenCalledTimes(1);
|
||||||
|
const sent = mockChannel.send.mock.calls[0][0];
|
||||||
|
expect(sent.files).toBeUndefined();
|
||||||
|
expect(sent.content).toContain('샘플을 첨부합니다.');
|
||||||
|
expect(sent.content).toContain('첨부 1건을 전송하지 못했습니다');
|
||||||
|
expect(sent.content).toContain('missing-sample.wav (파일을 찾을 수 없음)');
|
||||||
|
});
|
||||||
|
|
||||||
it('uses legacy image tags as Discord attachment fallback', async () => {
|
it('uses legacy image tags as Discord attachment fallback', async () => {
|
||||||
const opts = createTestOpts();
|
const opts = createTestOpts();
|
||||||
const channel = new DiscordChannel('test-token', opts);
|
const channel = new DiscordChannel('test-token', opts);
|
||||||
@@ -1142,127 +1160,3 @@ describe('channel properties', () => {
|
|||||||
expect(channel.name).toBe('discord');
|
expect(channel.name).toBe('discord');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// --- chunkForDiscord ---
|
|
||||||
|
|
||||||
describe('chunkForDiscord', () => {
|
|
||||||
it('returns the input unchanged when within the limit', () => {
|
|
||||||
const text = 'short message';
|
|
||||||
expect(chunkForDiscord(text, 2000)).toEqual([text]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns empty array for empty input', () => {
|
|
||||||
expect(chunkForDiscord('', 2000)).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('splits on newline boundaries when possible', () => {
|
|
||||||
const text = 'aaa\nbbb\nccc\nddd';
|
|
||||||
const chunks = chunkForDiscord(text, 8);
|
|
||||||
// Every chunk must be ≤ 8 chars and joining them must rebuild the text
|
|
||||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8);
|
|
||||||
expect(chunks.join('')).toBe(text);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('reopens a fenced code block across chunks (with language)', () => {
|
|
||||||
const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n');
|
|
||||||
const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter';
|
|
||||||
const chunks = chunkForDiscord(text, 500);
|
|
||||||
expect(chunks.length).toBeGreaterThan(1);
|
|
||||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500);
|
|
||||||
// Every chunk must have a balanced number of ``` fences (so it renders
|
|
||||||
// as valid markdown on its own).
|
|
||||||
for (const c of chunks) {
|
|
||||||
const fenceMatches = c.match(/^```/gm) ?? [];
|
|
||||||
expect(fenceMatches.length % 2).toBe(0);
|
|
||||||
}
|
|
||||||
// The middle chunks must reopen the same language.
|
|
||||||
for (let i = 1; i < chunks.length; i++) {
|
|
||||||
// It should either start with the reopened fence, or never have been
|
|
||||||
// inside the fence at this point.
|
|
||||||
if (chunks[i].startsWith('```')) {
|
|
||||||
expect(chunks[i].startsWith('```ts\n')).toBe(true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it('reopens a language-less fenced code block', () => {
|
|
||||||
const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join(
|
|
||||||
'\n',
|
|
||||||
);
|
|
||||||
const text = '```\n' + code + '\n```';
|
|
||||||
const chunks = chunkForDiscord(text, 400);
|
|
||||||
expect(chunks.length).toBeGreaterThan(1);
|
|
||||||
for (const c of chunks) {
|
|
||||||
const fenceMatches = c.match(/^```/gm) ?? [];
|
|
||||||
expect(fenceMatches.length % 2).toBe(0);
|
|
||||||
}
|
|
||||||
// Joining back should rebuild a string whose un-fenced content equals
|
|
||||||
// the original code (allowing for extra fence pairs in the middle).
|
|
||||||
const reconstructed = chunks
|
|
||||||
.join('\n')
|
|
||||||
.replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams
|
|
||||||
.replace(/```\n```$/g, '');
|
|
||||||
expect(reconstructed.includes('payload-0')).toBe(true);
|
|
||||||
expect(reconstructed.includes('payload-199')).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('handles multiple fenced blocks in the same document', () => {
|
|
||||||
const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n');
|
|
||||||
const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n');
|
|
||||||
const text =
|
|
||||||
'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```';
|
|
||||||
const chunks = chunkForDiscord(text, 200);
|
|
||||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200);
|
|
||||||
for (const c of chunks) {
|
|
||||||
const fenceMatches = c.match(/^```/gm) ?? [];
|
|
||||||
expect(fenceMatches.length % 2).toBe(0);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it('does not split a surrogate pair', () => {
|
|
||||||
// Emoji 🦄 is a surrogate pair (length 2 in JS).
|
|
||||||
const emoji = '🦄';
|
|
||||||
const text = emoji.repeat(50);
|
|
||||||
const chunks = chunkForDiscord(text, 9);
|
|
||||||
for (const c of chunks) {
|
|
||||||
// No chunk should start or end mid surrogate pair.
|
|
||||||
expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe(
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
const last = c.charCodeAt(c.length - 1);
|
|
||||||
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
|
|
||||||
}
|
|
||||||
expect(chunks.join('')).toBe(text);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('falls back gracefully when a single line exceeds maxLength', () => {
|
|
||||||
const longLine = 'x'.repeat(5000);
|
|
||||||
const chunks = chunkForDiscord(longLine, 2000);
|
|
||||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000);
|
|
||||||
expect(chunks.join('')).toBe(longLine);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('keeps a closing fence with its block when there is room', () => {
|
|
||||||
const text = '```ts\nshort code\n```';
|
|
||||||
const chunks = chunkForDiscord(text, 2000);
|
|
||||||
expect(chunks).toEqual([text]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('preserves total content across chunk seams (modulo reopen overhead)', () => {
|
|
||||||
const code = Array.from(
|
|
||||||
{ length: 80 },
|
|
||||||
(_, i) => `console.log(${i});`,
|
|
||||||
).join('\n');
|
|
||||||
const text = '```js\n' + code + '\n```';
|
|
||||||
const chunks = chunkForDiscord(text, 300);
|
|
||||||
// After removing the reopen-close fence pairs that the chunker inserts
|
|
||||||
// between chunks, the result should match the original text.
|
|
||||||
const stripped = chunks.join('\n').replace(/```\n```js\n/g, '');
|
|
||||||
// The first chunk still has the original opener and the last still has
|
|
||||||
// the original closer.
|
|
||||||
expect(stripped.startsWith('```js\n')).toBe(true);
|
|
||||||
expect(stripped.endsWith('```')).toBe(true);
|
|
||||||
expect(stripped.includes('console.log(0);')).toBe(true);
|
|
||||||
expect(stripped.includes('console.log(79);')).toBe(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ import {
|
|||||||
import { CACHE_DIR } from '../config.js';
|
import { CACHE_DIR } from '../config.js';
|
||||||
import { getEnv } from '../env.js';
|
import { getEnv } from '../env.js';
|
||||||
import { logger } from '../logger.js';
|
import { logger } from '../logger.js';
|
||||||
import { validateOutboundAttachments } from '../outbound-attachments.js';
|
import {
|
||||||
|
appendRejectionNotice,
|
||||||
|
validateOutboundAttachments,
|
||||||
|
} from '../outbound-attachments.js';
|
||||||
import { sanitizeForOutbound } from '../router.js';
|
import { sanitizeForOutbound } from '../router.js';
|
||||||
import { hasReviewerLease } from '../service-routing.js';
|
import { hasReviewerLease } from '../service-routing.js';
|
||||||
import type {
|
import type {
|
||||||
@@ -39,113 +42,6 @@ const DISCORD_OWNER_TOKEN_KEY = 'DISCORD_OWNER_BOT_TOKEN';
|
|||||||
const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN';
|
const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN';
|
||||||
const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
|
const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
|
||||||
|
|
||||||
// Matches a line that opens or closes a markdown fenced code block.
|
|
||||||
// Captures the fence marker (``` or ```` etc.) and the optional language tag.
|
|
||||||
const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a
|
|
||||||
* valid standalone Markdown snippet for Discord. If a chunk has to end inside
|
|
||||||
* a ``` fenced code block, we append a closing fence to it and reopen the
|
|
||||||
* same fence (with the same language tag) at the start of the next chunk.
|
|
||||||
*
|
|
||||||
* Behaviour:
|
|
||||||
* - Prefers splitting at newline boundaries.
|
|
||||||
* - If a single line is itself longer than `maxLength`, falls back to a
|
|
||||||
* surrogate-pair-safe byte split for that line only.
|
|
||||||
* - Returns `[text]` unchanged when `text.length <= maxLength`.
|
|
||||||
*/
|
|
||||||
export function chunkForDiscord(text: string, maxLength: number): string[] {
|
|
||||||
if (text.length <= maxLength) return text.length > 0 ? [text] : [];
|
|
||||||
|
|
||||||
const lines = text.split('\n');
|
|
||||||
const chunks: string[] = [];
|
|
||||||
let current = '';
|
|
||||||
let openLang = ''; // language tag of currently open fence
|
|
||||||
let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence
|
|
||||||
|
|
||||||
const flush = () => {
|
|
||||||
if (current === '') return;
|
|
||||||
if (openMarker !== null) {
|
|
||||||
if (!current.endsWith('\n')) current += '\n';
|
|
||||||
current += openMarker;
|
|
||||||
chunks.push(current);
|
|
||||||
current = openMarker + openLang + '\n';
|
|
||||||
} else {
|
|
||||||
chunks.push(current);
|
|
||||||
current = '';
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const hardSplitInto = (segment: string) => {
|
|
||||||
// Split `segment` across chunks at code-unit boundaries while avoiding
|
|
||||||
// splitting surrogate pairs. Used when a single source line exceeds the
|
|
||||||
// budget on its own.
|
|
||||||
let s = segment;
|
|
||||||
while (s.length > 0) {
|
|
||||||
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
|
|
||||||
const room = maxLength - current.length - reserve;
|
|
||||||
if (room <= 0) {
|
|
||||||
flush();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let take = Math.min(s.length, room);
|
|
||||||
if (
|
|
||||||
take < s.length &&
|
|
||||||
take > 0 &&
|
|
||||||
s.charCodeAt(take - 1) >= 0xd800 &&
|
|
||||||
s.charCodeAt(take - 1) <= 0xdbff
|
|
||||||
) {
|
|
||||||
take--;
|
|
||||||
}
|
|
||||||
if (take <= 0) {
|
|
||||||
// Couldn't fit anything safely; flush and retry on an empty chunk.
|
|
||||||
flush();
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
current += s.slice(0, take);
|
|
||||||
s = s.slice(take);
|
|
||||||
if (s.length > 0) flush();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
for (let li = 0; li < lines.length; li++) {
|
|
||||||
const isLast = li === lines.length - 1;
|
|
||||||
const line = lines[li];
|
|
||||||
const lineWithNL = isLast ? line : line + '\n';
|
|
||||||
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
|
|
||||||
|
|
||||||
if (
|
|
||||||
current.length > 0 &&
|
|
||||||
current.length + lineWithNL.length + reserve > maxLength
|
|
||||||
) {
|
|
||||||
flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (lineWithNL.length + reserve > maxLength) {
|
|
||||||
hardSplitInto(lineWithNL);
|
|
||||||
} else {
|
|
||||||
current += lineWithNL;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update fence state AFTER appending the line so the very-next iteration
|
|
||||||
// accounts for it correctly.
|
|
||||||
const m = line.match(FENCE_LINE_RE);
|
|
||||||
if (m) {
|
|
||||||
if (openMarker === null) {
|
|
||||||
openMarker = m[1];
|
|
||||||
openLang = m[2] || '';
|
|
||||||
} else if (m[1] === openMarker) {
|
|
||||||
openMarker = null;
|
|
||||||
openLang = '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (current.length > 0) chunks.push(current);
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wait for a pending transcription from the other service (poll cache file).
|
* Wait for a pending transcription from the other service (poll cache file).
|
||||||
* Returns the cached text, or null if timeout.
|
* Returns the cached text, or null if timeout.
|
||||||
@@ -522,16 +418,18 @@ export class DiscordChannel implements Channel {
|
|||||||
.trim();
|
.trim();
|
||||||
|
|
||||||
// Convert @username mentions to Discord mention format
|
// Convert @username mentions to Discord mention format
|
||||||
const mentionMap: Record<string, string> = {
|
cleaned = cleaned.replaceAll('@눈쟁이', '<@216851709744513024>');
|
||||||
눈쟁이: '216851709744513024',
|
// Markdown escaping already happened in prepareDiscordOutbound (prose
|
||||||
};
|
// escaped, structured output / fenced code preserved). Here we only
|
||||||
for (const [name, id] of Object.entries(mentionMap)) {
|
// re-sanitize so we don't double-escape the delimiters.
|
||||||
cleaned = cleaned.replace(new RegExp(`@${name}`, 'g'), `<@${id}>`);
|
|
||||||
}
|
|
||||||
// Markdown escaping already happened in prepareDiscordOutbound (before
|
|
||||||
// the deliberate link→inline-code step). Here we only strip/redact.
|
|
||||||
cleaned = sanitizeForOutbound(cleaned);
|
cleaned = sanitizeForOutbound(cleaned);
|
||||||
|
|
||||||
|
// Surface rejected attachments in the visible message. The MEDIA:
|
||||||
|
// directive was already stripped from the text, so without this the user
|
||||||
|
// would see a message that claims a file was attached while none was
|
||||||
|
// delivered. Making the failure loud prevents that silent mismatch.
|
||||||
|
cleaned = appendRejectionNotice(cleaned, validation.rejected);
|
||||||
|
|
||||||
// Discord has a 2000 character limit per message and 10 attachments per message
|
// Discord has a 2000 character limit per message and 10 attachments per message
|
||||||
const MAX_LENGTH = 2000;
|
const MAX_LENGTH = 2000;
|
||||||
const MAX_ATTACHMENTS = 10;
|
const MAX_ATTACHMENTS = 10;
|
||||||
@@ -576,14 +474,10 @@ export class DiscordChannel implements Channel {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Send text in chunks. The chunker preserves fenced code blocks so a
|
// Send text in chunks, attach first batch to the first chunk
|
||||||
// long ``` block that straddles the 2000-char boundary doesn't get
|
|
||||||
// split mid-fence (which Discord would render as literal backticks
|
|
||||||
// and break syntax for following chunks). It is also surrogate-pair
|
|
||||||
// safe (emoji etc.).
|
|
||||||
const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH);
|
|
||||||
let fileBatchIndex = 0;
|
let fileBatchIndex = 0;
|
||||||
for (const chunk of messageChunks) {
|
for (let i = 0; i < cleaned.length; i += MAX_LENGTH) {
|
||||||
|
const chunk = cleaned.slice(i, i + MAX_LENGTH);
|
||||||
const batch = fileBatches[fileBatchIndex];
|
const batch = fileBatches[fileBatchIndex];
|
||||||
recordSentMessage(
|
recordSentMessage(
|
||||||
await textChannel.send({
|
await textChannel.send({
|
||||||
|
|||||||
135
src/claude-usage-backoff.test.ts
Normal file
135
src/claude-usage-backoff.test.ts
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
/**
|
||||||
|
* Regression tests for the Claude usage-API 429 `Retry-After` backoff.
|
||||||
|
*
|
||||||
|
* Root cause this guards against: the usage poller used to retry every
|
||||||
|
* MIN_FETCH_INTERVAL_MS (60s), but the endpoint hands back a longer rate-limit
|
||||||
|
* window (e.g. `retry-after: 93`). Retrying mid-cooldown re-trips the limit, so
|
||||||
|
* the 429s never clear. The fix records a `cooldownUntil` from the header and
|
||||||
|
* refuses to call the API until that window closes.
|
||||||
|
*/
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./logger.js', () => ({
|
||||||
|
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./config.js', () => ({ DATA_DIR: '/tmp/ejclaw-usage-backoff-data' }));
|
||||||
|
|
||||||
|
const { getAllTokensMock } = vi.hoisted(() => ({ getAllTokensMock: vi.fn() }));
|
||||||
|
vi.mock('./token-rotation.js', () => ({
|
||||||
|
getAllTokens: getAllTokensMock,
|
||||||
|
getCurrentToken: () => null,
|
||||||
|
getConfiguredClaudeTokens: () => [],
|
||||||
|
}));
|
||||||
|
|
||||||
|
const { readJsonFileMock } = vi.hoisted(() => ({ readJsonFileMock: vi.fn() }));
|
||||||
|
vi.mock('./utils.js', async () => {
|
||||||
|
const actual =
|
||||||
|
await vi.importActual<typeof import('./utils.js')>('./utils.js');
|
||||||
|
return { ...actual, readJsonFile: readJsonFileMock, writeJsonFile: vi.fn() };
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
function makeResponse(
|
||||||
|
status: number,
|
||||||
|
headers: Record<string, string> = {},
|
||||||
|
): Response {
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
ok: status >= 200 && status < 300,
|
||||||
|
headers: { get: (k: string) => headers[k.toLowerCase()] ?? null },
|
||||||
|
json: async () => ({}),
|
||||||
|
} as unknown as Response;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('parseRetryAfterMs', () => {
|
||||||
|
it('parses delta-seconds', async () => {
|
||||||
|
const { parseRetryAfterMs } = await import('./claude-usage.js');
|
||||||
|
expect(parseRetryAfterMs('93')).toBe(93_000);
|
||||||
|
expect(parseRetryAfterMs('0')).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses an HTTP-date relative to now', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date('2026-06-20T00:00:00Z'));
|
||||||
|
const { parseRetryAfterMs } = await import('./claude-usage.js');
|
||||||
|
expect(parseRetryAfterMs('Sat, 20 Jun 2026 00:00:30 GMT')).toBe(30_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null for missing or junk values', async () => {
|
||||||
|
const { parseRetryAfterMs } = await import('./claude-usage.js');
|
||||||
|
expect(parseRetryAfterMs(null)).toBeNull();
|
||||||
|
expect(parseRetryAfterMs('soon')).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Claude usage 429 Retry-After backoff', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
readJsonFileMock.mockReturnValue({}); // start from an empty disk cache
|
||||||
|
getAllTokensMock.mockReturnValue([
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
token: 'sk-ant-oat01-testTokenAAAAAAAA',
|
||||||
|
masked: 'sk-ant-oat01-test...AAAA',
|
||||||
|
isActive: true,
|
||||||
|
isRateLimited: false,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('records a cooldown on 429 and skips the next fetch within the window', async () => {
|
||||||
|
const fetchMock = vi.fn(async () =>
|
||||||
|
makeResponse(429, { 'retry-after': '93' }),
|
||||||
|
);
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
|
||||||
|
const { fetchAllClaudeUsage } = await import('./claude-usage.js');
|
||||||
|
|
||||||
|
const first = await fetchAllClaudeUsage();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(first[0].usageRateLimited).toBe(true);
|
||||||
|
expect(first[0].usage).toBeNull();
|
||||||
|
|
||||||
|
// Immediate re-call: cooldown gate must hold, no second network call.
|
||||||
|
const second = await fetchAllClaudeUsage();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(second[0].usageRateLimited).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps backing off after the 60s throttle elapses but before the cooldown closes', async () => {
|
||||||
|
// This is the core regression: a 93s Retry-After must outlast the 60s
|
||||||
|
// MIN_FETCH_INTERVAL throttle. Without the cooldown the poller would refetch
|
||||||
|
// at ~60s — mid rate-limit window — and re-trip the 429 forever.
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date('2026-06-20T00:00:00Z'));
|
||||||
|
|
||||||
|
const fetchMock = vi
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValueOnce(makeResponse(429, { 'retry-after': '93' }))
|
||||||
|
.mockResolvedValueOnce(makeResponse(200));
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
|
||||||
|
const { fetchAllClaudeUsage } = await import('./claude-usage.js');
|
||||||
|
|
||||||
|
await fetchAllClaudeUsage();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
// +70s: past the 60s throttle, still inside the ~98s cooldown → no refetch.
|
||||||
|
vi.setSystemTime(Date.now() + 70_000);
|
||||||
|
const held = await fetchAllClaudeUsage();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(held[0].usageRateLimited).toBe(true);
|
||||||
|
|
||||||
|
// +100s total: cooldown window closed → poller is allowed to fetch again.
|
||||||
|
vi.setSystemTime(Date.now() + 30_000);
|
||||||
|
const resumed = await fetchAllClaudeUsage();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||||
|
expect(resumed[0].usageRateLimited).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -55,6 +55,30 @@ interface UsageCacheEntry {
|
|||||||
fetchedAt: number;
|
fetchedAt: number;
|
||||||
/** Timestamp of last API *attempt* including failures (use for throttling). */
|
/** Timestamp of last API *attempt* including failures (use for throttling). */
|
||||||
lastAttemptAt?: number;
|
lastAttemptAt?: number;
|
||||||
|
/**
|
||||||
|
* HTTP status of the most recent *failed* attempt (e.g. 429), cleared on a
|
||||||
|
* successful fetch. Used so the dashboard can render a live error indicator
|
||||||
|
* instead of a stale value.
|
||||||
|
*/
|
||||||
|
lastErrorStatus?: number;
|
||||||
|
/**
|
||||||
|
* Epoch-ms before which we MUST NOT retry, derived from a 429 `Retry-After`
|
||||||
|
* header (or a default backoff when the header is absent). Cleared on a
|
||||||
|
* successful fetch. Without this we re-poll every MIN_FETCH_INTERVAL_MS (60s)
|
||||||
|
* even though the endpoint's rate-limit window is longer (~90s+), so the
|
||||||
|
* cooldown never clears and the 429s become self-sustaining.
|
||||||
|
*/
|
||||||
|
cooldownUntil?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Outcome of a usage fetch. `rateLimited` is true when the most recent attempt
|
||||||
|
* hit a 429 — callers use it to surface an error in the UI rather than the
|
||||||
|
* last cached value.
|
||||||
|
*/
|
||||||
|
export interface UsageFetchResult {
|
||||||
|
usage: ClaudeUsageData | null;
|
||||||
|
rateLimited: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
let usageDiskCache: Record<string, UsageCacheEntry> = {};
|
let usageDiskCache: Record<string, UsageCacheEntry> = {};
|
||||||
@@ -112,13 +136,40 @@ export function getUsageCacheReadKeys(
|
|||||||
return keys;
|
return keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rate limit: at most one API call per token per 5 minutes
|
// Rate limit: at most one API call per token per 1 minute.
|
||||||
const MIN_FETCH_INTERVAL_MS = 300_000;
|
// We poll usage every minute (was 5) so the 95% exhaustion gate has tight
|
||||||
|
// resolution — at worst the gate fires ~60s after a window crosses 95%.
|
||||||
|
const MIN_FETCH_INTERVAL_MS = 60_000;
|
||||||
|
|
||||||
|
// Small margin added on top of a server-provided Retry-After so we wait until
|
||||||
|
// just *after* the cooldown window closes rather than racing its edge.
|
||||||
|
const RATE_LIMIT_MARGIN_MS = 5_000;
|
||||||
|
// Fallback cooldown when a 429 carries no usable Retry-After header.
|
||||||
|
const DEFAULT_RATE_LIMIT_COOLDOWN_MS = 5 * 60_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse an HTTP `Retry-After` header into milliseconds from now.
|
||||||
|
* Accepts either a delta-seconds integer ("93") or an HTTP-date.
|
||||||
|
* Returns null when the header is missing or unparseable.
|
||||||
|
*/
|
||||||
|
export function parseRetryAfterMs(header: string | null): number | null {
|
||||||
|
if (!header) return null;
|
||||||
|
const trimmed = header.trim();
|
||||||
|
if (/^\d+$/.test(trimmed)) {
|
||||||
|
return Number(trimmed) * 1000;
|
||||||
|
}
|
||||||
|
const dateMs = Date.parse(trimmed);
|
||||||
|
if (Number.isFinite(dateMs)) {
|
||||||
|
return Math.max(0, dateMs - Date.now());
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchUsageForToken(
|
async function fetchUsageForToken(
|
||||||
token: string,
|
token: string,
|
||||||
accountIndex?: number,
|
accountIndex?: number,
|
||||||
): Promise<ClaudeUsageData | null> {
|
refreshAttempted = false,
|
||||||
|
): Promise<UsageFetchResult> {
|
||||||
loadUsageDiskCache();
|
loadUsageDiskCache();
|
||||||
|
|
||||||
// Return cached data if attempted recently (avoid API rate-limit)
|
// Return cached data if attempted recently (avoid API rate-limit)
|
||||||
@@ -147,9 +198,54 @@ async function fetchUsageForToken(
|
|||||||
cached = usageDiskCache[writeKey];
|
cached = usageDiskCache[writeKey];
|
||||||
saveUsageDiskCache();
|
saveUsageDiskCache();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record this attempt's timestamp (and error status) at the write key,
|
||||||
|
* creating the entry if it does not exist yet. Recording even when there is
|
||||||
|
* no prior cached usage is what makes the MIN_FETCH_INTERVAL_MS throttle
|
||||||
|
* apply to *failures* too — without it an empty cache means every dashboard
|
||||||
|
* refresh hits the API, which itself sustains the very 429s we then report.
|
||||||
|
*/
|
||||||
|
const recordAttempt = (
|
||||||
|
errorStatus?: number,
|
||||||
|
cooldownUntil?: number,
|
||||||
|
): void => {
|
||||||
|
const now = Date.now();
|
||||||
|
const entry: UsageCacheEntry = usageDiskCache[writeKey] ?? {
|
||||||
|
usage: {},
|
||||||
|
fetchedAt: 0,
|
||||||
|
};
|
||||||
|
entry.lastAttemptAt = now;
|
||||||
|
entry.lastErrorStatus = errorStatus;
|
||||||
|
entry.cooldownUntil = cooldownUntil;
|
||||||
|
usageDiskCache[writeKey] = entry;
|
||||||
|
saveUsageDiskCache();
|
||||||
|
};
|
||||||
|
|
||||||
|
// Honour an active rate-limit cooldown from a prior 429's Retry-After. This
|
||||||
|
// is what actually breaks the self-sustaining 429 loop: the endpoint hands
|
||||||
|
// back a ~90s+ window, so retrying on the 60s MIN_FETCH_INTERVAL would keep
|
||||||
|
// hitting it mid-cooldown and never recover.
|
||||||
|
if (
|
||||||
|
!refreshAttempted &&
|
||||||
|
cached?.cooldownUntil &&
|
||||||
|
Date.now() < cached.cooldownUntil
|
||||||
|
) {
|
||||||
|
return { usage: null, rateLimited: true };
|
||||||
|
}
|
||||||
|
|
||||||
const lastAttempt = cached?.lastAttemptAt ?? cached?.fetchedAt ?? 0;
|
const lastAttempt = cached?.lastAttemptAt ?? cached?.fetchedAt ?? 0;
|
||||||
if (cached && Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS) {
|
if (
|
||||||
return cached.usage;
|
!refreshAttempted &&
|
||||||
|
cached &&
|
||||||
|
Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS
|
||||||
|
) {
|
||||||
|
// Within the throttle window: replay the last outcome without an API call.
|
||||||
|
// On a 429 we report rateLimited and DO NOT serve the stale value.
|
||||||
|
if (cached.lastErrorStatus === 429) {
|
||||||
|
return { usage: null, rateLimited: true };
|
||||||
|
}
|
||||||
|
return { usage: cached.usage ?? null, rateLimited: false };
|
||||||
}
|
}
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
||||||
@@ -165,34 +261,67 @@ async function fetchUsageForToken(
|
|||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 401) {
|
if (res.status === 401 || res.status === 403) {
|
||||||
|
// 401 = token expired; 403 = token lacks `user:profile` scope.
|
||||||
|
// Both are recoverable by exchanging refresh_token for a fresh access
|
||||||
|
// token (mint includes current default scope set). Retry once.
|
||||||
|
if (accountIndex != null && !refreshAttempted) {
|
||||||
|
try {
|
||||||
|
const { forceRefreshToken } = await import('./token-refresh.js');
|
||||||
|
const newToken = await forceRefreshToken(accountIndex);
|
||||||
|
if (newToken && newToken !== token) {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
account: accountIndex + 1,
|
||||||
|
status: res.status,
|
||||||
|
cacheKey: writeKey,
|
||||||
|
},
|
||||||
|
`Claude usage API: ${res.status} → refreshed access token, retrying`,
|
||||||
|
);
|
||||||
|
return fetchUsageForToken(newToken, accountIndex, true);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, account: accountIndex + 1 },
|
||||||
|
'Claude usage API: force-refresh failed',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
account: accountIndex != null ? accountIndex + 1 : '?',
|
account: accountIndex != null ? accountIndex + 1 : '?',
|
||||||
tokenKey: legacyTokenCacheKey(token),
|
tokenKey: legacyTokenCacheKey(token),
|
||||||
cacheKey: writeKey,
|
cacheKey: writeKey,
|
||||||
},
|
},
|
||||||
'Claude usage API: token expired or invalid (401)',
|
res.status === 401
|
||||||
|
? 'Claude usage API: token expired or invalid (401)'
|
||||||
|
: 'Claude usage API: forbidden — token missing required scope (403)',
|
||||||
);
|
);
|
||||||
return null;
|
recordAttempt(res.status);
|
||||||
|
return { usage: cached?.usage ?? null, rateLimited: false };
|
||||||
}
|
}
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
const staleMs = cached ? Date.now() - cached.fetchedAt : 0;
|
const staleMs = cached ? Date.now() - cached.fetchedAt : 0;
|
||||||
|
const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after'));
|
||||||
|
const cooldownMs = retryAfterMs ?? DEFAULT_RATE_LIMIT_COOLDOWN_MS;
|
||||||
|
const cooldownUntil = Date.now() + cooldownMs + RATE_LIMIT_MARGIN_MS;
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
account: accountIndex != null ? accountIndex + 1 : '?',
|
account: accountIndex != null ? accountIndex + 1 : '?',
|
||||||
tokenKey: legacyTokenCacheKey(token),
|
tokenKey: legacyTokenCacheKey(token),
|
||||||
cacheKey: writeKey,
|
cacheKey: writeKey,
|
||||||
staleMinutes: Math.round(staleMs / 60_000),
|
staleMinutes: Math.round(staleMs / 60_000),
|
||||||
|
retryAfterSec:
|
||||||
|
retryAfterMs != null ? Math.round(retryAfterMs / 1000) : null,
|
||||||
|
cooldownSec: Math.round(cooldownMs / 1000),
|
||||||
},
|
},
|
||||||
'Claude usage API: rate limited (429), returning cached data',
|
'Claude usage API: rate limited (429)',
|
||||||
);
|
);
|
||||||
// Record attempt time so we don't retry for MIN_FETCH_INTERVAL_MS
|
// Back off until the server-provided Retry-After window closes (plus a
|
||||||
if (cached) {
|
// margin) so we stop re-tripping the limit. Surfaces a 429 indicator and
|
||||||
cached.lastAttemptAt = Date.now();
|
// does NOT serve the stale value.
|
||||||
saveUsageDiskCache();
|
recordAttempt(429, cooldownUntil);
|
||||||
}
|
return { usage: null, rateLimited: true };
|
||||||
return cached?.usage ?? null;
|
|
||||||
}
|
}
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@@ -204,11 +333,8 @@ async function fetchUsageForToken(
|
|||||||
},
|
},
|
||||||
`Claude usage API: unexpected status ${res.status}`,
|
`Claude usage API: unexpected status ${res.status}`,
|
||||||
);
|
);
|
||||||
if (cached) {
|
recordAttempt(res.status);
|
||||||
cached.lastAttemptAt = Date.now();
|
return { usage: cached?.usage ?? null, rateLimited: false };
|
||||||
saveUsageDiskCache();
|
|
||||||
}
|
|
||||||
return cached?.usage ?? null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = (await res.json()) as UsageApiResponse;
|
const data = (await res.json()) as UsageApiResponse;
|
||||||
@@ -220,12 +346,14 @@ async function fetchUsageForToken(
|
|||||||
seven_day_opus: mapWindow(data.seven_day_opus),
|
seven_day_opus: mapWindow(data.seven_day_opus),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Persist to disk cache — success: update both fetchedAt and lastAttemptAt
|
// Persist to disk cache — success: update fetchedAt + lastAttemptAt and
|
||||||
|
// clear any prior error status.
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
usageDiskCache[writeKey] = {
|
usageDiskCache[writeKey] = {
|
||||||
usage: result,
|
usage: result,
|
||||||
fetchedAt: now,
|
fetchedAt: now,
|
||||||
lastAttemptAt: now,
|
lastAttemptAt: now,
|
||||||
|
lastErrorStatus: undefined,
|
||||||
};
|
};
|
||||||
saveUsageDiskCache();
|
saveUsageDiskCache();
|
||||||
|
|
||||||
@@ -240,7 +368,7 @@ async function fetchUsageForToken(
|
|||||||
'Claude usage API: fetched successfully',
|
'Claude usage API: fetched successfully',
|
||||||
);
|
);
|
||||||
|
|
||||||
return result;
|
return { usage: result, rateLimited: false };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if ((err as Error).name === 'AbortError') {
|
if ((err as Error).name === 'AbortError') {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@@ -263,11 +391,8 @@ async function fetchUsageForToken(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
// Record attempt time so we back off on persistent failures
|
// Record attempt time so we back off on persistent failures
|
||||||
if (cached) {
|
recordAttempt();
|
||||||
cached.lastAttemptAt = Date.now();
|
return { usage: cached?.usage ?? null, rateLimited: false };
|
||||||
saveUsageDiskCache();
|
|
||||||
}
|
|
||||||
return cached?.usage ?? null;
|
|
||||||
} finally {
|
} finally {
|
||||||
clearTimeout(timer);
|
clearTimeout(timer);
|
||||||
}
|
}
|
||||||
@@ -278,12 +403,20 @@ async function fetchUsageForToken(
|
|||||||
* Uses the current active token from rotation.
|
* Uses the current active token from rotation.
|
||||||
*/
|
*/
|
||||||
export async function fetchClaudeUsage(): Promise<ClaudeUsageData | null> {
|
export async function fetchClaudeUsage(): Promise<ClaudeUsageData | null> {
|
||||||
const token = getCurrentToken() || getConfiguredClaudeTokens()[0];
|
// Prefer the access token in ~/.claude/.credentials.json when present.
|
||||||
|
// The static .env token (CLAUDE_CODE_OAUTH_TOKEN) was issued before the
|
||||||
|
// `user:profile` scope was required for /api/oauth/usage and so always 403s.
|
||||||
|
// The credentials file is the canonical source written by `claude auth
|
||||||
|
// login` and kept fresh by token-refresh.ts — its accessToken carries the
|
||||||
|
// full scope set including user:profile.
|
||||||
|
const credsToken = readCredentialsAccessToken(0);
|
||||||
|
const token =
|
||||||
|
credsToken || getCurrentToken() || getConfiguredClaudeTokens()[0];
|
||||||
if (!token) {
|
if (!token) {
|
||||||
logger.debug('No Claude OAuth token available for usage check');
|
logger.debug('No Claude OAuth token available for usage check');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return fetchUsageForToken(token, undefined);
|
return (await fetchUsageForToken(token, undefined)).usage;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClaudeAccountProfile {
|
export interface ClaudeAccountProfile {
|
||||||
@@ -425,6 +558,11 @@ export interface ClaudeAccountUsage {
|
|||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
isRateLimited: boolean;
|
isRateLimited: boolean;
|
||||||
usage: ClaudeUsageData | null;
|
usage: ClaudeUsageData | null;
|
||||||
|
/**
|
||||||
|
* True when the most recent usage-API attempt for this account hit a 429.
|
||||||
|
* The dashboard renders a "429" indicator instead of a stale value.
|
||||||
|
*/
|
||||||
|
usageRateLimited?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -435,9 +573,10 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
|||||||
const allTokens = getAllTokens();
|
const allTokens = getAllTokens();
|
||||||
logger.debug({ tokenCount: allTokens.length }, 'fetchAllClaudeUsage called');
|
logger.debug({ tokenCount: allTokens.length }, 'fetchAllClaudeUsage called');
|
||||||
if (allTokens.length === 0) {
|
if (allTokens.length === 0) {
|
||||||
const token = getConfiguredClaudeTokens()[0];
|
const credsToken = readCredentialsAccessToken(0);
|
||||||
|
const token = credsToken || getConfiguredClaudeTokens()[0];
|
||||||
if (!token) return [];
|
if (!token) return [];
|
||||||
const usage = await fetchUsageForToken(token, 0);
|
const { usage, rateLimited } = await fetchUsageForToken(token, 0);
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
index: 0,
|
index: 0,
|
||||||
@@ -445,19 +584,31 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
|||||||
isActive: true,
|
isActive: true,
|
||||||
isRateLimited: false,
|
isRateLimited: false,
|
||||||
usage,
|
usage,
|
||||||
|
usageRateLimited: rateLimited,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
const results: ClaudeAccountUsage[] = [];
|
const results: ClaudeAccountUsage[] = [];
|
||||||
for (const t of allTokens) {
|
for (const t of allTokens) {
|
||||||
const usage = await fetchUsageForToken(t.token, t.index);
|
// Prefer the per-account credentials.json access token when present.
|
||||||
|
// The static .env CLAUDE_CODE_OAUTH_TOKEN(S) values were issued without
|
||||||
|
// the `user:profile` scope and so are rejected by /api/oauth/usage.
|
||||||
|
// The credentials file is rewritten by `claude auth login` and refreshed
|
||||||
|
// by token-refresh.ts, so it always carries the full scope set.
|
||||||
|
const credsToken = readCredentialsAccessToken(t.index);
|
||||||
|
const tokenForFetch = credsToken || t.token;
|
||||||
|
const { usage, rateLimited } = await fetchUsageForToken(
|
||||||
|
tokenForFetch,
|
||||||
|
t.index,
|
||||||
|
);
|
||||||
results.push({
|
results.push({
|
||||||
index: t.index,
|
index: t.index,
|
||||||
masked: t.masked,
|
masked: t.masked,
|
||||||
isActive: t.isActive,
|
isActive: t.isActive,
|
||||||
isRateLimited: t.isRateLimited,
|
isRateLimited: t.isRateLimited,
|
||||||
usage,
|
usage,
|
||||||
|
usageRateLimited: rateLimited,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
@@ -465,3 +616,103 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
|||||||
|
|
||||||
// Legacy alias
|
// Legacy alias
|
||||||
export const fetchClaudeUsageViaCli = fetchClaudeUsage;
|
export const fetchClaudeUsageViaCli = fetchClaudeUsage;
|
||||||
|
|
||||||
|
// ── Exhaustion gate ───────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Threshold above which a Claude account is considered "out of budget" for
|
||||||
|
// the purpose of admitting new turns. A turn is blocked only when *every*
|
||||||
|
// Claude account is either at/over this threshold on either window or
|
||||||
|
// already in a rate-limit cooldown.
|
||||||
|
|
||||||
|
export const CLAUDE_EXHAUSTION_THRESHOLD_PCT = 95;
|
||||||
|
|
||||||
|
export interface UsageExhaustionInfo {
|
||||||
|
exhausted: boolean;
|
||||||
|
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
||||||
|
nextResetAt: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Normalise the API's utilization field (which can be fractional or percent). */
|
||||||
|
function normaliseUtilization(u: number | undefined): number {
|
||||||
|
if (u == null || !Number.isFinite(u)) return 0;
|
||||||
|
return u >= 0 && u < 1 ? u * 100 : u;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResetMs(s: string | undefined | null): number | null {
|
||||||
|
if (!s) return null;
|
||||||
|
const t = Date.parse(s);
|
||||||
|
return Number.isFinite(t) ? t : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reports whether every configured Claude account is either ≥ 95% on a
|
||||||
|
* window or in a rate-limit cooldown, and — when so — the soonest moment at
|
||||||
|
* which any one account is expected to recover.
|
||||||
|
*
|
||||||
|
* Reads the on-disk usage cache (refreshed every minute by the dashboard
|
||||||
|
* renderer); performs no network I/O. Returns `exhausted: false` when there
|
||||||
|
* are no configured tokens, or when usage data is missing for any account
|
||||||
|
* (err on letting work through).
|
||||||
|
*/
|
||||||
|
export function getClaudeUsageExhaustion(): UsageExhaustionInfo {
|
||||||
|
loadUsageDiskCache();
|
||||||
|
const allTokens = getAllTokens();
|
||||||
|
if (allTokens.length === 0) return { exhausted: false, nextResetAt: null };
|
||||||
|
|
||||||
|
let earliestRecoveryMs = Infinity;
|
||||||
|
|
||||||
|
for (const t of allTokens) {
|
||||||
|
if (t.isRateLimited) {
|
||||||
|
// Rate-limit recovery time is not exposed on the token snapshot here;
|
||||||
|
// we treat it as "unknown but eventually". Skip from the min calc.
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const credsToken = readCredentialsAccessToken(t.index);
|
||||||
|
const readKeys = getUsageCacheReadKeys(t.token, t.index, credsToken);
|
||||||
|
let entry: UsageCacheEntry | undefined;
|
||||||
|
for (const key of readKeys) {
|
||||||
|
if (usageDiskCache[key]) {
|
||||||
|
entry = usageDiskCache[key];
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!entry) return { exhausted: false, nextResetAt: null };
|
||||||
|
|
||||||
|
const h5 = normaliseUtilization(entry.usage.five_hour?.utilization);
|
||||||
|
const d7 = normaliseUtilization(entry.usage.seven_day?.utilization);
|
||||||
|
if (Math.max(h5, d7) < CLAUDE_EXHAUSTION_THRESHOLD_PCT) {
|
||||||
|
return { exhausted: false, nextResetAt: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
// This account is exhausted. It becomes available again when whichever
|
||||||
|
// window(s) are over the threshold drop back below it. We approximate
|
||||||
|
// that as max(reset of each window currently ≥ threshold).
|
||||||
|
const h5ResetMs =
|
||||||
|
h5 >= CLAUDE_EXHAUSTION_THRESHOLD_PCT
|
||||||
|
? parseResetMs(entry.usage.five_hour?.resets_at)
|
||||||
|
: null;
|
||||||
|
const d7ResetMs =
|
||||||
|
d7 >= CLAUDE_EXHAUSTION_THRESHOLD_PCT
|
||||||
|
? parseResetMs(entry.usage.seven_day?.resets_at)
|
||||||
|
: null;
|
||||||
|
const candidates = [h5ResetMs, d7ResetMs].filter(
|
||||||
|
(v): v is number => v != null,
|
||||||
|
);
|
||||||
|
if (candidates.length > 0) {
|
||||||
|
const accountRecovery = Math.max(...candidates);
|
||||||
|
if (accountRecovery < earliestRecoveryMs) {
|
||||||
|
earliestRecoveryMs = accountRecovery;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
||||||
|
? new Date(earliestRecoveryMs).toISOString()
|
||||||
|
: null;
|
||||||
|
return { exhausted: true, nextResetAt };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Backwards-compat boolean wrapper. */
|
||||||
|
export function isClaudeUsageExhausted(): boolean {
|
||||||
|
return getClaudeUsageExhaustion().exhausted;
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ export function readCodexFeatureFromContent(
|
|||||||
feature: CodexConfigFeature,
|
feature: CodexConfigFeature,
|
||||||
): boolean {
|
): boolean {
|
||||||
const lines = content.split('\n');
|
const lines = content.split('\n');
|
||||||
|
const featureLinePattern = new RegExp(`^${feature}\\s*=\\s*(true|false)$`);
|
||||||
let inFeatures = false;
|
let inFeatures = false;
|
||||||
|
|
||||||
for (const line of lines) {
|
for (const line of lines) {
|
||||||
@@ -28,9 +29,7 @@ export function readCodexFeatureFromContent(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (!inFeatures) continue;
|
if (!inFeatures) continue;
|
||||||
const match = trimmed.match(
|
const match = trimmed.match(featureLinePattern);
|
||||||
new RegExp(`^${feature}\\s*=\\s*(true|false)$`),
|
|
||||||
);
|
|
||||||
if (match) return match[1] === 'true';
|
if (match) return match[1] === 'true';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|||||||
vi.mock('./agent-error-detection.js', () => ({
|
vi.mock('./agent-error-detection.js', () => ({
|
||||||
classifyAgentError: vi.fn(() => ({ category: 'none', reason: '' })),
|
classifyAgentError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||||
classifyCodexAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
|
classifyCodexAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||||
|
isCodexPoolUnavailableError: vi.fn(
|
||||||
|
(error: string | null | undefined) =>
|
||||||
|
/all\s+codex(?:\s+rotation)?\s+accounts(?:\s+are)?\s+unavailable/i.test(
|
||||||
|
error ?? '',
|
||||||
|
) || /codex\s+rotation\s+pool\s+unavailable/i.test(error ?? ''),
|
||||||
|
),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
@@ -165,6 +171,236 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
|||||||
expect(mod.getCodexAccountCount()).toBe(4);
|
expect(mod.getCodexAccountCount()).toBe(4);
|
||||||
expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath);
|
expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not mutate account health for the internal all-accounts-unavailable sentinel', async () => {
|
||||||
|
const agentErrors = await import('./agent-error-detection.js');
|
||||||
|
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
expect(
|
||||||
|
mod.rotateCodexToken(
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
),
|
||||||
|
).toBe(false);
|
||||||
|
|
||||||
|
const accounts = mod.getAllCodexAccounts();
|
||||||
|
expect(accounts[0]).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
isActive: true,
|
||||||
|
isAuthDead: false,
|
||||||
|
authStatus: 'healthy',
|
||||||
|
isRateLimited: false,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(accounts[1].isActive).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('marks refresh-token reuse as dead auth instead of a recoverable cooldown', async () => {
|
||||||
|
const agentErrors = await import('./agent-error-detection.js');
|
||||||
|
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
|
||||||
|
|
||||||
|
const accounts = mod.getAllCodexAccounts();
|
||||||
|
expect(accounts[0]).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
isAuthDead: true,
|
||||||
|
authStatus: 'dead_auth',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(accounts[0].isRateLimited).toBe(false);
|
||||||
|
expect(accounts[1].isActive).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('codex-token-rotation auth synchronization', () => {
|
||||||
|
let tempHome: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-rot-'));
|
||||||
|
process.env.CODEX_ROT_TEST_HOME = tempHome;
|
||||||
|
createFakeAccounts(tempHome, 4);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
delete process.env.CODEX_ROT_TEST_HOME;
|
||||||
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('recovers a dead_auth account when canonical auth.json is refreshed before lease claim', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
|
||||||
|
try {
|
||||||
|
const agentErrors = await import('./agent-error-detection.js');
|
||||||
|
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||||
|
category: 'auth-expired',
|
||||||
|
reason: 'auth-expired',
|
||||||
|
});
|
||||||
|
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
const utils = await import('./utils.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
|
||||||
|
expect(mod.getAllCodexAccounts()[0].isAuthDead).toBe(true);
|
||||||
|
|
||||||
|
const refreshedAt = new Date('2026-01-01T00:00:10.000Z');
|
||||||
|
const authPath = path.join(tempHome, '.codex-accounts', '0', 'auth.json');
|
||||||
|
fs.writeFileSync(
|
||||||
|
authPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: { account_id: 'acct-0', access_token: 'refreshed-token' },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
fs.utimesSync(authPath, refreshedAt, refreshedAt);
|
||||||
|
|
||||||
|
mod.setCurrentCodexAccountIndex(0);
|
||||||
|
const lease = mod.claimCodexAuthLease();
|
||||||
|
try {
|
||||||
|
expect(lease).toEqual(
|
||||||
|
expect.objectContaining({ accountIndex: 0, authPath }),
|
||||||
|
);
|
||||||
|
expect(mod.getAllCodexAccounts()[0]).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
authStatus: 'healthy',
|
||||||
|
isAuthDead: false,
|
||||||
|
isActive: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(vi.mocked(utils.writeJsonFile)).toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
lease?.release();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('leases different accounts for concurrent Codex runs', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
const first = mod.claimCodexAuthLease();
|
||||||
|
const second = mod.claimCodexAuthLease();
|
||||||
|
|
||||||
|
try {
|
||||||
|
expect(first).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
accountIndex: 0,
|
||||||
|
authPath: expect.any(String),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(second).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
accountIndex: 1,
|
||||||
|
authPath: expect.any(String),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(second?.authPath).not.toBe(first?.authPath);
|
||||||
|
} finally {
|
||||||
|
second?.release();
|
||||||
|
first?.release();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('syncs a refreshed session auth.json back to the canonical slot atomically', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
const canonicalAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'.codex-accounts',
|
||||||
|
'0',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
const sessionAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'session',
|
||||||
|
'.codex',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
sessionAuthPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: {
|
||||||
|
account_id: 'acct-0',
|
||||||
|
access_token: 'new-access',
|
||||||
|
refresh_token: 'new-refresh',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const synced = mod.syncCodexSessionAuthBack({
|
||||||
|
canonicalAuthPath,
|
||||||
|
sessionAuthPath,
|
||||||
|
accountIndex: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(synced).toBe(true);
|
||||||
|
const canonical = JSON.parse(fs.readFileSync(canonicalAuthPath, 'utf-8'));
|
||||||
|
expect(canonical.tokens).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
account_id: 'acct-0',
|
||||||
|
access_token: 'new-access',
|
||||||
|
refresh_token: 'new-refresh',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('refuses to sync a session auth.json that belongs to another Codex account', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
|
||||||
|
const canonicalAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'.codex-accounts',
|
||||||
|
'0',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
const before = fs.readFileSync(canonicalAuthPath, 'utf-8');
|
||||||
|
const sessionAuthPath = path.join(
|
||||||
|
tempHome,
|
||||||
|
'wrong-account',
|
||||||
|
'.codex',
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
sessionAuthPath,
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: {
|
||||||
|
account_id: 'acct-other',
|
||||||
|
access_token: 'other-access',
|
||||||
|
refresh_token: 'other-refresh',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
mod.syncCodexSessionAuthBack({
|
||||||
|
canonicalAuthPath,
|
||||||
|
sessionAuthPath,
|
||||||
|
accountIndex: 0,
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
|
expect(fs.readFileSync(canonicalAuthPath, 'utf-8')).toBe(before);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('codex-token-rotation single-account fallback', () => {
|
describe('codex-token-rotation single-account fallback', () => {
|
||||||
|
|||||||
@@ -17,13 +17,13 @@ import path from 'path';
|
|||||||
import {
|
import {
|
||||||
classifyAgentError,
|
classifyAgentError,
|
||||||
classifyCodexAuthError,
|
classifyCodexAuthError,
|
||||||
|
isCodexPoolUnavailableError,
|
||||||
type CodexRotationReason,
|
type CodexRotationReason,
|
||||||
} from './agent-error-detection.js';
|
} from './agent-error-detection.js';
|
||||||
import { DATA_DIR } from './config.js';
|
import { DATA_DIR } from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import {
|
import {
|
||||||
computeCooldownUntil,
|
computeCooldownUntil,
|
||||||
findNextAvailable,
|
|
||||||
parseRetryAfterFromError,
|
parseRetryAfterFromError,
|
||||||
} from './token-rotation-base.js';
|
} from './token-rotation-base.js';
|
||||||
import { readJsonFile, writeJsonFile } from './utils.js';
|
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||||
@@ -34,15 +34,27 @@ interface CodexAccount {
|
|||||||
index: number;
|
index: number;
|
||||||
authPath: string;
|
authPath: string;
|
||||||
accountId: string;
|
accountId: string;
|
||||||
|
authFileMtimeMs: number;
|
||||||
planType: string;
|
planType: string;
|
||||||
subscriptionUntil: string | null;
|
subscriptionUntil: string | null;
|
||||||
rateLimitedUntil: number | null;
|
rateLimitedUntil: number | null;
|
||||||
|
authStatus: 'healthy' | 'dead_auth';
|
||||||
|
authDeadAt: number | null;
|
||||||
|
authDeadReason: string | null;
|
||||||
|
leasedUntil: number | null;
|
||||||
|
leaseId: string | null;
|
||||||
lastUsagePct?: number;
|
lastUsagePct?: number;
|
||||||
lastUsageD7Pct?: number;
|
lastUsageD7Pct?: number;
|
||||||
resetAt?: string;
|
resetAt?: string;
|
||||||
resetD7At?: string;
|
resetD7At?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CodexAuthLease {
|
||||||
|
accountIndex: number;
|
||||||
|
authPath: string;
|
||||||
|
release: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
export type CodexRotationTriggerResult =
|
export type CodexRotationTriggerResult =
|
||||||
| {
|
| {
|
||||||
shouldRotate: false;
|
shouldRotate: false;
|
||||||
@@ -77,10 +89,20 @@ const accounts: CodexAccount[] = [];
|
|||||||
let currentIndex = 0;
|
let currentIndex = 0;
|
||||||
let initialized = false;
|
let initialized = false;
|
||||||
|
|
||||||
|
const CODEX_AUTH_LEASE_MS = 2 * 60 * 60_000;
|
||||||
|
|
||||||
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||||
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
||||||
const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json');
|
const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json');
|
||||||
|
|
||||||
|
function readAuthFileMtimeMs(authPath: string): number {
|
||||||
|
try {
|
||||||
|
return fs.statSync(authPath).mtimeMs;
|
||||||
|
} catch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function loadCodexAccount(
|
function loadCodexAccount(
|
||||||
authPath: string,
|
authPath: string,
|
||||||
fallbackAccountId: string,
|
fallbackAccountId: string,
|
||||||
@@ -99,9 +121,15 @@ function loadCodexAccount(
|
|||||||
index: accounts.length,
|
index: accounts.length,
|
||||||
authPath,
|
authPath,
|
||||||
accountId,
|
accountId,
|
||||||
|
authFileMtimeMs: readAuthFileMtimeMs(authPath),
|
||||||
planType: jwt.planType,
|
planType: jwt.planType,
|
||||||
subscriptionUntil: jwt.expiresAt,
|
subscriptionUntil: jwt.expiresAt,
|
||||||
rateLimitedUntil: null,
|
rateLimitedUntil: null,
|
||||||
|
authStatus: 'healthy',
|
||||||
|
authDeadAt: null,
|
||||||
|
authDeadReason: null,
|
||||||
|
leasedUntil: null,
|
||||||
|
leaseId: null,
|
||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -149,6 +177,8 @@ function saveCodexState(): void {
|
|||||||
const state = {
|
const state = {
|
||||||
currentIndex,
|
currentIndex,
|
||||||
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
||||||
|
authDeadAts: accounts.map((a) => a.authDeadAt),
|
||||||
|
authDeadReasons: accounts.map((a) => a.authDeadReason),
|
||||||
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
|
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
|
||||||
usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null),
|
usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null),
|
||||||
resetAts: accounts.map((a) => a.resetAt ?? null),
|
resetAts: accounts.map((a) => a.resetAt ?? null),
|
||||||
@@ -167,6 +197,8 @@ function loadCodexState(quiet = false): void {
|
|||||||
const state = readJsonFile<{
|
const state = readJsonFile<{
|
||||||
currentIndex?: number;
|
currentIndex?: number;
|
||||||
rateLimits?: (number | null)[];
|
rateLimits?: (number | null)[];
|
||||||
|
authDeadAts?: (number | null)[];
|
||||||
|
authDeadReasons?: (string | null)[];
|
||||||
usagePcts?: (number | null)[];
|
usagePcts?: (number | null)[];
|
||||||
usageD7Pcts?: (number | null)[];
|
usageD7Pcts?: (number | null)[];
|
||||||
resetAts?: (string | null)[];
|
resetAts?: (string | null)[];
|
||||||
@@ -175,6 +207,7 @@ function loadCodexState(quiet = false): void {
|
|||||||
if (!state) return;
|
if (!state) return;
|
||||||
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
let restoredDeadAuth = false;
|
||||||
if (
|
if (
|
||||||
typeof state.currentIndex === 'number' &&
|
typeof state.currentIndex === 'number' &&
|
||||||
state.currentIndex < accounts.length
|
state.currentIndex < accounts.length
|
||||||
@@ -195,6 +228,39 @@ function loadCodexState(quiet = false): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (Array.isArray(state.authDeadAts)) {
|
||||||
|
for (
|
||||||
|
let i = 0;
|
||||||
|
i < Math.min(state.authDeadAts.length, accounts.length);
|
||||||
|
i++
|
||||||
|
) {
|
||||||
|
const deadAt = state.authDeadAts[i];
|
||||||
|
const acct = accounts[i];
|
||||||
|
if (typeof deadAt === 'number' && deadAt > 0) {
|
||||||
|
const currentMtime = readAuthFileMtimeMs(acct.authPath);
|
||||||
|
acct.authFileMtimeMs = currentMtime;
|
||||||
|
if (currentMtime > deadAt) {
|
||||||
|
acct.authStatus = 'dead_auth';
|
||||||
|
acct.authDeadAt = deadAt;
|
||||||
|
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
|
||||||
|
restoredDeadAuth =
|
||||||
|
restoreDeadAuthIfAuthFileChanged(acct) || restoredDeadAuth;
|
||||||
|
} else {
|
||||||
|
acct.authStatus = 'dead_auth';
|
||||||
|
acct.authDeadAt = deadAt;
|
||||||
|
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
currentIndex < accounts.length &&
|
||||||
|
accounts[currentIndex]?.authStatus === 'dead_auth'
|
||||||
|
) {
|
||||||
|
const nextIdx = findNextCodexAvailable(currentIndex);
|
||||||
|
if (nextIdx !== null) currentIndex = nextIdx;
|
||||||
|
}
|
||||||
if (Array.isArray(state.usagePcts)) {
|
if (Array.isArray(state.usagePcts)) {
|
||||||
for (
|
for (
|
||||||
let i = 0;
|
let i = 0;
|
||||||
@@ -239,6 +305,7 @@ function loadCodexState(quiet = false): void {
|
|||||||
'Codex rotation state restored',
|
'Codex rotation state restored',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (restoredDeadAuth) saveCodexState();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -309,10 +376,272 @@ export function getCodexAuthPath(
|
|||||||
return accounts[accountIndex]?.authPath ?? null;
|
return accounts[accountIndex]?.authPath ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function codexLockPath(authPath: string): string {
|
||||||
|
return path.join(path.dirname(authPath), '.ejclaw-auth.lock');
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPidAlive(pid: number): boolean {
|
||||||
|
try {
|
||||||
|
process.kill(pid, 0);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readExistingLease(lockPath: string): {
|
||||||
|
leaseId?: string;
|
||||||
|
pid?: number;
|
||||||
|
expiresAt?: number;
|
||||||
|
} | null {
|
||||||
|
const data = readJsonFile<{
|
||||||
|
leaseId?: string;
|
||||||
|
pid?: number;
|
||||||
|
expiresAt?: number;
|
||||||
|
}>(lockPath);
|
||||||
|
return data ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tryAcquireDiskLease(
|
||||||
|
acct: CodexAccount,
|
||||||
|
leaseId: string,
|
||||||
|
now: number,
|
||||||
|
): boolean {
|
||||||
|
const lockPath = codexLockPath(acct.authPath);
|
||||||
|
const payload = JSON.stringify(
|
||||||
|
{
|
||||||
|
leaseId,
|
||||||
|
pid: process.pid,
|
||||||
|
accountIndex: acct.index,
|
||||||
|
createdAt: now,
|
||||||
|
expiresAt: now + CODEX_AUTH_LEASE_MS,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = readExistingLease(lockPath);
|
||||||
|
const expired = Boolean(existing?.expiresAt && existing.expiresAt <= now);
|
||||||
|
const deadPid = Boolean(existing?.pid && !isPidAlive(existing.pid));
|
||||||
|
if (!expired && !deadPid) return false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function releaseDiskLease(authPath: string, leaseId: string): void {
|
||||||
|
const lockPath = codexLockPath(authPath);
|
||||||
|
const existing = readExistingLease(lockPath);
|
||||||
|
if (existing?.leaseId && existing.leaseId !== leaseId) return;
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
} catch {
|
||||||
|
// Best effort only. Expired/stale locks are cleared by the next claimant.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isCodexAccountUsable(
|
||||||
|
acct: CodexAccount,
|
||||||
|
now = Date.now(),
|
||||||
|
opts?: { ignoreRateLimits?: boolean; ignoreD7?: boolean },
|
||||||
|
): boolean {
|
||||||
|
if (restoreDeadAuthIfAuthFileChanged(acct)) saveCodexState();
|
||||||
|
if (acct.authStatus === 'dead_auth') return false;
|
||||||
|
if (
|
||||||
|
!opts?.ignoreRateLimits &&
|
||||||
|
acct.rateLimitedUntil &&
|
||||||
|
acct.rateLimitedUntil > now
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!opts?.ignoreD7 &&
|
||||||
|
acct.lastUsageD7Pct != null &&
|
||||||
|
acct.lastUsageD7Pct >= 100
|
||||||
|
) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (acct.leasedUntil && acct.leasedUntil > now) return false;
|
||||||
|
|
||||||
|
const existing = readExistingLease(codexLockPath(acct.authPath));
|
||||||
|
if (!existing) return true;
|
||||||
|
if (existing.expiresAt && existing.expiresAt <= now) return true;
|
||||||
|
if (existing.pid && !isPidAlive(existing.pid)) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function restoreDeadAuthIfAuthFileChanged(acct: CodexAccount): boolean {
|
||||||
|
if (acct.authStatus !== 'dead_auth' || !acct.authDeadAt) return false;
|
||||||
|
const currentMtime = readAuthFileMtimeMs(acct.authPath);
|
||||||
|
acct.authFileMtimeMs = currentMtime;
|
||||||
|
if (currentMtime <= acct.authDeadAt) return false;
|
||||||
|
|
||||||
|
const previousDeadAt = acct.authDeadAt;
|
||||||
|
acct.authStatus = 'healthy';
|
||||||
|
acct.authDeadAt = null;
|
||||||
|
acct.authDeadReason = null;
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
transition: 'rotation:auth-file-refreshed',
|
||||||
|
accountIndex: acct.index,
|
||||||
|
previousDeadAt: new Date(previousDeadAt).toISOString(),
|
||||||
|
authFileMtime: new Date(currentMtime).toISOString(),
|
||||||
|
},
|
||||||
|
`Codex account #${acct.index + 1}/${accounts.length} marked healthy after auth.json refresh`,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function markCodexAccountDeadAuth(acct: CodexAccount, reason?: string): void {
|
||||||
|
acct.authStatus = 'dead_auth';
|
||||||
|
acct.authDeadAt = Date.now();
|
||||||
|
acct.authDeadReason = reason || 'auth-expired';
|
||||||
|
acct.rateLimitedUntil = null;
|
||||||
|
acct.leasedUntil = null;
|
||||||
|
acct.leaseId = null;
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
transition: 'rotation:dead-auth',
|
||||||
|
accountIndex: acct.index,
|
||||||
|
accountId: acct.accountId,
|
||||||
|
reason: acct.authDeadReason,
|
||||||
|
},
|
||||||
|
`Codex account #${acct.index + 1}/${accounts.length} marked dead; re-auth required`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateAccountMetadataFromAuthFile(acct: CodexAccount): void {
|
||||||
|
const data = readJsonFile<{
|
||||||
|
tokens?: { account_id?: string; id_token?: string };
|
||||||
|
}>(acct.authPath);
|
||||||
|
if (data?.tokens?.account_id) acct.accountId = data.tokens.account_id;
|
||||||
|
const jwt = parseJwtAuth(data?.tokens?.id_token || '');
|
||||||
|
acct.planType = jwt.planType;
|
||||||
|
acct.subscriptionUntil = jwt.expiresAt;
|
||||||
|
acct.authFileMtimeMs = readAuthFileMtimeMs(acct.authPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function claimCodexAuthLease(): CodexAuthLease | null {
|
||||||
|
if (accounts.length === 0) return null;
|
||||||
|
const now = Date.now();
|
||||||
|
const attempts = accounts.length;
|
||||||
|
for (let offset = 0; offset < attempts; offset += 1) {
|
||||||
|
const idx = (currentIndex + offset) % accounts.length;
|
||||||
|
const acct = accounts[idx];
|
||||||
|
if (!isCodexAccountUsable(acct, now)) continue;
|
||||||
|
const leaseId = `${process.pid}-${now}-${idx}-${Math.random()
|
||||||
|
.toString(36)
|
||||||
|
.slice(2)}`;
|
||||||
|
if (!tryAcquireDiskLease(acct, leaseId, now)) continue;
|
||||||
|
currentIndex = idx;
|
||||||
|
acct.leasedUntil = now + CODEX_AUTH_LEASE_MS;
|
||||||
|
acct.leaseId = leaseId;
|
||||||
|
return {
|
||||||
|
accountIndex: idx,
|
||||||
|
authPath: acct.authPath,
|
||||||
|
release: () => {
|
||||||
|
if (acct.leaseId === leaseId) {
|
||||||
|
acct.leasedUntil = null;
|
||||||
|
acct.leaseId = null;
|
||||||
|
}
|
||||||
|
releaseDiskLease(acct.authPath, leaseId);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function syncCodexSessionAuthBack(args: {
|
||||||
|
canonicalAuthPath: string;
|
||||||
|
sessionAuthPath: string;
|
||||||
|
accountIndex?: number | null;
|
||||||
|
}): boolean {
|
||||||
|
if (!fs.existsSync(args.sessionAuthPath)) return false;
|
||||||
|
if (!fs.existsSync(args.canonicalAuthPath)) return false;
|
||||||
|
|
||||||
|
const canonical = readJsonFile<{
|
||||||
|
auth_mode?: string;
|
||||||
|
tokens?: { account_id?: string };
|
||||||
|
}>(args.canonicalAuthPath);
|
||||||
|
const session = readJsonFile<{
|
||||||
|
auth_mode?: string;
|
||||||
|
tokens?: { account_id?: string };
|
||||||
|
}>(args.sessionAuthPath);
|
||||||
|
if (!canonical || !session?.tokens) return false;
|
||||||
|
|
||||||
|
const canonicalAccount = canonical.tokens?.account_id;
|
||||||
|
const sessionAccount = session.tokens.account_id;
|
||||||
|
if (
|
||||||
|
canonicalAccount &&
|
||||||
|
sessionAccount &&
|
||||||
|
canonicalAccount !== sessionAccount
|
||||||
|
) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
canonicalAuthPath: args.canonicalAuthPath,
|
||||||
|
sessionAuthPath: args.sessionAuthPath,
|
||||||
|
accountIndex: args.accountIndex ?? null,
|
||||||
|
},
|
||||||
|
'Refusing to sync Codex session auth back: account mismatch',
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionRaw = fs.readFileSync(args.sessionAuthPath, 'utf-8');
|
||||||
|
const canonicalRaw = fs.readFileSync(args.canonicalAuthPath, 'utf-8');
|
||||||
|
if (sessionRaw === canonicalRaw) return false;
|
||||||
|
|
||||||
|
const tmpPath = `${args.canonicalAuthPath}.tmp-${process.pid}-${Date.now()}`;
|
||||||
|
fs.writeFileSync(tmpPath, sessionRaw, { mode: 0o600 });
|
||||||
|
fs.renameSync(tmpPath, args.canonicalAuthPath);
|
||||||
|
|
||||||
|
const idx =
|
||||||
|
args.accountIndex ??
|
||||||
|
findCodexAccountIndexByAuthPath(args.canonicalAuthPath);
|
||||||
|
if (idx != null && accounts[idx]) {
|
||||||
|
const acct = accounts[idx];
|
||||||
|
acct.authStatus = 'healthy';
|
||||||
|
acct.authDeadAt = null;
|
||||||
|
acct.authDeadReason = null;
|
||||||
|
updateAccountMetadataFromAuthFile(acct);
|
||||||
|
saveCodexState();
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
accountIndex: idx,
|
||||||
|
canonicalAuthPath: args.canonicalAuthPath,
|
||||||
|
},
|
||||||
|
'Synced refreshed Codex session auth back to canonical account slot',
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
export function detectCodexRotationTrigger(
|
export function detectCodexRotationTrigger(
|
||||||
error?: string | null,
|
error?: string | null,
|
||||||
): CodexRotationTriggerResult {
|
): CodexRotationTriggerResult {
|
||||||
if (!error) return { shouldRotate: false, reason: '' };
|
if (!error) return { shouldRotate: false, reason: '' };
|
||||||
|
if (isCodexPoolUnavailableError(error)) {
|
||||||
|
return { shouldRotate: false, reason: '' };
|
||||||
|
}
|
||||||
|
|
||||||
// Common patterns (429, 503, network) — delegated to SSOT
|
// Common patterns (429, 503, network) — delegated to SSOT
|
||||||
const common = classifyAgentError(error);
|
const common = classifyAgentError(error);
|
||||||
@@ -339,18 +668,38 @@ export function rotateCodexToken(
|
|||||||
): boolean {
|
): boolean {
|
||||||
if (accounts.length <= 1) return false;
|
if (accounts.length <= 1) return false;
|
||||||
|
|
||||||
const previousIndex = currentIndex;
|
if (isCodexPoolUnavailableError(errorMessage)) {
|
||||||
const acct = accounts[currentIndex];
|
logger.warn(
|
||||||
const cooldownUntil = computeCooldownUntil(errorMessage);
|
{
|
||||||
acct.rateLimitedUntil = cooldownUntil;
|
transition: 'rotation:skip-pool-unavailable-sentinel',
|
||||||
acct.lastUsagePct = 100;
|
currentIndex,
|
||||||
// Extract reset time string from error for display
|
totalAccounts: accounts.length,
|
||||||
const retryAt = parseRetryAfterFromError(errorMessage);
|
reason: errorMessage ?? null,
|
||||||
if (retryAt) {
|
},
|
||||||
acct.resetAt = new Date(retryAt).toISOString();
|
'Refusing to mark Codex accounts unhealthy from internal pool-unavailable sentinel',
|
||||||
|
);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const nextIdx = findNextAvailable(accounts, currentIndex, opts);
|
const previousIndex = currentIndex;
|
||||||
|
const acct = accounts[currentIndex];
|
||||||
|
const authFailure = classifyCodexAuthError(errorMessage);
|
||||||
|
let cooldownUntil: number | null = null;
|
||||||
|
|
||||||
|
if (authFailure.category === 'auth-expired') {
|
||||||
|
markCodexAccountDeadAuth(acct, errorMessage || authFailure.reason);
|
||||||
|
} else {
|
||||||
|
cooldownUntil = computeCooldownUntil(errorMessage);
|
||||||
|
acct.rateLimitedUntil = cooldownUntil;
|
||||||
|
acct.lastUsagePct = 100;
|
||||||
|
// Extract reset time string from error for display
|
||||||
|
const retryAt = parseRetryAfterFromError(errorMessage);
|
||||||
|
if (retryAt) {
|
||||||
|
acct.resetAt = new Date(retryAt).toISOString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextIdx = findNextCodexAvailable(currentIndex, opts);
|
||||||
if (nextIdx !== null) {
|
if (nextIdx !== null) {
|
||||||
accounts[nextIdx].rateLimitedUntil = null;
|
accounts[nextIdx].rateLimitedUntil = null;
|
||||||
currentIndex = nextIdx;
|
currentIndex = nextIdx;
|
||||||
@@ -364,6 +713,7 @@ export function rotateCodexToken(
|
|||||||
ignoreRL: opts?.ignoreRateLimits ?? false,
|
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||||
cooldownUntil:
|
cooldownUntil:
|
||||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||||
|
authDead: authFailure.category === 'auth-expired',
|
||||||
reason: errorMessage ?? null,
|
reason: errorMessage ?? null,
|
||||||
},
|
},
|
||||||
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
||||||
@@ -380,28 +730,40 @@ export function rotateCodexToken(
|
|||||||
ignoreRL: opts?.ignoreRateLimits ?? false,
|
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||||
cooldownUntil:
|
cooldownUntil:
|
||||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||||
|
authDead: authFailure.category === 'auth-expired',
|
||||||
reason: errorMessage ?? null,
|
reason: errorMessage ?? null,
|
||||||
},
|
},
|
||||||
'All Codex accounts are rate-limited',
|
authFailure.category === 'auth-expired'
|
||||||
|
? 'All Codex accounts unavailable after auth failure; re-auth required'
|
||||||
|
: 'All Codex accounts are rate-limited',
|
||||||
);
|
);
|
||||||
|
saveCodexState();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
||||||
*/
|
*/
|
||||||
function findNextCodexAvailable(fromIndex?: number): number | null {
|
function findNextCodexAvailable(
|
||||||
|
fromIndex?: number,
|
||||||
|
opts?: { ignoreRateLimits?: boolean },
|
||||||
|
): number | null {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const start = fromIndex ?? currentIndex;
|
const start = fromIndex ?? currentIndex;
|
||||||
for (let i = 1; i < accounts.length; i++) {
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
const idx = (start + i) % accounts.length;
|
const idx = (start + i) % accounts.length;
|
||||||
const acct = accounts[idx];
|
const acct = accounts[idx];
|
||||||
const rlOk = !acct.rateLimitedUntil || acct.rateLimitedUntil <= now;
|
if (isCodexAccountUsable(acct, now, opts)) return idx;
|
||||||
const usageOk = acct.lastUsageD7Pct == null || acct.lastUsageD7Pct < 100;
|
|
||||||
if (rlOk && usageOk) return idx;
|
|
||||||
}
|
}
|
||||||
// All exhausted — fall back to rate-limit-only check
|
// All d7-exhausted — fall back to rate-limit/dead/lease checks only.
|
||||||
return findNextAvailable(accounts, start);
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
|
const idx = (start + i) % accounts.length;
|
||||||
|
const acct = accounts[idx];
|
||||||
|
if (isCodexAccountUsable(acct, now, { ...opts, ignoreD7: true })) {
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -469,9 +831,17 @@ export function updateCodexAccountUsage(
|
|||||||
export function markCodexTokenHealthy(): void {
|
export function markCodexTokenHealthy(): void {
|
||||||
if (accounts.length === 0) return;
|
if (accounts.length === 0) return;
|
||||||
const acct = accounts[currentIndex];
|
const acct = accounts[currentIndex];
|
||||||
|
let changed = false;
|
||||||
|
if (acct?.authStatus === 'dead_auth') {
|
||||||
|
acct.authStatus = 'healthy';
|
||||||
|
acct.authDeadAt = null;
|
||||||
|
acct.authDeadReason = null;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
if (acct?.rateLimitedUntil) {
|
if (acct?.rateLimitedUntil) {
|
||||||
const previousCooldownUntil = acct.rateLimitedUntil;
|
const previousCooldownUntil = acct.rateLimitedUntil;
|
||||||
acct.rateLimitedUntil = null;
|
acct.rateLimitedUntil = null;
|
||||||
|
changed = true;
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
transition: 'rotation:clear-rate-limit',
|
transition: 'rotation:clear-rate-limit',
|
||||||
@@ -481,8 +851,8 @@ export function markCodexTokenHealthy(): void {
|
|||||||
},
|
},
|
||||||
'Cleared Codex account rate-limit state after successful response',
|
'Cleared Codex account rate-limit state after successful response',
|
||||||
);
|
);
|
||||||
saveCodexState();
|
|
||||||
}
|
}
|
||||||
|
if (changed) saveCodexState();
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getCodexAccountCount(): number {
|
export function getCodexAccountCount(): number {
|
||||||
@@ -495,6 +865,10 @@ export function getAllCodexAccounts(): {
|
|||||||
planType: string;
|
planType: string;
|
||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
isRateLimited: boolean;
|
isRateLimited: boolean;
|
||||||
|
isAuthDead?: boolean;
|
||||||
|
authStatus?: 'healthy' | 'dead_auth';
|
||||||
|
authDeadAt?: string;
|
||||||
|
isLeased?: boolean;
|
||||||
cachedUsagePct?: number;
|
cachedUsagePct?: number;
|
||||||
cachedUsageD7Pct?: number;
|
cachedUsageD7Pct?: number;
|
||||||
resetAt?: string;
|
resetAt?: string;
|
||||||
@@ -507,6 +881,10 @@ export function getAllCodexAccounts(): {
|
|||||||
planType: a.planType,
|
planType: a.planType,
|
||||||
isActive: i === currentIndex,
|
isActive: i === currentIndex,
|
||||||
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
||||||
|
isAuthDead: a.authStatus === 'dead_auth',
|
||||||
|
authStatus: a.authStatus,
|
||||||
|
authDeadAt: a.authDeadAt ? new Date(a.authDeadAt).toISOString() : undefined,
|
||||||
|
isLeased: Boolean(a.leasedUntil && a.leasedUntil > now),
|
||||||
cachedUsagePct: a.lastUsagePct,
|
cachedUsagePct: a.lastUsagePct,
|
||||||
cachedUsageD7Pct: a.lastUsageD7Pct,
|
cachedUsageD7Pct: a.lastUsageD7Pct,
|
||||||
resetAt: a.resetAt,
|
resetAt: a.resetAt,
|
||||||
|
|||||||
@@ -31,10 +31,120 @@ export interface CodexUsageRefreshResult {
|
|||||||
/** Full scan interval — exported so the orchestrator can schedule it. */
|
/** Full scan interval — exported so the orchestrator can schedule it. */
|
||||||
export const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
export const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Threshold above which a Codex account is considered "out of budget" for
|
||||||
|
* the purpose of admitting new turns. A turn is blocked only when *every*
|
||||||
|
* Codex account is either at/over this threshold on either window or
|
||||||
|
* already in a rate-limit cooldown.
|
||||||
|
*/
|
||||||
|
export const CODEX_EXHAUSTION_THRESHOLD_PCT = 95;
|
||||||
|
|
||||||
|
export interface UsageExhaustionInfo {
|
||||||
|
exhausted: boolean;
|
||||||
|
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
||||||
|
nextResetAt: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResetMs(s: string | undefined | null): number | null {
|
||||||
|
if (!s) return null;
|
||||||
|
const t = Date.parse(s);
|
||||||
|
return Number.isFinite(t) ? t : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Coerce a string-or-number resetsAt (numeric epoch in seconds) to ISO. */
|
||||||
|
function toIsoMaybe(value: string | number | undefined): string | undefined {
|
||||||
|
if (value == null) return undefined;
|
||||||
|
if (typeof value === 'number') {
|
||||||
|
return new Date(value * 1000).toISOString();
|
||||||
|
}
|
||||||
|
// Already a string — pass through if it parses, else drop.
|
||||||
|
const ms = Date.parse(value);
|
||||||
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reports whether every configured Codex account is either ≥ 95% on a
|
||||||
|
* window or in a rate-limit cooldown, and — when so — the soonest moment
|
||||||
|
* at which any one account is expected to recover.
|
||||||
|
*
|
||||||
|
* Reads the in-process rotation snapshot; performs no I/O. Returns
|
||||||
|
* `exhausted: false` when there are no configured accounts, or when usage
|
||||||
|
* data is missing for any account (err on letting work through).
|
||||||
|
*/
|
||||||
|
export function getCodexUsageExhaustion(): UsageExhaustionInfo {
|
||||||
|
const accounts = getAllCodexAccounts();
|
||||||
|
if (accounts.length === 0) return { exhausted: false, nextResetAt: null };
|
||||||
|
|
||||||
|
let earliestRecoveryMs = Infinity;
|
||||||
|
|
||||||
|
for (const a of accounts) {
|
||||||
|
if (a.isRateLimited) continue; // counts as exhausted, recovery unknown here
|
||||||
|
const h5 = a.cachedUsagePct;
|
||||||
|
const d7 = a.cachedUsageD7Pct;
|
||||||
|
// -1 / undefined means "unknown" → treat as headroom
|
||||||
|
if (h5 == null || h5 < 0 || d7 == null || d7 < 0) {
|
||||||
|
return { exhausted: false, nextResetAt: null };
|
||||||
|
}
|
||||||
|
if (Math.max(h5, d7) < CODEX_EXHAUSTION_THRESHOLD_PCT) {
|
||||||
|
return { exhausted: false, nextResetAt: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
const h5ResetMs =
|
||||||
|
h5 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetAt) : null;
|
||||||
|
const d7ResetMs =
|
||||||
|
d7 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetD7At) : null;
|
||||||
|
const candidates = [h5ResetMs, d7ResetMs].filter(
|
||||||
|
(v): v is number => v != null,
|
||||||
|
);
|
||||||
|
if (candidates.length > 0) {
|
||||||
|
const accountRecovery = Math.max(...candidates);
|
||||||
|
if (accountRecovery < earliestRecoveryMs) {
|
||||||
|
earliestRecoveryMs = accountRecovery;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
||||||
|
? new Date(earliestRecoveryMs).toISOString()
|
||||||
|
: null;
|
||||||
|
return { exhausted: true, nextResetAt };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Backwards-compat boolean wrapper. */
|
||||||
|
export function isCodexUsageExhausted(): boolean {
|
||||||
|
return getCodexUsageExhaustion().exhausted;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFnmCodexBinDirs(): string[] {
|
||||||
|
const fnmRoot = path.join(os.homedir(), '.local', 'share', 'fnm');
|
||||||
|
const dirs: string[] = [];
|
||||||
|
// Prefer the alias `default` first if it exists.
|
||||||
|
const defaultBin = path.join(fnmRoot, 'aliases', 'default', 'bin');
|
||||||
|
if (fs.existsSync(defaultBin)) dirs.push(defaultBin);
|
||||||
|
// Then fall back to scanning every installed node version.
|
||||||
|
const versionsRoot = path.join(fnmRoot, 'node-versions');
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(versionsRoot)) {
|
||||||
|
for (const entry of fs.readdirSync(versionsRoot)) {
|
||||||
|
const bin = path.join(versionsRoot, entry, 'installation', 'bin');
|
||||||
|
if (fs.existsSync(bin)) dirs.push(bin);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
return dirs;
|
||||||
|
}
|
||||||
|
|
||||||
function getPreferredCodexPathEntries(): string[] {
|
function getPreferredCodexPathEntries(): string[] {
|
||||||
const entries = [
|
const entries = [
|
||||||
path.dirname(process.execPath),
|
path.dirname(process.execPath),
|
||||||
path.join(os.homedir(), '.npm-global', 'bin'),
|
path.join(os.homedir(), '.npm-global', 'bin'),
|
||||||
|
// fnm-managed node installs (where `npm i -g @openai/codex` actually
|
||||||
|
// lands when the host uses fnm). Without these, ejclaw running under
|
||||||
|
// bun/systemd cannot find the `codex` binary even though the user has
|
||||||
|
// it installed via fnm's default node.
|
||||||
|
...getFnmCodexBinDirs(),
|
||||||
];
|
];
|
||||||
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
||||||
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
||||||
@@ -42,6 +152,18 @@ function getPreferredCodexPathEntries(): string[] {
|
|||||||
return [...new Set(entries)];
|
return [...new Set(entries)];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function findCodexBinary(): string {
|
||||||
|
const candidates = [
|
||||||
|
path.join(os.homedir(), '.npm-global', 'bin', 'codex'),
|
||||||
|
...getFnmCodexBinDirs().map((dir) => path.join(dir, 'codex')),
|
||||||
|
];
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (fs.existsSync(candidate)) return candidate;
|
||||||
|
}
|
||||||
|
// Last resort: rely on PATH (we extend it via getPreferredCodexPathEntries).
|
||||||
|
return 'codex';
|
||||||
|
}
|
||||||
|
|
||||||
function getCodexHomeForAccount(accountIndex?: number): string | null {
|
function getCodexHomeForAccount(accountIndex?: number): string | null {
|
||||||
const authPath = getCodexAuthPath(accountIndex);
|
const authPath = getCodexAuthPath(accountIndex);
|
||||||
if (!authPath || !fs.existsSync(authPath)) return null;
|
if (!authPath || !fs.existsSync(authPath)) return null;
|
||||||
@@ -51,8 +173,7 @@ function getCodexHomeForAccount(accountIndex?: number): string | null {
|
|||||||
export async function fetchCodexUsage(
|
export async function fetchCodexUsage(
|
||||||
codexHomeOverride?: string,
|
codexHomeOverride?: string,
|
||||||
): Promise<CodexRateLimit[] | null> {
|
): Promise<CodexRateLimit[] | null> {
|
||||||
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
const codexBin = findCodexBinary();
|
||||||
const codexBin = fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
|
||||||
|
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
let done = false;
|
let done = false;
|
||||||
@@ -193,20 +314,19 @@ export function applyCodexUsageToAccount(
|
|||||||
|
|
||||||
const pct = Math.round(effective.primary.usedPercent);
|
const pct = Math.round(effective.primary.usedPercent);
|
||||||
const d7Pct = Math.round(effective.secondary.usedPercent);
|
const d7Pct = Math.round(effective.secondary.usedPercent);
|
||||||
const resetStr = effective.primary.resetsAt
|
// Store raw ISO timestamps (not pre-formatted strings) so the exhaustion
|
||||||
? formatResetRemaining(effective.primary.resetsAt)
|
// gate can compute "minutes until reset" later. The dashboard render path
|
||||||
: undefined;
|
// formats these at display time via `formatResetRemaining`.
|
||||||
const resetD7Str = effective.secondary.resetsAt
|
const resetIso = toIsoMaybe(effective.primary.resetsAt);
|
||||||
? formatResetRemaining(effective.secondary.resetsAt)
|
const resetD7Iso = toIsoMaybe(effective.secondary.resetsAt);
|
||||||
: undefined;
|
updateCodexAccountUsage(pct, resetIso, accountIndex, d7Pct, resetD7Iso);
|
||||||
updateCodexAccountUsage(pct, resetStr, accountIndex, d7Pct, resetD7Str);
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
account: accountIndex + 1,
|
account: accountIndex + 1,
|
||||||
bucket: effective.limitId,
|
bucket: effective.limitId,
|
||||||
h5: pct,
|
h5: pct,
|
||||||
d7: d7Pct,
|
d7: d7Pct,
|
||||||
reset: resetStr,
|
reset: resetIso,
|
||||||
},
|
},
|
||||||
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
||||||
);
|
);
|
||||||
@@ -229,9 +349,9 @@ export function buildCodexUsageRowsFromState(): UsageRow[] {
|
|||||||
return {
|
return {
|
||||||
name: label,
|
name: label,
|
||||||
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
||||||
h5reset: acct.resetAt || '',
|
h5reset: acct.resetAt ? formatResetRemaining(acct.resetAt) : '',
|
||||||
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
||||||
d7reset: acct.resetD7At || '',
|
d7reset: acct.resetD7At ? formatResetRemaining(acct.resetD7At) : '',
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -168,6 +168,66 @@ describe('Codex warm-up scheduler', () => {
|
|||||||
expect(state.consecutiveFailures).toBe(0);
|
expect(state.consecutiveFailures).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('spawns the bundled codex JS launcher via the JS runtime instead of a bare "codex" PATH lookup', async () => {
|
||||||
|
// Regression: under bun/systemd there is usually no global `codex` on PATH,
|
||||||
|
// so spawning bare `codex` failed with ENOENT and the primer never fired.
|
||||||
|
const childProcess = await import('child_process');
|
||||||
|
const rotation = await import('./codex-token-rotation.js');
|
||||||
|
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||||
|
const now = new Date('2026-04-24T09:00:00Z').getTime();
|
||||||
|
let capturedCmd: string | undefined;
|
||||||
|
let capturedArgs: readonly string[] | undefined;
|
||||||
|
|
||||||
|
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
accountId: 'fresh-account',
|
||||||
|
planType: 'pro',
|
||||||
|
isActive: false,
|
||||||
|
isRateLimited: false,
|
||||||
|
cachedUsagePct: 0,
|
||||||
|
cachedUsageD7Pct: 0,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||||
|
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||||
|
);
|
||||||
|
vi.mocked(childProcess.spawn).mockImplementation(((
|
||||||
|
cmd: string,
|
||||||
|
args?: readonly string[],
|
||||||
|
) => {
|
||||||
|
capturedCmd = cmd;
|
||||||
|
capturedArgs = args;
|
||||||
|
return createFakeCodexProcess(0) as never;
|
||||||
|
}) as unknown as typeof childProcess.spawn);
|
||||||
|
|
||||||
|
await runCodexWarmupCycle(
|
||||||
|
{
|
||||||
|
enabled: true,
|
||||||
|
prompt: 'Reply exactly OK. Do not run tools.',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
intervalMs: 300_000,
|
||||||
|
minIntervalMs: 18_300_000,
|
||||||
|
staggerMs: 1_800_000,
|
||||||
|
maxUsagePct: 0,
|
||||||
|
maxD7UsagePct: 0,
|
||||||
|
commandTimeoutMs: 120_000,
|
||||||
|
failureCooldownMs: 21_600_000,
|
||||||
|
maxConsecutiveFailures: 2,
|
||||||
|
},
|
||||||
|
{ nowMs: now, statePath },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(childProcess.spawn).toHaveBeenCalledTimes(1);
|
||||||
|
// Command is the JS runtime (bun/node), never a bare "codex" PATH lookup.
|
||||||
|
expect(capturedCmd).toBe(process.execPath);
|
||||||
|
expect(capturedCmd).not.toBe('codex');
|
||||||
|
// First arg is the resolvable codex JS launcher; 'exec' follows after it.
|
||||||
|
expect(capturedArgs?.[0]).toMatch(/codex\.js$/);
|
||||||
|
expect(fs.existsSync(capturedArgs?.[0] as string)).toBe(true);
|
||||||
|
expect(capturedArgs?.[1]).toBe('exec');
|
||||||
|
});
|
||||||
|
|
||||||
it('does not repeat warm-up while the same zero-usage quota window is already marked warmed', async () => {
|
it('does not repeat warm-up while the same zero-usage quota window is already marked warmed', async () => {
|
||||||
const childProcess = await import('child_process');
|
const childProcess = await import('child_process');
|
||||||
const rotation = await import('./codex-token-rotation.js');
|
const rotation = await import('./codex-token-rotation.js');
|
||||||
@@ -393,3 +453,118 @@ describe('Codex warm-up scheduler', () => {
|
|||||||
expect(childProcess.spawn).not.toHaveBeenCalled();
|
expect(childProcess.spawn).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Codex warm-up fixed-slot forceAttempt', () => {
|
||||||
|
let tempHome: string;
|
||||||
|
let statePath: string;
|
||||||
|
// Two consecutive slots 5h apart, but the primer fires a beat after the slot,
|
||||||
|
// so the gap is just under minIntervalMs (5h).
|
||||||
|
const t13 = new Date('2026-06-09T04:00:00.486Z').getTime();
|
||||||
|
const t18 = new Date('2026-06-09T09:00:00.000Z').getTime();
|
||||||
|
const config = {
|
||||||
|
enabled: true as const,
|
||||||
|
prompt: 'Reply exactly OK. Do not run tools.',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
intervalMs: 300_000,
|
||||||
|
minIntervalMs: 18_000_000, // 5h
|
||||||
|
staggerMs: 1_800_000,
|
||||||
|
maxUsagePct: 100,
|
||||||
|
maxD7UsagePct: 100,
|
||||||
|
commandTimeoutMs: 120_000,
|
||||||
|
failureCooldownMs: 21_600_000,
|
||||||
|
maxConsecutiveFailures: 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-force-'));
|
||||||
|
statePath = path.join(tempHome, 'codex-warmup-state.json');
|
||||||
|
fs.mkdirSync(path.dirname(authPathFor(tempHome, 0)), { recursive: true });
|
||||||
|
fs.writeFileSync(authPathFor(tempHome, 0), '{}');
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
async function setup() {
|
||||||
|
const childProcess = await import('child_process');
|
||||||
|
const rotation = await import('./codex-token-rotation.js');
|
||||||
|
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||||
|
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
accountId: 'acct',
|
||||||
|
planType: 'pro',
|
||||||
|
isActive: true,
|
||||||
|
isRateLimited: false,
|
||||||
|
cachedUsagePct: 0,
|
||||||
|
cachedUsageD7Pct: 0,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||||
|
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||||
|
);
|
||||||
|
vi.mocked(childProcess.spawn).mockImplementation(
|
||||||
|
(() =>
|
||||||
|
createFakeCodexProcess(
|
||||||
|
0,
|
||||||
|
) as never) as unknown as typeof childProcess.spawn,
|
||||||
|
);
|
||||||
|
return { childProcess, runCodexWarmupCycle };
|
||||||
|
}
|
||||||
|
|
||||||
|
it('fires at the next slot even when it is just under the 5h min-interval', async () => {
|
||||||
|
const { childProcess, runCodexWarmupCycle } = await setup();
|
||||||
|
const r1 = await runCodexWarmupCycle(config, {
|
||||||
|
nowMs: t13,
|
||||||
|
statePath,
|
||||||
|
ignoreZeroUsageWindow: true,
|
||||||
|
forceAttempt: true,
|
||||||
|
});
|
||||||
|
expect(r1).toEqual({ status: 'warmed', accountIndex: 0 });
|
||||||
|
const r2 = await runCodexWarmupCycle(config, {
|
||||||
|
nowMs: t18,
|
||||||
|
statePath,
|
||||||
|
ignoreZeroUsageWindow: true,
|
||||||
|
forceAttempt: true,
|
||||||
|
});
|
||||||
|
expect(r2).toEqual({ status: 'warmed', accountIndex: 0 });
|
||||||
|
expect(childProcess.spawn).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('without forceAttempt, the sub-5h second slot is skipped (regression guard)', async () => {
|
||||||
|
const { runCodexWarmupCycle } = await setup();
|
||||||
|
await runCodexWarmupCycle(config, {
|
||||||
|
nowMs: t13,
|
||||||
|
statePath,
|
||||||
|
ignoreZeroUsageWindow: true,
|
||||||
|
});
|
||||||
|
const r2 = await runCodexWarmupCycle(config, {
|
||||||
|
nowMs: t18,
|
||||||
|
statePath,
|
||||||
|
ignoreZeroUsageWindow: true,
|
||||||
|
});
|
||||||
|
expect(r2).toEqual({ status: 'skipped', reason: 'no_eligible_accounts' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('forceAttempt bypasses post-failure cooldown and stagger', async () => {
|
||||||
|
const { runCodexWarmupCycle } = await setup();
|
||||||
|
fs.writeFileSync(
|
||||||
|
statePath,
|
||||||
|
JSON.stringify({
|
||||||
|
disabledUntil: new Date(t18 + 3_600_000).toISOString(),
|
||||||
|
lastWarmupAt: new Date(t18 - 1_000).toISOString(),
|
||||||
|
accounts: {},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const result = await runCodexWarmupCycle(config, {
|
||||||
|
nowMs: t18,
|
||||||
|
statePath,
|
||||||
|
ignoreZeroUsageWindow: true,
|
||||||
|
forceAttempt: true,
|
||||||
|
});
|
||||||
|
expect(result).toEqual({ status: 'warmed', accountIndex: 0 });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import { ChildProcess, spawn } from 'child_process';
|
import { ChildProcess, spawn } from 'child_process';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
import { createRequire } from 'module';
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
import { DATA_DIR } from './config.js';
|
import { DATA_DIR } from './config.js';
|
||||||
import type { AppConfig } from './config/schema.js';
|
import type { AppConfig } from './config/schema.js';
|
||||||
@@ -34,6 +36,15 @@ interface CodexWarmupRuntimeOptions {
|
|||||||
statePath?: string;
|
statePath?: string;
|
||||||
shouldSkip?: () => boolean;
|
shouldSkip?: () => boolean;
|
||||||
ignoreZeroUsageWindow?: boolean;
|
ignoreZeroUsageWindow?: boolean;
|
||||||
|
/**
|
||||||
|
* Fixed-slot primer mode: bypass the opportunistic warm-up skip gates
|
||||||
|
* (stagger, post-failure cooldown, min-interval, per-account usage/rate-limit
|
||||||
|
* filters) so a real Codex command is attempted at every 08/13/18/23 slot and
|
||||||
|
* its success/failure recorded. Without this, two slots exactly 5h apart fall
|
||||||
|
* just under minIntervalMs (the primer fires a few hundred ms after the slot)
|
||||||
|
* and the later slot is silently skipped as `no_eligible_accounts`.
|
||||||
|
*/
|
||||||
|
forceAttempt?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type CodexWarmupCycleResult =
|
export type CodexWarmupCycleResult =
|
||||||
@@ -87,9 +98,63 @@ function getPreferredCodexPathEntries(): string[] {
|
|||||||
return [...new Set(entries)];
|
return [...new Set(entries)];
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveCodexBinary(): string {
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
|
interface CodexLauncher {
|
||||||
|
command: string;
|
||||||
|
prefixArgs: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve how to spawn the Codex CLI for warm-up.
|
||||||
|
*
|
||||||
|
* The bot runs under bun/systemd where a global `codex` binary usually is NOT
|
||||||
|
* on PATH, so the old bare-`codex` fallback failed with ENOENT and the primer
|
||||||
|
* never fired. Prefer the bundled `@openai/codex` JS launcher (the same path
|
||||||
|
* the codex-runner spawns) and run it through the current JS runtime, falling
|
||||||
|
* back to a globally installed binary only when the package is unavailable.
|
||||||
|
*/
|
||||||
|
function resolveCodexLauncher(): CodexLauncher {
|
||||||
|
const launcherCandidates: string[] = [];
|
||||||
|
try {
|
||||||
|
launcherCandidates.push(
|
||||||
|
path.join(
|
||||||
|
path.dirname(require.resolve('@openai/codex/package.json')),
|
||||||
|
'bin',
|
||||||
|
'codex.js',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
/* @openai/codex not resolvable from this module — try explicit paths */
|
||||||
|
}
|
||||||
|
// The codex-runner workspace always vendors @openai/codex.
|
||||||
|
const repoRoot = path.resolve(
|
||||||
|
path.dirname(fileURLToPath(import.meta.url)),
|
||||||
|
'..',
|
||||||
|
);
|
||||||
|
launcherCandidates.push(
|
||||||
|
path.join(
|
||||||
|
repoRoot,
|
||||||
|
'runners',
|
||||||
|
'codex-runner',
|
||||||
|
'node_modules',
|
||||||
|
'@openai',
|
||||||
|
'codex',
|
||||||
|
'bin',
|
||||||
|
'codex.js',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
for (const launcher of launcherCandidates) {
|
||||||
|
if (fs.existsSync(launcher)) {
|
||||||
|
return { command: process.execPath, prefixArgs: [launcher] };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Fall back to a globally installed codex binary, then bare PATH lookup.
|
||||||
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
||||||
return fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
if (fs.existsSync(npmGlobalBin)) {
|
||||||
|
return { command: npmGlobalBin, prefixArgs: [] };
|
||||||
|
}
|
||||||
|
return { command: 'codex', prefixArgs: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
function ensureAccountState(
|
function ensureAccountState(
|
||||||
@@ -106,15 +171,20 @@ function selectWarmupCandidate(
|
|||||||
config: CodexWarmupConfig,
|
config: CodexWarmupConfig,
|
||||||
state: CodexWarmupState,
|
state: CodexWarmupState,
|
||||||
nowMs: number,
|
nowMs: number,
|
||||||
options: { ignoreZeroUsageWindow?: boolean } = {},
|
options: { ignoreZeroUsageWindow?: boolean; forceAttempt?: boolean } = {},
|
||||||
): { accountIndex: number; zeroUsageWarmupUntil: string } | { reason: string } {
|
): { accountIndex: number; zeroUsageWarmupUntil: string } | { reason: string } {
|
||||||
const disabledUntilMs = parseTimestamp(state.disabledUntil);
|
const disabledUntilMs = parseTimestamp(state.disabledUntil);
|
||||||
if (disabledUntilMs != null && disabledUntilMs > nowMs) {
|
if (
|
||||||
|
!options.forceAttempt &&
|
||||||
|
disabledUntilMs != null &&
|
||||||
|
disabledUntilMs > nowMs
|
||||||
|
) {
|
||||||
return { reason: 'disabled_cooldown' };
|
return { reason: 'disabled_cooldown' };
|
||||||
}
|
}
|
||||||
|
|
||||||
const lastGlobalWarmupMs = parseTimestamp(state.lastWarmupAt);
|
const lastGlobalWarmupMs = parseTimestamp(state.lastWarmupAt);
|
||||||
if (
|
if (
|
||||||
|
!options.forceAttempt &&
|
||||||
lastGlobalWarmupMs != null &&
|
lastGlobalWarmupMs != null &&
|
||||||
config.staggerMs > 0 &&
|
config.staggerMs > 0 &&
|
||||||
nowMs - lastGlobalWarmupMs < config.staggerMs
|
nowMs - lastGlobalWarmupMs < config.staggerMs
|
||||||
@@ -125,6 +195,16 @@ function selectWarmupCandidate(
|
|||||||
const accounts = getAllCodexAccounts();
|
const accounts = getAllCodexAccounts();
|
||||||
if (accounts.length === 0) return { reason: 'no_accounts' };
|
if (accounts.length === 0) return { reason: 'no_accounts' };
|
||||||
|
|
||||||
|
// Fixed-slot primer: attempt a real command on the active account every slot,
|
||||||
|
// bypassing the per-account usage/rate-limit/min-interval filters below.
|
||||||
|
if (options.forceAttempt) {
|
||||||
|
const active = accounts.find((a) => a.isActive) ?? accounts[0];
|
||||||
|
return {
|
||||||
|
accountIndex: active.index,
|
||||||
|
zeroUsageWarmupUntil: new Date(nowMs).toISOString(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
for (const account of accounts) {
|
for (const account of accounts) {
|
||||||
if (account.isRateLimited) continue;
|
if (account.isRateLimited) continue;
|
||||||
if (typeof account.cachedUsagePct !== 'number') continue;
|
if (typeof account.cachedUsagePct !== 'number') continue;
|
||||||
@@ -219,7 +299,8 @@ function runCodexWarmupCommand(
|
|||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
proc = spawn(resolveCodexBinary(), args, {
|
const launcher = resolveCodexLauncher();
|
||||||
|
proc = spawn(launcher.command, [...launcher.prefixArgs, ...args], {
|
||||||
stdio: ['ignore', 'pipe', 'pipe'],
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
env: spawnEnv,
|
env: spawnEnv,
|
||||||
});
|
});
|
||||||
@@ -265,6 +346,7 @@ export async function runCodexWarmupCycle(
|
|||||||
const state = readWarmupState(statePath);
|
const state = readWarmupState(statePath);
|
||||||
const selected = selectWarmupCandidate(config, state, nowMs, {
|
const selected = selectWarmupCandidate(config, state, nowMs, {
|
||||||
ignoreZeroUsageWindow: runtime.ignoreZeroUsageWindow,
|
ignoreZeroUsageWindow: runtime.ignoreZeroUsageWindow,
|
||||||
|
forceAttempt: runtime.forceAttempt,
|
||||||
});
|
});
|
||||||
if ('reason' in selected) {
|
if ('reason' in selected) {
|
||||||
return { status: 'skipped', reason: selected.reason };
|
return { status: 'skipped', reason: selected.reason };
|
||||||
|
|||||||
@@ -95,6 +95,14 @@ export const AGENT_LANGUAGE = CONFIG.paired.agentLanguage;
|
|||||||
export const ARBITER_DEADLOCK_THRESHOLD =
|
export const ARBITER_DEADLOCK_THRESHOLD =
|
||||||
CONFIG.paired.arbiterDeadlockThreshold;
|
CONFIG.paired.arbiterDeadlockThreshold;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum number of times the arbiter may intervene on a single task before the
|
||||||
|
* task is escalated to the user instead of re-invoking the arbiter. This bounds
|
||||||
|
* the owner↔reviewer↔arbiter loop: once the arbiter has ruled this many times
|
||||||
|
* and the loop still deadlocks, a human is asked to step in.
|
||||||
|
*/
|
||||||
|
export const ARBITER_MAX_INTERVENTIONS = CONFIG.paired.arbiterMaxInterventions;
|
||||||
|
|
||||||
export function isArbiterEnabled(): boolean {
|
export function isArbiterEnabled(): boolean {
|
||||||
return ARBITER_AGENT_TYPE !== undefined;
|
return ARBITER_AGENT_TYPE !== undefined;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -244,6 +244,7 @@ export function loadConfig(): AppConfig {
|
|||||||
),
|
),
|
||||||
agentLanguage: readText('AGENT_LANGUAGE') ?? '',
|
agentLanguage: readText('AGENT_LANGUAGE') ?? '',
|
||||||
arbiterDeadlockThreshold: readInteger('ARBITER_DEADLOCK_THRESHOLD', 2),
|
arbiterDeadlockThreshold: readInteger('ARBITER_DEADLOCK_THRESHOLD', 2),
|
||||||
|
arbiterMaxInterventions: readInteger('ARBITER_MAX_INTERVENTIONS', 1),
|
||||||
maxRoundTrips,
|
maxRoundTrips,
|
||||||
},
|
},
|
||||||
models: {
|
models: {
|
||||||
@@ -259,7 +260,7 @@ export function loadConfig(): AppConfig {
|
|||||||
status: {
|
status: {
|
||||||
channelId: readText('STATUS_CHANNEL_ID') ?? '',
|
channelId: readText('STATUS_CHANNEL_ID') ?? '',
|
||||||
updateInterval: 10000,
|
updateInterval: 10000,
|
||||||
usageUpdateInterval: 300000,
|
usageUpdateInterval: 60000,
|
||||||
showRooms: readBooleanUnlessFalse('STATUS_SHOW_ROOMS', true),
|
showRooms: readBooleanUnlessFalse('STATUS_SHOW_ROOMS', true),
|
||||||
showRoomDetails: readBooleanUnlessFalse('STATUS_SHOW_ROOM_DETAILS', true),
|
showRoomDetails: readBooleanUnlessFalse('STATUS_SHOW_ROOM_DETAILS', true),
|
||||||
usageDashboardEnabled: readText('USAGE_DASHBOARD') === 'true',
|
usageDashboardEnabled: readText('USAGE_DASHBOARD') === 'true',
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ export interface AppConfig {
|
|||||||
forceFreshClaudeReviewerSessionInUnsafeHostMode: boolean;
|
forceFreshClaudeReviewerSessionInUnsafeHostMode: boolean;
|
||||||
agentLanguage: string;
|
agentLanguage: string;
|
||||||
arbiterDeadlockThreshold: number;
|
arbiterDeadlockThreshold: number;
|
||||||
|
arbiterMaxInterventions: number;
|
||||||
maxRoundTrips: number;
|
maxRoundTrips: number;
|
||||||
};
|
};
|
||||||
models: {
|
models: {
|
||||||
|
|||||||
@@ -133,6 +133,44 @@ describe('dashboard Claude usage rows', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('marks a rate-limited account with a 429 error and does not fall back to a cached value', () => {
|
||||||
|
const cachedAccounts: ClaudeAccountUsage[] = [
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
masked: 'tok-a',
|
||||||
|
isActive: true,
|
||||||
|
isRateLimited: false,
|
||||||
|
usage: {
|
||||||
|
five_hour: {
|
||||||
|
utilization: 0.4,
|
||||||
|
resets_at: '2026-03-24T04:00:00+09:00',
|
||||||
|
},
|
||||||
|
seven_day: {
|
||||||
|
utilization: 0.7,
|
||||||
|
resets_at: '2026-03-29T04:00:00+09:00',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const liveAccounts: ClaudeAccountUsage[] = [
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
masked: 'tok-a',
|
||||||
|
isActive: true,
|
||||||
|
isRateLimited: false,
|
||||||
|
usage: null,
|
||||||
|
usageRateLimited: true,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const merged = mergeClaudeDashboardAccounts(liveAccounts, cachedAccounts);
|
||||||
|
// The 429 must win over the cached value — no stale fallback.
|
||||||
|
expect(merged[0].usage).toBeNull();
|
||||||
|
|
||||||
|
const rows = buildClaudeUsageRows(merged);
|
||||||
|
expect(rows[0]).toMatchObject({ h5pct: -1, d7pct: -1, error: '429' });
|
||||||
|
});
|
||||||
|
|
||||||
it('preserves the last successful usage per account instead of collapsing to one cache entry', () => {
|
it('preserves the last successful usage per account instead of collapsing to one cache entry', () => {
|
||||||
const cachedAccounts: ClaudeAccountUsage[] = [
|
const cachedAccounts: ClaudeAccountUsage[] = [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ export type UsageRow = {
|
|||||||
h5reset: string;
|
h5reset: string;
|
||||||
d7pct: number;
|
d7pct: number;
|
||||||
d7reset: string;
|
d7reset: string;
|
||||||
|
/** When set, the renderer shows this text (e.g. "429") in place of the bars. */
|
||||||
|
error?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function formatResetRemaining(value: string | number): string {
|
export function formatResetRemaining(value: string | number): string {
|
||||||
@@ -42,7 +44,11 @@ export function mergeClaudeDashboardAccounts(
|
|||||||
|
|
||||||
return liveAccounts.map((account) => ({
|
return liveAccounts.map((account) => ({
|
||||||
...account,
|
...account,
|
||||||
usage: account.usage || cachedByIndex.get(account.index)?.usage || null,
|
// On a 429 we deliberately do NOT fall back to the last cached value —
|
||||||
|
// the dashboard surfaces a "429" indicator instead.
|
||||||
|
usage: account.usageRateLimited
|
||||||
|
? null
|
||||||
|
: account.usage || cachedByIndex.get(account.index)?.usage || null,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,6 +81,7 @@ export function buildClaudeUsageRows(
|
|||||||
: Math.round(d7.utilization * 100)
|
: Math.round(d7.utilization * 100)
|
||||||
: -1,
|
: -1,
|
||||||
d7reset: d7 ? formatResetRemaining(d7.resets_at) : '',
|
d7reset: d7 ? formatResetRemaining(d7.resets_at) : '',
|
||||||
|
error: account.usageRateLimited ? '429' : undefined,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ describe('listUnexpectedDataStateFiles', () => {
|
|||||||
fs.writeFileSync(path.join(dataDir, 'codex-warmup-state.json'), '{}');
|
fs.writeFileSync(path.join(dataDir, 'codex-warmup-state.json'), '{}');
|
||||||
fs.writeFileSync(path.join(dataDir, 'claude-usage-cache.json'), '{}');
|
fs.writeFileSync(path.join(dataDir, 'claude-usage-cache.json'), '{}');
|
||||||
fs.writeFileSync(path.join(dataDir, 'restart-context.abc.json'), '{}');
|
fs.writeFileSync(path.join(dataDir, 'restart-context.abc.json'), '{}');
|
||||||
|
fs.writeFileSync(path.join(dataDir, 'auto-paired-mode-state.json'), '{}');
|
||||||
|
|
||||||
expect(listUnexpectedDataStateFiles(dataDir)).toEqual([]);
|
expect(listUnexpectedDataStateFiles(dataDir)).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ const ALLOWED_DATA_JSON_PATTERNS = [
|
|||||||
/^codex-warmup-state\.json$/,
|
/^codex-warmup-state\.json$/,
|
||||||
/^claude-usage-cache\.json$/,
|
/^claude-usage-cache\.json$/,
|
||||||
/^restart-context\..+\.json$/,
|
/^restart-context\..+\.json$/,
|
||||||
|
/^auto-paired-mode-state\.json$/,
|
||||||
];
|
];
|
||||||
|
|
||||||
function isAllowedDataStateJson(filename: string): boolean {
|
function isAllowedDataStateJson(filename: string): boolean {
|
||||||
|
|||||||
@@ -143,6 +143,43 @@ describe('paired turn attempt restart recovery', () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('clears stale running attempts for already completed tasks without replaying them', () => {
|
||||||
|
const task = makeTask({
|
||||||
|
id: 'completed-stale-running-task',
|
||||||
|
status: 'completed',
|
||||||
|
completion_reason: 'done',
|
||||||
|
});
|
||||||
|
createPairedTask(task);
|
||||||
|
const turnIdentity = buildPairedTurnIdentity({
|
||||||
|
taskId: task.id,
|
||||||
|
taskUpdatedAt: task.updated_at,
|
||||||
|
intentKind: 'owner-turn',
|
||||||
|
role: 'owner',
|
||||||
|
});
|
||||||
|
|
||||||
|
markPairedTurnRunning({
|
||||||
|
turnIdentity,
|
||||||
|
executorServiceId: CODEX_MAIN_SERVICE_ID,
|
||||||
|
executorAgentType: 'codex',
|
||||||
|
runId: 'completed-run-before-restart',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(
|
||||||
|
recoverInterruptedPairedTurnAttemptsForService({
|
||||||
|
serviceIds: [CODEX_MAIN_SERVICE_ID],
|
||||||
|
now: '2026-05-27T00:11:00.000Z',
|
||||||
|
}),
|
||||||
|
).toEqual([]);
|
||||||
|
expect(getPairedTurnAttempts(turnIdentity.turnId)).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
state: 'failed',
|
||||||
|
active_run_id: null,
|
||||||
|
completed_at: '2026-05-27T00:11:00.000Z',
|
||||||
|
last_error: 'Interrupted by service restart before completion.',
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it('recovers codex executor attempts even when the orchestration lease used the claude service id', () => {
|
it('recovers codex executor attempts even when the orchestration lease used the claude service id', () => {
|
||||||
const task = makeTask({ id: 'cross-service-lease-task' });
|
const task = makeTask({ id: 'cross-service-lease-task' });
|
||||||
createPairedTask(task);
|
createPairedTask(task);
|
||||||
|
|||||||
@@ -2724,7 +2724,10 @@ describe('room assignment writes', () => {
|
|||||||
modeSource: 'inferred',
|
modeSource: 'inferred',
|
||||||
name: 'Legacy SQL Room',
|
name: 'Legacy SQL Room',
|
||||||
folder: 'legacy-sql-room',
|
folder: 'legacy-sql-room',
|
||||||
ownerAgentType: 'codex',
|
// Owner inference picks OWNER_AGENT_TYPE (env-driven, default codex but
|
||||||
|
// currently claude-code in this deployment) when both agent types are
|
||||||
|
// registered. See inferOwnerAgentTypeFromRegisteredAgentTypes.
|
||||||
|
ownerAgentType: OWNER_AGENT_TYPE,
|
||||||
});
|
});
|
||||||
expect(getEffectiveRoomMode('dc:legacy-sql')).toBe('tribunal');
|
expect(getEffectiveRoomMode('dc:legacy-sql')).toBe('tribunal');
|
||||||
expect(getEffectiveRuntimeRoomMode('dc:legacy-sql')).toBe('tribunal');
|
expect(getEffectiveRuntimeRoomMode('dc:legacy-sql')).toBe('tribunal');
|
||||||
@@ -3340,7 +3343,8 @@ describe('paired room registration', () => {
|
|||||||
name: 'Room Settings Test',
|
name: 'Room Settings Test',
|
||||||
folder: 'room-settings-test',
|
folder: 'room-settings-test',
|
||||||
trigger: '@Codex',
|
trigger: '@Codex',
|
||||||
ownerAgentType: 'codex',
|
// Both agent types registered → owner inference picks OWNER_AGENT_TYPE.
|
||||||
|
ownerAgentType: OWNER_AGENT_TYPE,
|
||||||
});
|
});
|
||||||
|
|
||||||
setExplicitRoomMode('dc:room-settings', 'single');
|
setExplicitRoomMode('dc:room-settings', 'single');
|
||||||
@@ -3352,7 +3356,7 @@ describe('paired room registration', () => {
|
|||||||
name: 'Room Settings Test',
|
name: 'Room Settings Test',
|
||||||
folder: 'room-settings-test',
|
folder: 'room-settings-test',
|
||||||
trigger: '@Codex',
|
trigger: '@Codex',
|
||||||
ownerAgentType: 'codex',
|
ownerAgentType: OWNER_AGENT_TYPE,
|
||||||
});
|
});
|
||||||
|
|
||||||
updateRegisteredGroupName('dc:room-settings', 'Room Settings Renamed');
|
updateRegisteredGroupName('dc:room-settings', 'Room Settings Renamed');
|
||||||
@@ -3364,7 +3368,7 @@ describe('paired room registration', () => {
|
|||||||
name: 'Room Settings Renamed',
|
name: 'Room Settings Renamed',
|
||||||
folder: 'room-settings-test',
|
folder: 'room-settings-test',
|
||||||
trigger: '@Codex',
|
trigger: '@Codex',
|
||||||
ownerAgentType: 'codex',
|
ownerAgentType: OWNER_AGENT_TYPE,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -147,6 +147,7 @@ export function applyBaseSchema(database: Database): void {
|
|||||||
finalize_step_done_count INTEGER NOT NULL DEFAULT 0,
|
finalize_step_done_count INTEGER NOT NULL DEFAULT 0,
|
||||||
task_done_then_user_reopen_count INTEGER NOT NULL DEFAULT 0,
|
task_done_then_user_reopen_count INTEGER NOT NULL DEFAULT 0,
|
||||||
empty_step_done_streak INTEGER NOT NULL DEFAULT 0,
|
empty_step_done_streak INTEGER NOT NULL DEFAULT 0,
|
||||||
|
arbiter_intervention_count INTEGER NOT NULL DEFAULT 0,
|
||||||
status TEXT NOT NULL DEFAULT 'active',
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
arbiter_verdict TEXT,
|
arbiter_verdict TEXT,
|
||||||
arbiter_requested_at TEXT,
|
arbiter_requested_at TEXT,
|
||||||
|
|||||||
@@ -6,8 +6,17 @@ import { afterEach, describe, expect, it } from 'vitest';
|
|||||||
|
|
||||||
import { applyBaseSchema } from './base-schema.js';
|
import { applyBaseSchema } from './base-schema.js';
|
||||||
import { initializeDatabaseSchema } from './bootstrap.js';
|
import { initializeDatabaseSchema } from './bootstrap.js';
|
||||||
|
import { applyVersionedSchemaMigrations } from './migrations/index.js';
|
||||||
import { applyLegacySchemaMigrations } from './schema.js';
|
import { applyLegacySchemaMigrations } from './schema.js';
|
||||||
|
|
||||||
|
function tableColumns(database: Database, table: string): string[] {
|
||||||
|
return (
|
||||||
|
database.prepare(`PRAGMA table_info(${table})`).all() as Array<{
|
||||||
|
name: string;
|
||||||
|
}>
|
||||||
|
).map((column) => column.name);
|
||||||
|
}
|
||||||
|
|
||||||
function getAppliedSchemaMigrations(
|
function getAppliedSchemaMigrations(
|
||||||
database: Database,
|
database: Database,
|
||||||
): Array<{ version: number; name: string }> {
|
): Array<{ version: number; name: string }> {
|
||||||
@@ -43,7 +52,10 @@ function getExpectedSchemaMigrations(): Array<{
|
|||||||
{ version: 16, name: 'room_skill_overrides' },
|
{ version: 16, name: 'room_skill_overrides' },
|
||||||
{ version: 17, name: 'scheduled_task_room_role' },
|
{ version: 17, name: 'scheduled_task_room_role' },
|
||||||
{ version: 18, name: 'paired_turn_output_attachments' },
|
{ version: 18, name: 'paired_turn_output_attachments' },
|
||||||
{ version: 19, name: 'reviewer_failure_count' },
|
{ version: 19, name: 'turn_progress_text_recovery' },
|
||||||
|
{ version: 20, name: 'arbiter_intervention_count' },
|
||||||
|
{ version: 21, name: 'reviewer_failure_count' },
|
||||||
|
{ version: 22, name: 'turn_progress_text_compat' },
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -114,4 +126,57 @@ describe('initializeDatabaseSchema', () => {
|
|||||||
reopened.close();
|
reopened.close();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('backfills turn_progress_text columns when version 15 was recorded under a different name', () => {
|
||||||
|
const database = new Database(':memory:');
|
||||||
|
|
||||||
|
try {
|
||||||
|
applyBaseSchema(database);
|
||||||
|
|
||||||
|
// Reproduce a deployment that first shipped reviewer_failure_count as a
|
||||||
|
// local migration numbered 15, before turn_progress_text claimed that
|
||||||
|
// version upstream. The runner skips by version number, so migration 015
|
||||||
|
// (turn_progress_text) would otherwise never run on this database.
|
||||||
|
database.exec(`
|
||||||
|
CREATE TABLE schema_migrations (
|
||||||
|
version INTEGER PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
)
|
||||||
|
`);
|
||||||
|
const collidedNames: Record<number, string> = {
|
||||||
|
15: 'reviewer_failure_count',
|
||||||
|
};
|
||||||
|
for (let version = 1; version <= 15; version += 1) {
|
||||||
|
database
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO schema_migrations (version, name) VALUES (?, ?)',
|
||||||
|
)
|
||||||
|
.run(version, collidedNames[version] ?? `legacy_${version}`);
|
||||||
|
}
|
||||||
|
// Precondition: the collided database is missing the progress columns.
|
||||||
|
expect(tableColumns(database, 'paired_turns')).not.toContain(
|
||||||
|
'progress_text',
|
||||||
|
);
|
||||||
|
|
||||||
|
applyVersionedSchemaMigrations(database, { assistantName: 'Andy' });
|
||||||
|
|
||||||
|
const columns = tableColumns(database, 'paired_turns');
|
||||||
|
expect(columns).toContain('progress_text');
|
||||||
|
expect(columns).toContain('progress_updated_at');
|
||||||
|
|
||||||
|
// The pre-existing version-15 row is left untouched; the compat migration
|
||||||
|
// (version 22 after merge-renumbering) is what reconciles the schema.
|
||||||
|
const version15 = database
|
||||||
|
.prepare('SELECT name FROM schema_migrations WHERE version = 15')
|
||||||
|
.get() as { name: string };
|
||||||
|
expect(version15.name).toBe('reviewer_failure_count');
|
||||||
|
const version22 = database
|
||||||
|
.prepare('SELECT name FROM schema_migrations WHERE version = 22')
|
||||||
|
.get() as { name: string } | null;
|
||||||
|
expect(version22?.name).toBe('turn_progress_text_compat');
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
36
src/db/migrations/019_turn-progress-text-recovery.ts
Normal file
36
src/db/migrations/019_turn-progress-text-recovery.ts
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
import type { Database } from 'bun:sqlite';
|
||||||
|
|
||||||
|
import { tableHasColumn } from './helpers.js';
|
||||||
|
import type { SchemaMigrationDefinition } from './types.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recovery for a migration-version collision introduced by merging
|
||||||
|
* origin/main into the long-lived deployment branch.
|
||||||
|
*
|
||||||
|
* The deployment DB recorded schema version 15 as `reviewer_failure_count`
|
||||||
|
* (a downstream-only migration), while upstream defines version 15 as
|
||||||
|
* `turn_progress_text` (migration 015). Because the migration runner tracks
|
||||||
|
* applied migrations by version number alone, the upstream `turn_progress_text`
|
||||||
|
* migration is skipped on the live DB and `paired_turns.progress_text` /
|
||||||
|
* `progress_updated_at` are never created — even though merged runtime code
|
||||||
|
* (src/db/paired-turns.ts, src/web-dashboard-data.ts) reads and writes them.
|
||||||
|
*
|
||||||
|
* This migration re-applies those columns idempotently under a fresh version
|
||||||
|
* number so the live deployment gets them. It is a no-op on databases where
|
||||||
|
* migration 015 already ran (fresh installs), thanks to the column guards.
|
||||||
|
*/
|
||||||
|
export const TURN_PROGRESS_TEXT_RECOVERY_MIGRATION: SchemaMigrationDefinition =
|
||||||
|
{
|
||||||
|
version: 19,
|
||||||
|
name: 'turn_progress_text_recovery',
|
||||||
|
apply(database: Database) {
|
||||||
|
if (!tableHasColumn(database, 'paired_turns', 'progress_text')) {
|
||||||
|
database.exec(`ALTER TABLE paired_turns ADD COLUMN progress_text TEXT`);
|
||||||
|
}
|
||||||
|
if (!tableHasColumn(database, 'paired_turns', 'progress_updated_at')) {
|
||||||
|
database.exec(
|
||||||
|
`ALTER TABLE paired_turns ADD COLUMN progress_updated_at TEXT`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
25
src/db/migrations/020_arbiter-intervention-count.ts
Normal file
25
src/db/migrations/020_arbiter-intervention-count.ts
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import type { Database } from 'bun:sqlite';
|
||||||
|
|
||||||
|
import { tableHasColumn } from './helpers.js';
|
||||||
|
import type { SchemaMigrationDefinition } from './types.js';
|
||||||
|
|
||||||
|
export const ARBITER_INTERVENTION_COUNT_MIGRATION: SchemaMigrationDefinition = {
|
||||||
|
version: 20,
|
||||||
|
name: 'arbiter_intervention_count',
|
||||||
|
apply(database: Database) {
|
||||||
|
if (
|
||||||
|
!tableHasColumn(database, 'paired_tasks', 'arbiter_intervention_count')
|
||||||
|
) {
|
||||||
|
database.exec(`
|
||||||
|
ALTER TABLE paired_tasks
|
||||||
|
ADD COLUMN arbiter_intervention_count INTEGER NOT NULL DEFAULT 0
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
database.exec(`
|
||||||
|
UPDATE paired_tasks
|
||||||
|
SET arbiter_intervention_count = 0
|
||||||
|
WHERE arbiter_intervention_count IS NULL
|
||||||
|
`);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -4,7 +4,7 @@ import { tableHasColumn } from './helpers.js';
|
|||||||
import type { SchemaMigrationDefinition } from './types.js';
|
import type { SchemaMigrationDefinition } from './types.js';
|
||||||
|
|
||||||
export const REVIEWER_FAILURE_COUNT_MIGRATION: SchemaMigrationDefinition = {
|
export const REVIEWER_FAILURE_COUNT_MIGRATION: SchemaMigrationDefinition = {
|
||||||
version: 19,
|
version: 21,
|
||||||
name: 'reviewer_failure_count',
|
name: 'reviewer_failure_count',
|
||||||
apply(database: Database) {
|
apply(database: Database) {
|
||||||
if (!tableHasColumn(database, 'paired_tasks', 'reviewer_failure_count')) {
|
if (!tableHasColumn(database, 'paired_tasks', 'reviewer_failure_count')) {
|
||||||
32
src/db/migrations/022_turn-progress-text-compat.ts
Normal file
32
src/db/migrations/022_turn-progress-text-compat.ts
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
import type { Database } from 'bun:sqlite';
|
||||||
|
|
||||||
|
import { tableHasColumn } from './helpers.js';
|
||||||
|
import type { SchemaMigrationDefinition } from './types.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compatibility backfill for `turn_progress_text` (migration 015).
|
||||||
|
*
|
||||||
|
* Some deployments first shipped `reviewer_failure_count` as a local migration
|
||||||
|
* numbered 15, so their `schema_migrations` table records version 15 under a
|
||||||
|
* different name than the canonical `turn_progress_text` migration. The runner
|
||||||
|
* skips migrations purely by version number, so on those databases the real
|
||||||
|
* version-15 migration never runs and `paired_turns.progress_text` /
|
||||||
|
* `progress_updated_at` are missing — yet the runtime reads them. This migration
|
||||||
|
* re-adds the columns idempotently so collided databases converge with fresh
|
||||||
|
* ones. On a fresh database migration 015 already created the columns, so the
|
||||||
|
* `tableHasColumn` guards make this a no-op.
|
||||||
|
*/
|
||||||
|
export const TURN_PROGRESS_TEXT_COMPAT_MIGRATION: SchemaMigrationDefinition = {
|
||||||
|
version: 22,
|
||||||
|
name: 'turn_progress_text_compat',
|
||||||
|
apply(database: Database) {
|
||||||
|
if (!tableHasColumn(database, 'paired_turns', 'progress_text')) {
|
||||||
|
database.exec(`ALTER TABLE paired_turns ADD COLUMN progress_text TEXT`);
|
||||||
|
}
|
||||||
|
if (!tableHasColumn(database, 'paired_turns', 'progress_updated_at')) {
|
||||||
|
database.exec(
|
||||||
|
`ALTER TABLE paired_turns ADD COLUMN progress_updated_at TEXT`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -18,7 +18,10 @@ import { TURN_PROGRESS_TEXT_MIGRATION } from './015_turn-progress-text.js';
|
|||||||
import { ROOM_SKILL_OVERRIDES_MIGRATION } from './016_room-skill-overrides.js';
|
import { ROOM_SKILL_OVERRIDES_MIGRATION } from './016_room-skill-overrides.js';
|
||||||
import { SCHEDULED_TASK_ROOM_ROLE_MIGRATION } from './017_scheduled-task-room-role.js';
|
import { SCHEDULED_TASK_ROOM_ROLE_MIGRATION } from './017_scheduled-task-room-role.js';
|
||||||
import { PAIRED_TURN_OUTPUT_ATTACHMENTS_MIGRATION } from './018_paired-turn-output-attachments.js';
|
import { PAIRED_TURN_OUTPUT_ATTACHMENTS_MIGRATION } from './018_paired-turn-output-attachments.js';
|
||||||
import { REVIEWER_FAILURE_COUNT_MIGRATION } from './019_reviewer-failure-count.js';
|
import { TURN_PROGRESS_TEXT_RECOVERY_MIGRATION } from './019_turn-progress-text-recovery.js';
|
||||||
|
import { ARBITER_INTERVENTION_COUNT_MIGRATION } from './020_arbiter-intervention-count.js';
|
||||||
|
import { REVIEWER_FAILURE_COUNT_MIGRATION } from './021_reviewer-failure-count.js';
|
||||||
|
import { TURN_PROGRESS_TEXT_COMPAT_MIGRATION } from './022_turn-progress-text-compat.js';
|
||||||
import type {
|
import type {
|
||||||
SchemaMigrationArgs,
|
SchemaMigrationArgs,
|
||||||
SchemaMigrationDefinition,
|
SchemaMigrationDefinition,
|
||||||
@@ -45,7 +48,10 @@ const ORDERED_SCHEMA_MIGRATIONS: readonly SchemaMigrationDefinition[] = [
|
|||||||
ROOM_SKILL_OVERRIDES_MIGRATION,
|
ROOM_SKILL_OVERRIDES_MIGRATION,
|
||||||
SCHEDULED_TASK_ROOM_ROLE_MIGRATION,
|
SCHEDULED_TASK_ROOM_ROLE_MIGRATION,
|
||||||
PAIRED_TURN_OUTPUT_ATTACHMENTS_MIGRATION,
|
PAIRED_TURN_OUTPUT_ATTACHMENTS_MIGRATION,
|
||||||
|
TURN_PROGRESS_TEXT_RECOVERY_MIGRATION,
|
||||||
|
ARBITER_INTERVENTION_COUNT_MIGRATION,
|
||||||
REVIEWER_FAILURE_COUNT_MIGRATION,
|
REVIEWER_FAILURE_COUNT_MIGRATION,
|
||||||
|
TURN_PROGRESS_TEXT_COMPAT_MIGRATION,
|
||||||
];
|
];
|
||||||
|
|
||||||
function ensureSchemaMigrationsTable(database: Database): void {
|
function ensureSchemaMigrationsTable(database: Database): void {
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ export type PairedTaskUpdates = Partial<
|
|||||||
| 'finalize_step_done_count'
|
| 'finalize_step_done_count'
|
||||||
| 'task_done_then_user_reopen_count'
|
| 'task_done_then_user_reopen_count'
|
||||||
| 'empty_step_done_streak'
|
| 'empty_step_done_streak'
|
||||||
|
| 'arbiter_intervention_count'
|
||||||
| 'status'
|
| 'status'
|
||||||
| 'arbiter_verdict'
|
| 'arbiter_verdict'
|
||||||
| 'arbiter_requested_at'
|
| 'arbiter_requested_at'
|
||||||
@@ -182,6 +183,7 @@ export function createPairedTaskInDatabase(
|
|||||||
finalize_step_done_count,
|
finalize_step_done_count,
|
||||||
task_done_then_user_reopen_count,
|
task_done_then_user_reopen_count,
|
||||||
empty_step_done_streak,
|
empty_step_done_streak,
|
||||||
|
arbiter_intervention_count,
|
||||||
status,
|
status,
|
||||||
arbiter_verdict,
|
arbiter_verdict,
|
||||||
arbiter_requested_at,
|
arbiter_requested_at,
|
||||||
@@ -189,7 +191,7 @@ export function createPairedTaskInDatabase(
|
|||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
)
|
)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
`,
|
`,
|
||||||
)
|
)
|
||||||
.run(
|
.run(
|
||||||
@@ -212,6 +214,7 @@ export function createPairedTaskInDatabase(
|
|||||||
task.finalize_step_done_count ?? 0,
|
task.finalize_step_done_count ?? 0,
|
||||||
task.task_done_then_user_reopen_count ?? 0,
|
task.task_done_then_user_reopen_count ?? 0,
|
||||||
task.empty_step_done_streak ?? 0,
|
task.empty_step_done_streak ?? 0,
|
||||||
|
task.arbiter_intervention_count ?? 0,
|
||||||
task.status,
|
task.status,
|
||||||
task.arbiter_verdict,
|
task.arbiter_verdict,
|
||||||
task.arbiter_requested_at,
|
task.arbiter_requested_at,
|
||||||
@@ -362,6 +365,10 @@ export function updatePairedTaskInDatabase(
|
|||||||
fields.push('empty_step_done_streak = ?');
|
fields.push('empty_step_done_streak = ?');
|
||||||
values.push(updates.empty_step_done_streak);
|
values.push(updates.empty_step_done_streak);
|
||||||
}
|
}
|
||||||
|
if (updates.arbiter_intervention_count !== undefined) {
|
||||||
|
fields.push('arbiter_intervention_count = ?');
|
||||||
|
values.push(updates.arbiter_intervention_count);
|
||||||
|
}
|
||||||
if (updates.status !== undefined) {
|
if (updates.status !== undefined) {
|
||||||
fields.push('status = ?');
|
fields.push('status = ?');
|
||||||
values.push(updates.status);
|
values.push(updates.status);
|
||||||
@@ -445,6 +452,10 @@ export function updatePairedTaskIfUnchangedInDatabase(
|
|||||||
fields.push('empty_step_done_streak = ?');
|
fields.push('empty_step_done_streak = ?');
|
||||||
values.push(updates.empty_step_done_streak);
|
values.push(updates.empty_step_done_streak);
|
||||||
}
|
}
|
||||||
|
if (updates.arbiter_intervention_count !== undefined) {
|
||||||
|
fields.push('arbiter_intervention_count = ?');
|
||||||
|
values.push(updates.arbiter_intervention_count);
|
||||||
|
}
|
||||||
if (updates.status !== undefined) {
|
if (updates.status !== undefined) {
|
||||||
fields.push('status = ?');
|
fields.push('status = ?');
|
||||||
values.push(updates.status);
|
values.push(updates.status);
|
||||||
|
|||||||
@@ -462,6 +462,24 @@ export function recoverInterruptedPairedTurnAttemptsForServiceInDatabase(
|
|||||||
const error =
|
const error =
|
||||||
args.error ?? 'Interrupted by service restart before completion.';
|
args.error ?? 'Interrupted by service restart before completion.';
|
||||||
return database.transaction(() => {
|
return database.transaction(() => {
|
||||||
|
database
|
||||||
|
.prepare(
|
||||||
|
`
|
||||||
|
UPDATE paired_turn_attempts
|
||||||
|
SET state = 'failed',
|
||||||
|
active_run_id = NULL,
|
||||||
|
updated_at = ?,
|
||||||
|
completed_at = ?,
|
||||||
|
last_error = COALESCE(last_error, ?)
|
||||||
|
WHERE state = 'running'
|
||||||
|
AND executor_service_id IN (${servicePlaceholders})
|
||||||
|
AND task_id IN (
|
||||||
|
SELECT id FROM paired_tasks WHERE status = 'completed'
|
||||||
|
)
|
||||||
|
`,
|
||||||
|
)
|
||||||
|
.run(now, now, error, ...serviceIds);
|
||||||
|
|
||||||
const rows = database
|
const rows = database
|
||||||
.prepare(
|
.prepare(
|
||||||
`
|
`
|
||||||
|
|||||||
@@ -194,7 +194,7 @@ export function assignRoomInDatabase(
|
|||||||
input: AssignRoomInput,
|
input: AssignRoomInput,
|
||||||
): (RegisteredGroup & { jid: string }) | undefined {
|
): (RegisteredGroup & { jid: string }) | undefined {
|
||||||
const existing = getStoredRoomSettingsRowFromDatabase(database, chatJid);
|
const existing = getStoredRoomSettingsRowFromDatabase(database, chatJid);
|
||||||
const roomMode = input.roomMode || existing?.roomMode || 'single';
|
const roomMode = input.roomMode || existing?.roomMode || 'tribunal';
|
||||||
const ownerAgentType =
|
const ownerAgentType =
|
||||||
input.ownerAgentType || existing?.ownerAgentType || OWNER_AGENT_TYPE;
|
input.ownerAgentType || existing?.ownerAgentType || OWNER_AGENT_TYPE;
|
||||||
const folder = resolveAssignedRoomFolder(
|
const folder = resolveAssignedRoomFolder(
|
||||||
|
|||||||
@@ -211,6 +211,7 @@ function parseLastAgentSeqState(
|
|||||||
`Invalid last_agent_seq JSON for ${serviceId}: ${
|
`Invalid last_agent_seq JSON for ${serviceId}: ${
|
||||||
err instanceof Error ? err.message : String(err)
|
err instanceof Error ? err.message : String(err)
|
||||||
}`,
|
}`,
|
||||||
|
{ cause: err },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -273,6 +273,34 @@ export function getOpenWorkItemFromDatabase(
|
|||||||
return row ? hydrateWorkItemRow(row) : undefined;
|
return row ? hydrateWorkItemRow(row) : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recent delivered owner-side work items for a chat, newest last.
|
||||||
|
* Used by single-mode prompt building to surface the bot's own prior
|
||||||
|
* answers (which only live in work_items.result_payload) so a new turn
|
||||||
|
* can reference what it previously said.
|
||||||
|
*/
|
||||||
|
export function getRecentDeliveredOwnerWorkItemsForChatFromDatabase(
|
||||||
|
database: Database,
|
||||||
|
chatJid: string,
|
||||||
|
limit: number,
|
||||||
|
): WorkItem[] {
|
||||||
|
if (limit <= 0) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const rows = database
|
||||||
|
.prepare(
|
||||||
|
`SELECT *
|
||||||
|
FROM work_items
|
||||||
|
WHERE chat_jid = ?
|
||||||
|
AND status = 'delivered'
|
||||||
|
AND (delivery_role = 'owner' OR delivery_role IS NULL)
|
||||||
|
ORDER BY id DESC
|
||||||
|
LIMIT ?`,
|
||||||
|
)
|
||||||
|
.all(chatJid, limit) as StoredWorkItemRow[];
|
||||||
|
return rows.map(hydrateWorkItemRow).reverse();
|
||||||
|
}
|
||||||
|
|
||||||
export function getOpenWorkItemForChatFromDatabase(
|
export function getOpenWorkItemForChatFromDatabase(
|
||||||
database: Database,
|
database: Database,
|
||||||
chatJid: string,
|
chatJid: string,
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ import {
|
|||||||
import { createMessageRuntime } from './message-runtime.js';
|
import { createMessageRuntime } from './message-runtime.js';
|
||||||
import { nudgeSchedulerLoop, startSchedulerLoop } from './task-scheduler.js';
|
import { nudgeSchedulerLoop, startSchedulerLoop } from './task-scheduler.js';
|
||||||
import { startUnifiedDashboard } from './unified-dashboard.js';
|
import { startUnifiedDashboard } from './unified-dashboard.js';
|
||||||
import { startWebDashboardServer } from './web-dashboard-server.js';
|
|
||||||
import { startUsagePrimer } from './usage-primer.js';
|
import { startUsagePrimer } from './usage-primer.js';
|
||||||
|
import { startWebDashboardServer } from './web-dashboard-server.js';
|
||||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
||||||
|
|||||||
@@ -1131,7 +1131,7 @@ describe('assign_room success', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('assign_room auto-fills missing folder for single rooms without exposing trigger metadata', async () => {
|
it('assign_room auto-fills missing folder and defaults to tribunal without exposing trigger metadata', async () => {
|
||||||
await processTaskIpc(
|
await processTaskIpc(
|
||||||
{
|
{
|
||||||
type: 'assign_room',
|
type: 'assign_room',
|
||||||
@@ -1146,7 +1146,7 @@ describe('assign_room success', () => {
|
|||||||
const group = getRegisteredGroup('partial@g.us');
|
const group = getRegisteredGroup('partial@g.us');
|
||||||
expect(group).toBeDefined();
|
expect(group).toBeDefined();
|
||||||
expect(group!.folder).toMatch(/^grp_whatsapp_/);
|
expect(group!.folder).toMatch(/^grp_whatsapp_/);
|
||||||
expect(getStoredRoomSettings('partial@g.us')?.roomMode).toBe('single');
|
expect(getStoredRoomSettings('partial@g.us')?.roomMode).toBe('tribunal');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('assign_room preserves an existing tribunal room mode when room_mode is omitted', async () => {
|
it('assign_room preserves an existing tribunal room mode when room_mode is omitted', async () => {
|
||||||
|
|||||||
71
src/message-agent-executor-attempt-runner.test.ts
Normal file
71
src/message-agent-executor-attempt-runner.test.ts
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const mockRunAgentProcess = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock('./agent-runner.js', () => ({
|
||||||
|
runAgentProcess: mockRunAgentProcess,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
|
getCodexAccountCount: vi.fn(() => 2),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
|
||||||
|
import type { AgentOutput } from './agent-runner.js';
|
||||||
|
|
||||||
|
describe('runMessageAgentAttempt', () => {
|
||||||
|
it('stores pre-stream Codex launch errors in paired summary', async () => {
|
||||||
|
const error = new Error(
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
);
|
||||||
|
mockRunAgentProcess.mockRejectedValueOnce(error);
|
||||||
|
const updateSummary = vi.fn();
|
||||||
|
|
||||||
|
const attempt = await runMessageAgentAttempt({
|
||||||
|
provider: 'codex',
|
||||||
|
currentSessionId: undefined,
|
||||||
|
isClaudeCodeAgent: false,
|
||||||
|
canRetryClaudeCredentials: false,
|
||||||
|
shouldPersistSession: false,
|
||||||
|
effectiveGroup: {
|
||||||
|
name: 'Test',
|
||||||
|
folder: 'test',
|
||||||
|
trigger: '@test',
|
||||||
|
added_at: new Date().toISOString(),
|
||||||
|
agentType: 'codex',
|
||||||
|
},
|
||||||
|
agentInput: {
|
||||||
|
prompt: 'arbiter prompt',
|
||||||
|
groupFolder: 'test',
|
||||||
|
chatJid: 'dc:test',
|
||||||
|
runId: 'run-test',
|
||||||
|
isMain: false,
|
||||||
|
assistantName: 'Andy',
|
||||||
|
},
|
||||||
|
activeRole: 'arbiter',
|
||||||
|
effectiveServiceId: 'codex-review',
|
||||||
|
effectiveAgentType: 'codex',
|
||||||
|
sessionFolder: 'test:arbiter',
|
||||||
|
onPersistSession: vi.fn(),
|
||||||
|
registerProcess: vi.fn(),
|
||||||
|
onOutput: vi.fn(async (_output: AgentOutput) => undefined),
|
||||||
|
pairedExecutionLifecycle: {
|
||||||
|
updateSummary,
|
||||||
|
recordFinalOutputBeforeDelivery: vi.fn(() => false),
|
||||||
|
},
|
||||||
|
log: {
|
||||||
|
child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(attempt.error).toBe(error);
|
||||||
|
expect(attempt.sawOutput).toBe(false);
|
||||||
|
expect(updateSummary).toHaveBeenCalledWith({
|
||||||
|
errorText:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -14,6 +14,7 @@ import {
|
|||||||
shouldResetCodexSessionOnAgentFailure,
|
shouldResetCodexSessionOnAgentFailure,
|
||||||
shouldResetSessionOnAgentFailure,
|
shouldResetSessionOnAgentFailure,
|
||||||
} from './session-recovery.js';
|
} from './session-recovery.js';
|
||||||
|
import { getErrorMessage } from './utils.js';
|
||||||
import type {
|
import type {
|
||||||
AgentType,
|
AgentType,
|
||||||
OutboundAttachment,
|
OutboundAttachment,
|
||||||
@@ -162,6 +163,9 @@ class MessageAgentAttemptRunner {
|
|||||||
);
|
);
|
||||||
return this.buildAttempt({ output });
|
return this.buildAttempt({ output });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
this.args.pairedExecutionLifecycle.updateSummary({
|
||||||
|
errorText: getErrorMessage(error),
|
||||||
|
});
|
||||||
return this.buildAttempt({ error });
|
return this.buildAttempt({ error });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ vi.mock('./db.js', () => ({
|
|||||||
getLastHumanMessageSender: vi.fn(() => '216851709744513024'),
|
getLastHumanMessageSender: vi.fn(() => '216851709744513024'),
|
||||||
getLatestTurnNumber: vi.fn(() => 0),
|
getLatestTurnNumber: vi.fn(() => 0),
|
||||||
getPairedTaskById: vi.fn(),
|
getPairedTaskById: vi.fn(),
|
||||||
|
getPairedTurnOutputs: vi.fn(() => []),
|
||||||
insertPairedTurnOutput: vi.fn(),
|
insertPairedTurnOutput: vi.fn(),
|
||||||
refreshPairedTaskExecutionLease: vi.fn(() => true),
|
refreshPairedTaskExecutionLease: vi.fn(() => true),
|
||||||
releasePairedTaskExecutionLease: vi.fn(),
|
releasePairedTaskExecutionLease: vi.fn(),
|
||||||
@@ -26,7 +27,11 @@ vi.mock('./message-runtime-follow-up.js', () => ({
|
|||||||
import type { AgentOutput } from './agent-runner.js';
|
import type { AgentOutput } from './agent-runner.js';
|
||||||
import * as db from './db.js';
|
import * as db from './db.js';
|
||||||
import * as pairedExecutionContextModule from './paired-execution-context.js';
|
import * as pairedExecutionContextModule from './paired-execution-context.js';
|
||||||
import { createPairedExecutionLifecycle } from './message-agent-executor-paired.js';
|
import {
|
||||||
|
createPairedExecutionLifecycle,
|
||||||
|
resolveOwnerNextStepNotice,
|
||||||
|
} from './message-agent-executor-paired.js';
|
||||||
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
const log = {
|
const log = {
|
||||||
info: vi.fn(),
|
info: vi.fn(),
|
||||||
@@ -235,6 +240,96 @@ describe('createPairedExecutionLifecycle completion handling', () => {
|
|||||||
expect(outputs).toEqual([]);
|
expect(outputs).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('delivers the held owner answer and a notice when the reviewer Codex is unavailable', async () => {
|
||||||
|
const outputs: AgentOutput[] = [];
|
||||||
|
|
||||||
|
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
task_id: 'paired-task-reviewer-down',
|
||||||
|
turn_number: 1,
|
||||||
|
role: 'owner',
|
||||||
|
output_text: 'TASK_DONE\n\n원하시던 .env 위치는 여기에 있습니다.',
|
||||||
|
created_at: '2026-04-09T00:00:00.000Z',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue({
|
||||||
|
id: 'paired-task-reviewer-down',
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
group_folder: 'test-group',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
round_trip_count: 1,
|
||||||
|
review_requested_at: '2026-04-09T00:00:00.000Z',
|
||||||
|
status: 'completed',
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'reviewer_codex_unavailable',
|
||||||
|
created_at: '2026-04-09T00:00:00.000Z',
|
||||||
|
updated_at: '2026-04-09T00:00:01.000Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
const lifecycle = createPairedExecutionLifecycle({
|
||||||
|
pairedExecutionContext: {
|
||||||
|
task: {
|
||||||
|
id: 'paired-task-reviewer-down',
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
group_folder: 'test-group',
|
||||||
|
owner_service_id: 'claude',
|
||||||
|
reviewer_service_id: 'codex-main',
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
round_trip_count: 1,
|
||||||
|
review_requested_at: '2026-04-09T00:00:00.000Z',
|
||||||
|
status: 'in_review',
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
created_at: '2026-04-09T00:00:00.000Z',
|
||||||
|
updated_at: '2026-04-09T00:00:00.000Z',
|
||||||
|
},
|
||||||
|
workspace: null,
|
||||||
|
envOverrides: {},
|
||||||
|
},
|
||||||
|
pairedTurnIdentity: {
|
||||||
|
turnId:
|
||||||
|
'paired-task-reviewer-down:2026-04-09T00:00:00.000Z:reviewer-turn',
|
||||||
|
taskId: 'paired-task-reviewer-down',
|
||||||
|
taskUpdatedAt: '2026-04-09T00:00:00.000Z',
|
||||||
|
intentKind: 'reviewer-turn',
|
||||||
|
role: 'reviewer',
|
||||||
|
},
|
||||||
|
completedRole: 'reviewer',
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-reviewer-down',
|
||||||
|
enqueueMessageCheck: vi.fn(),
|
||||||
|
onOutput: async (output) => {
|
||||||
|
outputs.push(output);
|
||||||
|
},
|
||||||
|
log,
|
||||||
|
});
|
||||||
|
|
||||||
|
lifecycle.markStatus('failed');
|
||||||
|
lifecycle.updateSummary({
|
||||||
|
errorText:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||||
|
});
|
||||||
|
await lifecycle.asyncFinalize();
|
||||||
|
|
||||||
|
const texts = outputs.map((o) =>
|
||||||
|
o.output?.visibility === 'public' ? o.output.text : o.result,
|
||||||
|
);
|
||||||
|
expect(texts).toEqual([
|
||||||
|
'원하시던 .env 위치는 여기에 있습니다.',
|
||||||
|
'ℹ️ 리뷰어 일시 사용 불가 — 리뷰 없이 owner 답변으로 진행했습니다.',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it('releases an owner turn interrupted by a human message without counting an owner failure', async () => {
|
it('releases an owner turn interrupted by a human message without counting an owner failure', async () => {
|
||||||
const enqueueMessageCheck = vi.fn();
|
const enqueueMessageCheck = vi.fn();
|
||||||
vi.mocked(db.getPairedTaskById).mockReturnValue({
|
vi.mocked(db.getPairedTaskById).mockReturnValue({
|
||||||
@@ -325,3 +420,42 @@ describe('createPairedExecutionLifecycle completion handling', () => {
|
|||||||
expect(enqueueMessageCheck).not.toHaveBeenCalled();
|
expect(enqueueMessageCheck).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('resolveOwnerNextStepNotice routing indicator', () => {
|
||||||
|
const task = (over: Partial<PairedTask>): PairedTask =>
|
||||||
|
({
|
||||||
|
status: 'active',
|
||||||
|
reviewer_agent_type: 'claude-code',
|
||||||
|
completion_reason: null,
|
||||||
|
...over,
|
||||||
|
}) as PairedTask;
|
||||||
|
|
||||||
|
it('announces reviewer request while waiting', () => {
|
||||||
|
expect(resolveOwnerNextStepNotice(task({ status: 'review_ready' }))).toBe(
|
||||||
|
'🔁 리뷰어 검토 요청됨 — 리뷰어 응답을 기다리는 중입니다.',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('announces arbiter call when arbiter is requested', () => {
|
||||||
|
expect(
|
||||||
|
resolveOwnerNextStepNotice(task({ status: 'arbiter_requested' })),
|
||||||
|
).toBe('⚖️ 의견 차이로 중재자 호출됨 — 중재 판정을 기다리는 중입니다.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stays silent when the task finishes (no redundant done line)', () => {
|
||||||
|
expect(
|
||||||
|
resolveOwnerNextStepNotice(
|
||||||
|
task({ status: 'completed', completion_reason: 'done' }),
|
||||||
|
),
|
||||||
|
).toBeNull();
|
||||||
|
expect(
|
||||||
|
resolveOwnerNextStepNotice(
|
||||||
|
task({ status: 'completed', completion_reason: 'escalated' }),
|
||||||
|
),
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stays silent for in-progress owner work', () => {
|
||||||
|
expect(resolveOwnerNextStepNotice(task({ status: 'active' }))).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
getLastHumanMessageSender,
|
getLastHumanMessageSender,
|
||||||
getLatestTurnNumber,
|
getLatestTurnNumber,
|
||||||
getPairedTaskById,
|
getPairedTaskById,
|
||||||
|
getPairedTurnOutputs,
|
||||||
insertPairedTurnOutput,
|
insertPairedTurnOutput,
|
||||||
refreshPairedTaskExecutionLease,
|
refreshPairedTaskExecutionLease,
|
||||||
releasePairedTaskExecutionLease,
|
releasePairedTaskExecutionLease,
|
||||||
@@ -14,7 +15,10 @@ import {
|
|||||||
completePairedExecutionContext,
|
completePairedExecutionContext,
|
||||||
type PreparedPairedExecutionContext,
|
type PreparedPairedExecutionContext,
|
||||||
} from './paired-execution-context.js';
|
} from './paired-execution-context.js';
|
||||||
import { parseVisibleVerdict } from './paired-verdict.js';
|
import {
|
||||||
|
parseVisibleVerdict,
|
||||||
|
stripLeadingStatusLine,
|
||||||
|
} from './paired-verdict.js';
|
||||||
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
||||||
import { enqueuePairedFollowUpAfterEvent } from './message-runtime-follow-up.js';
|
import { enqueuePairedFollowUpAfterEvent } from './message-runtime-follow-up.js';
|
||||||
import type { PairedTurnIdentity } from './paired-turn-identity.js';
|
import type { PairedTurnIdentity } from './paired-turn-identity.js';
|
||||||
@@ -64,25 +68,92 @@ function completeStoredExecution(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Routing indicator shown to the user after an owner turn when the turn does
|
||||||
|
* NOT end the task — i.e. when more is still pending — so a same-looking status
|
||||||
|
* line (e.g. TASK_DONE) no longer reads as inconsistent: the user can always
|
||||||
|
* tell whether the reviewer was requested or the arbiter was called, vs. the
|
||||||
|
* task simply finishing (no line — the owner's final message is the ending).
|
||||||
|
* Purely additive — it reflects the routing the state machine already chose; it
|
||||||
|
* does not change any routing, and it never duplicates the completion message.
|
||||||
|
*/
|
||||||
|
export function resolveOwnerNextStepNotice(
|
||||||
|
task: PairedTaskRecord,
|
||||||
|
): string | null {
|
||||||
|
switch (task.status) {
|
||||||
|
case 'review_ready':
|
||||||
|
case 'in_review':
|
||||||
|
return '🔁 리뷰어 검토 요청됨 — 리뷰어 응답을 기다리는 중입니다.';
|
||||||
|
case 'arbiter_requested':
|
||||||
|
case 'in_arbitration':
|
||||||
|
return '⚖️ 의견 차이로 중재자 호출됨 — 중재 판정을 기다리는 중입니다.';
|
||||||
|
default:
|
||||||
|
// 'completed' included: the owner's final message is itself the ending,
|
||||||
|
// so no extra done/escalation line is appended here (escalation has its
|
||||||
|
// own dedicated notice above).
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getLatestOwnerAnswer(taskId: string): string | null {
|
||||||
|
const outputs = getPairedTurnOutputs(taskId);
|
||||||
|
for (let i = outputs.length - 1; i >= 0; i -= 1) {
|
||||||
|
const output = outputs[i];
|
||||||
|
if (output.role !== 'owner') continue;
|
||||||
|
const text = stripLeadingStatusLine(output.output_text ?? '').trim();
|
||||||
|
if (text.length > 0) return text;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
async function notifyPairedCompletionIfNeeded(args: {
|
async function notifyPairedCompletionIfNeeded(args: {
|
||||||
task: PairedTaskRecord | null | undefined;
|
task: PairedTaskRecord | null | undefined;
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
|
completedRole: PairedRoomRole;
|
||||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
if (args.task?.status !== 'completed' || !args.task.completion_reason) return;
|
const task = args.task;
|
||||||
const sender = getLastHumanMessageSender(args.chatJid);
|
if (!task) return;
|
||||||
const mention = sender ? `<@${sender}>` : '';
|
|
||||||
const notifications: Record<string, string> = {
|
const emit = (text: string): Promise<void> | undefined =>
|
||||||
escalated: `${mention} ⚠️ 자동 해결 불가 — 확인이 필요합니다.`,
|
args.onOutput?.({
|
||||||
};
|
status: 'success',
|
||||||
const message = notifications[args.task.completion_reason];
|
result: text,
|
||||||
if (!message) return;
|
output: { visibility: 'public', text },
|
||||||
await args.onOutput?.({
|
phase: 'final',
|
||||||
status: 'success',
|
});
|
||||||
result: message,
|
|
||||||
output: { visibility: 'public', text: message },
|
// Escalation notice fires for any role's completion that escalated to the user.
|
||||||
phase: 'final',
|
if (task.status === 'completed' && task.completion_reason === 'escalated') {
|
||||||
});
|
const sender = getLastHumanMessageSender(args.chatJid);
|
||||||
|
const mention = sender ? `<@${sender}> ` : '';
|
||||||
|
await emit(`${mention}⚠️ 자동 해결 불가 — 확인이 필요합니다.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reviewer (Codex) was unavailable, so the turn ended before the reviewer
|
||||||
|
// could approve. Instead of stopping silently, deliver the owner's already
|
||||||
|
// produced answer and tell the user the review was skipped — otherwise the
|
||||||
|
// user sees nothing at all (the owner answer is held until review completes).
|
||||||
|
if (
|
||||||
|
task.status === 'completed' &&
|
||||||
|
task.completion_reason === 'reviewer_codex_unavailable'
|
||||||
|
) {
|
||||||
|
const ownerAnswer = getLatestOwnerAnswer(task.id);
|
||||||
|
if (ownerAnswer) {
|
||||||
|
await emit(ownerAnswer);
|
||||||
|
}
|
||||||
|
await emit(
|
||||||
|
'ℹ️ 리뷰어 일시 사용 불가 — 리뷰 없이 owner 답변으로 진행했습니다.',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Owner turn: surface the next routing step so the ending is unambiguous.
|
||||||
|
if (args.completedRole !== 'owner') return;
|
||||||
|
const notice = resolveOwnerNextStepNotice(task);
|
||||||
|
if (!notice) return;
|
||||||
|
await emit(notice);
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PairedExecutionLifecycle {
|
export interface PairedExecutionLifecycle {
|
||||||
@@ -505,8 +576,11 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
|||||||
if (!missingVisibleVerdict) {
|
if (!missingVisibleVerdict) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
this.pairedExecutionSummary =
|
const noVerdictSummary =
|
||||||
'Execution completed without a visible terminal verdict.';
|
'Execution completed without a visible terminal verdict.';
|
||||||
|
this.pairedExecutionSummary = this.pairedExecutionSummary
|
||||||
|
? `${this.pairedExecutionSummary}\n${noVerdictSummary}`
|
||||||
|
: noVerdictSummary;
|
||||||
log.warn(
|
log.warn(
|
||||||
{
|
{
|
||||||
pairedTaskId: pairedExecutionContext?.task.id ?? null,
|
pairedTaskId: pairedExecutionContext?.task.id ?? null,
|
||||||
@@ -578,7 +652,8 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
|||||||
private async notifyCompletionAndQueueFollowUp(
|
private async notifyCompletionAndQueueFollowUp(
|
||||||
state: FinalizeState,
|
state: FinalizeState,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const { chatJid, onOutput, pairedExecutionContext } = this.args;
|
const { chatJid, completedRole, onOutput, pairedExecutionContext } =
|
||||||
|
this.args;
|
||||||
if (!pairedExecutionContext || state.interruptedByHumanMessage) {
|
if (!pairedExecutionContext || state.interruptedByHumanMessage) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -586,6 +661,7 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
|||||||
await notifyPairedCompletionIfNeeded({
|
await notifyPairedCompletionIfNeeded({
|
||||||
task: finishedTask,
|
task: finishedTask,
|
||||||
chatJid,
|
chatJid,
|
||||||
|
completedRole,
|
||||||
onOutput,
|
onOutput,
|
||||||
});
|
});
|
||||||
this.queueFollowUpIfNeeded(state, finishedTask);
|
this.queueFollowUpIfNeeded(state, finishedTask);
|
||||||
@@ -620,6 +696,7 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
|||||||
sawOutput: state.sawOutputForFollowUp,
|
sawOutput: state.sawOutputForFollowUp,
|
||||||
taskStatus: finishedTask?.status ?? null,
|
taskStatus: finishedTask?.status ?? null,
|
||||||
outputSummary: this.pairedExecutionSummary,
|
outputSummary: this.pairedExecutionSummary,
|
||||||
|
ownerFailureCount: finishedTask?.owner_failure_count ?? null,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -172,6 +172,19 @@ describe('message-agent-executor-rules', () => {
|
|||||||
).toBe('pending');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('requeues owner follow-up after a silent arbiter Codex account failure returns the task active', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'arbiter',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'active',
|
||||||
|
outputSummary:
|
||||||
|
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.\nExecution completed without a visible terminal verdict.',
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
|
||||||
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
||||||
expect(
|
expect(
|
||||||
resolvePairedFollowUpQueueAction({
|
resolvePairedFollowUpQueueAction({
|
||||||
@@ -194,6 +207,19 @@ describe('message-agent-executor-rules', () => {
|
|||||||
).toBe('pending');
|
).toBe('pending');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('requeues a silent owner failure caused by Codex pool unavailability', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'owner',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'active',
|
||||||
|
outputSummary:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
|
||||||
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
|
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
|
||||||
expect(
|
expect(
|
||||||
resolvePairedFollowUpQueueAction({
|
resolvePairedFollowUpQueueAction({
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import {
|
|||||||
resolveNextTurnAction,
|
resolveNextTurnAction,
|
||||||
shouldRetrySilentOwnerExecution,
|
shouldRetrySilentOwnerExecution,
|
||||||
} from './message-runtime-rules.js';
|
} from './message-runtime-rules.js';
|
||||||
|
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||||
import { parseVisibleVerdict } from './paired-verdict.js';
|
import { parseVisibleVerdict } from './paired-verdict.js';
|
||||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||||
export {
|
export {
|
||||||
@@ -17,13 +18,31 @@ export {
|
|||||||
|
|
||||||
export type PairedFollowUpQueueAction = 'pending' | 'none';
|
export type PairedFollowUpQueueAction = 'pending' | 'none';
|
||||||
|
|
||||||
|
function isSilentCodexAccountFailure(summary?: string | null): boolean {
|
||||||
|
return isTerminalCodexAccountFailure(summary);
|
||||||
|
}
|
||||||
|
|
||||||
export function resolvePairedFollowUpQueueAction(args: {
|
export function resolvePairedFollowUpQueueAction(args: {
|
||||||
completedRole: PairedRoomRole;
|
completedRole: PairedRoomRole;
|
||||||
executionStatus: 'succeeded' | 'failed';
|
executionStatus: 'succeeded' | 'failed';
|
||||||
sawOutput: boolean;
|
sawOutput: boolean;
|
||||||
taskStatus: PairedTaskStatus | null;
|
taskStatus: PairedTaskStatus | null;
|
||||||
outputSummary?: string | null;
|
outputSummary?: string | null;
|
||||||
|
ownerFailureCount?: number | null;
|
||||||
}): PairedFollowUpQueueAction {
|
}): PairedFollowUpQueueAction {
|
||||||
|
if (
|
||||||
|
args.executionStatus === 'failed' &&
|
||||||
|
args.sawOutput === false &&
|
||||||
|
isSilentCodexAccountFailure(args.outputSummary)
|
||||||
|
) {
|
||||||
|
if (args.completedRole === 'arbiter' && args.taskStatus === 'active') {
|
||||||
|
return 'pending';
|
||||||
|
}
|
||||||
|
if (args.completedRole !== 'owner') {
|
||||||
|
return 'none';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
shouldRetrySilentOwnerExecution({
|
shouldRetrySilentOwnerExecution({
|
||||||
completedRole: args.completedRole,
|
completedRole: args.completedRole,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
import * as config from './config.js';
|
import * as config from './config.js';
|
||||||
|
|
||||||
|
const NO_VISIBLE_VERDICT_SUMMARY = `검토 중입니다.\nExecution completed without a visible terminal verdict.`;
|
||||||
vi.mock('./agent-runner.js', () => ({
|
vi.mock('./agent-runner.js', () => ({
|
||||||
runAgentProcess: vi.fn(),
|
runAgentProcess: vi.fn(),
|
||||||
writeGroupsSnapshot: vi.fn(),
|
writeGroupsSnapshot: vi.fn(),
|
||||||
@@ -1190,7 +1191,7 @@ describe('runAgentForGroup room memory', () => {
|
|||||||
taskId: 'paired-task-review-no-verdict',
|
taskId: 'paired-task-review-no-verdict',
|
||||||
role: 'reviewer',
|
role: 'reviewer',
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
summary: 'Execution completed without a visible terminal verdict.',
|
summary: NO_VISIBLE_VERDICT_SUMMARY,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
expect(db.failPairedTurn).toHaveBeenCalledWith({
|
expect(db.failPairedTurn).toHaveBeenCalledWith({
|
||||||
@@ -1202,7 +1203,7 @@ describe('runAgentForGroup room memory', () => {
|
|||||||
intentKind: 'reviewer-turn',
|
intentKind: 'reviewer-turn',
|
||||||
role: 'reviewer',
|
role: 'reviewer',
|
||||||
},
|
},
|
||||||
error: 'Execution completed without a visible terminal verdict.',
|
error: NO_VISIBLE_VERDICT_SUMMARY,
|
||||||
});
|
});
|
||||||
expect(db.completePairedTurn).not.toHaveBeenCalled();
|
expect(db.completePairedTurn).not.toHaveBeenCalled();
|
||||||
expect(db.insertPairedTurnOutput).not.toHaveBeenCalled();
|
expect(db.insertPairedTurnOutput).not.toHaveBeenCalled();
|
||||||
|
|||||||
@@ -21,6 +21,50 @@ export type PairedFollowUpSource = Parameters<
|
|||||||
typeof resolveFollowUpDispatch
|
typeof resolveFollowUpDispatch
|
||||||
>[0]['source'];
|
>[0]['source'];
|
||||||
|
|
||||||
|
type PairedTurnOutputContext = ReturnType<typeof getPairedTurnOutputs>[number];
|
||||||
|
|
||||||
|
type PairedFollowUpTaskContext = Pick<
|
||||||
|
PairedTask,
|
||||||
|
'id' | 'status' | 'round_trip_count' | 'updated_at' | 'owner_failure_count'
|
||||||
|
>;
|
||||||
|
|
||||||
|
function timestampMs(value: string | null | undefined): number | null {
|
||||||
|
if (!value) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const parsed = Date.parse(value);
|
||||||
|
return Number.isFinite(parsed) ? parsed : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPersistedOutputOlderThanTaskRevision(args: {
|
||||||
|
output: PairedTurnOutputContext;
|
||||||
|
task: Partial<Pick<PairedTask, 'updated_at'>> | null | undefined;
|
||||||
|
}): boolean {
|
||||||
|
const outputCreatedAt = timestampMs(args.output.created_at);
|
||||||
|
const taskUpdatedAt = timestampMs(args.task?.updated_at);
|
||||||
|
if (outputCreatedAt == null || taskUpdatedAt == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return outputCreatedAt < taskUpdatedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldPreferFallbackTurnOutputContext(args: {
|
||||||
|
output: PairedTurnOutputContext | null | undefined;
|
||||||
|
task: Partial<Pick<PairedTask, 'updated_at'>> | null | undefined;
|
||||||
|
fallbackRole: PairedRoomRole | null | undefined;
|
||||||
|
}): boolean {
|
||||||
|
if (!args.output || !args.fallbackRole) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (args.output.role === args.fallbackRole) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return isPersistedOutputOlderThanTaskRevision({
|
||||||
|
output: args.output,
|
||||||
|
task: args.task,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export interface PairedFollowUpDecision {
|
export interface PairedFollowUpDecision {
|
||||||
taskId: string | null;
|
taskId: string | null;
|
||||||
taskStatus: PairedTaskStatus | null;
|
taskStatus: PairedTaskStatus | null;
|
||||||
@@ -44,7 +88,10 @@ export type PairedFollowUpDispatchResult =
|
|||||||
});
|
});
|
||||||
|
|
||||||
export function resolveLatestPairedTurnOutputContext(args: {
|
export function resolveLatestPairedTurnOutputContext(args: {
|
||||||
task: Pick<PairedTask, 'id'> | null | undefined;
|
task:
|
||||||
|
| (Pick<PairedTask, 'id'> & Partial<Pick<PairedTask, 'updated_at'>>)
|
||||||
|
| null
|
||||||
|
| undefined;
|
||||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||||
fallbackLastTurnOutputVerdict?: VisibleVerdict | null;
|
fallbackLastTurnOutputVerdict?: VisibleVerdict | null;
|
||||||
}): {
|
}): {
|
||||||
@@ -54,12 +101,19 @@ export function resolveLatestPairedTurnOutputContext(args: {
|
|||||||
const latestOutput = args.task
|
const latestOutput = args.task
|
||||||
? getPairedTurnOutputs(args.task.id).at(-1)
|
? getPairedTurnOutputs(args.task.id).at(-1)
|
||||||
: null;
|
: null;
|
||||||
|
const output = shouldPreferFallbackTurnOutputContext({
|
||||||
|
output: latestOutput,
|
||||||
|
task: args.task,
|
||||||
|
fallbackRole: args.fallbackLastTurnOutputRole,
|
||||||
|
})
|
||||||
|
? null
|
||||||
|
: latestOutput;
|
||||||
return {
|
return {
|
||||||
role: latestOutput?.role ?? args.fallbackLastTurnOutputRole ?? null,
|
role: output?.role ?? args.fallbackLastTurnOutputRole ?? null,
|
||||||
verdict:
|
verdict:
|
||||||
resolveStoredVisibleVerdict({
|
resolveStoredVisibleVerdict({
|
||||||
verdict: latestOutput?.verdict ?? null,
|
verdict: output?.verdict ?? null,
|
||||||
outputText: latestOutput?.output_text ?? null,
|
outputText: output?.output_text ?? null,
|
||||||
}) ??
|
}) ??
|
||||||
args.fallbackLastTurnOutputVerdict ??
|
args.fallbackLastTurnOutputVerdict ??
|
||||||
null,
|
null,
|
||||||
@@ -67,7 +121,13 @@ export function resolveLatestPairedTurnOutputContext(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function resolvePairedFollowUpDecision(args: {
|
export function resolvePairedFollowUpDecision(args: {
|
||||||
task: Pick<PairedTask, 'id' | 'status'> | null | undefined;
|
task:
|
||||||
|
| Pick<
|
||||||
|
PairedFollowUpTaskContext,
|
||||||
|
'id' | 'status' | 'updated_at' | 'owner_failure_count'
|
||||||
|
>
|
||||||
|
| null
|
||||||
|
| undefined;
|
||||||
source: PairedFollowUpSource;
|
source: PairedFollowUpSource;
|
||||||
completedRole?: PairedRoomRole;
|
completedRole?: PairedRoomRole;
|
||||||
executionStatus?: 'succeeded' | 'failed';
|
executionStatus?: 'succeeded' | 'failed';
|
||||||
@@ -93,6 +153,7 @@ export function resolvePairedFollowUpDecision(args: {
|
|||||||
taskStatus: args.task?.status ?? null,
|
taskStatus: args.task?.status ?? null,
|
||||||
lastTurnOutputRole,
|
lastTurnOutputRole,
|
||||||
lastTurnOutputVerdict,
|
lastTurnOutputVerdict,
|
||||||
|
ownerFailureCount: args.task?.owner_failure_count ?? null,
|
||||||
});
|
});
|
||||||
const dispatch = resolveFollowUpDispatch({
|
const dispatch = resolveFollowUpDispatch({
|
||||||
source: args.source,
|
source: args.source,
|
||||||
@@ -115,10 +176,7 @@ export function resolvePairedFollowUpDecision(args: {
|
|||||||
export function schedulePairedFollowUpIntent(args: {
|
export function schedulePairedFollowUpIntent(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
intentKind: ScheduledPairedFollowUpIntentKind;
|
intentKind: ScheduledPairedFollowUpIntentKind;
|
||||||
enqueue: () => void;
|
enqueue: () => void;
|
||||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||||
@@ -148,6 +206,7 @@ export function schedulePairedFollowUpIntent(args: {
|
|||||||
taskStatus: args.task.status,
|
taskStatus: args.task.status,
|
||||||
lastTurnOutputRole: latestOutputContext.role,
|
lastTurnOutputRole: latestOutputContext.role,
|
||||||
lastTurnOutputVerdict: latestOutputContext.verdict,
|
lastTurnOutputVerdict: latestOutputContext.verdict,
|
||||||
|
ownerFailureCount: args.task.owner_failure_count ?? null,
|
||||||
intentKind: args.intentKind,
|
intentKind: args.intentKind,
|
||||||
}) &&
|
}) &&
|
||||||
!(
|
!(
|
||||||
@@ -171,10 +230,7 @@ export function schedulePairedFollowUpIntent(args: {
|
|||||||
export function schedulePairedFollowUpWithMessageCheck(args: {
|
export function schedulePairedFollowUpWithMessageCheck(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
intentKind: ScheduledPairedFollowUpIntentKind;
|
intentKind: ScheduledPairedFollowUpIntentKind;
|
||||||
enqueueMessageCheck: () => void;
|
enqueueMessageCheck: () => void;
|
||||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||||
@@ -198,10 +254,7 @@ export function schedulePairedFollowUpWithMessageCheck(args: {
|
|||||||
export function dispatchPairedFollowUpForEvent(args: {
|
export function dispatchPairedFollowUpForEvent(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
source: PairedFollowUpSource;
|
source: PairedFollowUpSource;
|
||||||
completedRole?: PairedRoomRole;
|
completedRole?: PairedRoomRole;
|
||||||
executionStatus?: 'succeeded' | 'failed';
|
executionStatus?: 'succeeded' | 'failed';
|
||||||
@@ -270,10 +323,7 @@ export function dispatchPairedFollowUpForEvent(args: {
|
|||||||
export function enqueuePairedFollowUpAfterEvent(args: {
|
export function enqueuePairedFollowUpAfterEvent(args: {
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
runId: string;
|
runId: string;
|
||||||
task:
|
task: PairedFollowUpTaskContext | null | undefined;
|
||||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
|
||||||
| null
|
|
||||||
| undefined;
|
|
||||||
source: PairedFollowUpSource;
|
source: PairedFollowUpSource;
|
||||||
completedRole?: PairedRoomRole;
|
completedRole?: PairedRoomRole;
|
||||||
executionStatus?: 'succeeded' | 'failed';
|
executionStatus?: 'succeeded' | 'failed';
|
||||||
|
|||||||
@@ -11,9 +11,12 @@ vi.mock('./session-commands.js', () => ({
|
|||||||
|
|
||||||
import { handleQueuedRunGates } from './message-runtime-gating.js';
|
import { handleQueuedRunGates } from './message-runtime-gating.js';
|
||||||
|
|
||||||
describe('message-runtime-gating', () => {
|
function makeArgs(overrides: {
|
||||||
const baseArgs = {
|
chatJid?: string;
|
||||||
chatJid: 'room-1',
|
sendMessage?: () => Promise<void>;
|
||||||
|
}) {
|
||||||
|
return {
|
||||||
|
chatJid: overrides.chatJid ?? 'room-1',
|
||||||
group: {
|
group: {
|
||||||
folder: 'room-folder',
|
folder: 'room-folder',
|
||||||
name: 'room',
|
name: 'room',
|
||||||
@@ -26,9 +29,13 @@ describe('message-runtime-gating', () => {
|
|||||||
triggerPattern: /^코덱스/,
|
triggerPattern: /^코덱스/,
|
||||||
timezone: 'Asia/Seoul',
|
timezone: 'Asia/Seoul',
|
||||||
hasImplicitContinuationWindow: () => false,
|
hasImplicitContinuationWindow: () => false,
|
||||||
sessionCommandDeps: {} as never,
|
sessionCommandDeps: {
|
||||||
|
sendMessage: overrides.sendMessage ?? (async () => {}),
|
||||||
|
} as never,
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('message-runtime-gating', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
handleSessionCommandMock.mockReset();
|
handleSessionCommandMock.mockReset();
|
||||||
});
|
});
|
||||||
@@ -39,15 +46,15 @@ describe('message-runtime-gating', () => {
|
|||||||
success: false,
|
success: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = await handleQueuedRunGates(baseArgs);
|
const result = await handleQueuedRunGates(makeArgs({}));
|
||||||
|
|
||||||
expect(result).toEqual({ handled: true, success: false });
|
expect(result).toEqual({ handled: true, success: false });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('falls through when no session command is handled', async () => {
|
it('falls through when no session command matched', async () => {
|
||||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||||
|
|
||||||
const result = await handleQueuedRunGates(baseArgs);
|
const result = await handleQueuedRunGates(makeArgs({}));
|
||||||
|
|
||||||
expect(result).toEqual({ handled: false });
|
expect(result).toEqual({ handled: false });
|
||||||
});
|
});
|
||||||
|
|||||||
125
src/message-runtime-queue-owner-retry.test.ts
Normal file
125
src/message-runtime-queue-owner-retry.test.ts
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import { _initTestDatabase, createPairedTask } from './db.js';
|
||||||
|
import { runQueuedGroupTurn } from './message-runtime-queue.js';
|
||||||
|
import { resetPairedFollowUpScheduleState } from './paired-follow-up-scheduler.js';
|
||||||
|
import type { Channel, PairedTask, RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
|
function makeGroup(): RegisteredGroup {
|
||||||
|
return {
|
||||||
|
name: 'Test Group',
|
||||||
|
folder: 'test-group',
|
||||||
|
trigger: '@Andy',
|
||||||
|
added_at: '2026-03-30T00:00:00.000Z',
|
||||||
|
requiresTrigger: false,
|
||||||
|
agentType: 'codex',
|
||||||
|
workDir: '/repo',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||||
|
return {
|
||||||
|
id: 'task-owner-retry',
|
||||||
|
chat_jid: 'group@test',
|
||||||
|
group_folder: 'test-group',
|
||||||
|
owner_service_id: 'codex-main',
|
||||||
|
reviewer_service_id: 'claude',
|
||||||
|
owner_agent_type: 'codex',
|
||||||
|
reviewer_agent_type: 'claude-code',
|
||||||
|
arbiter_agent_type: null,
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
review_requested_at: null,
|
||||||
|
round_trip_count: 0,
|
||||||
|
owner_failure_count: 1,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
task_done_then_user_reopen_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
status: 'active',
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
created_at: '2026-03-30T00:00:00.000Z',
|
||||||
|
updated_at: '2026-03-30T00:00:10.000Z',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeChannel(): Channel {
|
||||||
|
return {
|
||||||
|
name: 'discord-owner',
|
||||||
|
connect: vi.fn(),
|
||||||
|
sendMessage: vi.fn(),
|
||||||
|
isConnected: vi.fn(() => true),
|
||||||
|
ownsJid: vi.fn(() => true),
|
||||||
|
disconnect: vi.fn(),
|
||||||
|
} as unknown as Channel;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('message-runtime-queue owner retry turns', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
_initTestDatabase();
|
||||||
|
resetPairedFollowUpScheduleState();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps stale human messages in failed owner retries as owner follow-up', async () => {
|
||||||
|
const task = makeTask();
|
||||||
|
createPairedTask(task);
|
||||||
|
const executeTurn = vi.fn(async () => ({
|
||||||
|
outputStatus: 'error' as const,
|
||||||
|
deliverySucceeded: false,
|
||||||
|
visiblePhase: 'silent',
|
||||||
|
}));
|
||||||
|
|
||||||
|
const outcome = await runQueuedGroupTurn({
|
||||||
|
chatJid: task.chat_jid,
|
||||||
|
group: makeGroup(),
|
||||||
|
runId: 'run-owner-retry-stale-human',
|
||||||
|
log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as any,
|
||||||
|
timezone: 'UTC',
|
||||||
|
missedMessages: [
|
||||||
|
{
|
||||||
|
id: 'human-owner-stale-1',
|
||||||
|
chat_jid: task.chat_jid,
|
||||||
|
sender: 'user@test',
|
||||||
|
sender_name: 'User',
|
||||||
|
content: '원 요청입니다',
|
||||||
|
timestamp: '2026-03-30T00:00:03.000Z',
|
||||||
|
seq: 46,
|
||||||
|
is_bot_message: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
task,
|
||||||
|
roleToChannel: {
|
||||||
|
owner: null,
|
||||||
|
reviewer: makeChannel(),
|
||||||
|
arbiter: null,
|
||||||
|
},
|
||||||
|
ownerChannel: makeChannel(),
|
||||||
|
lastAgentTimestamps: {},
|
||||||
|
saveState: vi.fn(),
|
||||||
|
executeTurn,
|
||||||
|
getFixedRoleChannelName: () => 'discord-review',
|
||||||
|
labelPairedSenders: (_chatJid, messages) => messages,
|
||||||
|
formatMessages: () => 'formatted prompt',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(outcome).toBe(false);
|
||||||
|
expect(executeTurn).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
hasHumanMessage: false,
|
||||||
|
deliveryRole: 'owner',
|
||||||
|
forcedRole: 'owner',
|
||||||
|
pairedTurnIdentity: {
|
||||||
|
turnId: 'task-owner-retry:2026-03-30T00:00:10.000Z:owner-follow-up',
|
||||||
|
taskId: 'task-owner-retry',
|
||||||
|
taskUpdatedAt: '2026-03-30T00:00:10.000Z',
|
||||||
|
intentKind: 'owner-follow-up',
|
||||||
|
role: 'owner',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -56,6 +56,33 @@ function resolveQueuedTurnReservationIntent(args: {
|
|||||||
return 'owner-follow-up';
|
return 'owner-follow-up';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function timestampMs(value: string | null | undefined): number | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const parsed = Date.parse(value);
|
||||||
|
return Number.isFinite(parsed) ? parsed : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHumanMessageFreshForTask(
|
||||||
|
task: PairedTask | null | undefined,
|
||||||
|
message: NewMessage,
|
||||||
|
): boolean {
|
||||||
|
if (!isExternalHumanMessage(message)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!task) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (task.status !== 'active' || (task.owner_failure_count ?? 0) <= 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const messageTime = timestampMs(message.timestamp);
|
||||||
|
const taskUpdatedAt = timestampMs(task.updated_at);
|
||||||
|
if (messageTime == null || taskUpdatedAt == null) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return messageTime > taskUpdatedAt;
|
||||||
|
}
|
||||||
|
|
||||||
function buildQueuedGroupTurnPrompt(args: {
|
function buildQueuedGroupTurnPrompt(args: {
|
||||||
turnRole: 'owner' | 'reviewer' | 'arbiter';
|
turnRole: 'owner' | 'reviewer' | 'arbiter';
|
||||||
currentTask: PairedTask | null | undefined;
|
currentTask: PairedTask | null | undefined;
|
||||||
@@ -248,7 +275,9 @@ export async function runQueuedGroupTurn(args: {
|
|||||||
args;
|
args;
|
||||||
let currentTask = task;
|
let currentTask = task;
|
||||||
const hasHumanMsg = task
|
const hasHumanMsg = task
|
||||||
? missedMessages.some(isExternalHumanMessage)
|
? missedMessages.some((message) =>
|
||||||
|
isHumanMessageFreshForTask(task, message),
|
||||||
|
)
|
||||||
: !isBotOnlyPairedRoomTurn(chatJid, missedMessages);
|
: !isBotOnlyPairedRoomTurn(chatJid, missedMessages);
|
||||||
let fallbackMessages = missedMessages;
|
let fallbackMessages = missedMessages;
|
||||||
if (currentTask !== undefined && hasHumanMsg) {
|
if (currentTask !== undefined && hasHumanMsg) {
|
||||||
|
|||||||
@@ -2727,9 +2727,20 @@ describe('createMessageRuntime', () => {
|
|||||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockImplementation(
|
vi.mocked(db.getLatestOpenPairedTaskForChat).mockImplementation(
|
||||||
() => pairedTask,
|
() => pairedTask,
|
||||||
);
|
);
|
||||||
|
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
task_id: pairedTask.id,
|
||||||
|
turn_number: 1,
|
||||||
|
role: 'owner',
|
||||||
|
output_text: 'STEP_DONE\nowner work ready for review',
|
||||||
|
created_at: '2026-03-30T00:00:00.500Z',
|
||||||
|
},
|
||||||
|
] as any);
|
||||||
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
||||||
async (_group, _input, _onProcess, onOutput) => {
|
async (_group, _input, _onProcess, onOutput) => {
|
||||||
pairedTask.status = 'active';
|
pairedTask.status = 'active';
|
||||||
|
pairedTask.updated_at = '2026-03-30T00:00:01.000Z';
|
||||||
await onOutput?.({
|
await onOutput?.({
|
||||||
status: 'success',
|
status: 'success',
|
||||||
phase: 'final',
|
phase: 'final',
|
||||||
|
|||||||
@@ -802,6 +802,32 @@ describe('MessageTurnController outbound audit logging', () => {
|
|||||||
replaceMessageId: 'progress-1',
|
replaceMessageId: 'progress-1',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('publishes a failure final when an error finishes before any visible output', async () => {
|
||||||
|
const channel = makeChannel();
|
||||||
|
const deliverFinalText = vi.fn().mockResolvedValue(true);
|
||||||
|
const controller = new MessageTurnController({
|
||||||
|
chatJid: 'dc:test-room',
|
||||||
|
group: makeGroup(),
|
||||||
|
runId: 'run-silent-error-final',
|
||||||
|
channel,
|
||||||
|
idleTimeout: 1_000,
|
||||||
|
failureFinalText: '실패',
|
||||||
|
isClaudeCodeAgent: true,
|
||||||
|
clearSession: vi.fn(),
|
||||||
|
requestClose: vi.fn(),
|
||||||
|
deliverFinalText,
|
||||||
|
deliveryRole: 'owner',
|
||||||
|
});
|
||||||
|
|
||||||
|
await controller.start();
|
||||||
|
const finishResult = await controller.finish('error');
|
||||||
|
|
||||||
|
expect(finishResult.visiblePhase).toBe('final');
|
||||||
|
expect(deliverFinalText).toHaveBeenCalledWith('실패', {
|
||||||
|
replaceMessageId: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('MessageTurnController human interruptions', () => {
|
describe('MessageTurnController human interruptions', () => {
|
||||||
|
|||||||
@@ -385,6 +385,12 @@ export class MessageTurnController {
|
|||||||
this.hadError
|
this.hadError
|
||||||
) {
|
) {
|
||||||
await this.publishFailureFinal();
|
await this.publishFailureFinal();
|
||||||
|
} else if (
|
||||||
|
outputStatus === 'error' &&
|
||||||
|
this.visiblePhase === 'silent' &&
|
||||||
|
!this.terminalObserved()
|
||||||
|
) {
|
||||||
|
await this.publishFailureFinal();
|
||||||
}
|
}
|
||||||
|
|
||||||
this.clearProgressTicker();
|
this.clearProgressTicker();
|
||||||
@@ -615,15 +621,19 @@ export class MessageTurnController {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Capture the value before the await: a concurrent resetProgressState()
|
||||||
|
// can null `latestProgressText` while editMessage is in flight, which used
|
||||||
|
// to throw "null is not an object (evaluating 'this.latestProgressText.length')".
|
||||||
|
const progressText = this.latestProgressText;
|
||||||
if (
|
if (
|
||||||
!this.progressMessageId ||
|
!this.progressMessageId ||
|
||||||
!this.options.channel.editMessage ||
|
!this.options.channel.editMessage ||
|
||||||
!this.latestProgressText
|
!progressText
|
||||||
) {
|
) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const rendered = this.renderProgressMessage(this.latestProgressText);
|
const rendered = this.renderProgressMessage(progressText);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.options.channel.editMessage(
|
await this.options.channel.editMessage(
|
||||||
@@ -635,7 +645,7 @@ export class MessageTurnController {
|
|||||||
this.progressEditFailCount = 0;
|
this.progressEditFailCount = 0;
|
||||||
this.logOutboundAudit('progress-edit', {
|
this.logOutboundAudit('progress-edit', {
|
||||||
messageId: this.progressMessageId,
|
messageId: this.progressMessageId,
|
||||||
textLength: this.latestProgressText.length,
|
textLength: progressText.length,
|
||||||
renderedLength: rendered.length,
|
renderedLength: rendered.length,
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -4,7 +4,11 @@ import path from 'path';
|
|||||||
|
|
||||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
import { validateOutboundAttachments } from './outbound-attachments.js';
|
import {
|
||||||
|
appendRejectionNotice,
|
||||||
|
describeRejectedAttachments,
|
||||||
|
validateOutboundAttachments,
|
||||||
|
} from './outbound-attachments.js';
|
||||||
|
|
||||||
const ONE_PIXEL_PNG = Buffer.from(
|
const ONE_PIXEL_PNG = Buffer.from(
|
||||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
||||||
@@ -38,6 +42,7 @@ function writeFile(
|
|||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vi.unstubAllEnvs();
|
vi.unstubAllEnvs();
|
||||||
|
vi.restoreAllMocks();
|
||||||
for (const dir of cleanupDirs.splice(0)) {
|
for (const dir of cleanupDirs.splice(0)) {
|
||||||
fs.rmSync(dir, { force: true, recursive: true });
|
fs.rmSync(dir, { force: true, recursive: true });
|
||||||
}
|
}
|
||||||
@@ -176,7 +181,13 @@ describe('validateOutboundAttachments', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('validateOutboundAttachments policy checks', () => {
|
describe('validateOutboundAttachments policy checks', () => {
|
||||||
|
function useIsolatedDefaultTempDir(): void {
|
||||||
|
const tempDir = makeTempDir(os.tmpdir(), 'ejclaw-policy-temp-');
|
||||||
|
vi.spyOn(os, 'tmpdir').mockReturnValue(tempDir);
|
||||||
|
}
|
||||||
|
|
||||||
it('requires workspace paths to be explicitly allowlisted', () => {
|
it('requires workspace paths to be explicitly allowlisted', () => {
|
||||||
|
useIsolatedDefaultTempDir();
|
||||||
const dir = makeTempDir(process.cwd(), '.ejclaw-attachment-');
|
const dir = makeTempDir(process.cwd(), '.ejclaw-attachment-');
|
||||||
const imagePath = writeFile(dir, 'workspace-shot.png', ONE_PIXEL_PNG);
|
const imagePath = writeFile(dir, 'workspace-shot.png', ONE_PIXEL_PNG);
|
||||||
|
|
||||||
@@ -223,6 +234,7 @@ describe('validateOutboundAttachments policy checks', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('rejects symlink attempts that escape the allowed directory', () => {
|
it('rejects symlink attempts that escape the allowed directory', () => {
|
||||||
|
useIsolatedDefaultTempDir();
|
||||||
const workspaceDir = makeTempDir(process.cwd(), '.ejclaw-attachment-');
|
const workspaceDir = makeTempDir(process.cwd(), '.ejclaw-attachment-');
|
||||||
const targetPath = writeFile(
|
const targetPath = writeFile(
|
||||||
workspaceDir,
|
workspaceDir,
|
||||||
@@ -242,6 +254,7 @@ describe('validateOutboundAttachments policy checks', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('rejects symlink attempts that escape a user-configured directory', () => {
|
it('rejects symlink attempts that escape a user-configured directory', () => {
|
||||||
|
useIsolatedDefaultTempDir();
|
||||||
const allowedDir = makeTempDir(process.cwd(), '.ejclaw-user-images-');
|
const allowedDir = makeTempDir(process.cwd(), '.ejclaw-user-images-');
|
||||||
const secretDir = makeTempDir(process.cwd(), '.ejclaw-secret-images-');
|
const secretDir = makeTempDir(process.cwd(), '.ejclaw-secret-images-');
|
||||||
const targetPath = writeFile(secretDir, 'secret-shot.png', ONE_PIXEL_PNG);
|
const targetPath = writeFile(secretDir, 'secret-shot.png', ONE_PIXEL_PNG);
|
||||||
@@ -316,3 +329,59 @@ describe('validateOutboundAttachments policy checks', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('describeRejectedAttachments', () => {
|
||||||
|
it('returns an empty string when nothing was rejected', () => {
|
||||||
|
expect(describeRejectedAttachments([])).toBe('');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('summarizes rejected attachments by basename and reason', () => {
|
||||||
|
const note = describeRejectedAttachments([
|
||||||
|
{
|
||||||
|
path: '/home/claude/jarvis-tts/sample.wav',
|
||||||
|
reason: 'outside-allowed-dirs',
|
||||||
|
},
|
||||||
|
{ path: '/tmp/missing.png', reason: 'not-found' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(note).toContain('첨부 2건을 전송하지 못했습니다');
|
||||||
|
expect(note).toContain('sample.wav (허용된 폴더 밖의 경로)');
|
||||||
|
expect(note).toContain('missing.png (파일을 찾을 수 없음)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to the raw reason code when no label is mapped', () => {
|
||||||
|
const note = describeRejectedAttachments([
|
||||||
|
{ path: '/tmp/x.bin', reason: 'some-new-reason' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(note).toContain('x.bin (some-new-reason)');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('appendRejectionNotice', () => {
|
||||||
|
it('leaves the body untouched when nothing was rejected', () => {
|
||||||
|
expect(appendRejectionNotice('보고서입니다', [])).toBe('보고서입니다');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('appends the failure notice to the visible body', () => {
|
||||||
|
const body = appendRejectionNotice('샘플을 첨부합니다', [
|
||||||
|
{ path: '/home/claude/jarvis-tts/a.wav', reason: 'outside-allowed-dirs' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(body).toContain('샘플을 첨부합니다');
|
||||||
|
expect(body).toContain('첨부 1건을 전송하지 못했습니다');
|
||||||
|
expect(body).toContain('a.wav (허용된 폴더 밖의 경로)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns the notice alone when the body would otherwise be empty', () => {
|
||||||
|
const body = appendRejectionNotice('', [
|
||||||
|
{ path: '/tmp/missing.png', reason: 'not-found' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(body).toBe(
|
||||||
|
describeRejectedAttachments([
|
||||||
|
{ path: '/tmp/missing.png', reason: 'not-found' },
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -92,6 +92,13 @@ export function getDefaultAttachmentBaseDirs(): string[] {
|
|||||||
.filter((dir): dir is string => Boolean(dir));
|
.filter((dir): dir is string => Boolean(dir));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveAllowedBaseDirs(baseDirs?: string[]): string[] {
|
||||||
|
return unique([
|
||||||
|
...getDefaultAttachmentBaseDirs(),
|
||||||
|
...(baseDirs ?? []).map(resolveExistingDir),
|
||||||
|
]).filter((dir): dir is string => Boolean(dir));
|
||||||
|
}
|
||||||
|
|
||||||
function isWithinBaseDir(realPath: string, baseDir: string): boolean {
|
function isWithinBaseDir(realPath: string, baseDir: string): boolean {
|
||||||
const relative = path.relative(baseDir, realPath);
|
const relative = path.relative(baseDir, realPath);
|
||||||
return (
|
return (
|
||||||
@@ -235,14 +242,56 @@ function normalizeAttachmentName(
|
|||||||
return candidate || 'attachment';
|
return candidate || 'attachment';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const REJECTION_REASON_LABELS: Record<string, string> = {
|
||||||
|
'outside-allowed-dirs': '허용된 폴더 밖의 경로',
|
||||||
|
'not-found': '파일을 찾을 수 없음',
|
||||||
|
'not-absolute': '절대경로가 아님',
|
||||||
|
'not-file': '파일이 아님',
|
||||||
|
'too-large': '8MB 용량 초과',
|
||||||
|
'unsupported-extension': '지원하지 않는 형식',
|
||||||
|
'invalid-image-signature': '이미지 형식 손상 또는 불일치',
|
||||||
|
'invalid-file-signature': '파일 형식 손상 또는 불일치',
|
||||||
|
'mime-mismatch': '형식 불일치',
|
||||||
|
'validation-error': '검증 오류',
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds a concise, user-facing note describing attachments that failed to
|
||||||
|
* send. Returning a non-empty string lets the delivery layer surface the
|
||||||
|
* failure in the visible message instead of dropping it silently — otherwise a
|
||||||
|
* stripped MEDIA: directive leaves the user with a message that claims a file
|
||||||
|
* was attached when none was delivered.
|
||||||
|
*/
|
||||||
|
export function describeRejectedAttachments(
|
||||||
|
rejected: Array<{ path: string; reason: string }>,
|
||||||
|
): string {
|
||||||
|
if (rejected.length === 0) return '';
|
||||||
|
const lines = rejected.map(({ path: rejectedPath, reason }) => {
|
||||||
|
const label = REJECTION_REASON_LABELS[reason] ?? reason;
|
||||||
|
return `• ${path.basename(rejectedPath)} (${label})`;
|
||||||
|
});
|
||||||
|
return `⚠️ 첨부 ${rejected.length}건을 전송하지 못했습니다:\n${lines.join('\n')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Folds a rejected-attachment notice into the visible outbound text so the
|
||||||
|
* delivery layer never sends a message that silently dropped its attachments.
|
||||||
|
* Returns the notice on its own when the body would otherwise be empty.
|
||||||
|
*/
|
||||||
|
export function appendRejectionNotice(
|
||||||
|
cleanText: string,
|
||||||
|
rejected: Array<{ path: string; reason: string }>,
|
||||||
|
): string {
|
||||||
|
const note = describeRejectedAttachments(rejected);
|
||||||
|
if (!note) return cleanText;
|
||||||
|
return cleanText ? `${cleanText}\n\n${note}` : note;
|
||||||
|
}
|
||||||
|
|
||||||
export function validateOutboundAttachments(
|
export function validateOutboundAttachments(
|
||||||
attachments: OutboundAttachment[] | undefined,
|
attachments: OutboundAttachment[] | undefined,
|
||||||
options: { baseDirs?: string[] } = {},
|
options: { baseDirs?: string[] } = {},
|
||||||
): ValidateOutboundAttachmentsResult {
|
): ValidateOutboundAttachmentsResult {
|
||||||
const baseDirs = unique([
|
const baseDirs = resolveAllowedBaseDirs(options.baseDirs);
|
||||||
...getDefaultAttachmentBaseDirs(),
|
|
||||||
...(options.baseDirs ?? []).map(resolveExistingDir),
|
|
||||||
]).filter((dir): dir is string => Boolean(dir));
|
|
||||||
const defaultTempDir = resolveExistingDir(os.tmpdir());
|
const defaultTempDir = resolveExistingDir(os.tmpdir());
|
||||||
const files: ValidatedOutboundAttachment[] = [];
|
const files: ValidatedOutboundAttachment[] = [];
|
||||||
const rejected: Array<{ path: string; reason: string }> = [];
|
const rejected: Array<{ path: string; reason: string }> = [];
|
||||||
|
|||||||
207
src/owner-final-codex-unavailable.test.ts
Normal file
207
src/owner-final-codex-unavailable.test.ts
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./db.js', () => {
|
||||||
|
const updatePairedTask = vi.fn();
|
||||||
|
return {
|
||||||
|
getPairedTaskById: vi.fn(),
|
||||||
|
getPairedWorkspace: vi.fn(),
|
||||||
|
updatePairedTask,
|
||||||
|
updatePairedTaskIfUnchanged: vi.fn((id, _expectedUpdatedAt, updates) => {
|
||||||
|
updatePairedTask(id, updates);
|
||||||
|
return true;
|
||||||
|
}),
|
||||||
|
releasePairedTaskExecutionLease: vi.fn(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock('./paired-workspace-manager.js', () => ({
|
||||||
|
isOwnerWorkspaceRepairNeededError: vi.fn(() => false),
|
||||||
|
markPairedTaskReviewReady: vi.fn(),
|
||||||
|
prepareReviewerWorkspaceForExecution: vi.fn(),
|
||||||
|
provisionOwnerWorkspaceForPairedTask: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./logger.js', () => ({
|
||||||
|
logger: {
|
||||||
|
debug: vi.fn(),
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import * as config from './config.js';
|
||||||
|
import * as db from './db.js';
|
||||||
|
import { completePairedExecutionContext } from './paired-execution-context.js';
|
||||||
|
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
||||||
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
|
const CODEX_UNAVAILABLE_SUMMARY =
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.';
|
||||||
|
|
||||||
|
function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||||
|
return {
|
||||||
|
id: 'task-1',
|
||||||
|
chat_jid: 'dc:test',
|
||||||
|
group_folder: 'ejclaw',
|
||||||
|
owner_service_id: config.CODEX_MAIN_SERVICE_ID,
|
||||||
|
reviewer_service_id: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
review_requested_at: null,
|
||||||
|
round_trip_count: 0,
|
||||||
|
owner_failure_count: 0,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
task_done_then_user_reopen_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
status: 'merge_ready',
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
created_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
updated_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('owner final Codex unavailable handling', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(db.getPairedWorkspace).mockReturnValue(undefined);
|
||||||
|
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves active tasks when the first owner turn cannot start Codex', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
const updates = vi.mocked(db.updatePairedTask).mock.calls[0]?.[1];
|
||||||
|
expect(updates).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
owner_failure_count: 1,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(updates?.status).not.toBe('completed');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves merge_ready when owner finalization cannot start Codex', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
owner_failure_count: 1,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requests arbiter after repeated owner Codex account failures', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
owner_failure_count: 1,
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'arbiter_requested',
|
||||||
|
owner_failure_count: 2,
|
||||||
|
arbiter_requested_at: expect.any(String),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('escalates to the user after persistent owner and arbiter Codex account failures', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
owner_failure_count: 3,
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'completed',
|
||||||
|
owner_failure_count: 4,
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'escalated',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requeues owner finalization once after preserving merge_ready', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'owner',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'merge_ready',
|
||||||
|
ownerFailureCount: 1,
|
||||||
|
outputSummary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('queues arbiter after repeated owner finalization failures', () => {
|
||||||
|
expect(
|
||||||
|
resolvePairedFollowUpQueueAction({
|
||||||
|
completedRole: 'owner',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
taskStatus: 'arbiter_requested',
|
||||||
|
ownerFailureCount: 2,
|
||||||
|
outputSummary: CODEX_UNAVAILABLE_SUMMARY,
|
||||||
|
}),
|
||||||
|
).toBe('pending');
|
||||||
|
});
|
||||||
|
});
|
||||||
81
src/paired-arbiter-codex-owner-wake.test.ts
Normal file
81
src/paired-arbiter-codex-owner-wake.test.ts
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./db.js', () => ({
|
||||||
|
getPairedTurnOutputs: vi.fn(() => []),
|
||||||
|
reservePairedTurnReservation: vi.fn(() => true),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { getPairedTurnOutputs } from './db.js';
|
||||||
|
import { dispatchPairedFollowUpForEvent } from './message-runtime-follow-up.js';
|
||||||
|
import {
|
||||||
|
matchesExpectedPairedFollowUpIntent,
|
||||||
|
resolveNextTurnAction,
|
||||||
|
} from './message-runtime-rules.js';
|
||||||
|
|
||||||
|
describe('arbiter Codex failure owner wake', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
vi.mocked(getPairedTurnOutputs).mockReturnValue([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps owner follow-ups schedulable when active task has owner failure count', () => {
|
||||||
|
const followUpContext = {
|
||||||
|
taskStatus: 'active' as const,
|
||||||
|
lastTurnOutputRole: 'owner' as const,
|
||||||
|
lastTurnOutputVerdict: 'step_done' as const,
|
||||||
|
ownerFailureCount: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(resolveNextTurnAction(followUpContext)).toEqual({
|
||||||
|
kind: 'owner-follow-up',
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
matchesExpectedPairedFollowUpIntent({
|
||||||
|
...followUpContext,
|
||||||
|
intentKind: 'owner-follow-up',
|
||||||
|
}),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requeues owner follow-up when arbiter Codex failure returns the task active', () => {
|
||||||
|
const enqueue = vi.fn();
|
||||||
|
vi.mocked(getPairedTurnOutputs).mockReturnValue([
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
task_id: 'task-arbiter-codex-unavailable',
|
||||||
|
turn_number: 1,
|
||||||
|
role: 'owner',
|
||||||
|
output_text: 'STEP_DONE\nwaiting for CI',
|
||||||
|
verdict: 'step_done',
|
||||||
|
created_at: '2026-03-30T00:00:01.000Z',
|
||||||
|
},
|
||||||
|
] as any);
|
||||||
|
|
||||||
|
const result = dispatchPairedFollowUpForEvent({
|
||||||
|
chatJid: 'group@test',
|
||||||
|
runId: 'run-arbiter-codex-unavailable',
|
||||||
|
task: {
|
||||||
|
id: 'task-arbiter-codex-unavailable',
|
||||||
|
status: 'active',
|
||||||
|
round_trip_count: 1,
|
||||||
|
owner_failure_count: 1,
|
||||||
|
updated_at: '2026-03-30T00:00:02.000Z',
|
||||||
|
},
|
||||||
|
source: 'executor-recovery',
|
||||||
|
completedRole: 'arbiter',
|
||||||
|
executionStatus: 'failed',
|
||||||
|
sawOutput: false,
|
||||||
|
enqueue,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({
|
||||||
|
kind: 'paired-follow-up',
|
||||||
|
intentKind: 'owner-follow-up',
|
||||||
|
scheduled: true,
|
||||||
|
taskStatus: 'active',
|
||||||
|
lastTurnOutputRole: 'owner',
|
||||||
|
nextTurnAction: { kind: 'owner-follow-up' },
|
||||||
|
});
|
||||||
|
expect(enqueue).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { isArbiterEnabled } from './config.js';
|
import { ARBITER_MAX_INTERVENTIONS, isArbiterEnabled } from './config.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
||||||
import type { PairedTaskStatus } from './types.js';
|
import type { PairedTaskStatus } from './types.js';
|
||||||
@@ -11,6 +11,13 @@ export function requestArbiterOrEscalate(args: {
|
|||||||
arbiterLogMessage: string;
|
arbiterLogMessage: string;
|
||||||
escalateLogMessage: string;
|
escalateLogMessage: string;
|
||||||
logContext?: Record<string, unknown>;
|
logContext?: Record<string, unknown>;
|
||||||
|
/**
|
||||||
|
* How many times the arbiter has already intervened on this task. When this
|
||||||
|
* reaches {@link maxArbiterInterventions} the task is escalated to the user
|
||||||
|
* instead of re-invoking the arbiter, bounding the owner↔reviewer↔arbiter loop.
|
||||||
|
*/
|
||||||
|
arbiterInterventionCount?: number;
|
||||||
|
maxArbiterInterventions?: number;
|
||||||
patch?: {
|
patch?: {
|
||||||
title?: string | null;
|
title?: string | null;
|
||||||
source_ref?: string | null;
|
source_ref?: string | null;
|
||||||
@@ -23,6 +30,7 @@ export function requestArbiterOrEscalate(args: {
|
|||||||
finalize_step_done_count?: number;
|
finalize_step_done_count?: number;
|
||||||
task_done_then_user_reopen_count?: number;
|
task_done_then_user_reopen_count?: number;
|
||||||
empty_step_done_streak?: number;
|
empty_step_done_streak?: number;
|
||||||
|
arbiter_intervention_count?: number;
|
||||||
arbiter_verdict?: string | null;
|
arbiter_verdict?: string | null;
|
||||||
arbiter_requested_at?: string | null;
|
arbiter_requested_at?: string | null;
|
||||||
completion_reason?: string | null;
|
completion_reason?: string | null;
|
||||||
@@ -37,7 +45,10 @@ export function requestArbiterOrEscalate(args: {
|
|||||||
escalateLogMessage,
|
escalateLogMessage,
|
||||||
logContext,
|
logContext,
|
||||||
} = args;
|
} = args;
|
||||||
if (isArbiterEnabled()) {
|
const interventionCount = args.arbiterInterventionCount ?? 0;
|
||||||
|
const maxInterventions =
|
||||||
|
args.maxArbiterInterventions ?? ARBITER_MAX_INTERVENTIONS;
|
||||||
|
if (isArbiterEnabled() && interventionCount < maxInterventions) {
|
||||||
const transitioned = transitionPairedTaskStatus({
|
const transitioned = transitionPairedTaskStatus({
|
||||||
taskId,
|
taskId,
|
||||||
currentStatus,
|
currentStatus,
|
||||||
@@ -55,6 +66,33 @@ export function requestArbiterOrEscalate(args: {
|
|||||||
return transitioned;
|
return transitioned;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isArbiterEnabled() && interventionCount >= maxInterventions) {
|
||||||
|
const transitioned = transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus,
|
||||||
|
nextStatus: 'completed',
|
||||||
|
expectedUpdatedAt,
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
...args.patch,
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'escalated',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (transitioned) {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
...(logContext ?? { taskId }),
|
||||||
|
arbiterInterventionCount: interventionCount,
|
||||||
|
maxArbiterInterventions: maxInterventions,
|
||||||
|
},
|
||||||
|
'Arbiter intervention budget exhausted — escalating to user instead of re-invoking arbiter',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return transitioned;
|
||||||
|
}
|
||||||
|
|
||||||
const transitioned = transitionPairedTaskStatus({
|
const transitioned = transitionPairedTaskStatus({
|
||||||
taskId,
|
taskId,
|
||||||
currentStatus,
|
currentStatus,
|
||||||
|
|||||||
@@ -135,6 +135,16 @@ describe('paired completion signals', () => {
|
|||||||
visibleVerdict: 'done',
|
visibleVerdict: 'done',
|
||||||
}),
|
}),
|
||||||
).toEqual({ kind: 'request_owner_finalize' });
|
).toEqual({ kind: 'request_owner_finalize' });
|
||||||
|
expect(
|
||||||
|
resolveReviewerFailureSignal({
|
||||||
|
visibleVerdict: 'step_done',
|
||||||
|
}),
|
||||||
|
).toEqual({ kind: 'request_owner_changes' });
|
||||||
|
expect(
|
||||||
|
resolveReviewerFailureSignal({
|
||||||
|
visibleVerdict: 'done_with_concerns',
|
||||||
|
}),
|
||||||
|
).toEqual({ kind: 'request_owner_changes' });
|
||||||
expect(
|
expect(
|
||||||
resolveReviewerFailureSignal({
|
resolveReviewerFailureSignal({
|
||||||
visibleVerdict: 'needs_context',
|
visibleVerdict: 'needs_context',
|
||||||
|
|||||||
@@ -98,13 +98,15 @@ export function resolveReviewerFailureSignal(args: {
|
|||||||
case 'task_done':
|
case 'task_done':
|
||||||
case 'done':
|
case 'done':
|
||||||
return { kind: 'request_owner_finalize' };
|
return { kind: 'request_owner_finalize' };
|
||||||
|
case 'step_done':
|
||||||
|
case 'done_with_concerns':
|
||||||
|
return { kind: 'request_owner_changes' };
|
||||||
case 'blocked':
|
case 'blocked':
|
||||||
case 'needs_context':
|
case 'needs_context':
|
||||||
return {
|
return {
|
||||||
kind: 'complete',
|
kind: 'complete',
|
||||||
completionReason: 'escalated',
|
completionReason: 'escalated',
|
||||||
};
|
};
|
||||||
case 'done_with_concerns':
|
|
||||||
case 'continue':
|
case 'continue':
|
||||||
default:
|
default:
|
||||||
return { kind: 'preserve_review_ready' };
|
return { kind: 'preserve_review_ready' };
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
|
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||||
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
||||||
import { classifyArbiterVerdict } from './paired-verdict.js';
|
import { classifyArbiterVerdict } from './paired-verdict.js';
|
||||||
import type { PairedTask } from './types.js';
|
import type { PairedTask } from './types.js';
|
||||||
@@ -8,8 +9,42 @@ const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0;
|
|||||||
export function handleFailedArbiterExecution(args: {
|
export function handleFailedArbiterExecution(args: {
|
||||||
task: PairedTask;
|
task: PairedTask;
|
||||||
taskId: string;
|
taskId: string;
|
||||||
|
summary?: string | null;
|
||||||
}): void {
|
}): void {
|
||||||
const { task, taskId } = args;
|
const { task, taskId, summary } = args;
|
||||||
|
if (isTerminalCodexAccountFailure(summary)) {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const nextOwnerFailureCount = Math.max(
|
||||||
|
(task.owner_failure_count ?? 0) + 1,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'active',
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
owner_failure_count: nextOwnerFailureCount,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'arbiter',
|
||||||
|
status: task.status,
|
||||||
|
nextOwnerFailureCount,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
|
'Returned arbiter task to owner after terminal Codex account failure',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const fallbackStatus =
|
const fallbackStatus =
|
||||||
task.status === 'in_arbitration' || task.status === 'arbiter_requested'
|
task.status === 'in_arbitration' || task.status === 'arbiter_requested'
|
||||||
@@ -43,9 +78,19 @@ export function handleArbiterCompletion(args: {
|
|||||||
const { task, taskId, summary } = args;
|
const { task, taskId, summary } = args;
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const arbiterVerdict = classifyArbiterVerdict(summary);
|
const arbiterVerdict = classifyArbiterVerdict(summary);
|
||||||
|
// Persist a running count of arbiter interventions so the loop can be bounded.
|
||||||
|
// This must NOT be reset by the round_trip_count reset below — otherwise the
|
||||||
|
// owner↔reviewer loop could re-trigger the arbiter without limit.
|
||||||
|
const nextArbiterInterventionCount =
|
||||||
|
(task.arbiter_intervention_count ?? 0) + 1;
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, arbiterVerdict, summary: summary?.slice(0, 200) },
|
{
|
||||||
|
taskId,
|
||||||
|
arbiterVerdict,
|
||||||
|
arbiterInterventionCount: nextArbiterInterventionCount,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
'Arbiter verdict rendered',
|
'Arbiter verdict rendered',
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -63,6 +108,7 @@ export function handleArbiterCompletion(args: {
|
|||||||
owner_step_done_streak: 0,
|
owner_step_done_streak: 0,
|
||||||
finalize_step_done_count: 0,
|
finalize_step_done_count: 0,
|
||||||
empty_step_done_streak: 0,
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_intervention_count: nextArbiterInterventionCount,
|
||||||
arbiter_verdict: arbiterVerdict,
|
arbiter_verdict: arbiterVerdict,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
},
|
},
|
||||||
@@ -86,12 +132,17 @@ export function handleArbiterCompletion(args: {
|
|||||||
owner_step_done_streak: 0,
|
owner_step_done_streak: 0,
|
||||||
finalize_step_done_count: 0,
|
finalize_step_done_count: 0,
|
||||||
empty_step_done_streak: 0,
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_intervention_count: nextArbiterInterventionCount,
|
||||||
arbiter_verdict: arbiterVerdict,
|
arbiter_verdict: arbiterVerdict,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
logger.info(
|
logger.info(
|
||||||
{ taskId, arbiterVerdict },
|
{
|
||||||
|
taskId,
|
||||||
|
arbiterVerdict,
|
||||||
|
arbiterInterventionCount: nextArbiterInterventionCount,
|
||||||
|
},
|
||||||
'Arbiter requested owner changes — resuming owner flow',
|
'Arbiter requested owner changes — resuming owner flow',
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
|
|||||||
138
src/paired-execution-context-owner-follow-up.test.ts
Normal file
138
src/paired-execution-context-owner-follow-up.test.ts
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./db.js', () => {
|
||||||
|
const updatePairedTask = vi.fn();
|
||||||
|
return {
|
||||||
|
createPairedTask: vi.fn(),
|
||||||
|
getLatestPairedTaskForChat: vi.fn(),
|
||||||
|
getLatestOpenPairedTaskForChat: vi.fn(),
|
||||||
|
getPairedTaskById: vi.fn(),
|
||||||
|
getPairedTurnOutputs: vi.fn(() => []),
|
||||||
|
insertPairedTurnOutput: vi.fn(),
|
||||||
|
updatePairedTask,
|
||||||
|
updatePairedTaskIfUnchanged: vi.fn((id, _expectedUpdatedAt, updates) => {
|
||||||
|
updatePairedTask(id, updates);
|
||||||
|
return true;
|
||||||
|
}),
|
||||||
|
upsertPairedProject: vi.fn(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock('./paired-workspace-manager.js', () => ({
|
||||||
|
isOwnerWorkspaceRepairNeededError: vi.fn(() => false),
|
||||||
|
markPairedTaskReviewReady: vi.fn(),
|
||||||
|
prepareReviewerWorkspaceForExecution: vi.fn(),
|
||||||
|
provisionOwnerWorkspaceForPairedTask: vi.fn(() => ({
|
||||||
|
id: 'task-1:owner',
|
||||||
|
task_id: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
workspace_dir: '/tmp/paired/task-1/owner',
|
||||||
|
snapshot_source_dir: null,
|
||||||
|
snapshot_ref: null,
|
||||||
|
status: 'ready',
|
||||||
|
snapshot_refreshed_at: null,
|
||||||
|
created_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
updated_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./logger.js', () => ({
|
||||||
|
logger: {
|
||||||
|
debug: vi.fn(),
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import * as config from './config.js';
|
||||||
|
import * as db from './db.js';
|
||||||
|
import { preparePairedExecutionContext } from './paired-execution-context.js';
|
||||||
|
import type { PairedTask, RegisteredGroup, RoomRoleContext } from './types.js';
|
||||||
|
|
||||||
|
const group: RegisteredGroup = {
|
||||||
|
name: 'Paired Room',
|
||||||
|
folder: 'paired-room',
|
||||||
|
trigger: '@codex',
|
||||||
|
added_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
agentType: 'codex',
|
||||||
|
workDir: '/repo/canonical',
|
||||||
|
};
|
||||||
|
|
||||||
|
const ownerContext: RoomRoleContext = {
|
||||||
|
serviceId: config.CODEX_MAIN_SERVICE_ID,
|
||||||
|
role: 'owner',
|
||||||
|
ownerServiceId: config.CODEX_MAIN_SERVICE_ID,
|
||||||
|
reviewerServiceId: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||||
|
failoverOwner: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||||
|
return {
|
||||||
|
id: 'task-1',
|
||||||
|
chat_jid: 'dc:test',
|
||||||
|
group_folder: group.folder,
|
||||||
|
owner_service_id: config.CODEX_MAIN_SERVICE_ID,
|
||||||
|
reviewer_service_id: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||||
|
title: null,
|
||||||
|
source_ref: 'HEAD',
|
||||||
|
plan_notes: null,
|
||||||
|
review_requested_at: null,
|
||||||
|
round_trip_count: 0,
|
||||||
|
owner_failure_count: 0,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
task_done_then_user_reopen_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
status: 'active',
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
created_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
updated_at: '2026-03-28T00:00:00.000Z',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('paired owner follow-up preparation', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetAllMocks();
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not reset owner failure counters for scheduled owner follow-up turns', () => {
|
||||||
|
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
owner_failure_count: 1,
|
||||||
|
owner_step_done_streak: 2,
|
||||||
|
empty_step_done_streak: 2,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
preparePairedExecutionContext({
|
||||||
|
group,
|
||||||
|
chatJid: 'dc:test',
|
||||||
|
runId: 'run-owner-follow-up-no-reset',
|
||||||
|
roomRoleContext: ownerContext,
|
||||||
|
hasHumanMessage: true,
|
||||||
|
pairedTurnIdentity: {
|
||||||
|
turnId: 'task-1:2026-03-28T00:00:00.000Z:owner-follow-up',
|
||||||
|
taskId: 'task-1',
|
||||||
|
taskUpdatedAt: '2026-03-28T00:00:00.000Z',
|
||||||
|
intentKind: 'owner-follow-up',
|
||||||
|
role: 'owner',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
round_trip_count: 0,
|
||||||
|
owner_failure_count: 0,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
getPairedWorkspace,
|
getPairedWorkspace,
|
||||||
hasActiveCiWatcherForChat,
|
hasActiveCiWatcherForChat,
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
|
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
||||||
import { requestArbiterOrEscalate } from './paired-arbiter-request.js';
|
import { requestArbiterOrEscalate } from './paired-arbiter-request.js';
|
||||||
@@ -21,6 +22,7 @@ import type { PairedTask } from './types.js';
|
|||||||
|
|
||||||
type OwnerFinalizeOutcome = 'stop' | 're_review';
|
type OwnerFinalizeOutcome = 'stop' | 're_review';
|
||||||
const OWNER_FAILURE_ESCALATION_THRESHOLD = 2;
|
const OWNER_FAILURE_ESCALATION_THRESHOLD = 2;
|
||||||
|
const OWNER_CODEX_UNAVAILABLE_USER_ESCALATION_THRESHOLD = 4;
|
||||||
const EMPTY_STEP_DONE_THRESHOLD = 2;
|
const EMPTY_STEP_DONE_THRESHOLD = 2;
|
||||||
|
|
||||||
export function handleFailedOwnerExecution(args: {
|
export function handleFailedOwnerExecution(args: {
|
||||||
@@ -31,6 +33,107 @@ export function handleFailedOwnerExecution(args: {
|
|||||||
const { task, taskId, summary } = args;
|
const { task, taskId, summary } = args;
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const nextFailureCount = (task.owner_failure_count ?? 0) + 1;
|
const nextFailureCount = (task.owner_failure_count ?? 0) + 1;
|
||||||
|
if (isTerminalCodexAccountFailure(summary)) {
|
||||||
|
if (nextFailureCount >= OWNER_CODEX_UNAVAILABLE_USER_ESCALATION_THRESHOLD) {
|
||||||
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'completed',
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
owner_failure_count: nextFailureCount,
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'escalated',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'owner',
|
||||||
|
previousStatus: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
|
summary: summary?.slice(0, 160),
|
||||||
|
},
|
||||||
|
'Escalated owner task after persistent Codex account failures',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
||||||
|
requestArbiterOrEscalate({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
now,
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
|
arbiterLogMessage:
|
||||||
|
'Owner Codex unavailable repeatedly — requesting arbiter',
|
||||||
|
escalateLogMessage:
|
||||||
|
'Owner Codex unavailable repeatedly — escalating to user',
|
||||||
|
logContext: {
|
||||||
|
taskId,
|
||||||
|
role: 'owner',
|
||||||
|
previousStatus: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
|
summary: summary?.slice(0, 160),
|
||||||
|
},
|
||||||
|
patch: {
|
||||||
|
owner_failure_count: nextFailureCount,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
completion_reason: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const patch = {
|
||||||
|
owner_failure_count: nextFailureCount,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
};
|
||||||
|
if (task.status === 'active' || task.status === 'merge_ready') {
|
||||||
|
applyPairedTaskPatch({
|
||||||
|
taskId,
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch,
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'owner',
|
||||||
|
status: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
|
'Preserved owner task after terminal Codex account failure',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'active',
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch,
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'owner',
|
||||||
|
status: task.status,
|
||||||
|
ownerFailureCount: nextFailureCount,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
|
'Reset owner task to active after terminal Codex account failure',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
||||||
requestArbiterOrEscalate({
|
requestArbiterOrEscalate({
|
||||||
@@ -38,6 +141,7 @@ export function handleFailedOwnerExecution(args: {
|
|||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
now,
|
now,
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
arbiterLogMessage:
|
arbiterLogMessage:
|
||||||
'Owner failed repeatedly without a visible verdict — requesting arbiter',
|
'Owner failed repeatedly without a visible verdict — requesting arbiter',
|
||||||
escalateLogMessage:
|
escalateLogMessage:
|
||||||
@@ -135,6 +239,7 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
now,
|
now,
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
arbiterLogMessage,
|
arbiterLogMessage,
|
||||||
escalateLogMessage,
|
escalateLogMessage,
|
||||||
logContext: {
|
logContext: {
|
||||||
@@ -164,6 +269,7 @@ function handleOwnerFinalizeCompletion(args: {
|
|||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
now,
|
now,
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
arbiterLogMessage:
|
arbiterLogMessage:
|
||||||
'Owner repeated STEP_DONE during finalize without code changes — requesting arbiter',
|
'Owner repeated STEP_DONE during finalize without code changes — requesting arbiter',
|
||||||
escalateLogMessage:
|
escalateLogMessage:
|
||||||
@@ -363,6 +469,7 @@ export function handleOwnerCompletion(args: {
|
|||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
now,
|
now,
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter',
|
arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter',
|
||||||
escalateLogMessage: 'Owner blocked/needs_context — escalating to user',
|
escalateLogMessage: 'Owner blocked/needs_context — escalating to user',
|
||||||
logContext: {
|
logContext: {
|
||||||
@@ -389,6 +496,7 @@ export function handleOwnerCompletion(args: {
|
|||||||
currentStatus: task.status,
|
currentStatus: task.status,
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
now,
|
now,
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
arbiterLogMessage:
|
arbiterLogMessage:
|
||||||
'Owner repeated STEP_DONE without code changes — requesting arbiter',
|
'Owner repeated STEP_DONE without code changes — requesting arbiter',
|
||||||
escalateLogMessage:
|
escalateLogMessage:
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
|
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
|
||||||
import { getPairedWorkspace } from './db.js';
|
import { getPairedWorkspace } from './db.js';
|
||||||
|
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||||
import { logger } from './logger.js';
|
import { logger } from './logger.js';
|
||||||
import { requestArbiterOrEscalate } from './paired-arbiter-request.js';
|
import { requestArbiterOrEscalate } from './paired-arbiter-request.js';
|
||||||
import {
|
import {
|
||||||
@@ -11,7 +12,7 @@ import {
|
|||||||
resolveReviewerFailureSignal,
|
resolveReviewerFailureSignal,
|
||||||
} from './paired-completion-signals.js';
|
} from './paired-completion-signals.js';
|
||||||
import { resolveCanonicalSourceRef } from './paired-source-ref.js';
|
import { resolveCanonicalSourceRef } from './paired-source-ref.js';
|
||||||
import { parseVisibleVerdict } from './paired-verdict.js';
|
import { parseReviewerVerdict } from './paired-verdict.js';
|
||||||
import type { PairedTask } from './types.js';
|
import type { PairedTask } from './types.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -33,14 +34,40 @@ export function handleFailedReviewerExecution(args: {
|
|||||||
const { task, taskId, summary } = args;
|
const { task, taskId, summary } = args;
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
|
|
||||||
|
if (isTerminalCodexAccountFailure(summary)) {
|
||||||
|
transitionPairedTaskStatus({
|
||||||
|
taskId,
|
||||||
|
currentStatus: task.status,
|
||||||
|
nextStatus: 'completed',
|
||||||
|
expectedUpdatedAt: task.updated_at,
|
||||||
|
updatedAt: now,
|
||||||
|
patch: {
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'reviewer_codex_unavailable',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role: 'reviewer',
|
||||||
|
status: task.status,
|
||||||
|
summary: summary?.slice(0, 200),
|
||||||
|
},
|
||||||
|
'Completed reviewer task after terminal Codex account failure instead of preserving review loop',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (summary) {
|
if (summary) {
|
||||||
const verdict = parseVisibleVerdict(summary);
|
const verdict = parseReviewerVerdict(summary);
|
||||||
const signal = resolveReviewerFailureSignal({
|
const signal = resolveReviewerFailureSignal({
|
||||||
visibleVerdict: verdict,
|
visibleVerdict: verdict,
|
||||||
});
|
});
|
||||||
if (
|
if (
|
||||||
signal.kind === 'request_owner_finalize' ||
|
signal.kind === 'request_owner_finalize' ||
|
||||||
signal.kind === 'complete'
|
signal.kind === 'complete' ||
|
||||||
|
signal.kind === 'request_owner_changes'
|
||||||
) {
|
) {
|
||||||
const ownerWs =
|
const ownerWs =
|
||||||
signal.kind === 'request_owner_finalize'
|
signal.kind === 'request_owner_finalize'
|
||||||
@@ -56,7 +83,9 @@ export function handleFailedReviewerExecution(args: {
|
|||||||
nextStatus:
|
nextStatus:
|
||||||
signal.kind === 'request_owner_finalize'
|
signal.kind === 'request_owner_finalize'
|
||||||
? 'merge_ready'
|
? 'merge_ready'
|
||||||
: 'completed',
|
: signal.kind === 'request_owner_changes'
|
||||||
|
? 'active'
|
||||||
|
: 'completed',
|
||||||
expectedUpdatedAt: task.updated_at,
|
expectedUpdatedAt: task.updated_at,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
patch: {
|
patch: {
|
||||||
@@ -162,7 +191,7 @@ export function handleReviewerCompletion(args: {
|
|||||||
}): void {
|
}): void {
|
||||||
const { task, taskId, summary } = args;
|
const { task, taskId, summary } = args;
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const verdict = parseVisibleVerdict(summary);
|
const verdict = parseReviewerVerdict(summary);
|
||||||
const signal = resolveReviewerCompletionSignal({
|
const signal = resolveReviewerCompletionSignal({
|
||||||
visibleVerdict: verdict,
|
visibleVerdict: verdict,
|
||||||
roundTripCount: task.round_trip_count,
|
roundTripCount: task.round_trip_count,
|
||||||
@@ -221,6 +250,7 @@ export function handleReviewerCompletion(args: {
|
|||||||
arbiterLogMessage,
|
arbiterLogMessage,
|
||||||
escalateLogMessage,
|
escalateLogMessage,
|
||||||
logContext: { taskId, verdict, summary: summary?.slice(0, 100) },
|
logContext: { taskId, verdict, summary: summary?.slice(0, 100) },
|
||||||
|
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||||
patch: { reviewer_failure_count: 0 },
|
patch: { reviewer_failure_count: 0 },
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -104,6 +104,34 @@ describe('paired execution routing loop guards', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('treats a reviewer PROCEED as approval and routes to owner finalize', () => {
|
||||||
|
// Regression: Codex reviewers approve with an arbiter-style "PROCEED" line.
|
||||||
|
// It used to parse as 'continue' (a change request), so the reviewer's
|
||||||
|
// approval bounced the task back to active and owner↔reviewer ping-ponged
|
||||||
|
// (owner TASK_DONE ↔ reviewer PROCEED) until the deadlock cap. A PROCEED
|
||||||
|
// approval must move the task to merge_ready so the owner can finalize.
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'in_review',
|
||||||
|
source_ref: 'approved-ref',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'reviewer',
|
||||||
|
status: 'succeeded',
|
||||||
|
summary: 'PROCEED\n위치와 첨부는 맞습니다.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'merge_ready',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('clears stale owner loop state when reviewer approval is recovered from a failed run', () => {
|
it('clears stale owner loop state when reviewer approval is recovered from a failed run', () => {
|
||||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
buildPairedTask({
|
buildPairedTask({
|
||||||
@@ -197,6 +225,96 @@ describe('paired execution routing loop guards', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('returns arbiter terminal Codex account failures to owner without re-arming arbiter', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'in_arbitration',
|
||||||
|
owner_failure_count: 1,
|
||||||
|
owner_step_done_streak: 3,
|
||||||
|
finalize_step_done_count: 1,
|
||||||
|
empty_step_done_streak: 2,
|
||||||
|
arbiter_verdict: 'revise',
|
||||||
|
arbiter_requested_at: '2026-03-28T00:00:04.000Z',
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'arbiter',
|
||||||
|
status: 'failed',
|
||||||
|
summary:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'active',
|
||||||
|
owner_failure_count: 2,
|
||||||
|
owner_step_done_streak: 0,
|
||||||
|
finalize_step_done_count: 0,
|
||||||
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('completes reviewer task after terminal Codex account failure instead of preserving review_ready loop', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'in_review',
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'reviewer',
|
||||||
|
status: 'failed',
|
||||||
|
summary:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'completed',
|
||||||
|
arbiter_verdict: 'escalate',
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: 'reviewer_codex_unavailable',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves owner task after terminal Codex account failure instead of completing silently', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'active',
|
||||||
|
updated_at: '2026-03-28T00:00:05.000Z',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary:
|
||||||
|
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
owner_failure_count: 1,
|
||||||
|
arbiter_verdict: null,
|
||||||
|
arbiter_requested_at: null,
|
||||||
|
completion_reason: null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => {
|
it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => {
|
||||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
buildPairedTask({
|
buildPairedTask({
|
||||||
|
|||||||
@@ -148,6 +148,7 @@ function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
|||||||
finalize_step_done_count: 0,
|
finalize_step_done_count: 0,
|
||||||
task_done_then_user_reopen_count: 0,
|
task_done_then_user_reopen_count: 0,
|
||||||
empty_step_done_streak: 0,
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_intervention_count: 0,
|
||||||
status: 'active',
|
status: 'active',
|
||||||
arbiter_verdict: null,
|
arbiter_verdict: null,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
@@ -1225,6 +1226,81 @@ describe('paired execution context', () => {
|
|||||||
).not.toHaveBeenCalled();
|
).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('escalates to the user instead of re-invoking the arbiter once the intervention budget is exhausted', () => {
|
||||||
|
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(true);
|
||||||
|
|
||||||
|
const repoDir = createCanonicalRepoWithCommit('reviewed');
|
||||||
|
const approvedSourceRef = resolveTreeRef(repoDir);
|
||||||
|
fs.writeFileSync(path.join(repoDir, 'README.md'), 'changed again\n');
|
||||||
|
execFileSync('git', ['add', 'README.md'], {
|
||||||
|
cwd: repoDir,
|
||||||
|
stdio: 'ignore',
|
||||||
|
});
|
||||||
|
execFileSync('git', ['commit', '-m', 'code change'], {
|
||||||
|
cwd: repoDir,
|
||||||
|
stdio: 'ignore',
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'merge_ready',
|
||||||
|
source_ref: approvedSourceRef,
|
||||||
|
round_trip_count: config.ARBITER_DEADLOCK_THRESHOLD,
|
||||||
|
// Arbiter has already intervened the maximum allowed number of times.
|
||||||
|
arbiter_intervention_count: config.ARBITER_MAX_INTERVENTIONS,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
vi.mocked(db.getPairedWorkspace).mockImplementation((_taskId, role) =>
|
||||||
|
role === 'owner' ? buildWorkspace('owner', repoDir) : undefined,
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'succeeded',
|
||||||
|
summary: 'DONE',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'completed',
|
||||||
|
completion_reason: 'escalated',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({ status: 'arbiter_requested' }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('increments the arbiter intervention count and keeps it across the round-trip reset on REVISE', () => {
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'in_arbitration',
|
||||||
|
round_trip_count: config.ARBITER_DEADLOCK_THRESHOLD,
|
||||||
|
arbiter_intervention_count: 0,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'arbiter',
|
||||||
|
status: 'succeeded',
|
||||||
|
summary: 'VERDICT: REVISE\nOwner should address the reviewer feedback.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
'task-1',
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'active',
|
||||||
|
round_trip_count: 0,
|
||||||
|
arbiter_intervention_count: 1,
|
||||||
|
arbiter_verdict: 'revise',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it.each(['BLOCKED', 'NEEDS_CONTEXT'])(
|
it.each(['BLOCKED', 'NEEDS_CONTEXT'])(
|
||||||
'escalates immediately when owner reports %s during finalize without arbiter',
|
'escalates immediately when owner reports %s during finalize without arbiter',
|
||||||
(summary) => {
|
(summary) => {
|
||||||
@@ -1473,3 +1549,64 @@ describe('paired execution context', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('paired execution context completion lease cleanup', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('propagates completion handler errors when no execution lease was reserved', () => {
|
||||||
|
const transitionError = new Error('transition failed without lease');
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||||
|
buildPairedTask({
|
||||||
|
status: 'merge_ready',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
vi.mocked(db.updatePairedTaskIfUnchanged).mockImplementationOnce(() => {
|
||||||
|
throw transitionError;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(() =>
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: 'task-1',
|
||||||
|
role: 'owner',
|
||||||
|
status: 'failed',
|
||||||
|
summary: 'push failed',
|
||||||
|
}),
|
||||||
|
).toThrow('transition failed without lease');
|
||||||
|
expect(db.releasePairedTaskExecutionLease).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('paired execution context reviewer failures', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('moves failed reviewer STEP_DONE output back to active for owner follow-up', () => {
|
||||||
|
const task = buildPairedTask({
|
||||||
|
status: 'in_review',
|
||||||
|
updated_at: '2026-03-28T00:05:00.000Z',
|
||||||
|
});
|
||||||
|
vi.mocked(db.getPairedTaskById).mockReturnValue(task);
|
||||||
|
|
||||||
|
completePairedExecutionContext({
|
||||||
|
taskId: task.id,
|
||||||
|
role: 'reviewer',
|
||||||
|
status: 'failed',
|
||||||
|
runId: 'run-failed-reviewer-step-done',
|
||||||
|
summary: 'STEP_DONE\n오너 수정이 더 필요합니다.',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||||
|
task.id,
|
||||||
|
expect.objectContaining({
|
||||||
|
status: 'active',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(db.releasePairedTaskExecutionLease).toHaveBeenCalledWith({
|
||||||
|
taskId: task.id,
|
||||||
|
runId: 'run-failed-reviewer-step-done',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -146,6 +146,7 @@ function createActiveTaskForRoom(args: {
|
|||||||
finalize_step_done_count: 0,
|
finalize_step_done_count: 0,
|
||||||
task_done_then_user_reopen_count: 0,
|
task_done_then_user_reopen_count: 0,
|
||||||
empty_step_done_streak: 0,
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_intervention_count: 0,
|
||||||
status: 'active',
|
status: 'active',
|
||||||
arbiter_verdict: null,
|
arbiter_verdict: null,
|
||||||
arbiter_requested_at: null,
|
arbiter_requested_at: null,
|
||||||
@@ -413,6 +414,24 @@ export interface PairedExecutionRecoveryPlan {
|
|||||||
prompt: string;
|
prompt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isOwnerContinuationTurn(
|
||||||
|
turnIdentity: PairedTurnIdentity | undefined,
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
turnIdentity?.role === 'owner' && turnIdentity.intentKind !== 'owner-turn'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldResetOwnerLoopCounters(args: {
|
||||||
|
hasHumanMessage?: boolean;
|
||||||
|
pairedTurnIdentity?: PairedTurnIdentity;
|
||||||
|
}): boolean {
|
||||||
|
if (args.hasHumanMessage !== true) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return !isOwnerContinuationTurn(args.pairedTurnIdentity);
|
||||||
|
}
|
||||||
|
|
||||||
export function preparePairedExecutionContext(args: {
|
export function preparePairedExecutionContext(args: {
|
||||||
group: RegisteredGroup;
|
group: RegisteredGroup;
|
||||||
chatJid: string;
|
chatJid: string;
|
||||||
@@ -459,7 +478,7 @@ export function preparePairedExecutionContext(args: {
|
|||||||
// merge_ready is split into a fresh task before this function runs.
|
// merge_ready is split into a fresh task before this function runs.
|
||||||
// Only reset round_trip_count when a human message is present —
|
// Only reset round_trip_count when a human message is present —
|
||||||
// bot-only ping-pong must accumulate the counter for loop detection.
|
// bot-only ping-pong must accumulate the counter for loop detection.
|
||||||
const hasHuman = args.hasHumanMessage === true;
|
const hasHuman = shouldResetOwnerLoopCounters(args);
|
||||||
const needsStatusReset =
|
const needsStatusReset =
|
||||||
latestTask.status === 'review_ready' || latestTask.status === 'in_review';
|
latestTask.status === 'review_ready' || latestTask.status === 'in_review';
|
||||||
if (hasHuman || needsStatusReset) {
|
if (hasHuman || needsStatusReset) {
|
||||||
@@ -478,6 +497,7 @@ export function preparePairedExecutionContext(args: {
|
|||||||
reviewer_failure_count: 0,
|
reviewer_failure_count: 0,
|
||||||
owner_step_done_streak: 0,
|
owner_step_done_streak: 0,
|
||||||
empty_step_done_streak: 0,
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_intervention_count: 0,
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
},
|
},
|
||||||
@@ -495,6 +515,7 @@ export function preparePairedExecutionContext(args: {
|
|||||||
reviewer_failure_count: 0,
|
reviewer_failure_count: 0,
|
||||||
owner_step_done_streak: 0,
|
owner_step_done_streak: 0,
|
||||||
empty_step_done_streak: 0,
|
empty_step_done_streak: 0,
|
||||||
|
arbiter_intervention_count: 0,
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
},
|
},
|
||||||
@@ -622,15 +643,18 @@ export function preparePairedExecutionContext(args: {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function completePairedExecutionContext(args: {
|
type CompletePairedExecutionContextArgs = {
|
||||||
taskId: string;
|
taskId: string;
|
||||||
role: PairedRoomRole;
|
role: PairedRoomRole;
|
||||||
status: 'succeeded' | 'failed';
|
status: 'succeeded' | 'failed';
|
||||||
runId?: string;
|
runId?: string;
|
||||||
summary?: string | null;
|
summary?: string | null;
|
||||||
}): void {
|
};
|
||||||
|
|
||||||
|
export function completePairedExecutionContext(
|
||||||
|
args: CompletePairedExecutionContextArgs,
|
||||||
|
): void {
|
||||||
const { taskId, role, status } = args;
|
const { taskId, role, status } = args;
|
||||||
let completionError: unknown;
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
taskId,
|
taskId,
|
||||||
@@ -642,96 +666,126 @@ export function completePairedExecutionContext(args: {
|
|||||||
);
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const task = getPairedTaskById(taskId);
|
completePairedExecutionContextBody(args);
|
||||||
if (!task) {
|
} catch (error) {
|
||||||
return;
|
releaseExecutionLeaseAfterCompletion({
|
||||||
}
|
taskId,
|
||||||
if (task.status === 'completed') {
|
role,
|
||||||
logger.info(
|
runId: args.runId,
|
||||||
{
|
completionError: error,
|
||||||
taskId,
|
});
|
||||||
role,
|
throw error;
|
||||||
status,
|
}
|
||||||
completionReason: task.completion_reason ?? null,
|
|
||||||
},
|
|
||||||
'Ignoring late paired execution completion for an already completed task',
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status !== 'succeeded') {
|
releaseExecutionLeaseAfterCompletion({
|
||||||
if (role === 'reviewer') {
|
taskId,
|
||||||
handleFailedReviewerExecution({
|
role,
|
||||||
|
runId: args.runId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function completePairedExecutionContextBody(
|
||||||
|
args: CompletePairedExecutionContextArgs,
|
||||||
|
): void {
|
||||||
|
const { taskId, role, status } = args;
|
||||||
|
const task = getPairedTaskById(taskId);
|
||||||
|
if (!task) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (task.status === 'completed') {
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role,
|
||||||
|
status,
|
||||||
|
completionReason: task.completion_reason ?? null,
|
||||||
|
},
|
||||||
|
'Ignoring late paired execution completion for an already completed task',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status !== 'succeeded') {
|
||||||
|
if (role === 'reviewer') {
|
||||||
|
handleFailedReviewerExecution({
|
||||||
|
task,
|
||||||
|
taskId,
|
||||||
|
summary: args.summary,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (role === 'arbiter') {
|
||||||
|
handleFailedArbiterExecution({
|
||||||
|
task,
|
||||||
|
taskId,
|
||||||
|
summary: args.summary,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
handleFailedOwnerExecution({ task, taskId, summary: args.summary });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (role === 'owner') {
|
||||||
|
try {
|
||||||
|
provisionOwnerWorkspaceForPairedTask(taskId);
|
||||||
|
} catch (error) {
|
||||||
|
if (isOwnerWorkspaceRepairNeededError(error)) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
taskId,
|
||||||
|
role,
|
||||||
|
repairMessage: error.blockMessage || error.message,
|
||||||
|
},
|
||||||
|
'Owner workspace post-run guard blocked completion handling',
|
||||||
|
);
|
||||||
|
handleFailedOwnerExecution({
|
||||||
task,
|
task,
|
||||||
taskId,
|
taskId,
|
||||||
summary: args.summary,
|
summary: error.blockMessage || error.message,
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (role === 'arbiter') {
|
throw error;
|
||||||
handleFailedArbiterExecution({ task, taskId });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
handleFailedOwnerExecution({ task, taskId, summary: args.summary });
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
handleOwnerCompletion({ task, taskId, summary: args.summary });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (role === 'owner') {
|
if (role === 'reviewer') {
|
||||||
try {
|
handleReviewerCompletion({ task, taskId, summary: args.summary });
|
||||||
provisionOwnerWorkspaceForPairedTask(taskId);
|
return;
|
||||||
} catch (error) {
|
}
|
||||||
if (isOwnerWorkspaceRepairNeededError(error)) {
|
|
||||||
logger.warn(
|
|
||||||
{
|
|
||||||
taskId,
|
|
||||||
role,
|
|
||||||
repairMessage: error.blockMessage || error.message,
|
|
||||||
},
|
|
||||||
'Owner workspace post-run guard blocked completion handling',
|
|
||||||
);
|
|
||||||
handleFailedOwnerExecution({
|
|
||||||
task,
|
|
||||||
taskId,
|
|
||||||
summary: error.blockMessage || error.message,
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
handleOwnerCompletion({ task, taskId, summary: args.summary });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (role === 'reviewer') {
|
if (role === 'arbiter') {
|
||||||
handleReviewerCompletion({ task, taskId, summary: args.summary });
|
handleArbiterCompletion({ task, taskId, summary: args.summary });
|
||||||
return;
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (role === 'arbiter') {
|
function releaseExecutionLeaseAfterCompletion(args: {
|
||||||
handleArbiterCompletion({ task, taskId, summary: args.summary });
|
taskId: string;
|
||||||
}
|
role: PairedRoomRole;
|
||||||
} catch (error) {
|
runId?: string;
|
||||||
completionError = error;
|
completionError?: unknown;
|
||||||
throw error;
|
}): void {
|
||||||
} finally {
|
if (!args.runId) {
|
||||||
if (!args.runId) {
|
return;
|
||||||
return;
|
}
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
releasePairedTaskExecutionLease({ taskId, runId: args.runId });
|
releasePairedTaskExecutionLease({ taskId: args.taskId, runId: args.runId });
|
||||||
} catch (releaseError) {
|
} catch (releaseError) {
|
||||||
logger.error(
|
logger.error(
|
||||||
{
|
{
|
||||||
taskId,
|
taskId: args.taskId,
|
||||||
role,
|
role: args.role,
|
||||||
runId: args.runId,
|
runId: args.runId,
|
||||||
releaseError,
|
releaseError,
|
||||||
},
|
},
|
||||||
'Failed to release paired task execution lease after completion',
|
'Failed to release paired task execution lease after completion',
|
||||||
);
|
);
|
||||||
if (!completionError) {
|
if (!args.completionError) {
|
||||||
throw releaseError;
|
throw releaseError;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user