Compare commits
69 Commits
453157f6c3
...
822ac34c0e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
822ac34c0e | ||
|
|
e14ac3dfca | ||
|
|
1d8358d1e1 | ||
|
|
be9f2379c0 | ||
|
|
a621e85432 | ||
|
|
80ddb1aa0d | ||
|
|
ee4c5559b1 | ||
|
|
73fd71b39a | ||
|
|
9c46cf6761 | ||
|
|
aec70bc388 | ||
|
|
1a85ebf7eb | ||
|
|
728a8c2ec9 | ||
|
|
f6ce3122bf | ||
|
|
4ddaa88f8a | ||
|
|
b3a927b241 | ||
|
|
ef2b26e5da | ||
|
|
0ca24debfd | ||
|
|
edeeed9476 | ||
|
|
e324a1baa1 | ||
|
|
efc8c00380 | ||
|
|
2313d6cf3e | ||
|
|
65510e0fc3 | ||
|
|
de54f13469 | ||
|
|
5f06673ac6 | ||
|
|
dca56d9493 | ||
|
|
1dd948a7ac | ||
|
|
7eb97dd921 | ||
|
|
8883cd166d | ||
|
|
5c47024e44 | ||
|
|
337d46461c | ||
|
|
29ca23a558 | ||
|
|
80bc078a24 | ||
|
|
184a8eff6a | ||
|
|
ac0cd3e41d | ||
|
|
1af074786b | ||
|
|
8cf351580b | ||
|
|
d7b5166862 | ||
|
|
c001905678 | ||
|
|
f1679ac686 | ||
|
|
807828e0ce | ||
|
|
879d16235f | ||
|
|
0cbb7e3b9c | ||
|
|
32468e0214 | ||
|
|
2dab27f443 | ||
|
|
22602da1ec | ||
|
|
63a4515241 | ||
|
|
b4197875fa | ||
|
|
3d36e15f67 | ||
|
|
3894d4f406 | ||
|
|
7fc558acb6 | ||
|
|
5648b61075 | ||
|
|
03d4c81192 | ||
|
|
f28399021a | ||
|
|
f4de795b1e | ||
|
|
db715c6329 | ||
|
|
90c17c6890 | ||
|
|
e3c8de61f1 | ||
|
|
0d87778b78 | ||
|
|
047f0d3b70 | ||
|
|
65059719ce | ||
|
|
32fbb00dd2 | ||
|
|
cdb59ce9fe | ||
|
|
85085e2782 | ||
|
|
cf9bf08197 | ||
|
|
890991394d | ||
|
|
4e90e1d7ec | ||
|
|
7ed0af98a2 | ||
|
|
51512c9aeb | ||
|
|
1509108e04 |
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"env": {
|
||||
"CLAUDE_AUTOCOMPACT_PCT_OVERRIDE": "50"
|
||||
"CLAUDE_AUTOCOMPACT_PCT_OVERRIDE": "70"
|
||||
}
|
||||
}
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -39,6 +39,7 @@ tmp/
|
||||
backups/
|
||||
dist-backups/
|
||||
.deploy-backups/
|
||||
recovery/
|
||||
cache/
|
||||
runners/*/node_modules/
|
||||
runners/codex-runner-backups/
|
||||
|
||||
@@ -204,7 +204,7 @@ function buildRoomOptions(snapshots: StatusSnapshot[]): RoomOption[] {
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...rooms.values()].sort((a, b) =>
|
||||
return Array.from(rooms.values()).sort((a, b) =>
|
||||
`${a.name} ${a.folder}`.localeCompare(`${b.name} ${b.folder}`),
|
||||
);
|
||||
}
|
||||
@@ -247,7 +247,7 @@ function DashboardErrorCard({
|
||||
<strong>{t.error.api}</strong>
|
||||
<small>{humanizeError(error, t)}</small>
|
||||
</span>
|
||||
<button disabled={refreshing} onClick={onRetry}>
|
||||
<button disabled={refreshing} onClick={onRetry} type="button">
|
||||
{t.actions.retry}
|
||||
</button>
|
||||
</section>
|
||||
|
||||
207
apps/dashboard/src/GlassesApp.tsx
Normal file
207
apps/dashboard/src/GlassesApp.tsx
Normal file
@@ -0,0 +1,207 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
import {
|
||||
type DashboardInboxAction,
|
||||
type DashboardOverview,
|
||||
type StatusSnapshot,
|
||||
fetchDashboardData,
|
||||
runInboxAction,
|
||||
sendRoomMessage,
|
||||
} from './api';
|
||||
import { isLocale, matchLocale, type Locale } from './i18n';
|
||||
import { GlassesPanel } from './GlassesPanel';
|
||||
import './styles.css';
|
||||
import './glasses.css';
|
||||
|
||||
interface GlassesState {
|
||||
overview: DashboardOverview;
|
||||
snapshots: StatusSnapshot[];
|
||||
}
|
||||
|
||||
type InboxItem = DashboardOverview['inbox'][number];
|
||||
type InboxActionKey = `${string}:${DashboardInboxAction}`;
|
||||
|
||||
const REFRESH_INTERVAL_MS = 15_000;
|
||||
const DASHBOARD_STALE_MS = 75_000;
|
||||
const LOCALE_STORAGE_KEY = 'ejclaw.dashboard.locale.v2';
|
||||
|
||||
function makeClientRequestId(): string {
|
||||
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
function readInitialLocale(): Locale {
|
||||
const stored =
|
||||
typeof window === 'undefined'
|
||||
? null
|
||||
: window.localStorage.getItem(LOCALE_STORAGE_KEY);
|
||||
if (isLocale(stored)) return stored;
|
||||
|
||||
const languages =
|
||||
typeof navigator === 'undefined'
|
||||
? []
|
||||
: [...(navigator.languages || []), navigator.language];
|
||||
for (const language of languages) {
|
||||
const matched = matchLocale(language);
|
||||
if (matched) return matched;
|
||||
}
|
||||
|
||||
return 'en';
|
||||
}
|
||||
|
||||
function dashboardAgeMs(value: string | null | undefined): number | null {
|
||||
if (!value) return null;
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return null;
|
||||
return Math.max(0, Date.now() - date.getTime());
|
||||
}
|
||||
|
||||
function freshnessLabel(
|
||||
online: boolean,
|
||||
generatedAt: string | null | undefined,
|
||||
): string {
|
||||
if (!online) return 'offline';
|
||||
const age = dashboardAgeMs(generatedAt);
|
||||
if (age !== null && age > DASHBOARD_STALE_MS) return 'stale';
|
||||
return 'fresh';
|
||||
}
|
||||
|
||||
function readNickname(): string {
|
||||
if (typeof window === 'undefined') return '';
|
||||
return window.localStorage.getItem('ejclaw-nickname') ?? '';
|
||||
}
|
||||
|
||||
export function GlassesApp() {
|
||||
const [data, setData] = useState<GlassesState | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [online, setOnline] = useState(() =>
|
||||
typeof navigator === 'undefined' ? true : navigator.onLine,
|
||||
);
|
||||
const [locale] = useState<Locale>(readInitialLocale);
|
||||
const [inboxActionKey, setInboxActionKey] = useState<InboxActionKey | null>(
|
||||
null,
|
||||
);
|
||||
const [roomMessageKey, setRoomMessageKey] = useState<string | null>(null);
|
||||
|
||||
async function refresh(showSpinner = false) {
|
||||
if (showSpinner) setRefreshing(true);
|
||||
try {
|
||||
const nextData = await fetchDashboardData();
|
||||
setData({
|
||||
overview: nextData.overview,
|
||||
snapshots: nextData.snapshots,
|
||||
});
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
setRefreshing(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleInboxAction(
|
||||
item: InboxItem,
|
||||
action: DashboardInboxAction,
|
||||
) {
|
||||
const actionKey: InboxActionKey = `${item.id}:${action}`;
|
||||
setInboxActionKey(actionKey);
|
||||
try {
|
||||
await runInboxAction(item.id, action, {
|
||||
lastOccurredAt: item.lastOccurredAt,
|
||||
requestId: makeClientRequestId(),
|
||||
});
|
||||
await refresh(false);
|
||||
return true;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
return false;
|
||||
} finally {
|
||||
setInboxActionKey(null);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRoomMessage(
|
||||
roomJid: string,
|
||||
text: string,
|
||||
requestId: string,
|
||||
) {
|
||||
setRoomMessageKey(roomJid);
|
||||
try {
|
||||
await sendRoomMessage(roomJid, text, requestId, readNickname() || null);
|
||||
void refresh(false);
|
||||
return true;
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
return false;
|
||||
} finally {
|
||||
setRoomMessageKey(null);
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
document.documentElement.lang = locale;
|
||||
}, [locale]);
|
||||
|
||||
useEffect(() => {
|
||||
function handleOnline() {
|
||||
setOnline(true);
|
||||
}
|
||||
|
||||
function handleOffline() {
|
||||
setOnline(false);
|
||||
}
|
||||
|
||||
window.addEventListener('online', handleOnline);
|
||||
window.addEventListener('offline', handleOffline);
|
||||
return () => {
|
||||
window.removeEventListener('online', handleOnline);
|
||||
window.removeEventListener('offline', handleOffline);
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
const id = window.setInterval(() => {
|
||||
void refresh();
|
||||
}, REFRESH_INTERVAL_MS);
|
||||
return () => window.clearInterval(id);
|
||||
}, []);
|
||||
|
||||
if (loading && !data) {
|
||||
return (
|
||||
<main className="glasses-shell" aria-busy="true">
|
||||
<div className="glasses-empty">
|
||||
<strong>EJClaw</strong>
|
||||
<span>Loading</span>
|
||||
</div>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return (
|
||||
<main className="glasses-shell">
|
||||
<p className="glasses-error">{error ?? 'Dashboard unavailable'}</p>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<GlassesPanel
|
||||
createRequestId={makeClientRequestId}
|
||||
error={error}
|
||||
freshnessText={freshnessLabel(online, data.overview.generatedAt)}
|
||||
inboxActionKey={inboxActionKey}
|
||||
locale={locale}
|
||||
onInboxAction={handleInboxAction}
|
||||
onRefresh={() => void refresh(true)}
|
||||
onSendRoomMessage={handleRoomMessage}
|
||||
overview={data.overview}
|
||||
refreshing={refreshing}
|
||||
roomMessageKey={roomMessageKey}
|
||||
snapshots={data.snapshots}
|
||||
/>
|
||||
);
|
||||
}
|
||||
120
apps/dashboard/src/GlassesPanel.test.ts
Normal file
120
apps/dashboard/src/GlassesPanel.test.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import { createElement } from 'react';
|
||||
import { renderToStaticMarkup } from 'react-dom/server';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { GlassesPanel } from './GlassesPanel';
|
||||
import type { DashboardOverview, StatusSnapshot } from './api';
|
||||
|
||||
const overview: DashboardOverview = {
|
||||
generatedAt: '2026-06-07T13:00:00.000Z',
|
||||
services: [],
|
||||
rooms: {
|
||||
total: 1,
|
||||
active: 1,
|
||||
waiting: 0,
|
||||
inactive: 0,
|
||||
},
|
||||
tasks: {
|
||||
total: 1,
|
||||
active: 1,
|
||||
paused: 0,
|
||||
completed: 0,
|
||||
watchers: {
|
||||
active: 0,
|
||||
paused: 0,
|
||||
completed: 0,
|
||||
},
|
||||
},
|
||||
usage: {
|
||||
rows: [],
|
||||
fetchedAt: null,
|
||||
},
|
||||
inbox: [
|
||||
{
|
||||
createdAt: '2026-06-07T12:59:30.000Z',
|
||||
groupKey: 'task:review',
|
||||
id: 'inbox-1',
|
||||
kind: 'reviewer-request',
|
||||
lastOccurredAt: '2026-06-07T12:59:30.000Z',
|
||||
occurrences: 1,
|
||||
occurredAt: '2026-06-07T12:59:30.000Z',
|
||||
roomJid: 'dc:ops',
|
||||
roomName: 'Ops',
|
||||
severity: 'warn',
|
||||
source: 'paired-task',
|
||||
summary: 'Reviewer requested owner changes',
|
||||
taskId: 'task-1',
|
||||
taskStatus: 'active',
|
||||
title: 'Review follow-up',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const snapshots: StatusSnapshot[] = [
|
||||
{
|
||||
agentType: 'codex',
|
||||
assistantName: 'Codex',
|
||||
serviceId: 'codex-main',
|
||||
updatedAt: '2026-06-07T13:00:00.000Z',
|
||||
entries: [
|
||||
{
|
||||
agentType: 'codex',
|
||||
elapsedMs: 12000,
|
||||
folder: 'ejclaw',
|
||||
jid: 'dc:ops',
|
||||
name: 'Ops',
|
||||
pendingMessages: false,
|
||||
pendingTasks: 1,
|
||||
status: 'processing',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
describe('GlassesPanel', () => {
|
||||
it('renders a compact display queue and voice input surface', () => {
|
||||
const html = renderToStaticMarkup(
|
||||
createElement(GlassesPanel, {
|
||||
createRequestId: () => 'req-1',
|
||||
error: null,
|
||||
freshnessText: 'fresh',
|
||||
inboxActionKey: null,
|
||||
locale: 'ko',
|
||||
onInboxAction: async () => true,
|
||||
onRefresh: () => {},
|
||||
onSendRoomMessage: async () => true,
|
||||
overview,
|
||||
refreshing: false,
|
||||
roomMessageKey: null,
|
||||
snapshots,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(html).toContain('glasses-shell');
|
||||
expect(html).toContain('Review follow-up');
|
||||
expect(html).toContain('리뷰');
|
||||
expect(html).toContain('Voice');
|
||||
});
|
||||
|
||||
it('tolerates partial status snapshot payloads while data is refreshing', () => {
|
||||
const html = renderToStaticMarkup(
|
||||
createElement(GlassesPanel, {
|
||||
createRequestId: () => 'req-1',
|
||||
error: null,
|
||||
freshnessText: 'fresh',
|
||||
inboxActionKey: null,
|
||||
locale: 'ko',
|
||||
onInboxAction: async () => true,
|
||||
onRefresh: () => {},
|
||||
onSendRoomMessage: async () => true,
|
||||
overview,
|
||||
refreshing: false,
|
||||
roomMessageKey: null,
|
||||
snapshots: [{} as StatusSnapshot],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(html).toContain('glasses-shell');
|
||||
expect(html).toContain('Review follow-up');
|
||||
});
|
||||
});
|
||||
607
apps/dashboard/src/GlassesPanel.tsx
Normal file
607
apps/dashboard/src/GlassesPanel.tsx
Normal file
@@ -0,0 +1,607 @@
|
||||
import {
|
||||
type KeyboardEvent,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from 'react';
|
||||
import { Check, Inbox, Mic, RefreshCw, Send, X } from 'lucide-react';
|
||||
|
||||
import {
|
||||
type DashboardInboxAction,
|
||||
type DashboardOverview,
|
||||
type StatusSnapshot,
|
||||
} from './api';
|
||||
import { formatDate } from './dashboardHelpers';
|
||||
import type { Locale } from './i18n';
|
||||
|
||||
type InboxItem = DashboardOverview['inbox'][number];
|
||||
type GlassesMode = 'queue' | 'voice';
|
||||
|
||||
interface RoomChoice {
|
||||
jid: string;
|
||||
label: string;
|
||||
status: StatusSnapshot['entries'][number]['status'];
|
||||
pendingTasks: number;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionResultLike {
|
||||
readonly 0?: { transcript?: string };
|
||||
}
|
||||
|
||||
interface SpeechRecognitionEventLike {
|
||||
readonly results: ArrayLike<SpeechRecognitionResultLike>;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionLike {
|
||||
continuous: boolean;
|
||||
interimResults: boolean;
|
||||
lang: string;
|
||||
maxAlternatives: number;
|
||||
onend: (() => void) | null;
|
||||
onerror: (() => void) | null;
|
||||
onresult: ((event: SpeechRecognitionEventLike) => void) | null;
|
||||
start: () => void;
|
||||
}
|
||||
|
||||
type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
|
||||
|
||||
interface SpeechWindow extends Window {
|
||||
SpeechRecognition?: SpeechRecognitionConstructor;
|
||||
webkitSpeechRecognition?: SpeechRecognitionConstructor;
|
||||
}
|
||||
|
||||
export interface GlassesPanelProps {
|
||||
createRequestId: () => string;
|
||||
error: string | null;
|
||||
freshnessText: string;
|
||||
inboxActionKey: `${string}:${DashboardInboxAction}` | null;
|
||||
locale: Locale;
|
||||
onInboxAction: (
|
||||
item: InboxItem,
|
||||
action: DashboardInboxAction,
|
||||
) => Promise<boolean>;
|
||||
onRefresh: () => void;
|
||||
onSendRoomMessage: (
|
||||
roomJid: string,
|
||||
text: string,
|
||||
requestId: string,
|
||||
) => Promise<boolean>;
|
||||
overview: DashboardOverview;
|
||||
refreshing: boolean;
|
||||
roomMessageKey: string | null;
|
||||
snapshots: StatusSnapshot[];
|
||||
}
|
||||
|
||||
const QUEUE_KIND_ORDER: Record<InboxItem['kind'], number> = {
|
||||
approval: 0,
|
||||
'reviewer-request': 1,
|
||||
'arbiter-request': 2,
|
||||
'ci-failure': 3,
|
||||
mention: 4,
|
||||
'pending-room': 5,
|
||||
};
|
||||
|
||||
const SEVERITY_ORDER: Record<InboxItem['severity'], number> = {
|
||||
error: 0,
|
||||
warn: 1,
|
||||
info: 2,
|
||||
};
|
||||
|
||||
const SPEECH_LOCALES: Record<Locale, string> = {
|
||||
en: 'en-US',
|
||||
ja: 'ja-JP',
|
||||
ko: 'ko-KR',
|
||||
zh: 'zh-CN',
|
||||
};
|
||||
|
||||
function getSpeechRecognition(): SpeechRecognitionConstructor | null {
|
||||
if (typeof window === 'undefined') return null;
|
||||
const speechWindow = window as SpeechWindow;
|
||||
return (
|
||||
speechWindow.SpeechRecognition ??
|
||||
speechWindow.webkitSpeechRecognition ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function inboxActionsFor(item: InboxItem): DashboardInboxAction[] {
|
||||
if (
|
||||
item.source === 'paired-task' &&
|
||||
(item.kind === 'reviewer-request' ||
|
||||
item.kind === 'approval' ||
|
||||
item.kind === 'arbiter-request')
|
||||
) {
|
||||
return ['run', 'decline', 'dismiss'];
|
||||
}
|
||||
return ['dismiss'];
|
||||
}
|
||||
|
||||
function actionLabel(item: InboxItem, action: DashboardInboxAction): string {
|
||||
if (action === 'dismiss') return '닫기';
|
||||
if (action === 'decline') return '거절';
|
||||
if (item.kind === 'approval') return '최종화';
|
||||
if (item.kind === 'reviewer-request') return '리뷰';
|
||||
if (item.kind === 'arbiter-request') return '중재';
|
||||
return '실행';
|
||||
}
|
||||
|
||||
function actionIcon(action: DashboardInboxAction) {
|
||||
if (action === 'decline') return <X size={16} aria-hidden />;
|
||||
if (action === 'dismiss') return <Check size={16} aria-hidden />;
|
||||
return <Send size={16} aria-hidden />;
|
||||
}
|
||||
|
||||
function sortInboxItems(items: InboxItem[]): InboxItem[] {
|
||||
return [...items].sort((a, b) => {
|
||||
const severity = SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity];
|
||||
if (severity !== 0) return severity;
|
||||
const kind = QUEUE_KIND_ORDER[a.kind] - QUEUE_KIND_ORDER[b.kind];
|
||||
if (kind !== 0) return kind;
|
||||
return b.lastOccurredAt.localeCompare(a.lastOccurredAt);
|
||||
});
|
||||
}
|
||||
|
||||
function buildRoomChoices(snapshots: StatusSnapshot[]): RoomChoice[] {
|
||||
const rooms = new Map<string, RoomChoice>();
|
||||
for (const snapshot of snapshots) {
|
||||
const entries = Array.isArray(snapshot.entries) ? snapshot.entries : [];
|
||||
for (const entry of entries) {
|
||||
const existing = rooms.get(entry.jid);
|
||||
const label = entry.name || entry.folder || entry.jid;
|
||||
if (!existing) {
|
||||
rooms.set(entry.jid, {
|
||||
jid: entry.jid,
|
||||
label,
|
||||
pendingTasks: entry.pendingTasks,
|
||||
status: entry.status,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
entry.status === 'processing' ||
|
||||
entry.pendingTasks > existing.pendingTasks
|
||||
) {
|
||||
rooms.set(entry.jid, {
|
||||
jid: entry.jid,
|
||||
label,
|
||||
pendingTasks: entry.pendingTasks,
|
||||
status: entry.status,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...rooms.values()].sort((a, b) => {
|
||||
if (a.status === 'processing' && b.status !== 'processing') return -1;
|
||||
if (b.status === 'processing' && a.status !== 'processing') return 1;
|
||||
return b.pendingTasks - a.pendingTasks || a.label.localeCompare(b.label);
|
||||
});
|
||||
}
|
||||
|
||||
function clampIndex(index: number, length: number): number {
|
||||
if (length <= 0) return 0;
|
||||
return Math.max(0, Math.min(index, length - 1));
|
||||
}
|
||||
|
||||
function nextIndex(index: number, length: number, delta: number): number {
|
||||
if (length <= 0) return 0;
|
||||
return (index + delta + length) % length;
|
||||
}
|
||||
|
||||
function readTranscript(event: SpeechRecognitionEventLike): string {
|
||||
return Array.from(event.results)
|
||||
.map((result) => result[0]?.transcript?.trim() ?? '')
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
interface QueueCardProps {
|
||||
inboxActionKey: `${string}:${DashboardInboxAction}` | null;
|
||||
inboxActions: DashboardInboxAction[];
|
||||
inboxItems: InboxItem[];
|
||||
locale: Locale;
|
||||
onInboxAction: (
|
||||
item: InboxItem,
|
||||
action: DashboardInboxAction,
|
||||
) => Promise<boolean>;
|
||||
selectedActionIndex: number;
|
||||
selectedInbox: InboxItem | undefined;
|
||||
selectedInboxIndex: number;
|
||||
setSelectedActionIndex: (index: number) => void;
|
||||
}
|
||||
|
||||
function QueueCard({
|
||||
inboxActionKey,
|
||||
inboxActions,
|
||||
inboxItems,
|
||||
locale,
|
||||
onInboxAction,
|
||||
selectedActionIndex,
|
||||
selectedInbox,
|
||||
selectedInboxIndex,
|
||||
setSelectedActionIndex,
|
||||
}: QueueCardProps) {
|
||||
if (!selectedInbox) {
|
||||
return (
|
||||
<section className="glasses-card glasses-queue-card" aria-live="polite">
|
||||
<div className="glasses-empty">
|
||||
<strong>Queue clear</strong>
|
||||
<span>새 요청 없음</span>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<section className="glasses-card glasses-queue-card" aria-live="polite">
|
||||
<div className="glasses-card-head">
|
||||
<span className={`glasses-pill sev-${selectedInbox.severity}`}>
|
||||
{selectedInbox.kind}
|
||||
</span>
|
||||
<small>
|
||||
{selectedInboxIndex + 1}/{inboxItems.length}
|
||||
</small>
|
||||
</div>
|
||||
<h2>{selectedInbox.title}</h2>
|
||||
<p>{selectedInbox.summary}</p>
|
||||
<dl className="glasses-meta">
|
||||
<div>
|
||||
<dt>Target</dt>
|
||||
<dd>
|
||||
{selectedInbox.roomName ??
|
||||
selectedInbox.groupFolder ??
|
||||
selectedInbox.roomJid ??
|
||||
selectedInbox.taskId ??
|
||||
'-'}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Time</dt>
|
||||
<dd>{formatDate(selectedInbox.lastOccurredAt, locale)}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<div className="glasses-actions">
|
||||
{inboxActions.map((action, index) => {
|
||||
const actionKey = `${selectedInbox.id}:${action}`;
|
||||
const busy = inboxActionKey === actionKey;
|
||||
return (
|
||||
<button
|
||||
aria-busy={busy || undefined}
|
||||
aria-pressed={index === selectedActionIndex}
|
||||
className={
|
||||
index === selectedActionIndex ? 'is-active' : undefined
|
||||
}
|
||||
disabled={busy || Boolean(inboxActionKey)}
|
||||
key={action}
|
||||
onClick={() => {
|
||||
setSelectedActionIndex(index);
|
||||
void onInboxAction(selectedInbox, action);
|
||||
}}
|
||||
type="button"
|
||||
>
|
||||
{actionIcon(action)}
|
||||
{busy ? '처리 중' : actionLabel(selectedInbox, action)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
interface VoiceCardProps {
|
||||
canListen: boolean;
|
||||
listening: boolean;
|
||||
onSendVoiceText: () => void;
|
||||
onStartListening: () => void;
|
||||
selectedRoom: RoomChoice | undefined;
|
||||
sendingVoice: boolean;
|
||||
setSelectedRoomIndex: (index: number) => void;
|
||||
setVoiceText: (value: string) => void;
|
||||
rooms: RoomChoice[];
|
||||
voiceText: string;
|
||||
}
|
||||
|
||||
function VoiceCard({
|
||||
canListen,
|
||||
listening,
|
||||
onSendVoiceText,
|
||||
onStartListening,
|
||||
selectedRoom,
|
||||
sendingVoice,
|
||||
setSelectedRoomIndex,
|
||||
setVoiceText,
|
||||
rooms,
|
||||
voiceText,
|
||||
}: VoiceCardProps) {
|
||||
return (
|
||||
<section className="glasses-card glasses-voice-card">
|
||||
<div className="glasses-card-head">
|
||||
<span className="glasses-pill">voice</span>
|
||||
<small>{selectedRoom?.label ?? 'No room'}</small>
|
||||
</div>
|
||||
<textarea
|
||||
aria-label="음성 또는 키보드 입력"
|
||||
onChange={(event) => setVoiceText(event.target.value)}
|
||||
placeholder="말하거나 입력..."
|
||||
value={voiceText}
|
||||
/>
|
||||
<div className="glasses-room-strip" aria-label="room target">
|
||||
{rooms.slice(0, 4).map((room, index) => (
|
||||
<button
|
||||
aria-pressed={room.jid === selectedRoom?.jid}
|
||||
className={room.jid === selectedRoom?.jid ? 'is-active' : undefined}
|
||||
key={room.jid}
|
||||
onClick={() => setSelectedRoomIndex(index)}
|
||||
type="button"
|
||||
>
|
||||
{room.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="glasses-actions">
|
||||
<button
|
||||
disabled={!canListen || listening}
|
||||
onClick={onStartListening}
|
||||
type="button"
|
||||
>
|
||||
<Mic size={16} aria-hidden />
|
||||
{listening ? '듣는 중' : '말하기'}
|
||||
</button>
|
||||
<button
|
||||
className="is-active"
|
||||
disabled={!selectedRoom || !voiceText.trim() || sendingVoice}
|
||||
onClick={onSendVoiceText}
|
||||
type="button"
|
||||
>
|
||||
<Send size={16} aria-hidden />
|
||||
{sendingVoice ? '전송 중' : '전송'}
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
interface GlassesHeaderProps {
|
||||
onRefresh: () => void;
|
||||
refreshing: boolean;
|
||||
}
|
||||
|
||||
function GlassesHeader({ onRefresh, refreshing }: GlassesHeaderProps) {
|
||||
return (
|
||||
<header className="glasses-header">
|
||||
<div>
|
||||
<span className="glasses-kicker">EJClaw</span>
|
||||
<h1>Display</h1>
|
||||
</div>
|
||||
<button
|
||||
aria-busy={refreshing || undefined}
|
||||
aria-label="새로고침"
|
||||
className="glasses-icon-button"
|
||||
disabled={refreshing}
|
||||
onClick={onRefresh}
|
||||
type="button"
|
||||
>
|
||||
<RefreshCw size={18} aria-hidden />
|
||||
</button>
|
||||
</header>
|
||||
);
|
||||
}
|
||||
|
||||
interface ModeTabsProps {
|
||||
mode: GlassesMode;
|
||||
setMode: (mode: GlassesMode) => void;
|
||||
}
|
||||
|
||||
function ModeTabs({ mode, setMode }: ModeTabsProps) {
|
||||
return (
|
||||
<nav className="glasses-tabs" aria-label="display modes">
|
||||
<button
|
||||
aria-pressed={mode === 'queue'}
|
||||
className={mode === 'queue' ? 'is-active' : undefined}
|
||||
onClick={() => setMode('queue')}
|
||||
type="button"
|
||||
>
|
||||
<Inbox size={16} aria-hidden />
|
||||
Queue
|
||||
</button>
|
||||
<button
|
||||
aria-pressed={mode === 'voice'}
|
||||
className={mode === 'voice' ? 'is-active' : undefined}
|
||||
onClick={() => setMode('voice')}
|
||||
type="button"
|
||||
>
|
||||
<Mic size={16} aria-hidden />
|
||||
Voice
|
||||
</button>
|
||||
</nav>
|
||||
);
|
||||
}
|
||||
|
||||
function footerStatus(
|
||||
mode: GlassesMode,
|
||||
busyActionKey: string | null,
|
||||
selectedRoom: RoomChoice | undefined,
|
||||
): string {
|
||||
if (mode === 'queue') return busyActionKey ?? 'ready';
|
||||
return selectedRoom?.status ?? 'ready';
|
||||
}
|
||||
|
||||
export function GlassesPanel({
|
||||
createRequestId,
|
||||
error,
|
||||
freshnessText,
|
||||
inboxActionKey,
|
||||
locale,
|
||||
onInboxAction,
|
||||
onRefresh,
|
||||
onSendRoomMessage,
|
||||
overview,
|
||||
refreshing,
|
||||
roomMessageKey,
|
||||
snapshots,
|
||||
}: GlassesPanelProps) {
|
||||
const shellRef = useRef<HTMLElement | null>(null);
|
||||
const [mode, setMode] = useState<GlassesMode>('queue');
|
||||
const [selectedInboxIndex, setSelectedInboxIndex] = useState(0);
|
||||
const [selectedActionIndex, setSelectedActionIndex] = useState(0);
|
||||
const [selectedRoomIndex, setSelectedRoomIndex] = useState(0);
|
||||
const [voiceText, setVoiceText] = useState('');
|
||||
const [listening, setListening] = useState(false);
|
||||
const inboxItems = useMemo(
|
||||
() => sortInboxItems(Array.isArray(overview.inbox) ? overview.inbox : []),
|
||||
[overview.inbox],
|
||||
);
|
||||
const rooms = useMemo(
|
||||
() => buildRoomChoices(Array.isArray(snapshots) ? snapshots : []),
|
||||
[snapshots],
|
||||
);
|
||||
const selectedInbox =
|
||||
inboxItems[clampIndex(selectedInboxIndex, inboxItems.length)];
|
||||
const inboxActions = selectedInbox ? inboxActionsFor(selectedInbox) : [];
|
||||
const selectedAction =
|
||||
inboxActions[clampIndex(selectedActionIndex, inboxActions.length)];
|
||||
const selectedRoom = rooms[clampIndex(selectedRoomIndex, rooms.length)];
|
||||
const canListen = getSpeechRecognition() !== null;
|
||||
const busyActionKey =
|
||||
selectedInbox && selectedAction
|
||||
? `${selectedInbox.id}:${selectedAction}`
|
||||
: null;
|
||||
const sendingVoice = selectedRoom
|
||||
? roomMessageKey === selectedRoom.jid
|
||||
: false;
|
||||
|
||||
useEffect(() => {
|
||||
shellRef.current?.focus();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setSelectedInboxIndex((value) => clampIndex(value, inboxItems.length));
|
||||
}, [inboxItems.length]);
|
||||
|
||||
useEffect(() => {
|
||||
setSelectedActionIndex((value) => clampIndex(value, inboxActions.length));
|
||||
}, [inboxActions.length]);
|
||||
|
||||
useEffect(() => {
|
||||
setSelectedRoomIndex((value) => clampIndex(value, rooms.length));
|
||||
}, [rooms.length]);
|
||||
|
||||
function startListening() {
|
||||
const SpeechRecognition = getSpeechRecognition();
|
||||
if (!SpeechRecognition || listening) return;
|
||||
const recognition = new SpeechRecognition();
|
||||
recognition.lang = SPEECH_LOCALES[locale];
|
||||
recognition.continuous = false;
|
||||
recognition.interimResults = false;
|
||||
recognition.maxAlternatives = 1;
|
||||
recognition.onresult = (event) => {
|
||||
const nextText = readTranscript(event);
|
||||
if (nextText) setVoiceText(nextText);
|
||||
};
|
||||
recognition.onerror = () => setListening(false);
|
||||
recognition.onend = () => setListening(false);
|
||||
setListening(true);
|
||||
recognition.start();
|
||||
}
|
||||
|
||||
async function runSelectedAction() {
|
||||
if (!selectedInbox || !selectedAction || inboxActionKey) return;
|
||||
await onInboxAction(selectedInbox, selectedAction);
|
||||
}
|
||||
|
||||
async function sendVoiceText() {
|
||||
const text = voiceText.trim();
|
||||
if (!selectedRoom || !text || sendingVoice) return;
|
||||
const ok = await onSendRoomMessage(
|
||||
selectedRoom.jid,
|
||||
text,
|
||||
createRequestId(),
|
||||
);
|
||||
if (ok) setVoiceText('');
|
||||
}
|
||||
|
||||
function handleKeyDown(event: KeyboardEvent<HTMLElement>) {
|
||||
if (
|
||||
event.target instanceof HTMLInputElement ||
|
||||
event.target instanceof HTMLTextAreaElement
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') {
|
||||
event.preventDefault();
|
||||
setMode((value) => (value === 'queue' ? 'voice' : 'queue'));
|
||||
return;
|
||||
}
|
||||
if (event.key === 'ArrowUp' || event.key === 'ArrowDown') {
|
||||
event.preventDefault();
|
||||
const delta = event.key === 'ArrowDown' ? 1 : -1;
|
||||
if (mode === 'queue') {
|
||||
setSelectedInboxIndex((value) =>
|
||||
nextIndex(value, inboxItems.length, delta),
|
||||
);
|
||||
} else {
|
||||
setSelectedRoomIndex((value) => nextIndex(value, rooms.length, delta));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (event.key === 'Enter') {
|
||||
event.preventDefault();
|
||||
if (mode === 'queue') void runSelectedAction();
|
||||
else void sendVoiceText();
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<main
|
||||
aria-label="EJClaw Ray-Ban Display"
|
||||
className="glasses-shell"
|
||||
onKeyDown={handleKeyDown}
|
||||
ref={shellRef}
|
||||
tabIndex={0}
|
||||
>
|
||||
<GlassesHeader onRefresh={onRefresh} refreshing={refreshing} />
|
||||
|
||||
<section className="glasses-status-row" aria-label="상태">
|
||||
<span>{freshnessText}</span>
|
||||
<strong>{inboxItems.length} items</strong>
|
||||
</section>
|
||||
|
||||
{error ? <p className="glasses-error">{error}</p> : null}
|
||||
|
||||
<ModeTabs mode={mode} setMode={setMode} />
|
||||
|
||||
{mode === 'queue' ? (
|
||||
<QueueCard
|
||||
inboxActionKey={inboxActionKey}
|
||||
inboxActions={inboxActions}
|
||||
inboxItems={inboxItems}
|
||||
locale={locale}
|
||||
onInboxAction={onInboxAction}
|
||||
selectedActionIndex={selectedActionIndex}
|
||||
selectedInbox={selectedInbox}
|
||||
selectedInboxIndex={selectedInboxIndex}
|
||||
setSelectedActionIndex={setSelectedActionIndex}
|
||||
/>
|
||||
) : (
|
||||
<VoiceCard
|
||||
canListen={canListen}
|
||||
listening={listening}
|
||||
onSendVoiceText={() => void sendVoiceText()}
|
||||
onStartListening={startListening}
|
||||
rooms={rooms}
|
||||
selectedRoom={selectedRoom}
|
||||
sendingVoice={sendingVoice}
|
||||
setSelectedRoomIndex={setSelectedRoomIndex}
|
||||
setVoiceText={setVoiceText}
|
||||
voiceText={voiceText}
|
||||
/>
|
||||
)}
|
||||
|
||||
<footer className="glasses-footer">
|
||||
<span>{footerStatus(mode, busyActionKey, selectedRoom)}</span>
|
||||
</footer>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
@@ -300,7 +300,7 @@ export function RoomBoardV2({
|
||||
const filtered = allEntries.filter(
|
||||
(entry) => filter === 'all' || entry.status === filter,
|
||||
);
|
||||
const sorted = [...filtered].sort((a, b) => {
|
||||
const sorted = Array.from(filtered).sort((a, b) => {
|
||||
if (sort === 'name') return a.name.localeCompare(b.name);
|
||||
if (sort === 'queue') {
|
||||
const aQ = a.pendingTasks + (a.pendingMessages ? 1 : 0);
|
||||
@@ -321,10 +321,11 @@ export function RoomBoardV2({
|
||||
const nextJid = selectedEntry?.jid ?? null;
|
||||
if (nextJid !== selectedJid) {
|
||||
onSelectedJidChange(nextJid);
|
||||
setMobileDetailOpen(false);
|
||||
}
|
||||
}, [onSelectedJidChange, selectedEntry?.jid, selectedJid]);
|
||||
|
||||
const detailOpen = mobileDetailOpen && selectedEntry?.jid === selectedJid;
|
||||
|
||||
if (allEntries.length === 0) {
|
||||
return <EmptyState>{t.rooms.empty}</EmptyState>;
|
||||
}
|
||||
@@ -359,9 +360,7 @@ export function RoomBoardV2({
|
||||
{sorted.length === 0 ? (
|
||||
<EmptyState>{t.rooms.empty}</EmptyState>
|
||||
) : (
|
||||
<div
|
||||
className={`rooms-twopane${mobileDetailOpen ? ' is-detail-open' : ''}`}
|
||||
>
|
||||
<div className={`rooms-twopane${detailOpen ? ' is-detail-open' : ''}`}>
|
||||
<RoomsList
|
||||
entries={sorted}
|
||||
inbox={inbox}
|
||||
|
||||
@@ -91,6 +91,56 @@ interface TaskEditFormProps {
|
||||
t: Messages;
|
||||
}
|
||||
|
||||
const TASK_TIME_FORMATTERS: Record<Locale, Intl.DateTimeFormat> = {
|
||||
en: new Intl.DateTimeFormat(localeTags.en, {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
}),
|
||||
ja: new Intl.DateTimeFormat(localeTags.ja, {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
}),
|
||||
ko: new Intl.DateTimeFormat(localeTags.ko, {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
}),
|
||||
zh: new Intl.DateTimeFormat(localeTags.zh, {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
}),
|
||||
};
|
||||
|
||||
const EN_TASK_DATE_TIME_FORMATTER = new Intl.DateTimeFormat(localeTags.en, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
});
|
||||
|
||||
const RELATIVE_TIME_FORMATTERS: Record<Locale, Intl.RelativeTimeFormat> = {
|
||||
en: new Intl.RelativeTimeFormat(localeTags.en, {
|
||||
numeric: 'auto',
|
||||
style: 'short',
|
||||
}),
|
||||
ja: new Intl.RelativeTimeFormat(localeTags.ja, {
|
||||
numeric: 'auto',
|
||||
style: 'short',
|
||||
}),
|
||||
ko: new Intl.RelativeTimeFormat(localeTags.ko, {
|
||||
numeric: 'auto',
|
||||
style: 'short',
|
||||
}),
|
||||
zh: new Intl.RelativeTimeFormat(localeTags.zh, {
|
||||
numeric: 'auto',
|
||||
style: 'short',
|
||||
}),
|
||||
};
|
||||
|
||||
function formatTaskDate(
|
||||
value: string | null | undefined,
|
||||
locale: Locale,
|
||||
@@ -98,22 +148,12 @@ function formatTaskDate(
|
||||
if (!value) return '-';
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return value;
|
||||
const time = new Intl.DateTimeFormat(localeTags[locale], {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
}).format(date);
|
||||
const time = TASK_TIME_FORMATTERS[locale].format(date);
|
||||
if (locale === 'ko')
|
||||
return `${date.getMonth() + 1}월 ${date.getDate()}일 ${time}`;
|
||||
if (locale === 'ja' || locale === 'zh')
|
||||
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
||||
return new Intl.DateTimeFormat(localeTags[locale], {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
}).format(date);
|
||||
return EN_TASK_DATE_TIME_FORMATTER.format(date);
|
||||
}
|
||||
|
||||
function formatRelativeDate(
|
||||
@@ -135,10 +175,10 @@ function formatRelativeDate(
|
||||
];
|
||||
const [unit, unitMs] =
|
||||
units.find(([, threshold]) => absMs >= threshold) ?? units.at(-1)!;
|
||||
return new Intl.RelativeTimeFormat(localeTags[locale], {
|
||||
numeric: 'auto',
|
||||
style: 'short',
|
||||
}).format(Math.round(diffMs / unitMs), unit);
|
||||
return RELATIVE_TIME_FORMATTERS[locale].format(
|
||||
Math.round(diffMs / unitMs),
|
||||
unit,
|
||||
);
|
||||
}
|
||||
|
||||
function safePreview(
|
||||
|
||||
@@ -141,7 +141,7 @@ function UsageSpeed({ row, t }: { row: UsageRow; t: Messages }) {
|
||||
export function UsagePanel({ overview, t }: UsagePanelProps) {
|
||||
const rows = useMemo(
|
||||
() =>
|
||||
[...overview.usage.rows].sort((a, b) => {
|
||||
Array.from(overview.usage.rows).sort((a, b) => {
|
||||
if (usageActive(a) !== usageActive(b)) return usageActive(a) ? -1 : 1;
|
||||
return usagePeak(b) - usagePeak(a);
|
||||
}),
|
||||
@@ -194,24 +194,28 @@ export function UsagePanel({ overview, t }: UsagePanelProps) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="usage-matrix" role="table" aria-label={t.panels.usage}>
|
||||
<div className="usage-matrix-head" role="row">
|
||||
<span>{t.usage.usage}</span>
|
||||
<span>{t.usage.quota.h5}</span>
|
||||
<span>{t.usage.quota.d7}</span>
|
||||
<span>{t.usage.speed}</span>
|
||||
</div>
|
||||
<table className="usage-matrix" aria-label={t.panels.usage}>
|
||||
<thead>
|
||||
<tr className="usage-matrix-head">
|
||||
<th scope="col">{t.usage.usage}</th>
|
||||
<th scope="col">{t.usage.quota.h5}</th>
|
||||
<th scope="col">{t.usage.quota.d7}</th>
|
||||
<th scope="col">{t.usage.speed}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
{groups.map((group) => (
|
||||
<div className="usage-group" key={group.key} role="rowgroup">
|
||||
<div className="usage-group-label" role="row">
|
||||
<span>{group.label}</span>
|
||||
</div>
|
||||
<tbody className="usage-group" key={group.key}>
|
||||
<tr className="usage-group-label">
|
||||
<th colSpan={4} scope="colgroup">
|
||||
{group.label}
|
||||
</th>
|
||||
</tr>
|
||||
{group.rows.map((row) => {
|
||||
const risk = usageRiskLevel(row);
|
||||
const { account, plan } = usageNameParts(row);
|
||||
return (
|
||||
<section className={`usage-row usage-${risk}`} key={row.name}>
|
||||
<div className="usage-account">
|
||||
<tr className={`usage-row usage-${risk}`} key={row.name}>
|
||||
<th className="usage-account" scope="row">
|
||||
<strong>{account}</strong>
|
||||
<div>
|
||||
{usageActive(row) ? (
|
||||
@@ -224,26 +228,32 @@ export function UsagePanel({ overview, t }: UsagePanelProps) {
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
<UsageQuotaMeter
|
||||
row={row}
|
||||
rowName={account}
|
||||
window="h5"
|
||||
t={t}
|
||||
/>
|
||||
<UsageQuotaMeter
|
||||
row={row}
|
||||
rowName={account}
|
||||
window="d7"
|
||||
t={t}
|
||||
/>
|
||||
<UsageSpeed row={row} t={t} />
|
||||
</section>
|
||||
</th>
|
||||
<td>
|
||||
<UsageQuotaMeter
|
||||
row={row}
|
||||
rowName={account}
|
||||
window="h5"
|
||||
t={t}
|
||||
/>
|
||||
</td>
|
||||
<td>
|
||||
<UsageQuotaMeter
|
||||
row={row}
|
||||
rowName={account}
|
||||
window="d7"
|
||||
t={t}
|
||||
/>
|
||||
</td>
|
||||
<td>
|
||||
<UsageSpeed row={row} t={t} />
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</tbody>
|
||||
))}
|
||||
</div>
|
||||
</table>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,34 @@
|
||||
import type { DashboardTask, DashboardTaskAction } from './api';
|
||||
import { localeTags, type Locale, type Messages } from './i18n';
|
||||
|
||||
const SHORT_TIME_FORMAT_OPTIONS = {
|
||||
hour: '2-digit',
|
||||
hour12: false,
|
||||
minute: '2-digit',
|
||||
} as const;
|
||||
|
||||
const MONTH_DAY_TIME_FORMAT_OPTIONS = {
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
hour12: false,
|
||||
minute: '2-digit',
|
||||
month: 'short',
|
||||
} as const;
|
||||
|
||||
const SHORT_TIME_FORMATTERS: Record<Locale, Intl.DateTimeFormat> = {
|
||||
en: new Intl.DateTimeFormat(localeTags.en, SHORT_TIME_FORMAT_OPTIONS),
|
||||
ja: new Intl.DateTimeFormat(localeTags.ja, SHORT_TIME_FORMAT_OPTIONS),
|
||||
ko: new Intl.DateTimeFormat(localeTags.ko, SHORT_TIME_FORMAT_OPTIONS),
|
||||
zh: new Intl.DateTimeFormat(localeTags.zh, SHORT_TIME_FORMAT_OPTIONS),
|
||||
};
|
||||
|
||||
const MONTH_DAY_TIME_FORMATTERS: Record<Locale, Intl.DateTimeFormat> = {
|
||||
en: new Intl.DateTimeFormat(localeTags.en, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||
ja: new Intl.DateTimeFormat(localeTags.ja, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||
ko: new Intl.DateTimeFormat(localeTags.ko, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||
zh: new Intl.DateTimeFormat(localeTags.zh, MONTH_DAY_TIME_FORMAT_OPTIONS),
|
||||
};
|
||||
|
||||
export function formatDate(
|
||||
value: string | null | undefined,
|
||||
locale: Locale,
|
||||
@@ -31,30 +59,16 @@ export function formatDate(
|
||||
}
|
||||
const sameDay = new Date().toDateString() === date.toDateString();
|
||||
if (sameDay) {
|
||||
return new Intl.DateTimeFormat(localeTags[locale], {
|
||||
hour: '2-digit',
|
||||
hour12: false,
|
||||
minute: '2-digit',
|
||||
}).format(date);
|
||||
return SHORT_TIME_FORMATTERS[locale].format(date);
|
||||
}
|
||||
const time = new Intl.DateTimeFormat(localeTags[locale], {
|
||||
hour: '2-digit',
|
||||
hour12: false,
|
||||
minute: '2-digit',
|
||||
}).format(date);
|
||||
const time = SHORT_TIME_FORMATTERS[locale].format(date);
|
||||
if (locale === 'ko')
|
||||
return `${date.getMonth() + 1}월 ${date.getDate()}일 ${time}`;
|
||||
if (locale === 'ja')
|
||||
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
||||
if (locale === 'zh')
|
||||
return `${date.getMonth() + 1}月${date.getDate()}日 ${time}`;
|
||||
return new Intl.DateTimeFormat(localeTags[locale], {
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
hour12: false,
|
||||
minute: '2-digit',
|
||||
month: 'short',
|
||||
}).format(date);
|
||||
return MONTH_DAY_TIME_FORMATTERS[locale].format(date);
|
||||
}
|
||||
|
||||
export function taskActionsFor(task: DashboardTask): DashboardTaskAction[] {
|
||||
|
||||
293
apps/dashboard/src/glasses.css
Normal file
293
apps/dashboard/src/glasses.css
Normal file
@@ -0,0 +1,293 @@
|
||||
body:has(.glasses-shell) {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
background: #000;
|
||||
}
|
||||
|
||||
.glasses-shell {
|
||||
display: grid;
|
||||
grid-template-rows: auto auto auto minmax(0, 1fr) auto;
|
||||
gap: 8px;
|
||||
width: min(100vw, 600px);
|
||||
height: min(100vh, 600px);
|
||||
min-height: 0;
|
||||
margin: 0 auto;
|
||||
padding: 14px;
|
||||
overflow: hidden;
|
||||
background: #050706;
|
||||
color: #f4f1e9;
|
||||
}
|
||||
|
||||
.glasses-header,
|
||||
.glasses-status-row,
|
||||
.glasses-card-head,
|
||||
.glasses-actions,
|
||||
.glasses-footer,
|
||||
.glasses-tabs,
|
||||
.glasses-room-strip {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.glasses-header {
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.glasses-kicker,
|
||||
.glasses-status-row,
|
||||
.glasses-card-head small,
|
||||
.glasses-meta dt,
|
||||
.glasses-footer {
|
||||
color: #a8afa6;
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.glasses-header h1 {
|
||||
margin: 0;
|
||||
font-size: 26px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.glasses-icon-button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 42px;
|
||||
height: 42px;
|
||||
padding: 0;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.glasses-status-row {
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.glasses-status-row strong {
|
||||
color: #6cb087;
|
||||
}
|
||||
|
||||
.glasses-error {
|
||||
margin: 0;
|
||||
padding: 8px 10px;
|
||||
border: 1px solid rgba(224, 100, 75, 0.45);
|
||||
border-radius: 8px;
|
||||
background: rgba(224, 100, 75, 0.12);
|
||||
color: #ffb4a4;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.glasses-tabs {
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.glasses-tabs button,
|
||||
.glasses-actions button,
|
||||
.glasses-room-strip button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 6px;
|
||||
min-height: 42px;
|
||||
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||
border-radius: 8px;
|
||||
background: rgba(255, 255, 255, 0.06);
|
||||
color: #f4f1e9;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.glasses-tabs button:hover:not(:disabled),
|
||||
.glasses-actions button:hover:not(:disabled),
|
||||
.glasses-room-strip button:hover:not(:disabled) {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
.glasses-tabs button {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.glasses-tabs button.is-active,
|
||||
.glasses-tabs button[aria-pressed='true'],
|
||||
.glasses-actions button.is-active,
|
||||
.glasses-actions button[aria-pressed='true'],
|
||||
.glasses-room-strip button.is-active,
|
||||
.glasses-room-strip button[aria-pressed='true'] {
|
||||
border-color: rgba(232, 152, 112, 0.72);
|
||||
background: rgba(214, 130, 88, 0.2);
|
||||
color: #fff4ec;
|
||||
}
|
||||
|
||||
.glasses-tabs button.is-active:hover:not(:disabled),
|
||||
.glasses-tabs button[aria-pressed='true']:hover:not(:disabled),
|
||||
.glasses-actions button.is-active:hover:not(:disabled),
|
||||
.glasses-actions button[aria-pressed='true']:hover:not(:disabled),
|
||||
.glasses-room-strip button.is-active:hover:not(:disabled),
|
||||
.glasses-room-strip button[aria-pressed='true']:hover:not(:disabled) {
|
||||
background: rgba(214, 130, 88, 0.26);
|
||||
}
|
||||
|
||||
.glasses-card {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
gap: 10px;
|
||||
min-height: 0;
|
||||
overflow: hidden;
|
||||
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||
border-radius: 8px;
|
||||
background: #111512;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.glasses-card-head {
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.glasses-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
max-width: 100%;
|
||||
min-height: 24px;
|
||||
padding: 0 8px;
|
||||
overflow: hidden;
|
||||
border: 1px solid rgba(255, 255, 255, 0.12);
|
||||
border-radius: 999px;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
color: #d7ded2;
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
text-overflow: ellipsis;
|
||||
text-transform: uppercase;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.glasses-pill.sev-error {
|
||||
border-color: rgba(224, 100, 75, 0.5);
|
||||
color: #ffb4a4;
|
||||
}
|
||||
|
||||
.glasses-pill.sev-warn {
|
||||
border-color: rgba(212, 160, 74, 0.5);
|
||||
color: #ffd38b;
|
||||
}
|
||||
|
||||
.glasses-queue-card h2 {
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
font-size: 22px;
|
||||
line-height: 1.08;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.glasses-queue-card p {
|
||||
display: -webkit-box;
|
||||
min-height: 0;
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
color: #d2d5ce;
|
||||
font-size: 14px;
|
||||
line-height: 1.35;
|
||||
-webkit-box-orient: vertical;
|
||||
-webkit-line-clamp: 4;
|
||||
}
|
||||
|
||||
.glasses-meta {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 8px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.glasses-meta div {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.glasses-meta dt,
|
||||
.glasses-meta dd {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.glasses-meta dd {
|
||||
margin: 2px 0 0;
|
||||
color: #f4f1e9;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.glasses-actions {
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.glasses-actions button {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
padding: 0 10px;
|
||||
}
|
||||
|
||||
.glasses-empty {
|
||||
display: grid;
|
||||
place-items: center;
|
||||
gap: 6px;
|
||||
min-height: 220px;
|
||||
color: #a8afa6;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.glasses-empty strong {
|
||||
color: #f4f1e9;
|
||||
font-size: 24px;
|
||||
}
|
||||
|
||||
.glasses-voice-card textarea {
|
||||
width: 100%;
|
||||
min-height: 126px;
|
||||
resize: none;
|
||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||
border-radius: 8px;
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
color: #f4f1e9;
|
||||
font: inherit;
|
||||
font-size: 18px;
|
||||
line-height: 1.35;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.glasses-voice-card textarea:focus-visible {
|
||||
border-color: #d68258;
|
||||
outline: 2px solid rgba(214, 130, 88, 0.2);
|
||||
}
|
||||
|
||||
.glasses-room-strip {
|
||||
gap: 6px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.glasses-room-strip button {
|
||||
min-width: 0;
|
||||
max-width: 50%;
|
||||
overflow: hidden;
|
||||
padding: 0 10px;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.glasses-footer {
|
||||
min-height: 18px;
|
||||
justify-content: center;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.glasses-shell {
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,13 @@ import { StrictMode } from 'react';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
|
||||
import App from './App';
|
||||
import { GlassesApp } from './GlassesApp';
|
||||
|
||||
function isGlassesRoute(): boolean {
|
||||
const pathname = window.location.pathname.replace(/\/+$/, '') || '/';
|
||||
const search = new URLSearchParams(window.location.search);
|
||||
return pathname === '/glasses' || search.get('display') === 'rayban';
|
||||
}
|
||||
|
||||
const root = document.getElementById('root');
|
||||
|
||||
@@ -10,7 +17,5 @@ if (!root) {
|
||||
}
|
||||
|
||||
createRoot(root).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>,
|
||||
<StrictMode>{isGlassesRoute() ? <GlassesApp /> : <App />}</StrictMode>,
|
||||
);
|
||||
|
||||
@@ -169,6 +169,41 @@ function mergeAdjacentBotChunks(entries: RoomThreadEntry[]): RoomThreadEntry[] {
|
||||
return merged;
|
||||
}
|
||||
|
||||
function collectOutputEntries(outputs: RoomOutput[]): RoomThreadEntry[] {
|
||||
const entries: RoomThreadEntry[] = [];
|
||||
for (const output of outputs) {
|
||||
const entry = toOutputEntry(output);
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function collectChatEntries(
|
||||
messages: RoomMessage[],
|
||||
outputEntries: RoomThreadEntry[],
|
||||
): RoomThreadEntry[] {
|
||||
const entries: RoomThreadEntry[] = [];
|
||||
for (const message of messages) {
|
||||
if (!isThreadChatMessage(message)) continue;
|
||||
const entry = toMessageEntry(message);
|
||||
if (!isDuplicateOutputMessage(entry, outputEntries)) entries.push(entry);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function collectOptimisticPending(
|
||||
pendingMessages: RoomMessage[],
|
||||
confirmedSet: Set<string>,
|
||||
): RoomThreadEntry[] {
|
||||
const entries: RoomThreadEntry[] = [];
|
||||
for (const message of pendingMessages) {
|
||||
if (confirmedSet.has(messageKey(message))) continue;
|
||||
if (isInternalProtocolPayload(message.content)) continue;
|
||||
entries.push(toMessageEntry(message));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
export function buildRoomThreadEntries({
|
||||
messages,
|
||||
outputs,
|
||||
@@ -179,20 +214,14 @@ export function buildRoomThreadEntries({
|
||||
pendingMessages?: RoomMessage[];
|
||||
}): RoomThreadEntry[] {
|
||||
const confirmedSet = new Set(messages.map(messageKey));
|
||||
const outputEntries = outputs
|
||||
.map(toOutputEntry)
|
||||
.filter((entry): entry is RoomThreadEntry => Boolean(entry));
|
||||
const chatEntries = messages
|
||||
.filter(isThreadChatMessage)
|
||||
.map(toMessageEntry)
|
||||
.filter((entry) => !isDuplicateOutputMessage(entry, outputEntries));
|
||||
const optimisticPending = pendingMessages
|
||||
.filter((message) => !confirmedSet.has(messageKey(message)))
|
||||
.filter((message) => !isInternalProtocolPayload(message.content))
|
||||
.map(toMessageEntry);
|
||||
|
||||
const entries = [...chatEntries, ...optimisticPending, ...outputEntries].sort(
|
||||
(a, b) => a.timestamp.localeCompare(b.timestamp),
|
||||
const outputEntries = collectOutputEntries(outputs);
|
||||
const chatEntries = collectChatEntries(messages, outputEntries);
|
||||
const optimisticPending = collectOptimisticPending(
|
||||
pendingMessages,
|
||||
confirmedSet,
|
||||
);
|
||||
const entries = chatEntries
|
||||
.concat(optimisticPending, outputEntries)
|
||||
.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
||||
return mergeAdjacentBotChunks(entries);
|
||||
}
|
||||
|
||||
@@ -2227,6 +2227,19 @@ dd,
|
||||
.usage-matrix {
|
||||
display: grid;
|
||||
overflow: hidden;
|
||||
border-spacing: 0;
|
||||
}
|
||||
|
||||
.usage-matrix thead,
|
||||
.usage-matrix tbody {
|
||||
display: grid;
|
||||
}
|
||||
|
||||
.usage-matrix th,
|
||||
.usage-matrix td {
|
||||
padding: 0;
|
||||
border: 0;
|
||||
text-align: inherit;
|
||||
}
|
||||
|
||||
.usage-matrix-head,
|
||||
@@ -2258,7 +2271,6 @@ dd,
|
||||
}
|
||||
|
||||
.usage-group-label {
|
||||
padding: 7px 10px;
|
||||
color: var(--accent-strong);
|
||||
font-size: 11px;
|
||||
font-weight: 800;
|
||||
@@ -2267,6 +2279,14 @@ dd,
|
||||
background: rgba(191, 95, 44, 0.07);
|
||||
}
|
||||
|
||||
.usage-group-label th {
|
||||
padding: 7px 10px;
|
||||
color: inherit;
|
||||
font-weight: inherit;
|
||||
letter-spacing: inherit;
|
||||
text-transform: inherit;
|
||||
}
|
||||
|
||||
.usage-row {
|
||||
--meter: var(--green);
|
||||
padding: 8px 10px;
|
||||
|
||||
@@ -33,7 +33,6 @@ export function useSelectedRoomActivity({
|
||||
|
||||
useEffect(() => {
|
||||
if (!active || !selectedRoomJid) {
|
||||
setRoomActivityLoading(false);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -64,5 +63,10 @@ export function useSelectedRoomActivity({
|
||||
};
|
||||
}, [active, pollMs, refreshRoom, selectedRoomJid]);
|
||||
|
||||
return { refreshRoom, roomActivity, roomActivityLoading };
|
||||
return {
|
||||
refreshRoom,
|
||||
roomActivity,
|
||||
roomActivityLoading:
|
||||
active && selectedRoomJid !== null ? roomActivityLoading : false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,3 +5,99 @@ This file is for mutable memory shared across Claude groups.
|
||||
Use it for durable facts, preferences, and shared context that may change over time.
|
||||
|
||||
Do not store platform-wide operating rules here. Those now live in `prompts/claude-platform.md`.
|
||||
|
||||
## Stored credentials
|
||||
|
||||
Shared credentials live at `/home/claude/.config/ejclaw/secrets.json` (chmod 600, owner-only). Read with the Read tool when a session in any channel asks to "저장해둔 계정토큰으로 로그인" or otherwise needs a registered token.
|
||||
|
||||
Schema: `credentials.<host>.{type, host, token, note, added_at}`.
|
||||
|
||||
Currently stored:
|
||||
- `git.tkrmagid.kr` — Gitea personal access token. Use via `Authorization: token <value>` header, or embed in HTTPS URL as `https://<user>:<token>@git.tkrmagid.kr/...`. For `git clone/push`, prefer the URL form or `git -c http.extraHeader="Authorization: token <value>" clone ...`. Do not paste the raw token into chat replies.
|
||||
- `sudo` — local sudo password for the `claude` user on this host. Use via `echo "$PW" | sudo -S <cmd>` (read the password from `credentials.sudo.password` with the Read tool, then pipe). Do not paste the raw password into chat replies.
|
||||
|
||||
To add or rotate a credential, edit `secrets.json` and append a new entry under `credentials`; update this list with the host and intended use.
|
||||
|
||||
## Room mode policy
|
||||
|
||||
All rooms default to `tribunal` (paired) mode. Owner runs the work, reviewer/arbiter (claude-code) verifies. New rooms registered via `bun setup/index.ts --step register` are also `tribunal` by default.
|
||||
|
||||
If the user in a channel says any of these — "클로드 사용하지 말자", "paired 모드 끄자", "리뷰어 끄자", "이 방은 single 로 바꿔줘" — switch that channel back to `single` mode. Use:
|
||||
|
||||
```bash
|
||||
bun -e "import { initDatabase, setExplicitRoomMode } from './src/db.js'; initDatabase(); setExplicitRoomMode('<chatJid>', 'single');"
|
||||
```
|
||||
|
||||
The reverse phrase ("paired 켜자", "리뷰어 다시 켜자") flips it back to `'tribunal'`. Acknowledge the change in chat and confirm the new mode.
|
||||
|
||||
## Git backups
|
||||
|
||||
- 2026-05-27 10:53 KST: EJClaw 정상 동작 상태를 git commit `1509108` (`backup current stable ejclaw state`)로 백업했다. "이번 요청만 리뷰어 사용" 기능 작업은 이 백업 커밋 이후에 시작한 변경이다.
|
||||
|
||||
---
|
||||
|
||||
# CLAUDE.md
|
||||
|
||||
Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
|
||||
|
||||
**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
|
||||
|
||||
## 1. Think Before Coding
|
||||
|
||||
**Don't assume. Don't hide confusion. Surface tradeoffs.**
|
||||
|
||||
Before implementing:
|
||||
- State your assumptions explicitly. If uncertain, ask.
|
||||
- If multiple interpretations exist, present them - don't pick silently.
|
||||
- If a simpler approach exists, say so. Push back when warranted.
|
||||
- If something is unclear, stop. Name what's confusing. Ask.
|
||||
|
||||
## 2. Simplicity First
|
||||
|
||||
**Minimum code that solves the problem. Nothing speculative.**
|
||||
|
||||
- No features beyond what was asked.
|
||||
- No abstractions for single-use code.
|
||||
- No "flexibility" or "configurability" that wasn't requested.
|
||||
- No error handling for impossible scenarios.
|
||||
- If you write 200 lines and it could be 50, rewrite it.
|
||||
|
||||
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
|
||||
|
||||
## 3. Surgical Changes
|
||||
|
||||
**Touch only what you must. Clean up only your own mess.**
|
||||
|
||||
When editing existing code:
|
||||
- Don't "improve" adjacent code, comments, or formatting.
|
||||
- Don't refactor things that aren't broken.
|
||||
- Match existing style, even if you'd do it differently.
|
||||
- If you notice unrelated dead code, mention it - don't delete it.
|
||||
|
||||
When your changes create orphans:
|
||||
- Remove imports/variables/functions that YOUR changes made unused.
|
||||
- Don't remove pre-existing dead code unless asked.
|
||||
|
||||
The test: Every changed line should trace directly to the user's request.
|
||||
|
||||
## 4. Goal-Driven Execution
|
||||
|
||||
**Define success criteria. Loop until verified.**
|
||||
|
||||
Transform tasks into verifiable goals:
|
||||
- "Add validation" → "Write tests for invalid inputs, then make them pass"
|
||||
- "Fix the bug" → "Write a test that reproduces it, then make it pass"
|
||||
- "Refactor X" → "Ensure tests pass before and after"
|
||||
|
||||
For multi-step tasks, state a brief plan:
|
||||
```
|
||||
1. [Step] → verify: [check]
|
||||
2. [Step] → verify: [check]
|
||||
3. [Step] → verify: [check]
|
||||
```
|
||||
|
||||
Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
|
||||
|
||||
---
|
||||
|
||||
**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.
|
||||
|
||||
3859
package-lock.json
generated
Normal file
3859
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -51,6 +51,8 @@ If the first visible line is not one of these statuses, the output is invalid; d
|
||||
- **BLOCKED** — Cannot proceed without user decision
|
||||
- **NEEDS_CONTEXT** — Missing information from user
|
||||
|
||||
Do not start with non-status review labels like **APPROVE** or **REVISE**. Use **TASK_DONE** for approval and **DONE_WITH_CONCERNS** for change requests.
|
||||
|
||||
## Rules
|
||||
|
||||
- Judge completion only by verification output. "It should work now" means run it. "I'm confident" means nothing — confidence is not evidence. "I tested earlier" means test again if code changed since. "It's a trivial change" means verify anyway
|
||||
|
||||
@@ -9,6 +9,7 @@ You are the **owner** (implementer) in this paired room.
|
||||
## Critical review
|
||||
|
||||
Before accepting any proposal from the reviewer, run it through:
|
||||
|
||||
1. **Essence** — Is the stated problem the actual problem?
|
||||
2. **Root cause** — Are we fixing the root cause or treating a symptom?
|
||||
3. **Prerequisites** — What must be true before this approach can work?
|
||||
|
||||
1
restart.sh
Normal file
1
restart.sh
Normal file
@@ -0,0 +1 @@
|
||||
systemctl --user restart ejclaw
|
||||
@@ -12,7 +12,6 @@ import path from 'path';
|
||||
import { CronExpressionParser } from 'cron-parser';
|
||||
import {
|
||||
DEFAULT_SCHEDULE_TASK_CONTEXT_MODE,
|
||||
DEFAULT_TASK_CONTEXT_MODE,
|
||||
DEFAULT_WATCH_CI_CONTEXT_MODE,
|
||||
EJCLAW_ENV,
|
||||
TASK_CONTEXT_MODES,
|
||||
|
||||
@@ -153,6 +153,7 @@ export async function runVerificationRequestDirect(
|
||||
`Failed to parse verification response from ${pathToFileURL(helperPath).href}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}${detail ? `\n${detail}` : ''}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import {
|
||||
DEFAULT_WATCH_CI_CONTEXT_MODE,
|
||||
WATCH_CI_PROMPT_PREFIX,
|
||||
} from 'ejclaw-runners-shared';
|
||||
import { WATCH_CI_PROMPT_PREFIX } from 'ejclaw-runners-shared';
|
||||
|
||||
export const DEFAULT_WATCH_CI_INTERVAL_SECONDS = 60;
|
||||
export const MIN_WATCH_CI_INTERVAL_SECONDS = 30;
|
||||
|
||||
@@ -80,9 +80,7 @@ async function measure(page: Page, sel: string) {
|
||||
}
|
||||
|
||||
async function evalExpr(page: Page, expr: string) {
|
||||
const result = await page.evaluate((e) => {
|
||||
return eval(e);
|
||||
}, expr);
|
||||
const result = await page.evaluate(expr);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ const STEPS: Record<
|
||||
'restart-stack': () => import('./restart-stack.js'),
|
||||
service: () => import('./service.js'),
|
||||
verify: () => import('./verify.js'),
|
||||
login: () => import('./login.js'),
|
||||
};
|
||||
|
||||
async function main(): Promise<void> {
|
||||
|
||||
228
setup/login.ts
Normal file
228
setup/login.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
/**
|
||||
* Step: login — Sign in to Claude (Pro/Max) via OAuth PKCE manual flow.
|
||||
*
|
||||
* Mirrors the URL Claude Code's `claude auth login --claudeai` actually
|
||||
* builds (captured from the official CLI). This is the manual / paste-code
|
||||
* variant: the user visits the authorize URL, completes login in their
|
||||
* browser, and pastes the code back. We exchange it for tokens and write
|
||||
* `~/.claude/.credentials.json`.
|
||||
*
|
||||
* Usage:
|
||||
* bun setup/index.ts --step login # phase 1: print authorize URL
|
||||
* bun setup/index.ts --step login --code <c> # phase 2: exchange the code
|
||||
*
|
||||
* Phase 1 stashes the PKCE verifier + state in /tmp/ejclaw-claude-login.json
|
||||
* (mode 0600). Phase 2 reads it back, exchanges the code, writes credentials.
|
||||
*
|
||||
* This step is run by hand for re-auth. Once `.credentials.json` exists, the
|
||||
* existing token-refresh loop (src/token-refresh.ts) keeps it fresh.
|
||||
*/
|
||||
|
||||
import crypto from 'crypto';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
import { logger } from '../src/logger.js';
|
||||
import { emitStatus } from './status.js';
|
||||
|
||||
const CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
|
||||
const AUTHORIZE_URL = 'https://claude.com/cai/oauth/authorize';
|
||||
const TOKEN_URL = 'https://platform.claude.com/v1/oauth/token';
|
||||
const REDIRECT_URI = 'https://platform.claude.com/oauth/code/callback';
|
||||
const SCOPES = [
|
||||
'org:create_api_key',
|
||||
'user:profile',
|
||||
'user:inference',
|
||||
'user:sessions:claude_code',
|
||||
'user:mcp_servers',
|
||||
'user:file_upload',
|
||||
];
|
||||
|
||||
const STATE_FILE = path.join(os.tmpdir(), 'ejclaw-claude-login.json');
|
||||
const CREDS_PATH = path.join(os.homedir(), '.claude', '.credentials.json');
|
||||
|
||||
interface PendingState {
|
||||
verifier: string;
|
||||
state: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
interface ExchangeResponse {
|
||||
access_token: string;
|
||||
refresh_token?: string;
|
||||
expires_in: number;
|
||||
scope?: string;
|
||||
account?: {
|
||||
email_address?: string;
|
||||
has_claude_max?: boolean;
|
||||
has_claude_pro?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
function base64url(buf: Buffer): string {
|
||||
return buf
|
||||
.toString('base64')
|
||||
.replace(/=+$/g, '')
|
||||
.replace(/\+/g, '-')
|
||||
.replace(/\//g, '_');
|
||||
}
|
||||
|
||||
function generatePkce(): { verifier: string; challenge: string } {
|
||||
const verifier = base64url(crypto.randomBytes(32));
|
||||
const challenge = base64url(
|
||||
crypto.createHash('sha256').update(verifier).digest(),
|
||||
);
|
||||
return { verifier, challenge };
|
||||
}
|
||||
|
||||
function buildAuthorizeUrl(state: string, challenge: string): string {
|
||||
const url = new URL(AUTHORIZE_URL);
|
||||
url.searchParams.append('code', 'true');
|
||||
url.searchParams.append('client_id', CLIENT_ID);
|
||||
url.searchParams.append('response_type', 'code');
|
||||
url.searchParams.append('redirect_uri', REDIRECT_URI);
|
||||
url.searchParams.append('scope', SCOPES.join(' '));
|
||||
url.searchParams.append('code_challenge', challenge);
|
||||
url.searchParams.append('code_challenge_method', 'S256');
|
||||
url.searchParams.append('state', state);
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function writePendingState(p: PendingState): void {
|
||||
fs.writeFileSync(STATE_FILE, JSON.stringify(p), { mode: 0o600 });
|
||||
}
|
||||
|
||||
function readPendingState(): PendingState | null {
|
||||
if (!fs.existsSync(STATE_FILE)) return null;
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')) as PendingState;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function exchangeCode(
|
||||
code: string,
|
||||
verifier: string,
|
||||
state: string,
|
||||
): Promise<ExchangeResponse> {
|
||||
// The success page can hand the user a value of `code#state` or just
|
||||
// `code` — accept both.
|
||||
const [bareCode, embeddedState] = code.split('#');
|
||||
const stateForExchange = embeddedState || state;
|
||||
const body = JSON.stringify({
|
||||
grant_type: 'authorization_code',
|
||||
code: bareCode,
|
||||
redirect_uri: REDIRECT_URI,
|
||||
client_id: CLIENT_ID,
|
||||
code_verifier: verifier,
|
||||
state: stateForExchange,
|
||||
});
|
||||
|
||||
const res = await fetch(TOKEN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => '');
|
||||
throw new Error(
|
||||
`Token exchange failed (${res.status}): ${text.slice(0, 400)}`,
|
||||
);
|
||||
}
|
||||
return (await res.json()) as ExchangeResponse;
|
||||
}
|
||||
|
||||
function writeCredentials(resp: ExchangeResponse): void {
|
||||
const expiresAt = Date.now() + resp.expires_in * 1000;
|
||||
const creds = {
|
||||
claudeAiOauth: {
|
||||
accessToken: resp.access_token,
|
||||
refreshToken: resp.refresh_token || '',
|
||||
expiresAt,
|
||||
scopes: resp.scope ? resp.scope.split(' ') : SCOPES,
|
||||
subscriptionType: resp.account?.has_claude_max
|
||||
? 'max'
|
||||
: resp.account?.has_claude_pro
|
||||
? 'pro'
|
||||
: '',
|
||||
},
|
||||
};
|
||||
const dir = path.dirname(CREDS_PATH);
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
const tmp = `${CREDS_PATH}.tmp`;
|
||||
fs.writeFileSync(tmp, JSON.stringify(creds, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmp, CREDS_PATH);
|
||||
}
|
||||
|
||||
interface Args {
|
||||
code: string | null;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): Args {
|
||||
let code: string | null = null;
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--code' && args[i + 1]) {
|
||||
code = args[++i];
|
||||
}
|
||||
}
|
||||
return { code };
|
||||
}
|
||||
|
||||
export async function run(args: string[]): Promise<void> {
|
||||
const { code } = parseArgs(args);
|
||||
|
||||
if (!code) {
|
||||
// Phase 1: produce the authorize URL.
|
||||
const { verifier, challenge } = generatePkce();
|
||||
const state = base64url(crypto.randomBytes(32));
|
||||
writePendingState({ verifier, state, createdAt: Date.now() });
|
||||
const url = buildAuthorizeUrl(state, challenge);
|
||||
|
||||
console.log('AUTHORIZE_URL=' + url);
|
||||
console.log(
|
||||
'NEXT: open the URL above in your browser, complete the login,' +
|
||||
' and re-run this command with --code <pasted_code>.',
|
||||
);
|
||||
emitStatus('LOGIN_URL', { STATUS: 'awaiting_code', URL: url });
|
||||
return;
|
||||
}
|
||||
|
||||
// Phase 2: exchange the code.
|
||||
const pending = readPendingState();
|
||||
if (!pending) {
|
||||
emitStatus('LOGIN', {
|
||||
STATUS: 'failed',
|
||||
ERROR: 'no_pending_state — run phase 1 first',
|
||||
});
|
||||
process.exit(4);
|
||||
}
|
||||
if (Date.now() - pending.createdAt > 30 * 60 * 1000) {
|
||||
logger.warn(
|
||||
'Pending login state is older than 30 minutes — proceeding anyway',
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await exchangeCode(code, pending.verifier, pending.state);
|
||||
writeCredentials(resp);
|
||||
fs.unlinkSync(STATE_FILE);
|
||||
const newScopes = (resp.scope || SCOPES.join(' ')).split(' ').sort();
|
||||
logger.info(
|
||||
{ scopes: newScopes, expiresInMin: Math.round(resp.expires_in / 60) },
|
||||
'Wrote ~/.claude/.credentials.json',
|
||||
);
|
||||
emitStatus('LOGIN', {
|
||||
STATUS: 'success',
|
||||
CREDENTIALS_PATH: CREDS_PATH,
|
||||
SCOPES: newScopes.join(','),
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
logger.error({ err: message }, 'Token exchange failed');
|
||||
emitStatus('LOGIN', { STATUS: 'failed', ERROR: message });
|
||||
process.exit(4);
|
||||
}
|
||||
}
|
||||
@@ -99,7 +99,9 @@ export function restartStackServices(
|
||||
throw error;
|
||||
}
|
||||
if (managedServiceCaller) {
|
||||
throw new Error(MANAGED_SERVICE_CALLER_FALLBACK_MESSAGE);
|
||||
throw new Error(MANAGED_SERVICE_CALLER_FALLBACK_MESSAGE, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
return restartStackServicesDirect(projectRoot, deps);
|
||||
|
||||
@@ -113,6 +113,15 @@ WorkingDirectory=${projectRoot}
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
RestartPreventExitStatus=${STARTUP_PRECONDITION_EXIT_CODE}
|
||||
# Per-turn agent runners do heavy file I/O (git clone, builds, video/ffmpeg
|
||||
# scratch) inside this service cgroup. The kernel keeps those reads as
|
||||
# reclaimable page cache, which makes systemd's MemoryCurrent climb to several
|
||||
# GB even though real (anon) usage stays a few hundred MB. MemoryHigh applies
|
||||
# gentle reclaim pressure so that cache is dropped before the number balloons,
|
||||
# while MemoryMax stays unset (infinity) so a legitimate burst is never
|
||||
# OOM-killed — it is throttled at worst.
|
||||
MemoryAccounting=yes
|
||||
MemoryHigh=3G
|
||||
${envLines.join('\n')}
|
||||
StandardOutput=append:${projectRoot}/logs/${def.logName}.log
|
||||
StandardError=append:${projectRoot}/logs/${def.logName}.error.log
|
||||
|
||||
@@ -111,6 +111,20 @@ describe('systemd unit generation', () => {
|
||||
expect(unit).toContain('RestartPreventExitStatus=78');
|
||||
});
|
||||
|
||||
it('bounds reclaimable page cache with a soft memory limit', () => {
|
||||
const unit = buildSystemdUnit(
|
||||
baseServiceDef,
|
||||
'/home/user/ejclaw',
|
||||
'/usr/bin/node',
|
||||
'/home/user',
|
||||
false,
|
||||
);
|
||||
expect(unit).toContain('MemoryAccounting=yes');
|
||||
expect(unit).toContain('MemoryHigh=3G');
|
||||
// MemoryMax must stay unset so a real burst is throttled, never OOM-killed.
|
||||
expect(unit).not.toContain('MemoryMax=');
|
||||
});
|
||||
|
||||
it('sets correct ExecStart', () => {
|
||||
const unit = buildSystemdUnit(
|
||||
baseServiceDef,
|
||||
|
||||
29
src/agent-attempt-retry.test.ts
Normal file
29
src/agent-attempt-retry.test.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { resolveAttemptRetryAction } from './agent-attempt-retry.js';
|
||||
|
||||
describe('resolveAttemptRetryAction', () => {
|
||||
it('uses the streamed Codex trigger message as the rotation message', () => {
|
||||
const errorMessage =
|
||||
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
|
||||
|
||||
const action = resolveAttemptRetryAction({
|
||||
provider: 'codex',
|
||||
canRetryClaudeCredentials: false,
|
||||
canRetryCodex: true,
|
||||
attempt: {
|
||||
sawOutput: false,
|
||||
streamedTriggerReason: {
|
||||
reason: 'auth-expired',
|
||||
message: errorMessage,
|
||||
} as any,
|
||||
},
|
||||
});
|
||||
|
||||
expect(action).toEqual({
|
||||
kind: 'codex',
|
||||
trigger: { reason: 'auth-expired' },
|
||||
rotationMessage: errorMessage,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,7 @@ import { getErrorMessage } from './utils.js';
|
||||
export interface AttemptStreamedTrigger {
|
||||
reason: AgentTriggerReason;
|
||||
retryAfterMs?: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface AttemptRetryState {
|
||||
@@ -131,7 +132,10 @@ export function resolveAttemptRetryAction(args: {
|
||||
attempt: Pick<AttemptRetryState, 'sawOutput' | 'streamedTriggerReason'>;
|
||||
rotationMessage?: string | null;
|
||||
}): AttemptRetryAction {
|
||||
const normalizedRotationMessage = args.rotationMessage ?? undefined;
|
||||
const normalizedRotationMessage =
|
||||
args.rotationMessage ??
|
||||
args.attempt.streamedTriggerReason?.message ??
|
||||
undefined;
|
||||
|
||||
const claudeTrigger = resolveClaudeRetryTrigger({
|
||||
canRetryClaudeCredentials: args.canRetryClaudeCredentials,
|
||||
|
||||
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
classifyAgentError,
|
||||
classifyClaudeAuthError,
|
||||
classifyCodexAuthError,
|
||||
detectClaudeProviderFailureMessage,
|
||||
isClaudeOrgAccessDeniedMessage,
|
||||
shouldRotateClaudeToken,
|
||||
@@ -66,6 +67,45 @@ describe('agent-error-detection', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('classifies Codex reused refresh-token errors as auth-expired', () => {
|
||||
expect(
|
||||
classifyCodexAuthError(
|
||||
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.',
|
||||
),
|
||||
).toEqual({
|
||||
category: 'auth-expired',
|
||||
reason: 'auth-expired',
|
||||
});
|
||||
});
|
||||
|
||||
it('classifies a Codex auth-expired trigger reason as auth-expired', () => {
|
||||
expect(classifyCodexAuthError('auth-expired')).toEqual({
|
||||
category: 'auth-expired',
|
||||
reason: 'auth-expired',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not classify the internal Codex pool-unavailable sentinel as an auth failure', () => {
|
||||
expect(
|
||||
classifyCodexAuthError(
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||
),
|
||||
).toEqual({ category: 'none', reason: '' });
|
||||
expect(
|
||||
classifyCodexAuthError(
|
||||
'Codex rotation pool unavailable: all rotation accounts are currently dead, rate-limited, or locked',
|
||||
),
|
||||
).toEqual({ category: 'none', reason: '' });
|
||||
});
|
||||
|
||||
it('classifies Codex workspace credit exhaustion as rate-limit', () => {
|
||||
expect(classifyAgentError('Workspace out of credits')).toEqual({
|
||||
category: 'rate-limit',
|
||||
reason: '429',
|
||||
retryAfterMs: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('marks only Claude quota/auth reasons as Claude rotation reasons', () => {
|
||||
expect(shouldRotateClaudeToken('429')).toBe(true);
|
||||
expect(shouldRotateClaudeToken('usage-exhausted')).toBe(true);
|
||||
|
||||
@@ -231,6 +231,35 @@ const NONE: AgentErrorClassification = {
|
||||
reason: '',
|
||||
};
|
||||
|
||||
export function isCodexPoolUnavailableError(
|
||||
error: string | null | undefined,
|
||||
): boolean {
|
||||
if (!error) return false;
|
||||
return (
|
||||
/all\s+codex(?:\s+rotation)?\s+accounts(?:\s+are)?\s+unavailable/i.test(
|
||||
error,
|
||||
) || /codex\s+rotation\s+pool\s+unavailable/i.test(error)
|
||||
);
|
||||
}
|
||||
|
||||
export function isTerminalCodexAccountFailure(
|
||||
error: string | null | undefined,
|
||||
): boolean {
|
||||
if (!error) return false;
|
||||
if (isCodexPoolUnavailableError(error)) return true;
|
||||
if (classifyCodexAuthError(error).category !== 'none') return true;
|
||||
const lower = error.toLowerCase();
|
||||
if (
|
||||
classifyAgentError(error).category === 'rate-limit' &&
|
||||
(lower.includes('workspace out of credits') ||
|
||||
lower.includes('out of credits') ||
|
||||
lower.includes('codex'))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify an agent error string into a category.
|
||||
* Handles patterns common to both Claude and Codex: 429, 503, network.
|
||||
@@ -249,6 +278,8 @@ export function classifyAgentError(
|
||||
lower.includes('429') ||
|
||||
lower.includes('rate limit') ||
|
||||
lower.includes('usage limit') ||
|
||||
lower.includes('out of credits') ||
|
||||
lower.includes('workspace out of credits') ||
|
||||
lower.includes('hit your limit') ||
|
||||
lower.includes('too many requests') ||
|
||||
lower.includes('rate_limit')
|
||||
@@ -260,8 +291,11 @@ export function classifyAgentError(
|
||||
return { category: 'rate-limit', reason: '429', retryAfterMs };
|
||||
}
|
||||
|
||||
// 503 / Overloaded
|
||||
const hasApi5xx = /\bapi error:\s*5\d\d\b/i.test(error);
|
||||
|
||||
// 5xx / Overloaded
|
||||
if (
|
||||
hasApi5xx ||
|
||||
lower.includes('503') ||
|
||||
lower.includes('overloaded') ||
|
||||
lower.includes('selected model is at capacity') ||
|
||||
@@ -332,14 +366,21 @@ export function classifyCodexAuthError(
|
||||
error: string | null | undefined,
|
||||
): AgentErrorClassification {
|
||||
if (!error) return NONE;
|
||||
if (isCodexPoolUnavailableError(error)) return NONE;
|
||||
const lower = error.toLowerCase();
|
||||
|
||||
if (
|
||||
lower.includes('auth-expired') ||
|
||||
lower.includes('auth expired') ||
|
||||
lower.includes('401') ||
|
||||
lower.includes('authentication_error') ||
|
||||
lower.includes('failed to authenticate') ||
|
||||
lower.includes('access token could not be refreshed') ||
|
||||
lower.includes('oauth token has expired') ||
|
||||
lower.includes('refresh token was already used') ||
|
||||
lower.includes('refresh your existing token') ||
|
||||
lower.includes('log out and sign in again') ||
|
||||
lower.includes('app_session_terminated') ||
|
||||
lower.includes('unauthorized')
|
||||
) {
|
||||
return { category: 'auth-expired', reason: 'auth-expired' };
|
||||
|
||||
@@ -5,9 +5,24 @@ import path from 'path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { EJCLAW_ENV } from 'ejclaw-runners-shared';
|
||||
|
||||
const { mockReadEnvFile, mockGetActiveCodexAuthPath } = vi.hoisted(() => ({
|
||||
const {
|
||||
mockReadEnvFile,
|
||||
mockGetActiveCodexAuthPath,
|
||||
mockGetCodexAccountCount,
|
||||
mockClaimCodexAuthLease,
|
||||
mockFindCodexAccountIndexByAuthPath,
|
||||
} = vi.hoisted(() => ({
|
||||
mockReadEnvFile: vi.fn<() => Record<string, string>>(),
|
||||
mockGetActiveCodexAuthPath: vi.fn<() => string | null>(),
|
||||
mockGetCodexAccountCount: vi.fn<() => number>(),
|
||||
mockClaimCodexAuthLease: vi.fn<
|
||||
() => {
|
||||
authPath: string;
|
||||
accountIndex: number;
|
||||
release: () => void;
|
||||
} | null
|
||||
>(),
|
||||
mockFindCodexAccountIndexByAuthPath: vi.fn<() => number | null>(),
|
||||
}));
|
||||
|
||||
vi.mock('./config.js', () => ({
|
||||
@@ -29,6 +44,9 @@ vi.mock('./env.js', () => ({
|
||||
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
getActiveCodexAuthPath: mockGetActiveCodexAuthPath,
|
||||
getCodexAccountCount: mockGetCodexAccountCount,
|
||||
claimCodexAuthLease: mockClaimCodexAuthLease,
|
||||
findCodexAccountIndexByAuthPath: mockFindCodexAccountIndexByAuthPath,
|
||||
}));
|
||||
|
||||
vi.mock('./token-rotation.js', () => ({
|
||||
@@ -159,6 +177,12 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
||||
|
||||
mockReadEnvFile.mockReset();
|
||||
mockGetActiveCodexAuthPath.mockReset();
|
||||
mockGetCodexAccountCount.mockReset();
|
||||
mockGetCodexAccountCount.mockReturnValue(0);
|
||||
mockClaimCodexAuthLease.mockReset();
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -240,6 +264,84 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
||||
expect(auth).toEqual(rotatedAuth);
|
||||
});
|
||||
|
||||
it('fails fast instead of launching Codex without OAuth when rotation accounts exist but no lease is available', () => {
|
||||
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||
fs.writeFileSync(
|
||||
rotatedAuthPath,
|
||||
JSON.stringify({
|
||||
auth_mode: 'chatgpt',
|
||||
tokens: { access_token: 'locked-token' },
|
||||
}),
|
||||
);
|
||||
mockGetCodexAccountCount.mockReturnValue(1);
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
|
||||
mockReadEnvFile.mockReturnValue({});
|
||||
|
||||
expect(() => prepareGroupEnvironment(group, false, 'dc:test')).toThrow(
|
||||
/Codex rotation pool unavailable/,
|
||||
);
|
||||
|
||||
const authPath = path.join(
|
||||
tempRoot,
|
||||
'sessions',
|
||||
group.folder,
|
||||
'services',
|
||||
'codex-main',
|
||||
'.codex',
|
||||
'auth.json',
|
||||
);
|
||||
expect(fs.existsSync(authPath)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('prepareGroupEnvironment prompt stacks', () => {
|
||||
let tempRoot: string;
|
||||
let previousCwd: string;
|
||||
let previousOpenAiKey: string | undefined;
|
||||
let previousCodexOpenAiKey: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tempRoot = fs.mkdtempSync(path.join('/tmp', 'ejclaw-agent-env-'));
|
||||
previousCwd = process.cwd();
|
||||
process.chdir(tempRoot);
|
||||
|
||||
process.env.EJ_TEST_ROOT = tempRoot;
|
||||
process.env.EJ_TEST_HOME = path.join(tempRoot, 'home');
|
||||
previousOpenAiKey = process.env.OPENAI_API_KEY;
|
||||
previousCodexOpenAiKey = process.env.CODEX_OPENAI_API_KEY;
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
delete process.env.CODEX_OPENAI_API_KEY;
|
||||
|
||||
fs.mkdirSync(process.env.EJ_TEST_HOME, { recursive: true });
|
||||
fs.mkdirSync(path.join(process.env.EJ_TEST_HOME, '.codex'), {
|
||||
recursive: true,
|
||||
});
|
||||
|
||||
mockReadEnvFile.mockReset();
|
||||
mockGetActiveCodexAuthPath.mockReset();
|
||||
mockGetCodexAccountCount.mockReset();
|
||||
mockGetCodexAccountCount.mockReturnValue(0);
|
||||
mockClaimCodexAuthLease.mockReset();
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.chdir(previousCwd);
|
||||
delete process.env.EJ_TEST_ROOT;
|
||||
delete process.env.EJ_TEST_HOME;
|
||||
if (previousOpenAiKey) process.env.OPENAI_API_KEY = previousOpenAiKey;
|
||||
else delete process.env.OPENAI_API_KEY;
|
||||
if (previousCodexOpenAiKey) {
|
||||
process.env.CODEX_OPENAI_API_KEY = previousCodexOpenAiKey;
|
||||
} else {
|
||||
delete process.env.CODEX_OPENAI_API_KEY;
|
||||
}
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('uses the failover owner prompt pack for codex-review when it owns an explicit failover lease', () => {
|
||||
vi.mocked(config.isReviewService).mockReturnValue(true);
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||
@@ -411,6 +513,12 @@ describe('prepareGroupEnvironment Codex MCP room role env', () => {
|
||||
|
||||
mockReadEnvFile.mockReset();
|
||||
mockGetActiveCodexAuthPath.mockReset();
|
||||
mockGetCodexAccountCount.mockReset();
|
||||
mockGetCodexAccountCount.mockReturnValue(0);
|
||||
mockClaimCodexAuthLease.mockReset();
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||
resetRoutingMocks();
|
||||
});
|
||||
|
||||
@@ -484,6 +592,12 @@ describe('prepareGroupEnvironment room skill overrides', () => {
|
||||
});
|
||||
mockReadEnvFile.mockReset();
|
||||
mockGetActiveCodexAuthPath.mockReset();
|
||||
mockGetCodexAccountCount.mockReset();
|
||||
mockGetCodexAccountCount.mockReturnValue(0);
|
||||
mockClaimCodexAuthLease.mockReset();
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -616,6 +730,12 @@ describe('prepareGroupEnvironment Codex goals handling', () => {
|
||||
|
||||
mockReadEnvFile.mockReset();
|
||||
mockGetActiveCodexAuthPath.mockReset();
|
||||
mockGetCodexAccountCount.mockReset();
|
||||
mockGetCodexAccountCount.mockReturnValue(0);
|
||||
mockClaimCodexAuthLease.mockReset();
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||
vi.mocked(config.isReviewService).mockReturnValue(false);
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(false);
|
||||
vi.mocked(serviceRouting.getEffectiveChannelLease).mockReturnValue({
|
||||
@@ -706,6 +826,12 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
||||
|
||||
mockReadEnvFile.mockReset();
|
||||
mockGetActiveCodexAuthPath.mockReset();
|
||||
mockGetCodexAccountCount.mockReset();
|
||||
mockGetCodexAccountCount.mockReturnValue(0);
|
||||
mockClaimCodexAuthLease.mockReset();
|
||||
mockClaimCodexAuthLease.mockReturnValue(null);
|
||||
mockFindCodexAccountIndexByAuthPath.mockReset();
|
||||
mockFindCodexAccountIndexByAuthPath.mockReturnValue(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -715,6 +841,37 @@ describe('prepareReadonlySessionEnvironment codex compatibility', () => {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('does not claim a Codex auth lease for Claude read-only sessions', () => {
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||
mockGetCodexAccountCount.mockReturnValue(1);
|
||||
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||
fs.writeFileSync(rotatedAuthPath, '{"auth_mode":"chatgpt"}\n');
|
||||
const release = vi.fn();
|
||||
mockClaimCodexAuthLease.mockReturnValue({
|
||||
authPath: rotatedAuthPath,
|
||||
accountIndex: 0,
|
||||
release,
|
||||
});
|
||||
|
||||
const sessionDir = path.join(tempRoot, 'readonly-claude-session');
|
||||
const prepared = prepareReadonlySessionEnvironment({
|
||||
sessionDir,
|
||||
chatJid: 'dc:test',
|
||||
isMain: false,
|
||||
groupFolder: 'codex-test-group',
|
||||
agentType: 'claude-code',
|
||||
memoryBriefing: 'memory briefing',
|
||||
role: 'reviewer',
|
||||
});
|
||||
|
||||
expect(prepared.codexSessionAuth).toBeNull();
|
||||
expect(mockClaimCodexAuthLease).not.toHaveBeenCalled();
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
expect(fs.existsSync(path.join(sessionDir, '.codex', 'auth.json'))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('writes matching AGENTS.md and copies host codex auth/config into the role-scoped session', () => {
|
||||
vi.mocked(serviceRouting.hasReviewerLease).mockReturnValue(true);
|
||||
|
||||
|
||||
@@ -12,7 +12,13 @@ import {
|
||||
} from './config.js';
|
||||
import { logger } from './logger.js';
|
||||
import { readEnvFile } from './env.js';
|
||||
import { getActiveCodexAuthPath } from './codex-token-rotation.js';
|
||||
import {
|
||||
claimCodexAuthLease,
|
||||
findCodexAccountIndexByAuthPath,
|
||||
getActiveCodexAuthPath,
|
||||
getCodexAccountCount,
|
||||
type CodexAuthLease,
|
||||
} from './codex-token-rotation.js';
|
||||
import { readCodexFeatureFromFile } from './codex-config-features.js';
|
||||
import { ensureClaudeSessionSettings } from './claude-session-settings.js';
|
||||
import {
|
||||
@@ -140,17 +146,32 @@ function ensureClaudeGlobalSettingsFile(sessionDir: string): void {
|
||||
fs.writeFileSync(settingsFile, '{}\n');
|
||||
}
|
||||
|
||||
function syncHostCodexSessionFiles(sessionCodexDir: string): void {
|
||||
export interface PreparedCodexSessionAuth {
|
||||
canonicalAuthPath: string;
|
||||
sessionAuthPath: string;
|
||||
accountIndex: number | null;
|
||||
lease?: CodexAuthLease;
|
||||
}
|
||||
|
||||
function syncHostCodexSessionFiles(
|
||||
sessionCodexDir: string,
|
||||
): PreparedCodexSessionAuth | null {
|
||||
const hostCodexDir = path.join(os.homedir(), '.codex');
|
||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||
|
||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||
const rotatedAuthSrc = getActiveCodexAuthPath();
|
||||
const hasRotationAccounts = getCodexAccountCount() > 0;
|
||||
const lease = claimCodexAuthLease();
|
||||
const rotatedAuthSrc =
|
||||
lease?.authPath ?? (!hasRotationAccounts ? getActiveCodexAuthPath() : null);
|
||||
const fallbackAuthSrc = path.join(hostCodexDir, 'auth.json');
|
||||
const authSrc =
|
||||
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||
? rotatedAuthSrc
|
||||
: path.join(hostCodexDir, 'auth.json');
|
||||
if (fs.existsSync(authSrc)) {
|
||||
: !hasRotationAccounts && fs.existsSync(fallbackAuthSrc)
|
||||
? fallbackAuthSrc
|
||||
: null;
|
||||
if (authSrc) {
|
||||
fs.copyFileSync(authSrc, authDst);
|
||||
} else if (fs.existsSync(authDst)) {
|
||||
fs.unlinkSync(authDst);
|
||||
@@ -163,6 +184,28 @@ function syncHostCodexSessionFiles(sessionCodexDir: string): void {
|
||||
fs.copyFileSync(src, dst);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
!rotatedAuthSrc ||
|
||||
authSrc !== rotatedAuthSrc ||
|
||||
!fs.existsSync(authDst)
|
||||
) {
|
||||
lease?.release();
|
||||
if (hasRotationAccounts) {
|
||||
throw new Error(
|
||||
'Codex rotation pool unavailable: all rotation accounts are currently dead, rate-limited, or locked; re-auth or clear stale state before launching Codex',
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
canonicalAuthPath: rotatedAuthSrc,
|
||||
sessionAuthPath: authDst,
|
||||
accountIndex:
|
||||
lease?.accountIndex ?? findCodexAccountIndexByAuthPath(rotatedAuthSrc),
|
||||
...(lease ? { lease } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function upsertEjclawMcpServerSection(args: {
|
||||
@@ -357,7 +400,7 @@ function prepareCodexSessionEnvironment(args: {
|
||||
useFailoverPromptPack: boolean;
|
||||
memoryBriefing?: string;
|
||||
skillOverrides?: StoredRoomSkillOverride[];
|
||||
}): void {
|
||||
}): PreparedCodexSessionAuth | null {
|
||||
// API key auth intentionally removed — Codex uses OAuth only.
|
||||
// Never pass any API key to Codex child process to prevent API billing.
|
||||
delete args.env.OPENAI_API_KEY;
|
||||
@@ -376,7 +419,7 @@ function prepareCodexSessionEnvironment(args: {
|
||||
if (codexEffort) args.env.CODEX_EFFORT = codexEffort;
|
||||
|
||||
const sessionCodexDir = path.join(args.sessionRootDir, '.codex');
|
||||
syncHostCodexSessionFiles(sessionCodexDir);
|
||||
const codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||
|
||||
const overlayPath = path.join(args.groupDir, '.codex', 'config.toml');
|
||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||
@@ -506,16 +549,19 @@ function prepareCodexSessionEnvironment(args: {
|
||||
delete args.env.ANTHROPIC_BASE_URL;
|
||||
delete args.env.CLAUDE_CODE_OAUTH_TOKEN;
|
||||
args.env.CODEX_HOME = sessionCodexDir;
|
||||
return codexSessionAuth;
|
||||
}
|
||||
|
||||
export interface PreparedGroupEnvironment {
|
||||
env: Record<string, string>;
|
||||
groupDir: string;
|
||||
runnerDir: string;
|
||||
codexSessionAuth?: PreparedCodexSessionAuth | null;
|
||||
}
|
||||
|
||||
export interface PreparedReadonlySessionEnvironment {
|
||||
codexHomeDir?: string;
|
||||
codexSessionAuth?: PreparedCodexSessionAuth | null;
|
||||
}
|
||||
|
||||
export function prepareGroupEnvironment(
|
||||
@@ -657,8 +703,9 @@ export function prepareGroupEnvironment(
|
||||
runtimeTaskId,
|
||||
});
|
||||
|
||||
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
|
||||
if (agentType === 'codex') {
|
||||
prepareCodexSessionEnvironment({
|
||||
codexSessionAuth = prepareCodexSessionEnvironment({
|
||||
env,
|
||||
envVars,
|
||||
projectRoot,
|
||||
@@ -677,7 +724,7 @@ export function prepareGroupEnvironment(
|
||||
prepareClaudeEnvironment({ env, envVars, group });
|
||||
}
|
||||
|
||||
return { env, groupDir, runnerDir };
|
||||
return { env, groupDir, runnerDir, codexSessionAuth };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -716,6 +763,7 @@ export function prepareReadonlySessionEnvironment(args: {
|
||||
skillOverrides,
|
||||
} = args;
|
||||
const projectRoot = process.cwd();
|
||||
let codexSessionAuth: PreparedCodexSessionAuth | null = null;
|
||||
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
ensureClaudeSessionSettings(sessionDir);
|
||||
@@ -784,32 +832,39 @@ export function prepareReadonlySessionEnvironment(args: {
|
||||
const sessionClaudeMdPath = path.join(sessionDir, 'CLAUDE.md');
|
||||
if (sessionClaudeMd) {
|
||||
fs.writeFileSync(sessionClaudeMdPath, sessionClaudeMd + '\n');
|
||||
const sessionCodexDir = path.join(sessionDir, '.codex');
|
||||
syncHostCodexSessionFiles(sessionCodexDir);
|
||||
fs.writeFileSync(
|
||||
path.join(sessionCodexDir, 'AGENTS.md'),
|
||||
sessionClaudeMd + '\n',
|
||||
);
|
||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||
const mcpServerPath = path.join(
|
||||
projectRoot,
|
||||
'runners',
|
||||
'agent-runner',
|
||||
'dist',
|
||||
'ipc-mcp-stdio.js',
|
||||
);
|
||||
if (fs.existsSync(mcpServerPath)) {
|
||||
upsertEjclawMcpServerSection({
|
||||
sessionConfigPath,
|
||||
mcpServerPath,
|
||||
ipcDir,
|
||||
hostIpcDir,
|
||||
chatJid,
|
||||
groupFolder,
|
||||
isMain,
|
||||
agentType,
|
||||
roomRole: role,
|
||||
workDir,
|
||||
if (agentType === 'codex') {
|
||||
const sessionCodexDir = path.join(sessionDir, '.codex');
|
||||
codexSessionAuth = syncHostCodexSessionFiles(sessionCodexDir);
|
||||
fs.writeFileSync(
|
||||
path.join(sessionCodexDir, 'AGENTS.md'),
|
||||
sessionClaudeMd + '\n',
|
||||
);
|
||||
const sessionConfigPath = path.join(sessionCodexDir, 'config.toml');
|
||||
const mcpServerPath = path.join(
|
||||
projectRoot,
|
||||
'runners',
|
||||
'agent-runner',
|
||||
'dist',
|
||||
'ipc-mcp-stdio.js',
|
||||
);
|
||||
if (fs.existsSync(mcpServerPath)) {
|
||||
upsertEjclawMcpServerSection({
|
||||
sessionConfigPath,
|
||||
mcpServerPath,
|
||||
ipcDir,
|
||||
hostIpcDir,
|
||||
chatJid,
|
||||
groupFolder,
|
||||
isMain,
|
||||
agentType,
|
||||
roomRole: role,
|
||||
workDir,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
fs.rmSync(path.join(sessionDir, '.codex'), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
logger.info(
|
||||
@@ -826,5 +881,5 @@ export function prepareReadonlySessionEnvironment(args: {
|
||||
} else if (fs.existsSync(sessionClaudeMdPath)) {
|
||||
fs.unlinkSync(sessionClaudeMdPath);
|
||||
}
|
||||
return { codexHomeDir };
|
||||
return { codexHomeDir, codexSessionAuth };
|
||||
}
|
||||
|
||||
@@ -22,6 +22,11 @@ interface RunSpawnedAgentProcessArgs {
|
||||
logsDir: string;
|
||||
startTime: number;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
onTerminalStreamedOutputFlushed?: (output: AgentOutput) => void;
|
||||
}
|
||||
|
||||
function isTerminalStreamedOutput(output: AgentOutput): boolean {
|
||||
return (output.phase ?? 'final') !== 'progress';
|
||||
}
|
||||
|
||||
function parseLegacyAgentOutput(stdout: string): AgentOutput {
|
||||
@@ -55,6 +60,44 @@ function logStreamedOutputDeliveryError(
|
||||
);
|
||||
}
|
||||
|
||||
function logStreamedAgentErrorOutput(
|
||||
output: AgentOutput,
|
||||
group: RegisteredGroup,
|
||||
input: AgentInput,
|
||||
): void {
|
||||
if (output.status !== 'error') return;
|
||||
logger.warn(
|
||||
{
|
||||
group: group.name,
|
||||
chatJid: input.chatJid,
|
||||
runId: input.runId,
|
||||
error: output.error,
|
||||
newSessionId: output.newSessionId,
|
||||
},
|
||||
'Streamed agent error output',
|
||||
);
|
||||
}
|
||||
|
||||
function chainStreamedOutputDelivery(args: {
|
||||
outputChain: Promise<void>;
|
||||
parsed: AgentOutput;
|
||||
onOutput: (output: AgentOutput) => Promise<void>;
|
||||
onTerminalStreamedOutputFlushed?: (output: AgentOutput) => void;
|
||||
group: RegisteredGroup;
|
||||
input: AgentInput;
|
||||
}): Promise<void> {
|
||||
return args.outputChain.then(async () => {
|
||||
try {
|
||||
await args.onOutput(args.parsed);
|
||||
if (isTerminalStreamedOutput(args.parsed)) {
|
||||
args.onTerminalStreamedOutputFlushed?.(args.parsed);
|
||||
}
|
||||
} catch (err) {
|
||||
logStreamedOutputDeliveryError(err, args.group, args.input);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function writeTimeoutLog(args: {
|
||||
logsDir: string;
|
||||
input: AgentInput;
|
||||
@@ -105,6 +148,11 @@ export function runSpawnedAgentProcess(
|
||||
let stdoutTruncated = false;
|
||||
let stderrTruncated = false;
|
||||
|
||||
// Use UTF-8 string decoding on streams to avoid splitting multi-byte
|
||||
// characters (e.g. Korean) at chunk boundaries, which produces U+FFFD.
|
||||
stdoutStream.setEncoding('utf8');
|
||||
stderrStream.setEncoding('utf8');
|
||||
|
||||
// Streaming output: parse OUTPUT_START/END marker pairs as they arrive.
|
||||
let parseBuffer = '';
|
||||
let newSessionId: string | undefined;
|
||||
@@ -183,21 +231,16 @@ export function runSpawnedAgentProcess(
|
||||
}
|
||||
hadStreamingOutput = true;
|
||||
resetTimeout();
|
||||
if (parsed.status === 'error') {
|
||||
logger.warn(
|
||||
{
|
||||
group: group.name,
|
||||
chatJid: input.chatJid,
|
||||
runId: input.runId,
|
||||
error: parsed.error,
|
||||
newSessionId: parsed.newSessionId,
|
||||
},
|
||||
'Streamed agent error output',
|
||||
);
|
||||
}
|
||||
outputChain = outputChain
|
||||
.then(() => onOutput(parsed))
|
||||
.catch((err) => logStreamedOutputDeliveryError(err, group, input));
|
||||
logStreamedAgentErrorOutput(parsed, group, input);
|
||||
outputChain = chainStreamedOutputDelivery({
|
||||
outputChain,
|
||||
parsed,
|
||||
onOutput,
|
||||
onTerminalStreamedOutputFlushed:
|
||||
args.onTerminalStreamedOutputFlushed,
|
||||
group,
|
||||
input,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{
|
||||
|
||||
@@ -137,6 +137,72 @@ function emitOutputMarker(
|
||||
proc.stdout.push(`${OUTPUT_START_MARKER}\n${json}\n${OUTPUT_END_MARKER}\n`);
|
||||
}
|
||||
|
||||
function mockCodexLeaseEnvironment(releaseLease: () => void) {
|
||||
return vi
|
||||
.spyOn(agentRunnerEnvironment, 'prepareGroupEnvironment')
|
||||
.mockReturnValueOnce({
|
||||
env: {
|
||||
EJCLAW_IPC_DIR: '/tmp/ejclaw-test-data/ipc/test-group',
|
||||
},
|
||||
groupDir: '/tmp/ejclaw-test-groups/test-group',
|
||||
runnerDir: '/tmp/ejclaw-test-runners/codex-runner',
|
||||
codexSessionAuth: {
|
||||
canonicalAuthPath: '/tmp/codex-account/auth.json',
|
||||
sessionAuthPath: '/tmp/codex-session/auth.json',
|
||||
accountIndex: 0,
|
||||
lease: {
|
||||
accountIndex: 0,
|
||||
authPath: '/tmp/codex-account/auth.json',
|
||||
release: releaseLease,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function expectCodexLeaseReleasedAfterFinalOutputFlush() {
|
||||
const releaseLease = vi.fn();
|
||||
const prepareGroupEnvironmentSpy = mockCodexLeaseEnvironment(releaseLease);
|
||||
const onOutput = vi.fn(async () => {});
|
||||
const codexGroup: RegisteredGroup = {
|
||||
...testGroup,
|
||||
agentType: 'codex',
|
||||
};
|
||||
let resultPromise: Promise<AgentOutput> | undefined;
|
||||
|
||||
try {
|
||||
resultPromise = runAgentProcess(
|
||||
codexGroup,
|
||||
{
|
||||
...testInput,
|
||||
runId: 'run-codex-lease-final',
|
||||
},
|
||||
() => {},
|
||||
onOutput,
|
||||
);
|
||||
|
||||
emitOutputMarker(fakeProc, {
|
||||
status: 'success',
|
||||
result: 'TASK_DONE\n작업 완료',
|
||||
phase: 'final',
|
||||
newSessionId: 'codex-session-lease-final',
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
expect(onOutput).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
result: 'TASK_DONE\n작업 완료',
|
||||
phase: 'final',
|
||||
}),
|
||||
);
|
||||
expect(releaseLease).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
fakeProc.emit('close', 0);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
await resultPromise?.catch(() => undefined);
|
||||
prepareGroupEnvironmentSpy.mockRestore();
|
||||
}
|
||||
}
|
||||
|
||||
describe('agent-runner timeout behavior', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
@@ -157,23 +223,18 @@ describe('agent-runner timeout behavior', () => {
|
||||
onOutput,
|
||||
);
|
||||
|
||||
// Emit output with a result
|
||||
emitOutputMarker(fakeProc, {
|
||||
status: 'success',
|
||||
result: 'Here is my response',
|
||||
newSessionId: 'session-123',
|
||||
});
|
||||
|
||||
// Let output processing settle
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
// Fire the hard timeout (IDLE_TIMEOUT + 30s = 1830000ms)
|
||||
await vi.advanceTimersByTimeAsync(1830000);
|
||||
|
||||
// Emit close event (as if agent was stopped by the timeout)
|
||||
fakeProc.emit('close', 137);
|
||||
|
||||
// Let the promise resolve
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
const result = await resultPromise;
|
||||
@@ -193,10 +254,8 @@ describe('agent-runner timeout behavior', () => {
|
||||
onOutput,
|
||||
);
|
||||
|
||||
// No output emitted — fire the hard timeout
|
||||
await vi.advanceTimersByTimeAsync(1830000);
|
||||
|
||||
// Emit close event
|
||||
fakeProc.emit('close', 137);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
@@ -216,7 +275,6 @@ describe('agent-runner timeout behavior', () => {
|
||||
onOutput,
|
||||
);
|
||||
|
||||
// Emit output
|
||||
emitOutputMarker(fakeProc, {
|
||||
status: 'success',
|
||||
result: 'Done',
|
||||
@@ -225,7 +283,6 @@ describe('agent-runner timeout behavior', () => {
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
// Normal exit (no timeout)
|
||||
fakeProc.emit('close', 0);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
@@ -235,6 +292,10 @@ describe('agent-runner timeout behavior', () => {
|
||||
expect(result.newSessionId).toBe('session-456');
|
||||
});
|
||||
|
||||
it('releases Codex auth lease after final streamed output flushes even before process close', async () => {
|
||||
await expectCodexLeaseReleasedAfterFinalOutputFlush();
|
||||
});
|
||||
|
||||
it('preserves streamed progress phase metadata', async () => {
|
||||
const onOutput = vi.fn(async () => {});
|
||||
const resultPromise = runAgentProcess(
|
||||
@@ -276,6 +337,18 @@ describe('agent-runner timeout behavior', () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('agent-runner environment wiring', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
fakeProc = createFakeProcess();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('passes the actual chat JID into codex runner MCP env', async () => {
|
||||
vi.useRealTimers();
|
||||
@@ -413,6 +486,55 @@ describe('agent-runner timeout behavior', () => {
|
||||
expect(spawnEnv?.CODEX_HOME).toBe('/tmp/host-reviewer-session/.codex');
|
||||
});
|
||||
|
||||
it('releases the initial Codex auth lease when read-only session preparation fails', async () => {
|
||||
vi.useRealTimers();
|
||||
fakeProc = createFakeProcess();
|
||||
|
||||
const releaseLease = vi.fn();
|
||||
const prepareGroupEnvironmentSpy = mockCodexLeaseEnvironment(releaseLease);
|
||||
const readonlyError = new Error('Codex rotation pool unavailable');
|
||||
const prepareReadonlySessionEnvironmentSpy = vi
|
||||
.spyOn(agentRunnerEnvironment, 'prepareReadonlySessionEnvironment')
|
||||
.mockImplementationOnce(() => {
|
||||
throw readonlyError;
|
||||
});
|
||||
const codexGroup: RegisteredGroup = {
|
||||
...testGroup,
|
||||
agentType: 'codex',
|
||||
};
|
||||
|
||||
try {
|
||||
await expect(
|
||||
runAgentProcess(
|
||||
codexGroup,
|
||||
{
|
||||
...testInput,
|
||||
roomRoleContext: {
|
||||
serviceId: 'codex-review',
|
||||
role: 'reviewer',
|
||||
ownerServiceId: 'codex-main',
|
||||
reviewerServiceId: 'codex-review',
|
||||
failoverOwner: false,
|
||||
},
|
||||
},
|
||||
() => {},
|
||||
async () => {},
|
||||
{
|
||||
EJCLAW_UNSAFE_HOST_PAIRED_MODE: '1',
|
||||
CLAUDE_CONFIG_DIR: '/tmp/host-reviewer-session',
|
||||
EJCLAW_WORK_DIR: '/tmp/paired/task-1/reviewer',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow(readonlyError);
|
||||
|
||||
expect(prepareReadonlySessionEnvironmentSpy).toHaveBeenCalledTimes(1);
|
||||
expect(releaseLease).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
prepareReadonlySessionEnvironmentSpy.mockRestore();
|
||||
prepareGroupEnvironmentSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('serializes roomRoleContext into the runner stdin payload', async () => {
|
||||
vi.useRealTimers();
|
||||
fakeProc = createFakeProcess();
|
||||
@@ -634,6 +756,18 @@ OUROBOROS_LLM_BACKEND = "codex"
|
||||
expect(toml).toContain('OUROBOROS_AGENT_RUNTIME = "codex"');
|
||||
expect(toml).toContain('[mcp_servers.ejclaw]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('agent-runner output flushing', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
fakeProc = createFakeProcess();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('waits for queued streamed output before resolving an error exit', async () => {
|
||||
let releaseOutputs: (() => void) | undefined;
|
||||
|
||||
@@ -14,7 +14,9 @@ import {
|
||||
import {
|
||||
prepareReadonlySessionEnvironment,
|
||||
prepareGroupEnvironment,
|
||||
type PreparedCodexSessionAuth,
|
||||
} from './agent-runner-environment.js';
|
||||
import { syncCodexSessionAuthBack } from './codex-token-rotation.js';
|
||||
import { runSpawnedAgentProcess } from './agent-runner-process.js';
|
||||
import { getStoredRoomSkillOverrides } from './db.js';
|
||||
export {
|
||||
@@ -76,6 +78,47 @@ function readRoomSkillOverridesForRunner(
|
||||
}
|
||||
}
|
||||
|
||||
function releaseCodexAuthSession(
|
||||
auth: PreparedCodexSessionAuth | null | undefined,
|
||||
): void {
|
||||
auth?.lease?.release();
|
||||
}
|
||||
|
||||
function finalizeCodexAuthSession(
|
||||
auth: PreparedCodexSessionAuth | null | undefined,
|
||||
): void {
|
||||
if (!auth) return;
|
||||
try {
|
||||
syncCodexSessionAuthBack({
|
||||
canonicalAuthPath: auth.canonicalAuthPath,
|
||||
sessionAuthPath: auth.sessionAuthPath,
|
||||
accountIndex: auth.accountIndex,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{
|
||||
err,
|
||||
canonicalAuthPath: auth.canonicalAuthPath,
|
||||
accountIndex: auth.accountIndex,
|
||||
},
|
||||
'Failed to sync Codex session auth back to canonical slot',
|
||||
);
|
||||
} finally {
|
||||
auth.lease?.release();
|
||||
}
|
||||
}
|
||||
|
||||
function createCodexAuthSessionFinalizer(
|
||||
getAuth: () => PreparedCodexSessionAuth | null | undefined,
|
||||
): () => void {
|
||||
let finalized = false;
|
||||
return () => {
|
||||
if (finalized) return;
|
||||
finalized = true;
|
||||
finalizeCodexAuthSession(getAuth());
|
||||
};
|
||||
}
|
||||
|
||||
export async function runAgentProcess(
|
||||
group: RegisteredGroup,
|
||||
input: AgentInput,
|
||||
@@ -92,18 +135,15 @@ export async function runAgentProcess(
|
||||
// ── Host process mode (owner) ───────────────────────────────────
|
||||
const startTime = Date.now();
|
||||
const skillOverrides = readRoomSkillOverridesForRunner(input.chatJid);
|
||||
const { env, groupDir, runnerDir } = prepareGroupEnvironment(
|
||||
group,
|
||||
input.isMain,
|
||||
input.chatJid,
|
||||
{
|
||||
memoryBriefing: input.memoryBriefing,
|
||||
runtimeTaskId: input.runtimeTaskId,
|
||||
useTaskScopedSession: input.useTaskScopedSession,
|
||||
skillOverrides,
|
||||
roomRole: input.roomRoleContext?.role,
|
||||
},
|
||||
);
|
||||
const prepared = prepareGroupEnvironment(group, input.isMain, input.chatJid, {
|
||||
memoryBriefing: input.memoryBriefing,
|
||||
runtimeTaskId: input.runtimeTaskId,
|
||||
useTaskScopedSession: input.useTaskScopedSession,
|
||||
skillOverrides,
|
||||
roomRole: input.roomRoleContext?.role,
|
||||
});
|
||||
const { env, groupDir, runnerDir } = prepared;
|
||||
let codexSessionAuth = prepared.codexSessionAuth;
|
||||
|
||||
// Apply env overrides (caller-provided)
|
||||
if (envOverrides) {
|
||||
@@ -117,20 +157,32 @@ export async function runAgentProcess(
|
||||
(input.roomRoleContext?.role === 'reviewer' ||
|
||||
input.roomRoleContext?.role === 'arbiter')
|
||||
) {
|
||||
const readonlySession = prepareReadonlySessionEnvironment({
|
||||
sessionDir: envOverrides.CLAUDE_CONFIG_DIR,
|
||||
chatJid: input.chatJid,
|
||||
isMain: input.isMain,
|
||||
groupFolder: group.folder,
|
||||
agentType: group.agentType || 'claude-code',
|
||||
memoryBriefing: input.memoryBriefing,
|
||||
role: input.roomRoleContext.role,
|
||||
ipcDir: env[EJCLAW_ENV.ipcDir],
|
||||
hostIpcDir: env[EJCLAW_ENV.hostIpcDir],
|
||||
workDir: envOverrides[EJCLAW_ENV.workDir] || env[EJCLAW_ENV.workDir],
|
||||
skillOverrides,
|
||||
});
|
||||
if ((group.agentType || 'claude-code') === 'codex') {
|
||||
const isCodexGroup = (group.agentType || 'claude-code') === 'codex';
|
||||
const previousCodexSessionAuth = codexSessionAuth;
|
||||
let readonlySession: ReturnType<typeof prepareReadonlySessionEnvironment>;
|
||||
try {
|
||||
readonlySession = prepareReadonlySessionEnvironment({
|
||||
sessionDir: envOverrides.CLAUDE_CONFIG_DIR,
|
||||
chatJid: input.chatJid,
|
||||
isMain: input.isMain,
|
||||
groupFolder: group.folder,
|
||||
agentType: group.agentType || 'claude-code',
|
||||
memoryBriefing: input.memoryBriefing,
|
||||
role: input.roomRoleContext.role,
|
||||
ipcDir: env[EJCLAW_ENV.ipcDir],
|
||||
hostIpcDir: env[EJCLAW_ENV.hostIpcDir],
|
||||
workDir: envOverrides[EJCLAW_ENV.workDir] || env[EJCLAW_ENV.workDir],
|
||||
skillOverrides,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isCodexGroup) {
|
||||
releaseCodexAuthSession(previousCodexSessionAuth);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (isCodexGroup) {
|
||||
releaseCodexAuthSession(previousCodexSessionAuth);
|
||||
codexSessionAuth = readonlySession.codexSessionAuth;
|
||||
env.CODEX_HOME = path.join(envOverrides.CLAUDE_CONFIG_DIR, '.codex');
|
||||
if (readonlySession.codexHomeDir) {
|
||||
env.HOME = readonlySession.codexHomeDir;
|
||||
@@ -144,6 +196,9 @@ export async function runAgentProcess(
|
||||
const safeName = group.folder.replace(/[^a-zA-Z0-9-]/g, '-');
|
||||
const processSuffix = input.runId || `${Date.now()}`;
|
||||
const processName = `ejclaw-${safeName}-${processSuffix}`;
|
||||
const finalizeCodexAuthSessionOnce = createCodexAuthSessionFinalizer(
|
||||
() => codexSessionAuth,
|
||||
);
|
||||
|
||||
// Check if runner is built
|
||||
const distEntry = path.join(runnerDir, 'dist', 'index.js');
|
||||
@@ -152,6 +207,7 @@ export async function runAgentProcess(
|
||||
{ runnerDir, chatJid: input.chatJid, runId: input.runId },
|
||||
'Runner not built. Run: cd runners/agent-runner && bun install && bun run build',
|
||||
);
|
||||
releaseCodexAuthSession(codexSessionAuth);
|
||||
return {
|
||||
status: 'error',
|
||||
result: null,
|
||||
@@ -199,6 +255,23 @@ export async function runAgentProcess(
|
||||
logsDir,
|
||||
startTime,
|
||||
onOutput,
|
||||
}).then(resolve);
|
||||
onTerminalStreamedOutputFlushed: finalizeCodexAuthSessionOnce,
|
||||
})
|
||||
.then((output) => {
|
||||
finalizeCodexAuthSessionOnce();
|
||||
resolve(output);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
finalizeCodexAuthSessionOnce();
|
||||
logger.error(
|
||||
{ err, processName, chatJid: input.chatJid, runId: input.runId },
|
||||
'Spawned agent process runner failed',
|
||||
);
|
||||
resolve({
|
||||
status: 'error',
|
||||
result: null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
179
src/auto-paired-mode.ts
Normal file
179
src/auto-paired-mode.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Auto paired-mode degrade/restore.
|
||||
*
|
||||
* When all configured Claude accounts are at/over the 95% exhaustion threshold,
|
||||
* paired (tribunal) rooms cannot meaningfully run their reviewer/arbiter
|
||||
* because the gate blocks Claude turns. To keep the user-facing rooms
|
||||
* responsive, we temporarily flip selected rooms to `single` mode and notify
|
||||
* each affected channel. When usage recovers we flip them back to `tribunal`.
|
||||
*
|
||||
* Selection (Option B agreed with the user):
|
||||
* - The most-recently-active room (per chats.last_message_time)
|
||||
* - All rooms that are currently in-flight (queue status === 'processing')
|
||||
* - Deduplicated; rooms whose owner is `claude-code` are skipped because
|
||||
* degrading them does not help — the 95% gate blocks them either way.
|
||||
*
|
||||
* Restore behavior:
|
||||
* - Only restore a room if it is still in 'single' (don't override a manual
|
||||
* user change made during the degrade window).
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { getClaudeUsageExhaustion } from './claude-usage.js';
|
||||
import { DATA_DIR } from './config.js';
|
||||
import {
|
||||
getAllChats,
|
||||
getAllRoomBindings,
|
||||
getEffectiveRoomMode,
|
||||
getStoredRoomSettings,
|
||||
setExplicitRoomMode,
|
||||
} from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||
|
||||
const STATE_PATH = path.join(DATA_DIR, 'auto-paired-mode-state.json');
|
||||
|
||||
const DEGRADE_NOTICE =
|
||||
'클로드 사용량이 95%를 넘어 잠시 single 모드로 전환했어요. 사용량이 회복되면 paired 모드로 다시 켤게요.';
|
||||
const RESTORE_NOTICE = '클로드 사용량이 회복돼서 paired 모드로 다시 켰어요.';
|
||||
|
||||
interface DowngradeRecord {
|
||||
chatJid: string;
|
||||
atIso: string;
|
||||
}
|
||||
|
||||
interface AutoPairedModeState {
|
||||
exhausted: boolean;
|
||||
downgraded: DowngradeRecord[];
|
||||
}
|
||||
|
||||
export interface AutoPairedModeDeps {
|
||||
queue?: {
|
||||
getStatuses(jids: string[]): Array<{ jid: string; status: string }>;
|
||||
};
|
||||
sendNotice: (chatJid: string, text: string) => Promise<void>;
|
||||
}
|
||||
|
||||
function loadState(): AutoPairedModeState {
|
||||
const raw = readJsonFile<AutoPairedModeState>(STATE_PATH);
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
return { exhausted: false, downgraded: [] };
|
||||
}
|
||||
return {
|
||||
exhausted: Boolean(raw.exhausted),
|
||||
downgraded: Array.isArray(raw.downgraded) ? raw.downgraded : [],
|
||||
};
|
||||
}
|
||||
|
||||
function saveState(state: AutoPairedModeState): void {
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(STATE_PATH), { recursive: true });
|
||||
writeJsonFile(STATE_PATH, state);
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'auto-paired-mode: failed to persist state');
|
||||
}
|
||||
}
|
||||
|
||||
function pickCandidates(deps: AutoPairedModeDeps): string[] {
|
||||
const bindings = getAllRoomBindings();
|
||||
const allJids = Object.keys(bindings);
|
||||
if (allJids.length === 0) return [];
|
||||
|
||||
const candidates = new Set<string>();
|
||||
|
||||
// Most-recently-active room first (chats ordered DESC by last_message_time).
|
||||
const chats = getAllChats();
|
||||
const recent = chats.find((c) => allJids.includes(c.jid));
|
||||
if (recent) candidates.add(recent.jid);
|
||||
|
||||
// Currently in-flight rooms.
|
||||
if (deps.queue) {
|
||||
for (const status of deps.queue.getStatuses(allJids)) {
|
||||
if (status.status === 'processing') candidates.add(status.jid);
|
||||
}
|
||||
}
|
||||
|
||||
// Skip claude-owned rooms (Option B).
|
||||
const result: string[] = [];
|
||||
for (const jid of candidates) {
|
||||
const settings = getStoredRoomSettings(jid);
|
||||
if (settings?.ownerAgentType === 'claude-code') continue;
|
||||
if (getEffectiveRoomMode(jid) !== 'tribunal') continue;
|
||||
result.push(jid);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function evaluateAndApplyAutoPairedMode(
|
||||
deps: AutoPairedModeDeps,
|
||||
): Promise<void> {
|
||||
const exhaustion = getClaudeUsageExhaustion();
|
||||
const state = loadState();
|
||||
|
||||
if (exhaustion.exhausted && !state.exhausted) {
|
||||
// Transition: healthy → exhausted. Downgrade selected rooms.
|
||||
const targets = pickCandidates(deps);
|
||||
const downgraded: DowngradeRecord[] = [];
|
||||
for (const jid of targets) {
|
||||
try {
|
||||
setExplicitRoomMode(jid, 'single');
|
||||
downgraded.push({ chatJid: jid, atIso: new Date().toISOString() });
|
||||
logger.info(
|
||||
{ jid },
|
||||
'auto-paired-mode: downgraded room to single (claude exhausted)',
|
||||
);
|
||||
try {
|
||||
await deps.sendNotice(jid, DEGRADE_NOTICE);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, jid },
|
||||
'auto-paired-mode: failed to send degrade notice',
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ err, jid }, 'auto-paired-mode: failed to downgrade room');
|
||||
}
|
||||
}
|
||||
saveState({ exhausted: true, downgraded });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!exhaustion.exhausted && state.exhausted) {
|
||||
// Transition: exhausted → healthy. Restore previously-downgraded rooms.
|
||||
for (const record of state.downgraded) {
|
||||
try {
|
||||
if (getEffectiveRoomMode(record.chatJid) !== 'single') {
|
||||
// Respect manual user changes during the degrade window.
|
||||
logger.info(
|
||||
{ jid: record.chatJid },
|
||||
'auto-paired-mode: skip restore (room is no longer single)',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
setExplicitRoomMode(record.chatJid, 'tribunal');
|
||||
logger.info(
|
||||
{ jid: record.chatJid },
|
||||
'auto-paired-mode: restored room to tribunal',
|
||||
);
|
||||
try {
|
||||
await deps.sendNotice(record.chatJid, RESTORE_NOTICE);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, jid: record.chatJid },
|
||||
'auto-paired-mode: failed to send restore notice',
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, jid: record.chatJid },
|
||||
'auto-paired-mode: failed to restore room',
|
||||
);
|
||||
}
|
||||
}
|
||||
saveState({ exhausted: false, downgraded: [] });
|
||||
return;
|
||||
}
|
||||
|
||||
// Steady state — nothing to do.
|
||||
}
|
||||
@@ -129,11 +129,7 @@ vi.mock('discord.js', () => {
|
||||
};
|
||||
});
|
||||
|
||||
import {
|
||||
DiscordChannel,
|
||||
DiscordChannelOpts,
|
||||
chunkForDiscord,
|
||||
} from './discord.js';
|
||||
import { DiscordChannel, DiscordChannelOpts } from './discord.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
// --- Test helpers ---
|
||||
@@ -981,6 +977,28 @@ describe('sendMessage', () => {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('surfaces rejected attachments in the sent message instead of dropping them silently', async () => {
|
||||
const opts = createTestOpts();
|
||||
const channel = new DiscordChannel('test-token', opts);
|
||||
await channel.connect();
|
||||
const mockChannel = {
|
||||
send: vi.fn().mockResolvedValue({ id: 'discord-message-1' }),
|
||||
sendTyping: vi.fn(),
|
||||
};
|
||||
currentClient().channels.fetch.mockResolvedValue(mockChannel);
|
||||
|
||||
await channel.sendMessage('dc:1234567890123456', '샘플을 첨부합니다.', {
|
||||
attachments: [{ path: '/home/claude/missing-sample.wav' }],
|
||||
});
|
||||
|
||||
expect(mockChannel.send).toHaveBeenCalledTimes(1);
|
||||
const sent = mockChannel.send.mock.calls[0][0];
|
||||
expect(sent.files).toBeUndefined();
|
||||
expect(sent.content).toContain('샘플을 첨부합니다.');
|
||||
expect(sent.content).toContain('첨부 1건을 전송하지 못했습니다');
|
||||
expect(sent.content).toContain('missing-sample.wav (파일을 찾을 수 없음)');
|
||||
});
|
||||
|
||||
it('uses legacy image tags as Discord attachment fallback', async () => {
|
||||
const opts = createTestOpts();
|
||||
const channel = new DiscordChannel('test-token', opts);
|
||||
@@ -1142,127 +1160,3 @@ describe('channel properties', () => {
|
||||
expect(channel.name).toBe('discord');
|
||||
});
|
||||
});
|
||||
|
||||
// --- chunkForDiscord ---
|
||||
|
||||
describe('chunkForDiscord', () => {
|
||||
it('returns the input unchanged when within the limit', () => {
|
||||
const text = 'short message';
|
||||
expect(chunkForDiscord(text, 2000)).toEqual([text]);
|
||||
});
|
||||
|
||||
it('returns empty array for empty input', () => {
|
||||
expect(chunkForDiscord('', 2000)).toEqual([]);
|
||||
});
|
||||
|
||||
it('splits on newline boundaries when possible', () => {
|
||||
const text = 'aaa\nbbb\nccc\nddd';
|
||||
const chunks = chunkForDiscord(text, 8);
|
||||
// Every chunk must be ≤ 8 chars and joining them must rebuild the text
|
||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(8);
|
||||
expect(chunks.join('')).toBe(text);
|
||||
});
|
||||
|
||||
it('reopens a fenced code block across chunks (with language)', () => {
|
||||
const code = Array.from({ length: 200 }, (_, i) => `line ${i}`).join('\n');
|
||||
const text = 'intro paragraph.\n\n```ts\n' + code + '\n```\nafter';
|
||||
const chunks = chunkForDiscord(text, 500);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500);
|
||||
// Every chunk must have a balanced number of ``` fences (so it renders
|
||||
// as valid markdown on its own).
|
||||
for (const c of chunks) {
|
||||
const fenceMatches = c.match(/^```/gm) ?? [];
|
||||
expect(fenceMatches.length % 2).toBe(0);
|
||||
}
|
||||
// The middle chunks must reopen the same language.
|
||||
for (let i = 1; i < chunks.length; i++) {
|
||||
// It should either start with the reopened fence, or never have been
|
||||
// inside the fence at this point.
|
||||
if (chunks[i].startsWith('```')) {
|
||||
expect(chunks[i].startsWith('```ts\n')).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('reopens a language-less fenced code block', () => {
|
||||
const code = Array.from({ length: 200 }, (_, i) => `payload-${i}`).join(
|
||||
'\n',
|
||||
);
|
||||
const text = '```\n' + code + '\n```';
|
||||
const chunks = chunkForDiscord(text, 400);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) {
|
||||
const fenceMatches = c.match(/^```/gm) ?? [];
|
||||
expect(fenceMatches.length % 2).toBe(0);
|
||||
}
|
||||
// Joining back should rebuild a string whose un-fenced content equals
|
||||
// the original code (allowing for extra fence pairs in the middle).
|
||||
const reconstructed = chunks
|
||||
.join('\n')
|
||||
.replace(/```\n```\n/g, '') // drop reopen-close pairs at chunk seams
|
||||
.replace(/```\n```$/g, '');
|
||||
expect(reconstructed.includes('payload-0')).toBe(true);
|
||||
expect(reconstructed.includes('payload-199')).toBe(true);
|
||||
});
|
||||
|
||||
it('handles multiple fenced blocks in the same document', () => {
|
||||
const blockA = Array.from({ length: 50 }, () => 'AAAA').join('\n');
|
||||
const blockB = Array.from({ length: 50 }, () => 'BBBB').join('\n');
|
||||
const text =
|
||||
'intro\n```js\n' + blockA + '\n```\nbetween\n```py\n' + blockB + '\n```';
|
||||
const chunks = chunkForDiscord(text, 200);
|
||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(200);
|
||||
for (const c of chunks) {
|
||||
const fenceMatches = c.match(/^```/gm) ?? [];
|
||||
expect(fenceMatches.length % 2).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not split a surrogate pair', () => {
|
||||
// Emoji 🦄 is a surrogate pair (length 2 in JS).
|
||||
const emoji = '🦄';
|
||||
const text = emoji.repeat(50);
|
||||
const chunks = chunkForDiscord(text, 9);
|
||||
for (const c of chunks) {
|
||||
// No chunk should start or end mid surrogate pair.
|
||||
expect(c.charCodeAt(0) >= 0xdc00 && c.charCodeAt(0) <= 0xdfff).toBe(
|
||||
false,
|
||||
);
|
||||
const last = c.charCodeAt(c.length - 1);
|
||||
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
|
||||
}
|
||||
expect(chunks.join('')).toBe(text);
|
||||
});
|
||||
|
||||
it('falls back gracefully when a single line exceeds maxLength', () => {
|
||||
const longLine = 'x'.repeat(5000);
|
||||
const chunks = chunkForDiscord(longLine, 2000);
|
||||
for (const c of chunks) expect(c.length).toBeLessThanOrEqual(2000);
|
||||
expect(chunks.join('')).toBe(longLine);
|
||||
});
|
||||
|
||||
it('keeps a closing fence with its block when there is room', () => {
|
||||
const text = '```ts\nshort code\n```';
|
||||
const chunks = chunkForDiscord(text, 2000);
|
||||
expect(chunks).toEqual([text]);
|
||||
});
|
||||
|
||||
it('preserves total content across chunk seams (modulo reopen overhead)', () => {
|
||||
const code = Array.from(
|
||||
{ length: 80 },
|
||||
(_, i) => `console.log(${i});`,
|
||||
).join('\n');
|
||||
const text = '```js\n' + code + '\n```';
|
||||
const chunks = chunkForDiscord(text, 300);
|
||||
// After removing the reopen-close fence pairs that the chunker inserts
|
||||
// between chunks, the result should match the original text.
|
||||
const stripped = chunks.join('\n').replace(/```\n```js\n/g, '');
|
||||
// The first chunk still has the original opener and the last still has
|
||||
// the original closer.
|
||||
expect(stripped.startsWith('```js\n')).toBe(true);
|
||||
expect(stripped.endsWith('```')).toBe(true);
|
||||
expect(stripped.includes('console.log(0);')).toBe(true);
|
||||
expect(stripped.includes('console.log(79);')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,7 +14,10 @@ import {
|
||||
import { CACHE_DIR } from '../config.js';
|
||||
import { getEnv } from '../env.js';
|
||||
import { logger } from '../logger.js';
|
||||
import { validateOutboundAttachments } from '../outbound-attachments.js';
|
||||
import {
|
||||
appendRejectionNotice,
|
||||
validateOutboundAttachments,
|
||||
} from '../outbound-attachments.js';
|
||||
import { sanitizeForOutbound } from '../router.js';
|
||||
import { hasReviewerLease } from '../service-routing.js';
|
||||
import type {
|
||||
@@ -39,113 +42,6 @@ const DISCORD_OWNER_TOKEN_KEY = 'DISCORD_OWNER_BOT_TOKEN';
|
||||
const DISCORD_REVIEWER_TOKEN_KEY = 'DISCORD_REVIEWER_BOT_TOKEN';
|
||||
const DISCORD_ARBITER_TOKEN_KEY = 'DISCORD_ARBITER_BOT_TOKEN';
|
||||
|
||||
// Matches a line that opens or closes a markdown fenced code block.
|
||||
// Captures the fence marker (``` or ```` etc.) and the optional language tag.
|
||||
const FENCE_LINE_RE = /^(`{3,})([^\s`]*)\s*$/;
|
||||
|
||||
/**
|
||||
* Split `text` into chunks ≤ `maxLength` chars while keeping every chunk a
|
||||
* valid standalone Markdown snippet for Discord. If a chunk has to end inside
|
||||
* a ``` fenced code block, we append a closing fence to it and reopen the
|
||||
* same fence (with the same language tag) at the start of the next chunk.
|
||||
*
|
||||
* Behaviour:
|
||||
* - Prefers splitting at newline boundaries.
|
||||
* - If a single line is itself longer than `maxLength`, falls back to a
|
||||
* surrogate-pair-safe byte split for that line only.
|
||||
* - Returns `[text]` unchanged when `text.length <= maxLength`.
|
||||
*/
|
||||
export function chunkForDiscord(text: string, maxLength: number): string[] {
|
||||
if (text.length <= maxLength) return text.length > 0 ? [text] : [];
|
||||
|
||||
const lines = text.split('\n');
|
||||
const chunks: string[] = [];
|
||||
let current = '';
|
||||
let openLang = ''; // language tag of currently open fence
|
||||
let openMarker: string | null = null; // ``` or ```` etc; null when not in a fence
|
||||
|
||||
const flush = () => {
|
||||
if (current === '') return;
|
||||
if (openMarker !== null) {
|
||||
if (!current.endsWith('\n')) current += '\n';
|
||||
current += openMarker;
|
||||
chunks.push(current);
|
||||
current = openMarker + openLang + '\n';
|
||||
} else {
|
||||
chunks.push(current);
|
||||
current = '';
|
||||
}
|
||||
};
|
||||
|
||||
const hardSplitInto = (segment: string) => {
|
||||
// Split `segment` across chunks at code-unit boundaries while avoiding
|
||||
// splitting surrogate pairs. Used when a single source line exceeds the
|
||||
// budget on its own.
|
||||
let s = segment;
|
||||
while (s.length > 0) {
|
||||
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
|
||||
const room = maxLength - current.length - reserve;
|
||||
if (room <= 0) {
|
||||
flush();
|
||||
continue;
|
||||
}
|
||||
let take = Math.min(s.length, room);
|
||||
if (
|
||||
take < s.length &&
|
||||
take > 0 &&
|
||||
s.charCodeAt(take - 1) >= 0xd800 &&
|
||||
s.charCodeAt(take - 1) <= 0xdbff
|
||||
) {
|
||||
take--;
|
||||
}
|
||||
if (take <= 0) {
|
||||
// Couldn't fit anything safely; flush and retry on an empty chunk.
|
||||
flush();
|
||||
continue;
|
||||
}
|
||||
current += s.slice(0, take);
|
||||
s = s.slice(take);
|
||||
if (s.length > 0) flush();
|
||||
}
|
||||
};
|
||||
|
||||
for (let li = 0; li < lines.length; li++) {
|
||||
const isLast = li === lines.length - 1;
|
||||
const line = lines[li];
|
||||
const lineWithNL = isLast ? line : line + '\n';
|
||||
const reserve = openMarker !== null ? openMarker.length + 1 : 0;
|
||||
|
||||
if (
|
||||
current.length > 0 &&
|
||||
current.length + lineWithNL.length + reserve > maxLength
|
||||
) {
|
||||
flush();
|
||||
}
|
||||
|
||||
if (lineWithNL.length + reserve > maxLength) {
|
||||
hardSplitInto(lineWithNL);
|
||||
} else {
|
||||
current += lineWithNL;
|
||||
}
|
||||
|
||||
// Update fence state AFTER appending the line so the very-next iteration
|
||||
// accounts for it correctly.
|
||||
const m = line.match(FENCE_LINE_RE);
|
||||
if (m) {
|
||||
if (openMarker === null) {
|
||||
openMarker = m[1];
|
||||
openLang = m[2] || '';
|
||||
} else if (m[1] === openMarker) {
|
||||
openMarker = null;
|
||||
openLang = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (current.length > 0) chunks.push(current);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a pending transcription from the other service (poll cache file).
|
||||
* Returns the cached text, or null if timeout.
|
||||
@@ -522,16 +418,18 @@ export class DiscordChannel implements Channel {
|
||||
.trim();
|
||||
|
||||
// Convert @username mentions to Discord mention format
|
||||
const mentionMap: Record<string, string> = {
|
||||
눈쟁이: '216851709744513024',
|
||||
};
|
||||
for (const [name, id] of Object.entries(mentionMap)) {
|
||||
cleaned = cleaned.replace(new RegExp(`@${name}`, 'g'), `<@${id}>`);
|
||||
}
|
||||
// Markdown escaping already happened in prepareDiscordOutbound (before
|
||||
// the deliberate link→inline-code step). Here we only strip/redact.
|
||||
cleaned = cleaned.replaceAll('@눈쟁이', '<@216851709744513024>');
|
||||
// Markdown escaping already happened in prepareDiscordOutbound (prose
|
||||
// escaped, structured output / fenced code preserved). Here we only
|
||||
// re-sanitize so we don't double-escape the delimiters.
|
||||
cleaned = sanitizeForOutbound(cleaned);
|
||||
|
||||
// Surface rejected attachments in the visible message. The MEDIA:
|
||||
// directive was already stripped from the text, so without this the user
|
||||
// would see a message that claims a file was attached while none was
|
||||
// delivered. Making the failure loud prevents that silent mismatch.
|
||||
cleaned = appendRejectionNotice(cleaned, validation.rejected);
|
||||
|
||||
// Discord has a 2000 character limit per message and 10 attachments per message
|
||||
const MAX_LENGTH = 2000;
|
||||
const MAX_ATTACHMENTS = 10;
|
||||
@@ -576,14 +474,10 @@ export class DiscordChannel implements Channel {
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Send text in chunks. The chunker preserves fenced code blocks so a
|
||||
// long ``` block that straddles the 2000-char boundary doesn't get
|
||||
// split mid-fence (which Discord would render as literal backticks
|
||||
// and break syntax for following chunks). It is also surrogate-pair
|
||||
// safe (emoji etc.).
|
||||
const messageChunks = chunkForDiscord(cleaned, MAX_LENGTH);
|
||||
// Send text in chunks, attach first batch to the first chunk
|
||||
let fileBatchIndex = 0;
|
||||
for (const chunk of messageChunks) {
|
||||
for (let i = 0; i < cleaned.length; i += MAX_LENGTH) {
|
||||
const chunk = cleaned.slice(i, i + MAX_LENGTH);
|
||||
const batch = fileBatches[fileBatchIndex];
|
||||
recordSentMessage(
|
||||
await textChannel.send({
|
||||
|
||||
@@ -112,12 +112,15 @@ export function getUsageCacheReadKeys(
|
||||
return keys;
|
||||
}
|
||||
|
||||
// Rate limit: at most one API call per token per 5 minutes
|
||||
const MIN_FETCH_INTERVAL_MS = 300_000;
|
||||
// Rate limit: at most one API call per token per 1 minute.
|
||||
// We poll usage every minute (was 5) so the 95% exhaustion gate has tight
|
||||
// resolution — at worst the gate fires ~60s after a window crosses 95%.
|
||||
const MIN_FETCH_INTERVAL_MS = 60_000;
|
||||
|
||||
async function fetchUsageForToken(
|
||||
token: string,
|
||||
accountIndex?: number,
|
||||
refreshAttempted = false,
|
||||
): Promise<ClaudeUsageData | null> {
|
||||
loadUsageDiskCache();
|
||||
|
||||
@@ -148,7 +151,11 @@ async function fetchUsageForToken(
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
const lastAttempt = cached?.lastAttemptAt ?? cached?.fetchedAt ?? 0;
|
||||
if (cached && Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS) {
|
||||
if (
|
||||
!refreshAttempted &&
|
||||
cached &&
|
||||
Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS
|
||||
) {
|
||||
return cached.usage;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
@@ -165,16 +172,47 @@ async function fetchUsageForToken(
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (res.status === 401) {
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
// 401 = token expired; 403 = token lacks `user:profile` scope.
|
||||
// Both are recoverable by exchanging refresh_token for a fresh access
|
||||
// token (mint includes current default scope set). Retry once.
|
||||
if (accountIndex != null && !refreshAttempted) {
|
||||
try {
|
||||
const { forceRefreshToken } = await import('./token-refresh.js');
|
||||
const newToken = await forceRefreshToken(accountIndex);
|
||||
if (newToken && newToken !== token) {
|
||||
logger.info(
|
||||
{
|
||||
account: accountIndex + 1,
|
||||
status: res.status,
|
||||
cacheKey: writeKey,
|
||||
},
|
||||
`Claude usage API: ${res.status} → refreshed access token, retrying`,
|
||||
);
|
||||
return fetchUsageForToken(newToken, accountIndex, true);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, account: accountIndex + 1 },
|
||||
'Claude usage API: force-refresh failed',
|
||||
);
|
||||
}
|
||||
}
|
||||
logger.warn(
|
||||
{
|
||||
account: accountIndex != null ? accountIndex + 1 : '?',
|
||||
tokenKey: legacyTokenCacheKey(token),
|
||||
cacheKey: writeKey,
|
||||
},
|
||||
'Claude usage API: token expired or invalid (401)',
|
||||
res.status === 401
|
||||
? 'Claude usage API: token expired or invalid (401)'
|
||||
: 'Claude usage API: forbidden — token missing required scope (403)',
|
||||
);
|
||||
return null;
|
||||
if (cached) {
|
||||
cached.lastAttemptAt = Date.now();
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
return cached?.usage ?? null;
|
||||
}
|
||||
if (res.status === 429) {
|
||||
const staleMs = cached ? Date.now() - cached.fetchedAt : 0;
|
||||
@@ -278,7 +316,15 @@ async function fetchUsageForToken(
|
||||
* Uses the current active token from rotation.
|
||||
*/
|
||||
export async function fetchClaudeUsage(): Promise<ClaudeUsageData | null> {
|
||||
const token = getCurrentToken() || getConfiguredClaudeTokens()[0];
|
||||
// Prefer the access token in ~/.claude/.credentials.json when present.
|
||||
// The static .env token (CLAUDE_CODE_OAUTH_TOKEN) was issued before the
|
||||
// `user:profile` scope was required for /api/oauth/usage and so always 403s.
|
||||
// The credentials file is the canonical source written by `claude auth
|
||||
// login` and kept fresh by token-refresh.ts — its accessToken carries the
|
||||
// full scope set including user:profile.
|
||||
const credsToken = readCredentialsAccessToken(0);
|
||||
const token =
|
||||
credsToken || getCurrentToken() || getConfiguredClaudeTokens()[0];
|
||||
if (!token) {
|
||||
logger.debug('No Claude OAuth token available for usage check');
|
||||
return null;
|
||||
@@ -435,7 +481,8 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
const allTokens = getAllTokens();
|
||||
logger.debug({ tokenCount: allTokens.length }, 'fetchAllClaudeUsage called');
|
||||
if (allTokens.length === 0) {
|
||||
const token = getConfiguredClaudeTokens()[0];
|
||||
const credsToken = readCredentialsAccessToken(0);
|
||||
const token = credsToken || getConfiguredClaudeTokens()[0];
|
||||
if (!token) return [];
|
||||
const usage = await fetchUsageForToken(token, 0);
|
||||
return [
|
||||
@@ -451,7 +498,14 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
|
||||
const results: ClaudeAccountUsage[] = [];
|
||||
for (const t of allTokens) {
|
||||
const usage = await fetchUsageForToken(t.token, t.index);
|
||||
// Prefer the per-account credentials.json access token when present.
|
||||
// The static .env CLAUDE_CODE_OAUTH_TOKEN(S) values were issued without
|
||||
// the `user:profile` scope and so are rejected by /api/oauth/usage.
|
||||
// The credentials file is rewritten by `claude auth login` and refreshed
|
||||
// by token-refresh.ts, so it always carries the full scope set.
|
||||
const credsToken = readCredentialsAccessToken(t.index);
|
||||
const tokenForFetch = credsToken || t.token;
|
||||
const usage = await fetchUsageForToken(tokenForFetch, t.index);
|
||||
results.push({
|
||||
index: t.index,
|
||||
masked: t.masked,
|
||||
@@ -465,3 +519,103 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
|
||||
// Legacy alias
|
||||
export const fetchClaudeUsageViaCli = fetchClaudeUsage;
|
||||
|
||||
// ── Exhaustion gate ───────────────────────────────────────────────────────
|
||||
//
|
||||
// Threshold above which a Claude account is considered "out of budget" for
|
||||
// the purpose of admitting new turns. A turn is blocked only when *every*
|
||||
// Claude account is either at/over this threshold on either window or
|
||||
// already in a rate-limit cooldown.
|
||||
|
||||
export const CLAUDE_EXHAUSTION_THRESHOLD_PCT = 95;
|
||||
|
||||
export interface UsageExhaustionInfo {
|
||||
exhausted: boolean;
|
||||
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
||||
nextResetAt: string | null;
|
||||
}
|
||||
|
||||
/** Normalise the API's utilization field (which can be fractional or percent). */
|
||||
function normaliseUtilization(u: number | undefined): number {
|
||||
if (u == null || !Number.isFinite(u)) return 0;
|
||||
return u >= 0 && u < 1 ? u * 100 : u;
|
||||
}
|
||||
|
||||
function parseResetMs(s: string | undefined | null): number | null {
|
||||
if (!s) return null;
|
||||
const t = Date.parse(s);
|
||||
return Number.isFinite(t) ? t : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether every configured Claude account is either ≥ 95% on a
|
||||
* window or in a rate-limit cooldown, and — when so — the soonest moment at
|
||||
* which any one account is expected to recover.
|
||||
*
|
||||
* Reads the on-disk usage cache (refreshed every minute by the dashboard
|
||||
* renderer); performs no network I/O. Returns `exhausted: false` when there
|
||||
* are no configured tokens, or when usage data is missing for any account
|
||||
* (err on letting work through).
|
||||
*/
|
||||
export function getClaudeUsageExhaustion(): UsageExhaustionInfo {
|
||||
loadUsageDiskCache();
|
||||
const allTokens = getAllTokens();
|
||||
if (allTokens.length === 0) return { exhausted: false, nextResetAt: null };
|
||||
|
||||
let earliestRecoveryMs = Infinity;
|
||||
|
||||
for (const t of allTokens) {
|
||||
if (t.isRateLimited) {
|
||||
// Rate-limit recovery time is not exposed on the token snapshot here;
|
||||
// we treat it as "unknown but eventually". Skip from the min calc.
|
||||
continue;
|
||||
}
|
||||
const credsToken = readCredentialsAccessToken(t.index);
|
||||
const readKeys = getUsageCacheReadKeys(t.token, t.index, credsToken);
|
||||
let entry: UsageCacheEntry | undefined;
|
||||
for (const key of readKeys) {
|
||||
if (usageDiskCache[key]) {
|
||||
entry = usageDiskCache[key];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!entry) return { exhausted: false, nextResetAt: null };
|
||||
|
||||
const h5 = normaliseUtilization(entry.usage.five_hour?.utilization);
|
||||
const d7 = normaliseUtilization(entry.usage.seven_day?.utilization);
|
||||
if (Math.max(h5, d7) < CLAUDE_EXHAUSTION_THRESHOLD_PCT) {
|
||||
return { exhausted: false, nextResetAt: null };
|
||||
}
|
||||
|
||||
// This account is exhausted. It becomes available again when whichever
|
||||
// window(s) are over the threshold drop back below it. We approximate
|
||||
// that as max(reset of each window currently ≥ threshold).
|
||||
const h5ResetMs =
|
||||
h5 >= CLAUDE_EXHAUSTION_THRESHOLD_PCT
|
||||
? parseResetMs(entry.usage.five_hour?.resets_at)
|
||||
: null;
|
||||
const d7ResetMs =
|
||||
d7 >= CLAUDE_EXHAUSTION_THRESHOLD_PCT
|
||||
? parseResetMs(entry.usage.seven_day?.resets_at)
|
||||
: null;
|
||||
const candidates = [h5ResetMs, d7ResetMs].filter(
|
||||
(v): v is number => v != null,
|
||||
);
|
||||
if (candidates.length > 0) {
|
||||
const accountRecovery = Math.max(...candidates);
|
||||
if (accountRecovery < earliestRecoveryMs) {
|
||||
earliestRecoveryMs = accountRecovery;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
||||
? new Date(earliestRecoveryMs).toISOString()
|
||||
: null;
|
||||
return { exhausted: true, nextResetAt };
|
||||
}
|
||||
|
||||
/** Backwards-compat boolean wrapper. */
|
||||
export function isClaudeUsageExhausted(): boolean {
|
||||
return getClaudeUsageExhaustion().exhausted;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ export function readCodexFeatureFromContent(
|
||||
feature: CodexConfigFeature,
|
||||
): boolean {
|
||||
const lines = content.split('\n');
|
||||
const featureLinePattern = new RegExp(`^${feature}\\s*=\\s*(true|false)$`);
|
||||
let inFeatures = false;
|
||||
|
||||
for (const line of lines) {
|
||||
@@ -28,9 +29,7 @@ export function readCodexFeatureFromContent(
|
||||
break;
|
||||
}
|
||||
if (!inFeatures) continue;
|
||||
const match = trimmed.match(
|
||||
new RegExp(`^${feature}\\s*=\\s*(true|false)$`),
|
||||
);
|
||||
const match = trimmed.match(featureLinePattern);
|
||||
if (match) return match[1] === 'true';
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
vi.mock('./agent-error-detection.js', () => ({
|
||||
classifyAgentError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||
classifyCodexAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||
isCodexPoolUnavailableError: vi.fn(
|
||||
(error: string | null | undefined) =>
|
||||
/all\s+codex(?:\s+rotation)?\s+accounts(?:\s+are)?\s+unavailable/i.test(
|
||||
error ?? '',
|
||||
) || /codex\s+rotation\s+pool\s+unavailable/i.test(error ?? ''),
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock('./config.js', () => ({
|
||||
@@ -165,6 +171,236 @@ describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
||||
expect(mod.getCodexAccountCount()).toBe(4);
|
||||
expect(mod.getActiveCodexAuthPath()).not.toBe(fallbackAuthPath);
|
||||
});
|
||||
|
||||
it('does not mutate account health for the internal all-accounts-unavailable sentinel', async () => {
|
||||
const agentErrors = await import('./agent-error-detection.js');
|
||||
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||
category: 'auth-expired',
|
||||
reason: 'auth-expired',
|
||||
});
|
||||
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
expect(
|
||||
mod.rotateCodexToken(
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||
),
|
||||
).toBe(false);
|
||||
|
||||
const accounts = mod.getAllCodexAccounts();
|
||||
expect(accounts[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
isActive: true,
|
||||
isAuthDead: false,
|
||||
authStatus: 'healthy',
|
||||
isRateLimited: false,
|
||||
}),
|
||||
);
|
||||
expect(accounts[1].isActive).toBe(false);
|
||||
});
|
||||
|
||||
it('marks refresh-token reuse as dead auth instead of a recoverable cooldown', async () => {
|
||||
const agentErrors = await import('./agent-error-detection.js');
|
||||
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||
category: 'auth-expired',
|
||||
reason: 'auth-expired',
|
||||
});
|
||||
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
|
||||
|
||||
const accounts = mod.getAllCodexAccounts();
|
||||
expect(accounts[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
isAuthDead: true,
|
||||
authStatus: 'dead_auth',
|
||||
}),
|
||||
);
|
||||
expect(accounts[0].isRateLimited).toBe(false);
|
||||
expect(accounts[1].isActive).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('codex-token-rotation auth synchronization', () => {
|
||||
let tempHome: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-rot-'));
|
||||
process.env.CODEX_ROT_TEST_HOME = tempHome;
|
||||
createFakeAccounts(tempHome, 4);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.CODEX_ROT_TEST_HOME;
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('recovers a dead_auth account when canonical auth.json is refreshed before lease claim', async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
|
||||
try {
|
||||
const agentErrors = await import('./agent-error-detection.js');
|
||||
vi.mocked(agentErrors.classifyCodexAuthError).mockReturnValueOnce({
|
||||
category: 'auth-expired',
|
||||
reason: 'auth-expired',
|
||||
});
|
||||
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
const utils = await import('./utils.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
expect(mod.rotateCodexToken('refresh token was already used')).toBe(true);
|
||||
expect(mod.getAllCodexAccounts()[0].isAuthDead).toBe(true);
|
||||
|
||||
const refreshedAt = new Date('2026-01-01T00:00:10.000Z');
|
||||
const authPath = path.join(tempHome, '.codex-accounts', '0', 'auth.json');
|
||||
fs.writeFileSync(
|
||||
authPath,
|
||||
JSON.stringify({
|
||||
auth_mode: 'chatgpt',
|
||||
tokens: { account_id: 'acct-0', access_token: 'refreshed-token' },
|
||||
}),
|
||||
);
|
||||
fs.utimesSync(authPath, refreshedAt, refreshedAt);
|
||||
|
||||
mod.setCurrentCodexAccountIndex(0);
|
||||
const lease = mod.claimCodexAuthLease();
|
||||
try {
|
||||
expect(lease).toEqual(
|
||||
expect.objectContaining({ accountIndex: 0, authPath }),
|
||||
);
|
||||
expect(mod.getAllCodexAccounts()[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
authStatus: 'healthy',
|
||||
isAuthDead: false,
|
||||
isActive: true,
|
||||
}),
|
||||
);
|
||||
expect(vi.mocked(utils.writeJsonFile)).toHaveBeenCalled();
|
||||
} finally {
|
||||
lease?.release();
|
||||
}
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('leases different accounts for concurrent Codex runs', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
const first = mod.claimCodexAuthLease();
|
||||
const second = mod.claimCodexAuthLease();
|
||||
|
||||
try {
|
||||
expect(first).toEqual(
|
||||
expect.objectContaining({
|
||||
accountIndex: 0,
|
||||
authPath: expect.any(String),
|
||||
}),
|
||||
);
|
||||
expect(second).toEqual(
|
||||
expect.objectContaining({
|
||||
accountIndex: 1,
|
||||
authPath: expect.any(String),
|
||||
}),
|
||||
);
|
||||
expect(second?.authPath).not.toBe(first?.authPath);
|
||||
} finally {
|
||||
second?.release();
|
||||
first?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it('syncs a refreshed session auth.json back to the canonical slot atomically', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
const canonicalAuthPath = path.join(
|
||||
tempHome,
|
||||
'.codex-accounts',
|
||||
'0',
|
||||
'auth.json',
|
||||
);
|
||||
const sessionAuthPath = path.join(
|
||||
tempHome,
|
||||
'session',
|
||||
'.codex',
|
||||
'auth.json',
|
||||
);
|
||||
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
sessionAuthPath,
|
||||
JSON.stringify({
|
||||
auth_mode: 'chatgpt',
|
||||
tokens: {
|
||||
account_id: 'acct-0',
|
||||
access_token: 'new-access',
|
||||
refresh_token: 'new-refresh',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const synced = mod.syncCodexSessionAuthBack({
|
||||
canonicalAuthPath,
|
||||
sessionAuthPath,
|
||||
accountIndex: 0,
|
||||
});
|
||||
|
||||
expect(synced).toBe(true);
|
||||
const canonical = JSON.parse(fs.readFileSync(canonicalAuthPath, 'utf-8'));
|
||||
expect(canonical.tokens).toEqual(
|
||||
expect.objectContaining({
|
||||
account_id: 'acct-0',
|
||||
access_token: 'new-access',
|
||||
refresh_token: 'new-refresh',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('refuses to sync a session auth.json that belongs to another Codex account', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
|
||||
const canonicalAuthPath = path.join(
|
||||
tempHome,
|
||||
'.codex-accounts',
|
||||
'0',
|
||||
'auth.json',
|
||||
);
|
||||
const before = fs.readFileSync(canonicalAuthPath, 'utf-8');
|
||||
const sessionAuthPath = path.join(
|
||||
tempHome,
|
||||
'wrong-account',
|
||||
'.codex',
|
||||
'auth.json',
|
||||
);
|
||||
fs.mkdirSync(path.dirname(sessionAuthPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
sessionAuthPath,
|
||||
JSON.stringify({
|
||||
auth_mode: 'chatgpt',
|
||||
tokens: {
|
||||
account_id: 'acct-other',
|
||||
access_token: 'other-access',
|
||||
refresh_token: 'other-refresh',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
mod.syncCodexSessionAuthBack({
|
||||
canonicalAuthPath,
|
||||
sessionAuthPath,
|
||||
accountIndex: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(fs.readFileSync(canonicalAuthPath, 'utf-8')).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe('codex-token-rotation single-account fallback', () => {
|
||||
|
||||
@@ -17,13 +17,13 @@ import path from 'path';
|
||||
import {
|
||||
classifyAgentError,
|
||||
classifyCodexAuthError,
|
||||
isCodexPoolUnavailableError,
|
||||
type CodexRotationReason,
|
||||
} from './agent-error-detection.js';
|
||||
import { DATA_DIR } from './config.js';
|
||||
import { logger } from './logger.js';
|
||||
import {
|
||||
computeCooldownUntil,
|
||||
findNextAvailable,
|
||||
parseRetryAfterFromError,
|
||||
} from './token-rotation-base.js';
|
||||
import { readJsonFile, writeJsonFile } from './utils.js';
|
||||
@@ -34,15 +34,27 @@ interface CodexAccount {
|
||||
index: number;
|
||||
authPath: string;
|
||||
accountId: string;
|
||||
authFileMtimeMs: number;
|
||||
planType: string;
|
||||
subscriptionUntil: string | null;
|
||||
rateLimitedUntil: number | null;
|
||||
authStatus: 'healthy' | 'dead_auth';
|
||||
authDeadAt: number | null;
|
||||
authDeadReason: string | null;
|
||||
leasedUntil: number | null;
|
||||
leaseId: string | null;
|
||||
lastUsagePct?: number;
|
||||
lastUsageD7Pct?: number;
|
||||
resetAt?: string;
|
||||
resetD7At?: string;
|
||||
}
|
||||
|
||||
export interface CodexAuthLease {
|
||||
accountIndex: number;
|
||||
authPath: string;
|
||||
release: () => void;
|
||||
}
|
||||
|
||||
export type CodexRotationTriggerResult =
|
||||
| {
|
||||
shouldRotate: false;
|
||||
@@ -77,10 +89,20 @@ const accounts: CodexAccount[] = [];
|
||||
let currentIndex = 0;
|
||||
let initialized = false;
|
||||
|
||||
const CODEX_AUTH_LEASE_MS = 2 * 60 * 60_000;
|
||||
|
||||
const ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
||||
const DEFAULT_AUTH_PATH = path.join(DEFAULT_CODEX_DIR, 'auth.json');
|
||||
|
||||
function readAuthFileMtimeMs(authPath: string): number {
|
||||
try {
|
||||
return fs.statSync(authPath).mtimeMs;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function loadCodexAccount(
|
||||
authPath: string,
|
||||
fallbackAccountId: string,
|
||||
@@ -99,9 +121,15 @@ function loadCodexAccount(
|
||||
index: accounts.length,
|
||||
authPath,
|
||||
accountId,
|
||||
authFileMtimeMs: readAuthFileMtimeMs(authPath),
|
||||
planType: jwt.planType,
|
||||
subscriptionUntil: jwt.expiresAt,
|
||||
rateLimitedUntil: null,
|
||||
authStatus: 'healthy',
|
||||
authDeadAt: null,
|
||||
authDeadReason: null,
|
||||
leasedUntil: null,
|
||||
leaseId: null,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -149,6 +177,8 @@ function saveCodexState(): void {
|
||||
const state = {
|
||||
currentIndex,
|
||||
rateLimits: accounts.map((a) => a.rateLimitedUntil),
|
||||
authDeadAts: accounts.map((a) => a.authDeadAt),
|
||||
authDeadReasons: accounts.map((a) => a.authDeadReason),
|
||||
usagePcts: accounts.map((a) => a.lastUsagePct ?? null),
|
||||
usageD7Pcts: accounts.map((a) => a.lastUsageD7Pct ?? null),
|
||||
resetAts: accounts.map((a) => a.resetAt ?? null),
|
||||
@@ -167,6 +197,8 @@ function loadCodexState(quiet = false): void {
|
||||
const state = readJsonFile<{
|
||||
currentIndex?: number;
|
||||
rateLimits?: (number | null)[];
|
||||
authDeadAts?: (number | null)[];
|
||||
authDeadReasons?: (string | null)[];
|
||||
usagePcts?: (number | null)[];
|
||||
usageD7Pcts?: (number | null)[];
|
||||
resetAts?: (string | null)[];
|
||||
@@ -175,6 +207,7 @@ function loadCodexState(quiet = false): void {
|
||||
if (!state) return;
|
||||
|
||||
const now = Date.now();
|
||||
let restoredDeadAuth = false;
|
||||
if (
|
||||
typeof state.currentIndex === 'number' &&
|
||||
state.currentIndex < accounts.length
|
||||
@@ -195,6 +228,39 @@ function loadCodexState(quiet = false): void {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Array.isArray(state.authDeadAts)) {
|
||||
for (
|
||||
let i = 0;
|
||||
i < Math.min(state.authDeadAts.length, accounts.length);
|
||||
i++
|
||||
) {
|
||||
const deadAt = state.authDeadAts[i];
|
||||
const acct = accounts[i];
|
||||
if (typeof deadAt === 'number' && deadAt > 0) {
|
||||
const currentMtime = readAuthFileMtimeMs(acct.authPath);
|
||||
acct.authFileMtimeMs = currentMtime;
|
||||
if (currentMtime > deadAt) {
|
||||
acct.authStatus = 'dead_auth';
|
||||
acct.authDeadAt = deadAt;
|
||||
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
|
||||
restoredDeadAuth =
|
||||
restoreDeadAuthIfAuthFileChanged(acct) || restoredDeadAuth;
|
||||
} else {
|
||||
acct.authStatus = 'dead_auth';
|
||||
acct.authDeadAt = deadAt;
|
||||
acct.authDeadReason = state.authDeadReasons?.[i] ?? 'auth-expired';
|
||||
acct.rateLimitedUntil = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
currentIndex < accounts.length &&
|
||||
accounts[currentIndex]?.authStatus === 'dead_auth'
|
||||
) {
|
||||
const nextIdx = findNextCodexAvailable(currentIndex);
|
||||
if (nextIdx !== null) currentIndex = nextIdx;
|
||||
}
|
||||
if (Array.isArray(state.usagePcts)) {
|
||||
for (
|
||||
let i = 0;
|
||||
@@ -239,6 +305,7 @@ function loadCodexState(quiet = false): void {
|
||||
'Codex rotation state restored',
|
||||
);
|
||||
}
|
||||
if (restoredDeadAuth) saveCodexState();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -309,10 +376,272 @@ export function getCodexAuthPath(
|
||||
return accounts[accountIndex]?.authPath ?? null;
|
||||
}
|
||||
|
||||
function codexLockPath(authPath: string): string {
|
||||
return path.join(path.dirname(authPath), '.ejclaw-auth.lock');
|
||||
}
|
||||
|
||||
function isPidAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readExistingLease(lockPath: string): {
|
||||
leaseId?: string;
|
||||
pid?: number;
|
||||
expiresAt?: number;
|
||||
} | null {
|
||||
const data = readJsonFile<{
|
||||
leaseId?: string;
|
||||
pid?: number;
|
||||
expiresAt?: number;
|
||||
}>(lockPath);
|
||||
return data ?? null;
|
||||
}
|
||||
|
||||
function tryAcquireDiskLease(
|
||||
acct: CodexAccount,
|
||||
leaseId: string,
|
||||
now: number,
|
||||
): boolean {
|
||||
const lockPath = codexLockPath(acct.authPath);
|
||||
const payload = JSON.stringify(
|
||||
{
|
||||
leaseId,
|
||||
pid: process.pid,
|
||||
accountIndex: acct.index,
|
||||
createdAt: now,
|
||||
expiresAt: now + CODEX_AUTH_LEASE_MS,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
try {
|
||||
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
|
||||
return true;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') return false;
|
||||
}
|
||||
|
||||
const existing = readExistingLease(lockPath);
|
||||
const expired = Boolean(existing?.expiresAt && existing.expiresAt <= now);
|
||||
const deadPid = Boolean(existing?.pid && !isPidAlive(existing.pid));
|
||||
if (!expired && !deadPid) return false;
|
||||
|
||||
try {
|
||||
fs.unlinkSync(lockPath);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function releaseDiskLease(authPath: string, leaseId: string): void {
|
||||
const lockPath = codexLockPath(authPath);
|
||||
const existing = readExistingLease(lockPath);
|
||||
if (existing?.leaseId && existing.leaseId !== leaseId) return;
|
||||
try {
|
||||
fs.unlinkSync(lockPath);
|
||||
} catch {
|
||||
// Best effort only. Expired/stale locks are cleared by the next claimant.
|
||||
}
|
||||
}
|
||||
|
||||
function isCodexAccountUsable(
|
||||
acct: CodexAccount,
|
||||
now = Date.now(),
|
||||
opts?: { ignoreRateLimits?: boolean; ignoreD7?: boolean },
|
||||
): boolean {
|
||||
if (restoreDeadAuthIfAuthFileChanged(acct)) saveCodexState();
|
||||
if (acct.authStatus === 'dead_auth') return false;
|
||||
if (
|
||||
!opts?.ignoreRateLimits &&
|
||||
acct.rateLimitedUntil &&
|
||||
acct.rateLimitedUntil > now
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
!opts?.ignoreD7 &&
|
||||
acct.lastUsageD7Pct != null &&
|
||||
acct.lastUsageD7Pct >= 100
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (acct.leasedUntil && acct.leasedUntil > now) return false;
|
||||
|
||||
const existing = readExistingLease(codexLockPath(acct.authPath));
|
||||
if (!existing) return true;
|
||||
if (existing.expiresAt && existing.expiresAt <= now) return true;
|
||||
if (existing.pid && !isPidAlive(existing.pid)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function restoreDeadAuthIfAuthFileChanged(acct: CodexAccount): boolean {
|
||||
if (acct.authStatus !== 'dead_auth' || !acct.authDeadAt) return false;
|
||||
const currentMtime = readAuthFileMtimeMs(acct.authPath);
|
||||
acct.authFileMtimeMs = currentMtime;
|
||||
if (currentMtime <= acct.authDeadAt) return false;
|
||||
|
||||
const previousDeadAt = acct.authDeadAt;
|
||||
acct.authStatus = 'healthy';
|
||||
acct.authDeadAt = null;
|
||||
acct.authDeadReason = null;
|
||||
acct.rateLimitedUntil = null;
|
||||
logger.info(
|
||||
{
|
||||
transition: 'rotation:auth-file-refreshed',
|
||||
accountIndex: acct.index,
|
||||
previousDeadAt: new Date(previousDeadAt).toISOString(),
|
||||
authFileMtime: new Date(currentMtime).toISOString(),
|
||||
},
|
||||
`Codex account #${acct.index + 1}/${accounts.length} marked healthy after auth.json refresh`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
function markCodexAccountDeadAuth(acct: CodexAccount, reason?: string): void {
|
||||
acct.authStatus = 'dead_auth';
|
||||
acct.authDeadAt = Date.now();
|
||||
acct.authDeadReason = reason || 'auth-expired';
|
||||
acct.rateLimitedUntil = null;
|
||||
acct.leasedUntil = null;
|
||||
acct.leaseId = null;
|
||||
logger.warn(
|
||||
{
|
||||
transition: 'rotation:dead-auth',
|
||||
accountIndex: acct.index,
|
||||
accountId: acct.accountId,
|
||||
reason: acct.authDeadReason,
|
||||
},
|
||||
`Codex account #${acct.index + 1}/${accounts.length} marked dead; re-auth required`,
|
||||
);
|
||||
}
|
||||
|
||||
function updateAccountMetadataFromAuthFile(acct: CodexAccount): void {
|
||||
const data = readJsonFile<{
|
||||
tokens?: { account_id?: string; id_token?: string };
|
||||
}>(acct.authPath);
|
||||
if (data?.tokens?.account_id) acct.accountId = data.tokens.account_id;
|
||||
const jwt = parseJwtAuth(data?.tokens?.id_token || '');
|
||||
acct.planType = jwt.planType;
|
||||
acct.subscriptionUntil = jwt.expiresAt;
|
||||
acct.authFileMtimeMs = readAuthFileMtimeMs(acct.authPath);
|
||||
}
|
||||
|
||||
export function claimCodexAuthLease(): CodexAuthLease | null {
|
||||
if (accounts.length === 0) return null;
|
||||
const now = Date.now();
|
||||
const attempts = accounts.length;
|
||||
for (let offset = 0; offset < attempts; offset += 1) {
|
||||
const idx = (currentIndex + offset) % accounts.length;
|
||||
const acct = accounts[idx];
|
||||
if (!isCodexAccountUsable(acct, now)) continue;
|
||||
const leaseId = `${process.pid}-${now}-${idx}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
if (!tryAcquireDiskLease(acct, leaseId, now)) continue;
|
||||
currentIndex = idx;
|
||||
acct.leasedUntil = now + CODEX_AUTH_LEASE_MS;
|
||||
acct.leaseId = leaseId;
|
||||
return {
|
||||
accountIndex: idx,
|
||||
authPath: acct.authPath,
|
||||
release: () => {
|
||||
if (acct.leaseId === leaseId) {
|
||||
acct.leasedUntil = null;
|
||||
acct.leaseId = null;
|
||||
}
|
||||
releaseDiskLease(acct.authPath, leaseId);
|
||||
},
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function syncCodexSessionAuthBack(args: {
|
||||
canonicalAuthPath: string;
|
||||
sessionAuthPath: string;
|
||||
accountIndex?: number | null;
|
||||
}): boolean {
|
||||
if (!fs.existsSync(args.sessionAuthPath)) return false;
|
||||
if (!fs.existsSync(args.canonicalAuthPath)) return false;
|
||||
|
||||
const canonical = readJsonFile<{
|
||||
auth_mode?: string;
|
||||
tokens?: { account_id?: string };
|
||||
}>(args.canonicalAuthPath);
|
||||
const session = readJsonFile<{
|
||||
auth_mode?: string;
|
||||
tokens?: { account_id?: string };
|
||||
}>(args.sessionAuthPath);
|
||||
if (!canonical || !session?.tokens) return false;
|
||||
|
||||
const canonicalAccount = canonical.tokens?.account_id;
|
||||
const sessionAccount = session.tokens.account_id;
|
||||
if (
|
||||
canonicalAccount &&
|
||||
sessionAccount &&
|
||||
canonicalAccount !== sessionAccount
|
||||
) {
|
||||
logger.warn(
|
||||
{
|
||||
canonicalAuthPath: args.canonicalAuthPath,
|
||||
sessionAuthPath: args.sessionAuthPath,
|
||||
accountIndex: args.accountIndex ?? null,
|
||||
},
|
||||
'Refusing to sync Codex session auth back: account mismatch',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const sessionRaw = fs.readFileSync(args.sessionAuthPath, 'utf-8');
|
||||
const canonicalRaw = fs.readFileSync(args.canonicalAuthPath, 'utf-8');
|
||||
if (sessionRaw === canonicalRaw) return false;
|
||||
|
||||
const tmpPath = `${args.canonicalAuthPath}.tmp-${process.pid}-${Date.now()}`;
|
||||
fs.writeFileSync(tmpPath, sessionRaw, { mode: 0o600 });
|
||||
fs.renameSync(tmpPath, args.canonicalAuthPath);
|
||||
|
||||
const idx =
|
||||
args.accountIndex ??
|
||||
findCodexAccountIndexByAuthPath(args.canonicalAuthPath);
|
||||
if (idx != null && accounts[idx]) {
|
||||
const acct = accounts[idx];
|
||||
acct.authStatus = 'healthy';
|
||||
acct.authDeadAt = null;
|
||||
acct.authDeadReason = null;
|
||||
updateAccountMetadataFromAuthFile(acct);
|
||||
saveCodexState();
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
accountIndex: idx,
|
||||
canonicalAuthPath: args.canonicalAuthPath,
|
||||
},
|
||||
'Synced refreshed Codex session auth back to canonical account slot',
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function detectCodexRotationTrigger(
|
||||
error?: string | null,
|
||||
): CodexRotationTriggerResult {
|
||||
if (!error) return { shouldRotate: false, reason: '' };
|
||||
if (isCodexPoolUnavailableError(error)) {
|
||||
return { shouldRotate: false, reason: '' };
|
||||
}
|
||||
|
||||
// Common patterns (429, 503, network) — delegated to SSOT
|
||||
const common = classifyAgentError(error);
|
||||
@@ -339,18 +668,38 @@ export function rotateCodexToken(
|
||||
): boolean {
|
||||
if (accounts.length <= 1) return false;
|
||||
|
||||
const previousIndex = currentIndex;
|
||||
const acct = accounts[currentIndex];
|
||||
const cooldownUntil = computeCooldownUntil(errorMessage);
|
||||
acct.rateLimitedUntil = cooldownUntil;
|
||||
acct.lastUsagePct = 100;
|
||||
// Extract reset time string from error for display
|
||||
const retryAt = parseRetryAfterFromError(errorMessage);
|
||||
if (retryAt) {
|
||||
acct.resetAt = new Date(retryAt).toISOString();
|
||||
if (isCodexPoolUnavailableError(errorMessage)) {
|
||||
logger.warn(
|
||||
{
|
||||
transition: 'rotation:skip-pool-unavailable-sentinel',
|
||||
currentIndex,
|
||||
totalAccounts: accounts.length,
|
||||
reason: errorMessage ?? null,
|
||||
},
|
||||
'Refusing to mark Codex accounts unhealthy from internal pool-unavailable sentinel',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const nextIdx = findNextAvailable(accounts, currentIndex, opts);
|
||||
const previousIndex = currentIndex;
|
||||
const acct = accounts[currentIndex];
|
||||
const authFailure = classifyCodexAuthError(errorMessage);
|
||||
let cooldownUntil: number | null = null;
|
||||
|
||||
if (authFailure.category === 'auth-expired') {
|
||||
markCodexAccountDeadAuth(acct, errorMessage || authFailure.reason);
|
||||
} else {
|
||||
cooldownUntil = computeCooldownUntil(errorMessage);
|
||||
acct.rateLimitedUntil = cooldownUntil;
|
||||
acct.lastUsagePct = 100;
|
||||
// Extract reset time string from error for display
|
||||
const retryAt = parseRetryAfterFromError(errorMessage);
|
||||
if (retryAt) {
|
||||
acct.resetAt = new Date(retryAt).toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
const nextIdx = findNextCodexAvailable(currentIndex, opts);
|
||||
if (nextIdx !== null) {
|
||||
accounts[nextIdx].rateLimitedUntil = null;
|
||||
currentIndex = nextIdx;
|
||||
@@ -364,6 +713,7 @@ export function rotateCodexToken(
|
||||
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||
cooldownUntil:
|
||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||
authDead: authFailure.category === 'auth-expired',
|
||||
reason: errorMessage ?? null,
|
||||
},
|
||||
`Codex rotated to account #${currentIndex + 1}/${accounts.length}`,
|
||||
@@ -380,28 +730,40 @@ export function rotateCodexToken(
|
||||
ignoreRL: opts?.ignoreRateLimits ?? false,
|
||||
cooldownUntil:
|
||||
cooldownUntil != null ? new Date(cooldownUntil).toISOString() : null,
|
||||
authDead: authFailure.category === 'auth-expired',
|
||||
reason: errorMessage ?? null,
|
||||
},
|
||||
'All Codex accounts are rate-limited',
|
||||
authFailure.category === 'auth-expired'
|
||||
? 'All Codex accounts unavailable after auth failure; re-auth required'
|
||||
: 'All Codex accounts are rate-limited',
|
||||
);
|
||||
saveCodexState();
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
||||
*/
|
||||
function findNextCodexAvailable(fromIndex?: number): number | null {
|
||||
function findNextCodexAvailable(
|
||||
fromIndex?: number,
|
||||
opts?: { ignoreRateLimits?: boolean },
|
||||
): number | null {
|
||||
const now = Date.now();
|
||||
const start = fromIndex ?? currentIndex;
|
||||
for (let i = 1; i < accounts.length; i++) {
|
||||
const idx = (start + i) % accounts.length;
|
||||
const acct = accounts[idx];
|
||||
const rlOk = !acct.rateLimitedUntil || acct.rateLimitedUntil <= now;
|
||||
const usageOk = acct.lastUsageD7Pct == null || acct.lastUsageD7Pct < 100;
|
||||
if (rlOk && usageOk) return idx;
|
||||
if (isCodexAccountUsable(acct, now, opts)) return idx;
|
||||
}
|
||||
// All exhausted — fall back to rate-limit-only check
|
||||
return findNextAvailable(accounts, start);
|
||||
// All d7-exhausted — fall back to rate-limit/dead/lease checks only.
|
||||
for (let i = 1; i < accounts.length; i++) {
|
||||
const idx = (start + i) % accounts.length;
|
||||
const acct = accounts[idx];
|
||||
if (isCodexAccountUsable(acct, now, { ...opts, ignoreD7: true })) {
|
||||
return idx;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -469,9 +831,17 @@ export function updateCodexAccountUsage(
|
||||
export function markCodexTokenHealthy(): void {
|
||||
if (accounts.length === 0) return;
|
||||
const acct = accounts[currentIndex];
|
||||
let changed = false;
|
||||
if (acct?.authStatus === 'dead_auth') {
|
||||
acct.authStatus = 'healthy';
|
||||
acct.authDeadAt = null;
|
||||
acct.authDeadReason = null;
|
||||
changed = true;
|
||||
}
|
||||
if (acct?.rateLimitedUntil) {
|
||||
const previousCooldownUntil = acct.rateLimitedUntil;
|
||||
acct.rateLimitedUntil = null;
|
||||
changed = true;
|
||||
logger.info(
|
||||
{
|
||||
transition: 'rotation:clear-rate-limit',
|
||||
@@ -481,8 +851,8 @@ export function markCodexTokenHealthy(): void {
|
||||
},
|
||||
'Cleared Codex account rate-limit state after successful response',
|
||||
);
|
||||
saveCodexState();
|
||||
}
|
||||
if (changed) saveCodexState();
|
||||
}
|
||||
|
||||
export function getCodexAccountCount(): number {
|
||||
@@ -495,6 +865,10 @@ export function getAllCodexAccounts(): {
|
||||
planType: string;
|
||||
isActive: boolean;
|
||||
isRateLimited: boolean;
|
||||
isAuthDead?: boolean;
|
||||
authStatus?: 'healthy' | 'dead_auth';
|
||||
authDeadAt?: string;
|
||||
isLeased?: boolean;
|
||||
cachedUsagePct?: number;
|
||||
cachedUsageD7Pct?: number;
|
||||
resetAt?: string;
|
||||
@@ -507,6 +881,10 @@ export function getAllCodexAccounts(): {
|
||||
planType: a.planType,
|
||||
isActive: i === currentIndex,
|
||||
isRateLimited: Boolean(a.rateLimitedUntil && a.rateLimitedUntil > now),
|
||||
isAuthDead: a.authStatus === 'dead_auth',
|
||||
authStatus: a.authStatus,
|
||||
authDeadAt: a.authDeadAt ? new Date(a.authDeadAt).toISOString() : undefined,
|
||||
isLeased: Boolean(a.leasedUntil && a.leasedUntil > now),
|
||||
cachedUsagePct: a.lastUsagePct,
|
||||
cachedUsageD7Pct: a.lastUsageD7Pct,
|
||||
resetAt: a.resetAt,
|
||||
|
||||
@@ -31,10 +31,120 @@ export interface CodexUsageRefreshResult {
|
||||
/** Full scan interval — exported so the orchestrator can schedule it. */
|
||||
export const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
||||
|
||||
/**
|
||||
* Threshold above which a Codex account is considered "out of budget" for
|
||||
* the purpose of admitting new turns. A turn is blocked only when *every*
|
||||
* Codex account is either at/over this threshold on either window or
|
||||
* already in a rate-limit cooldown.
|
||||
*/
|
||||
export const CODEX_EXHAUSTION_THRESHOLD_PCT = 95;
|
||||
|
||||
export interface UsageExhaustionInfo {
|
||||
exhausted: boolean;
|
||||
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
||||
nextResetAt: string | null;
|
||||
}
|
||||
|
||||
function parseResetMs(s: string | undefined | null): number | null {
|
||||
if (!s) return null;
|
||||
const t = Date.parse(s);
|
||||
return Number.isFinite(t) ? t : null;
|
||||
}
|
||||
|
||||
/** Coerce a string-or-number resetsAt (numeric epoch in seconds) to ISO. */
|
||||
function toIsoMaybe(value: string | number | undefined): string | undefined {
|
||||
if (value == null) return undefined;
|
||||
if (typeof value === 'number') {
|
||||
return new Date(value * 1000).toISOString();
|
||||
}
|
||||
// Already a string — pass through if it parses, else drop.
|
||||
const ms = Date.parse(value);
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether every configured Codex account is either ≥ 95% on a
|
||||
* window or in a rate-limit cooldown, and — when so — the soonest moment
|
||||
* at which any one account is expected to recover.
|
||||
*
|
||||
* Reads the in-process rotation snapshot; performs no I/O. Returns
|
||||
* `exhausted: false` when there are no configured accounts, or when usage
|
||||
* data is missing for any account (err on letting work through).
|
||||
*/
|
||||
export function getCodexUsageExhaustion(): UsageExhaustionInfo {
|
||||
const accounts = getAllCodexAccounts();
|
||||
if (accounts.length === 0) return { exhausted: false, nextResetAt: null };
|
||||
|
||||
let earliestRecoveryMs = Infinity;
|
||||
|
||||
for (const a of accounts) {
|
||||
if (a.isRateLimited) continue; // counts as exhausted, recovery unknown here
|
||||
const h5 = a.cachedUsagePct;
|
||||
const d7 = a.cachedUsageD7Pct;
|
||||
// -1 / undefined means "unknown" → treat as headroom
|
||||
if (h5 == null || h5 < 0 || d7 == null || d7 < 0) {
|
||||
return { exhausted: false, nextResetAt: null };
|
||||
}
|
||||
if (Math.max(h5, d7) < CODEX_EXHAUSTION_THRESHOLD_PCT) {
|
||||
return { exhausted: false, nextResetAt: null };
|
||||
}
|
||||
|
||||
const h5ResetMs =
|
||||
h5 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetAt) : null;
|
||||
const d7ResetMs =
|
||||
d7 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetD7At) : null;
|
||||
const candidates = [h5ResetMs, d7ResetMs].filter(
|
||||
(v): v is number => v != null,
|
||||
);
|
||||
if (candidates.length > 0) {
|
||||
const accountRecovery = Math.max(...candidates);
|
||||
if (accountRecovery < earliestRecoveryMs) {
|
||||
earliestRecoveryMs = accountRecovery;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
||||
? new Date(earliestRecoveryMs).toISOString()
|
||||
: null;
|
||||
return { exhausted: true, nextResetAt };
|
||||
}
|
||||
|
||||
/** Backwards-compat boolean wrapper. */
|
||||
export function isCodexUsageExhausted(): boolean {
|
||||
return getCodexUsageExhaustion().exhausted;
|
||||
}
|
||||
|
||||
function getFnmCodexBinDirs(): string[] {
|
||||
const fnmRoot = path.join(os.homedir(), '.local', 'share', 'fnm');
|
||||
const dirs: string[] = [];
|
||||
// Prefer the alias `default` first if it exists.
|
||||
const defaultBin = path.join(fnmRoot, 'aliases', 'default', 'bin');
|
||||
if (fs.existsSync(defaultBin)) dirs.push(defaultBin);
|
||||
// Then fall back to scanning every installed node version.
|
||||
const versionsRoot = path.join(fnmRoot, 'node-versions');
|
||||
try {
|
||||
if (fs.existsSync(versionsRoot)) {
|
||||
for (const entry of fs.readdirSync(versionsRoot)) {
|
||||
const bin = path.join(versionsRoot, entry, 'installation', 'bin');
|
||||
if (fs.existsSync(bin)) dirs.push(bin);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
function getPreferredCodexPathEntries(): string[] {
|
||||
const entries = [
|
||||
path.dirname(process.execPath),
|
||||
path.join(os.homedir(), '.npm-global', 'bin'),
|
||||
// fnm-managed node installs (where `npm i -g @openai/codex` actually
|
||||
// lands when the host uses fnm). Without these, ejclaw running under
|
||||
// bun/systemd cannot find the `codex` binary even though the user has
|
||||
// it installed via fnm's default node.
|
||||
...getFnmCodexBinDirs(),
|
||||
];
|
||||
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
||||
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
||||
@@ -42,6 +152,18 @@ function getPreferredCodexPathEntries(): string[] {
|
||||
return [...new Set(entries)];
|
||||
}
|
||||
|
||||
function findCodexBinary(): string {
|
||||
const candidates = [
|
||||
path.join(os.homedir(), '.npm-global', 'bin', 'codex'),
|
||||
...getFnmCodexBinDirs().map((dir) => path.join(dir, 'codex')),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) return candidate;
|
||||
}
|
||||
// Last resort: rely on PATH (we extend it via getPreferredCodexPathEntries).
|
||||
return 'codex';
|
||||
}
|
||||
|
||||
function getCodexHomeForAccount(accountIndex?: number): string | null {
|
||||
const authPath = getCodexAuthPath(accountIndex);
|
||||
if (!authPath || !fs.existsSync(authPath)) return null;
|
||||
@@ -51,8 +173,7 @@ function getCodexHomeForAccount(accountIndex?: number): string | null {
|
||||
export async function fetchCodexUsage(
|
||||
codexHomeOverride?: string,
|
||||
): Promise<CodexRateLimit[] | null> {
|
||||
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
||||
const codexBin = fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
||||
const codexBin = findCodexBinary();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
@@ -193,20 +314,19 @@ export function applyCodexUsageToAccount(
|
||||
|
||||
const pct = Math.round(effective.primary.usedPercent);
|
||||
const d7Pct = Math.round(effective.secondary.usedPercent);
|
||||
const resetStr = effective.primary.resetsAt
|
||||
? formatResetRemaining(effective.primary.resetsAt)
|
||||
: undefined;
|
||||
const resetD7Str = effective.secondary.resetsAt
|
||||
? formatResetRemaining(effective.secondary.resetsAt)
|
||||
: undefined;
|
||||
updateCodexAccountUsage(pct, resetStr, accountIndex, d7Pct, resetD7Str);
|
||||
// Store raw ISO timestamps (not pre-formatted strings) so the exhaustion
|
||||
// gate can compute "minutes until reset" later. The dashboard render path
|
||||
// formats these at display time via `formatResetRemaining`.
|
||||
const resetIso = toIsoMaybe(effective.primary.resetsAt);
|
||||
const resetD7Iso = toIsoMaybe(effective.secondary.resetsAt);
|
||||
updateCodexAccountUsage(pct, resetIso, accountIndex, d7Pct, resetD7Iso);
|
||||
logger.info(
|
||||
{
|
||||
account: accountIndex + 1,
|
||||
bucket: effective.limitId,
|
||||
h5: pct,
|
||||
d7: d7Pct,
|
||||
reset: resetStr,
|
||||
reset: resetIso,
|
||||
},
|
||||
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
||||
);
|
||||
@@ -229,9 +349,9 @@ export function buildCodexUsageRowsFromState(): UsageRow[] {
|
||||
return {
|
||||
name: label,
|
||||
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
||||
h5reset: acct.resetAt || '',
|
||||
h5reset: acct.resetAt ? formatResetRemaining(acct.resetAt) : '',
|
||||
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
||||
d7reset: acct.resetD7At || '',
|
||||
d7reset: acct.resetD7At ? formatResetRemaining(acct.resetD7At) : '',
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -168,6 +168,66 @@ describe('Codex warm-up scheduler', () => {
|
||||
expect(state.consecutiveFailures).toBe(0);
|
||||
});
|
||||
|
||||
it('spawns the bundled codex JS launcher via the JS runtime instead of a bare "codex" PATH lookup', async () => {
|
||||
// Regression: under bun/systemd there is usually no global `codex` on PATH,
|
||||
// so spawning bare `codex` failed with ENOENT and the primer never fired.
|
||||
const childProcess = await import('child_process');
|
||||
const rotation = await import('./codex-token-rotation.js');
|
||||
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||
const now = new Date('2026-04-24T09:00:00Z').getTime();
|
||||
let capturedCmd: string | undefined;
|
||||
let capturedArgs: readonly string[] | undefined;
|
||||
|
||||
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'fresh-account',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 0,
|
||||
cachedUsageD7Pct: 0,
|
||||
},
|
||||
]);
|
||||
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||
);
|
||||
vi.mocked(childProcess.spawn).mockImplementation(((
|
||||
cmd: string,
|
||||
args?: readonly string[],
|
||||
) => {
|
||||
capturedCmd = cmd;
|
||||
capturedArgs = args;
|
||||
return createFakeCodexProcess(0) as never;
|
||||
}) as unknown as typeof childProcess.spawn);
|
||||
|
||||
await runCodexWarmupCycle(
|
||||
{
|
||||
enabled: true,
|
||||
prompt: 'Reply exactly OK. Do not run tools.',
|
||||
model: 'gpt-5.5',
|
||||
intervalMs: 300_000,
|
||||
minIntervalMs: 18_300_000,
|
||||
staggerMs: 1_800_000,
|
||||
maxUsagePct: 0,
|
||||
maxD7UsagePct: 0,
|
||||
commandTimeoutMs: 120_000,
|
||||
failureCooldownMs: 21_600_000,
|
||||
maxConsecutiveFailures: 2,
|
||||
},
|
||||
{ nowMs: now, statePath },
|
||||
);
|
||||
|
||||
expect(childProcess.spawn).toHaveBeenCalledTimes(1);
|
||||
// Command is the JS runtime (bun/node), never a bare "codex" PATH lookup.
|
||||
expect(capturedCmd).toBe(process.execPath);
|
||||
expect(capturedCmd).not.toBe('codex');
|
||||
// First arg is the resolvable codex JS launcher; 'exec' follows after it.
|
||||
expect(capturedArgs?.[0]).toMatch(/codex\.js$/);
|
||||
expect(fs.existsSync(capturedArgs?.[0] as string)).toBe(true);
|
||||
expect(capturedArgs?.[1]).toBe('exec');
|
||||
});
|
||||
|
||||
it('does not repeat warm-up while the same zero-usage quota window is already marked warmed', async () => {
|
||||
const childProcess = await import('child_process');
|
||||
const rotation = await import('./codex-token-rotation.js');
|
||||
@@ -393,3 +453,118 @@ describe('Codex warm-up scheduler', () => {
|
||||
expect(childProcess.spawn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Codex warm-up fixed-slot forceAttempt', () => {
|
||||
let tempHome: string;
|
||||
let statePath: string;
|
||||
// Two consecutive slots 5h apart, but the primer fires a beat after the slot,
|
||||
// so the gap is just under minIntervalMs (5h).
|
||||
const t13 = new Date('2026-06-09T04:00:00.486Z').getTime();
|
||||
const t18 = new Date('2026-06-09T09:00:00.000Z').getTime();
|
||||
const config = {
|
||||
enabled: true as const,
|
||||
prompt: 'Reply exactly OK. Do not run tools.',
|
||||
model: 'gpt-5.5',
|
||||
intervalMs: 300_000,
|
||||
minIntervalMs: 18_000_000, // 5h
|
||||
staggerMs: 1_800_000,
|
||||
maxUsagePct: 100,
|
||||
maxD7UsagePct: 100,
|
||||
commandTimeoutMs: 120_000,
|
||||
failureCooldownMs: 21_600_000,
|
||||
maxConsecutiveFailures: 2,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-force-'));
|
||||
statePath = path.join(tempHome, 'codex-warmup-state.json');
|
||||
fs.mkdirSync(path.dirname(authPathFor(tempHome, 0)), { recursive: true });
|
||||
fs.writeFileSync(authPathFor(tempHome, 0), '{}');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function setup() {
|
||||
const childProcess = await import('child_process');
|
||||
const rotation = await import('./codex-token-rotation.js');
|
||||
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'acct',
|
||||
planType: 'pro',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 0,
|
||||
cachedUsageD7Pct: 0,
|
||||
},
|
||||
]);
|
||||
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||
);
|
||||
vi.mocked(childProcess.spawn).mockImplementation(
|
||||
(() =>
|
||||
createFakeCodexProcess(
|
||||
0,
|
||||
) as never) as unknown as typeof childProcess.spawn,
|
||||
);
|
||||
return { childProcess, runCodexWarmupCycle };
|
||||
}
|
||||
|
||||
it('fires at the next slot even when it is just under the 5h min-interval', async () => {
|
||||
const { childProcess, runCodexWarmupCycle } = await setup();
|
||||
const r1 = await runCodexWarmupCycle(config, {
|
||||
nowMs: t13,
|
||||
statePath,
|
||||
ignoreZeroUsageWindow: true,
|
||||
forceAttempt: true,
|
||||
});
|
||||
expect(r1).toEqual({ status: 'warmed', accountIndex: 0 });
|
||||
const r2 = await runCodexWarmupCycle(config, {
|
||||
nowMs: t18,
|
||||
statePath,
|
||||
ignoreZeroUsageWindow: true,
|
||||
forceAttempt: true,
|
||||
});
|
||||
expect(r2).toEqual({ status: 'warmed', accountIndex: 0 });
|
||||
expect(childProcess.spawn).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('without forceAttempt, the sub-5h second slot is skipped (regression guard)', async () => {
|
||||
const { runCodexWarmupCycle } = await setup();
|
||||
await runCodexWarmupCycle(config, {
|
||||
nowMs: t13,
|
||||
statePath,
|
||||
ignoreZeroUsageWindow: true,
|
||||
});
|
||||
const r2 = await runCodexWarmupCycle(config, {
|
||||
nowMs: t18,
|
||||
statePath,
|
||||
ignoreZeroUsageWindow: true,
|
||||
});
|
||||
expect(r2).toEqual({ status: 'skipped', reason: 'no_eligible_accounts' });
|
||||
});
|
||||
|
||||
it('forceAttempt bypasses post-failure cooldown and stagger', async () => {
|
||||
const { runCodexWarmupCycle } = await setup();
|
||||
fs.writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify({
|
||||
disabledUntil: new Date(t18 + 3_600_000).toISOString(),
|
||||
lastWarmupAt: new Date(t18 - 1_000).toISOString(),
|
||||
accounts: {},
|
||||
}),
|
||||
);
|
||||
const result = await runCodexWarmupCycle(config, {
|
||||
nowMs: t18,
|
||||
statePath,
|
||||
ignoreZeroUsageWindow: true,
|
||||
forceAttempt: true,
|
||||
});
|
||||
expect(result).toEqual({ status: 'warmed', accountIndex: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { ChildProcess, spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import { createRequire } from 'module';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
import { DATA_DIR } from './config.js';
|
||||
import type { AppConfig } from './config/schema.js';
|
||||
@@ -34,6 +36,15 @@ interface CodexWarmupRuntimeOptions {
|
||||
statePath?: string;
|
||||
shouldSkip?: () => boolean;
|
||||
ignoreZeroUsageWindow?: boolean;
|
||||
/**
|
||||
* Fixed-slot primer mode: bypass the opportunistic warm-up skip gates
|
||||
* (stagger, post-failure cooldown, min-interval, per-account usage/rate-limit
|
||||
* filters) so a real Codex command is attempted at every 08/13/18/23 slot and
|
||||
* its success/failure recorded. Without this, two slots exactly 5h apart fall
|
||||
* just under minIntervalMs (the primer fires a few hundred ms after the slot)
|
||||
* and the later slot is silently skipped as `no_eligible_accounts`.
|
||||
*/
|
||||
forceAttempt?: boolean;
|
||||
}
|
||||
|
||||
export type CodexWarmupCycleResult =
|
||||
@@ -87,9 +98,63 @@ function getPreferredCodexPathEntries(): string[] {
|
||||
return [...new Set(entries)];
|
||||
}
|
||||
|
||||
function resolveCodexBinary(): string {
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
interface CodexLauncher {
|
||||
command: string;
|
||||
prefixArgs: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve how to spawn the Codex CLI for warm-up.
|
||||
*
|
||||
* The bot runs under bun/systemd where a global `codex` binary usually is NOT
|
||||
* on PATH, so the old bare-`codex` fallback failed with ENOENT and the primer
|
||||
* never fired. Prefer the bundled `@openai/codex` JS launcher (the same path
|
||||
* the codex-runner spawns) and run it through the current JS runtime, falling
|
||||
* back to a globally installed binary only when the package is unavailable.
|
||||
*/
|
||||
function resolveCodexLauncher(): CodexLauncher {
|
||||
const launcherCandidates: string[] = [];
|
||||
try {
|
||||
launcherCandidates.push(
|
||||
path.join(
|
||||
path.dirname(require.resolve('@openai/codex/package.json')),
|
||||
'bin',
|
||||
'codex.js',
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
/* @openai/codex not resolvable from this module — try explicit paths */
|
||||
}
|
||||
// The codex-runner workspace always vendors @openai/codex.
|
||||
const repoRoot = path.resolve(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
'..',
|
||||
);
|
||||
launcherCandidates.push(
|
||||
path.join(
|
||||
repoRoot,
|
||||
'runners',
|
||||
'codex-runner',
|
||||
'node_modules',
|
||||
'@openai',
|
||||
'codex',
|
||||
'bin',
|
||||
'codex.js',
|
||||
),
|
||||
);
|
||||
for (const launcher of launcherCandidates) {
|
||||
if (fs.existsSync(launcher)) {
|
||||
return { command: process.execPath, prefixArgs: [launcher] };
|
||||
}
|
||||
}
|
||||
// Fall back to a globally installed codex binary, then bare PATH lookup.
|
||||
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
||||
return fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
||||
if (fs.existsSync(npmGlobalBin)) {
|
||||
return { command: npmGlobalBin, prefixArgs: [] };
|
||||
}
|
||||
return { command: 'codex', prefixArgs: [] };
|
||||
}
|
||||
|
||||
function ensureAccountState(
|
||||
@@ -106,15 +171,20 @@ function selectWarmupCandidate(
|
||||
config: CodexWarmupConfig,
|
||||
state: CodexWarmupState,
|
||||
nowMs: number,
|
||||
options: { ignoreZeroUsageWindow?: boolean } = {},
|
||||
options: { ignoreZeroUsageWindow?: boolean; forceAttempt?: boolean } = {},
|
||||
): { accountIndex: number; zeroUsageWarmupUntil: string } | { reason: string } {
|
||||
const disabledUntilMs = parseTimestamp(state.disabledUntil);
|
||||
if (disabledUntilMs != null && disabledUntilMs > nowMs) {
|
||||
if (
|
||||
!options.forceAttempt &&
|
||||
disabledUntilMs != null &&
|
||||
disabledUntilMs > nowMs
|
||||
) {
|
||||
return { reason: 'disabled_cooldown' };
|
||||
}
|
||||
|
||||
const lastGlobalWarmupMs = parseTimestamp(state.lastWarmupAt);
|
||||
if (
|
||||
!options.forceAttempt &&
|
||||
lastGlobalWarmupMs != null &&
|
||||
config.staggerMs > 0 &&
|
||||
nowMs - lastGlobalWarmupMs < config.staggerMs
|
||||
@@ -125,6 +195,16 @@ function selectWarmupCandidate(
|
||||
const accounts = getAllCodexAccounts();
|
||||
if (accounts.length === 0) return { reason: 'no_accounts' };
|
||||
|
||||
// Fixed-slot primer: attempt a real command on the active account every slot,
|
||||
// bypassing the per-account usage/rate-limit/min-interval filters below.
|
||||
if (options.forceAttempt) {
|
||||
const active = accounts.find((a) => a.isActive) ?? accounts[0];
|
||||
return {
|
||||
accountIndex: active.index,
|
||||
zeroUsageWarmupUntil: new Date(nowMs).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
for (const account of accounts) {
|
||||
if (account.isRateLimited) continue;
|
||||
if (typeof account.cachedUsagePct !== 'number') continue;
|
||||
@@ -219,7 +299,8 @@ function runCodexWarmupCommand(
|
||||
);
|
||||
|
||||
try {
|
||||
proc = spawn(resolveCodexBinary(), args, {
|
||||
const launcher = resolveCodexLauncher();
|
||||
proc = spawn(launcher.command, [...launcher.prefixArgs, ...args], {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: spawnEnv,
|
||||
});
|
||||
@@ -265,6 +346,7 @@ export async function runCodexWarmupCycle(
|
||||
const state = readWarmupState(statePath);
|
||||
const selected = selectWarmupCandidate(config, state, nowMs, {
|
||||
ignoreZeroUsageWindow: runtime.ignoreZeroUsageWindow,
|
||||
forceAttempt: runtime.forceAttempt,
|
||||
});
|
||||
if ('reason' in selected) {
|
||||
return { status: 'skipped', reason: selected.reason };
|
||||
|
||||
@@ -95,6 +95,14 @@ export const AGENT_LANGUAGE = CONFIG.paired.agentLanguage;
|
||||
export const ARBITER_DEADLOCK_THRESHOLD =
|
||||
CONFIG.paired.arbiterDeadlockThreshold;
|
||||
|
||||
/**
|
||||
* Maximum number of times the arbiter may intervene on a single task before the
|
||||
* task is escalated to the user instead of re-invoking the arbiter. This bounds
|
||||
* the owner↔reviewer↔arbiter loop: once the arbiter has ruled this many times
|
||||
* and the loop still deadlocks, a human is asked to step in.
|
||||
*/
|
||||
export const ARBITER_MAX_INTERVENTIONS = CONFIG.paired.arbiterMaxInterventions;
|
||||
|
||||
export function isArbiterEnabled(): boolean {
|
||||
return ARBITER_AGENT_TYPE !== undefined;
|
||||
}
|
||||
|
||||
@@ -244,6 +244,7 @@ export function loadConfig(): AppConfig {
|
||||
),
|
||||
agentLanguage: readText('AGENT_LANGUAGE') ?? '',
|
||||
arbiterDeadlockThreshold: readInteger('ARBITER_DEADLOCK_THRESHOLD', 2),
|
||||
arbiterMaxInterventions: readInteger('ARBITER_MAX_INTERVENTIONS', 1),
|
||||
maxRoundTrips,
|
||||
},
|
||||
models: {
|
||||
@@ -259,7 +260,7 @@ export function loadConfig(): AppConfig {
|
||||
status: {
|
||||
channelId: readText('STATUS_CHANNEL_ID') ?? '',
|
||||
updateInterval: 10000,
|
||||
usageUpdateInterval: 300000,
|
||||
usageUpdateInterval: 60000,
|
||||
showRooms: readBooleanUnlessFalse('STATUS_SHOW_ROOMS', true),
|
||||
showRoomDetails: readBooleanUnlessFalse('STATUS_SHOW_ROOM_DETAILS', true),
|
||||
usageDashboardEnabled: readText('USAGE_DASHBOARD') === 'true',
|
||||
|
||||
@@ -57,6 +57,7 @@ export interface AppConfig {
|
||||
forceFreshClaudeReviewerSessionInUnsafeHostMode: boolean;
|
||||
agentLanguage: string;
|
||||
arbiterDeadlockThreshold: number;
|
||||
arbiterMaxInterventions: number;
|
||||
maxRoundTrips: number;
|
||||
};
|
||||
models: {
|
||||
|
||||
@@ -28,6 +28,7 @@ describe('listUnexpectedDataStateFiles', () => {
|
||||
fs.writeFileSync(path.join(dataDir, 'codex-warmup-state.json'), '{}');
|
||||
fs.writeFileSync(path.join(dataDir, 'claude-usage-cache.json'), '{}');
|
||||
fs.writeFileSync(path.join(dataDir, 'restart-context.abc.json'), '{}');
|
||||
fs.writeFileSync(path.join(dataDir, 'auto-paired-mode-state.json'), '{}');
|
||||
|
||||
expect(listUnexpectedDataStateFiles(dataDir)).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ const ALLOWED_DATA_JSON_PATTERNS = [
|
||||
/^codex-warmup-state\.json$/,
|
||||
/^claude-usage-cache\.json$/,
|
||||
/^restart-context\..+\.json$/,
|
||||
/^auto-paired-mode-state\.json$/,
|
||||
];
|
||||
|
||||
function isAllowedDataStateJson(filename: string): boolean {
|
||||
|
||||
@@ -143,6 +143,43 @@ describe('paired turn attempt restart recovery', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('clears stale running attempts for already completed tasks without replaying them', () => {
|
||||
const task = makeTask({
|
||||
id: 'completed-stale-running-task',
|
||||
status: 'completed',
|
||||
completion_reason: 'done',
|
||||
});
|
||||
createPairedTask(task);
|
||||
const turnIdentity = buildPairedTurnIdentity({
|
||||
taskId: task.id,
|
||||
taskUpdatedAt: task.updated_at,
|
||||
intentKind: 'owner-turn',
|
||||
role: 'owner',
|
||||
});
|
||||
|
||||
markPairedTurnRunning({
|
||||
turnIdentity,
|
||||
executorServiceId: CODEX_MAIN_SERVICE_ID,
|
||||
executorAgentType: 'codex',
|
||||
runId: 'completed-run-before-restart',
|
||||
});
|
||||
|
||||
expect(
|
||||
recoverInterruptedPairedTurnAttemptsForService({
|
||||
serviceIds: [CODEX_MAIN_SERVICE_ID],
|
||||
now: '2026-05-27T00:11:00.000Z',
|
||||
}),
|
||||
).toEqual([]);
|
||||
expect(getPairedTurnAttempts(turnIdentity.turnId)).toEqual([
|
||||
expect.objectContaining({
|
||||
state: 'failed',
|
||||
active_run_id: null,
|
||||
completed_at: '2026-05-27T00:11:00.000Z',
|
||||
last_error: 'Interrupted by service restart before completion.',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('recovers codex executor attempts even when the orchestration lease used the claude service id', () => {
|
||||
const task = makeTask({ id: 'cross-service-lease-task' });
|
||||
createPairedTask(task);
|
||||
|
||||
@@ -2724,7 +2724,10 @@ describe('room assignment writes', () => {
|
||||
modeSource: 'inferred',
|
||||
name: 'Legacy SQL Room',
|
||||
folder: 'legacy-sql-room',
|
||||
ownerAgentType: 'codex',
|
||||
// Owner inference picks OWNER_AGENT_TYPE (env-driven, default codex but
|
||||
// currently claude-code in this deployment) when both agent types are
|
||||
// registered. See inferOwnerAgentTypeFromRegisteredAgentTypes.
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
expect(getEffectiveRoomMode('dc:legacy-sql')).toBe('tribunal');
|
||||
expect(getEffectiveRuntimeRoomMode('dc:legacy-sql')).toBe('tribunal');
|
||||
@@ -3340,7 +3343,8 @@ describe('paired room registration', () => {
|
||||
name: 'Room Settings Test',
|
||||
folder: 'room-settings-test',
|
||||
trigger: '@Codex',
|
||||
ownerAgentType: 'codex',
|
||||
// Both agent types registered → owner inference picks OWNER_AGENT_TYPE.
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
|
||||
setExplicitRoomMode('dc:room-settings', 'single');
|
||||
@@ -3352,7 +3356,7 @@ describe('paired room registration', () => {
|
||||
name: 'Room Settings Test',
|
||||
folder: 'room-settings-test',
|
||||
trigger: '@Codex',
|
||||
ownerAgentType: 'codex',
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
|
||||
updateRegisteredGroupName('dc:room-settings', 'Room Settings Renamed');
|
||||
@@ -3364,7 +3368,7 @@ describe('paired room registration', () => {
|
||||
name: 'Room Settings Renamed',
|
||||
folder: 'room-settings-test',
|
||||
trigger: '@Codex',
|
||||
ownerAgentType: 'codex',
|
||||
ownerAgentType: OWNER_AGENT_TYPE,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -147,6 +147,7 @@ export function applyBaseSchema(database: Database): void {
|
||||
finalize_step_done_count INTEGER NOT NULL DEFAULT 0,
|
||||
task_done_then_user_reopen_count INTEGER NOT NULL DEFAULT 0,
|
||||
empty_step_done_streak INTEGER NOT NULL DEFAULT 0,
|
||||
arbiter_intervention_count INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
arbiter_verdict TEXT,
|
||||
arbiter_requested_at TEXT,
|
||||
|
||||
@@ -52,8 +52,10 @@ function getExpectedSchemaMigrations(): Array<{
|
||||
{ version: 16, name: 'room_skill_overrides' },
|
||||
{ version: 17, name: 'scheduled_task_room_role' },
|
||||
{ version: 18, name: 'paired_turn_output_attachments' },
|
||||
{ version: 19, name: 'reviewer_failure_count' },
|
||||
{ version: 20, name: 'turn_progress_text_compat' },
|
||||
{ version: 19, name: 'turn_progress_text_recovery' },
|
||||
{ version: 20, name: 'arbiter_intervention_count' },
|
||||
{ version: 21, name: 'reviewer_failure_count' },
|
||||
{ version: 22, name: 'turn_progress_text_compat' },
|
||||
];
|
||||
}
|
||||
|
||||
@@ -162,15 +164,15 @@ describe('initializeDatabaseSchema', () => {
|
||||
expect(columns).toContain('progress_updated_at');
|
||||
|
||||
// The pre-existing version-15 row is left untouched; the compat migration
|
||||
// (version 20) is what reconciles the schema.
|
||||
// (version 22 after merge-renumbering) is what reconciles the schema.
|
||||
const version15 = database
|
||||
.prepare('SELECT name FROM schema_migrations WHERE version = 15')
|
||||
.get() as { name: string };
|
||||
expect(version15.name).toBe('reviewer_failure_count');
|
||||
const version20 = database
|
||||
.prepare('SELECT name FROM schema_migrations WHERE version = 20')
|
||||
const version22 = database
|
||||
.prepare('SELECT name FROM schema_migrations WHERE version = 22')
|
||||
.get() as { name: string } | null;
|
||||
expect(version20?.name).toBe('turn_progress_text_compat');
|
||||
expect(version22?.name).toBe('turn_progress_text_compat');
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
36
src/db/migrations/019_turn-progress-text-recovery.ts
Normal file
36
src/db/migrations/019_turn-progress-text-recovery.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import type { Database } from 'bun:sqlite';
|
||||
|
||||
import { tableHasColumn } from './helpers.js';
|
||||
import type { SchemaMigrationDefinition } from './types.js';
|
||||
|
||||
/**
|
||||
* Recovery for a migration-version collision introduced by merging
|
||||
* origin/main into the long-lived deployment branch.
|
||||
*
|
||||
* The deployment DB recorded schema version 15 as `reviewer_failure_count`
|
||||
* (a downstream-only migration), while upstream defines version 15 as
|
||||
* `turn_progress_text` (migration 015). Because the migration runner tracks
|
||||
* applied migrations by version number alone, the upstream `turn_progress_text`
|
||||
* migration is skipped on the live DB and `paired_turns.progress_text` /
|
||||
* `progress_updated_at` are never created — even though merged runtime code
|
||||
* (src/db/paired-turns.ts, src/web-dashboard-data.ts) reads and writes them.
|
||||
*
|
||||
* This migration re-applies those columns idempotently under a fresh version
|
||||
* number so the live deployment gets them. It is a no-op on databases where
|
||||
* migration 015 already ran (fresh installs), thanks to the column guards.
|
||||
*/
|
||||
export const TURN_PROGRESS_TEXT_RECOVERY_MIGRATION: SchemaMigrationDefinition =
|
||||
{
|
||||
version: 19,
|
||||
name: 'turn_progress_text_recovery',
|
||||
apply(database: Database) {
|
||||
if (!tableHasColumn(database, 'paired_turns', 'progress_text')) {
|
||||
database.exec(`ALTER TABLE paired_turns ADD COLUMN progress_text TEXT`);
|
||||
}
|
||||
if (!tableHasColumn(database, 'paired_turns', 'progress_updated_at')) {
|
||||
database.exec(
|
||||
`ALTER TABLE paired_turns ADD COLUMN progress_updated_at TEXT`,
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
25
src/db/migrations/020_arbiter-intervention-count.ts
Normal file
25
src/db/migrations/020_arbiter-intervention-count.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import type { Database } from 'bun:sqlite';
|
||||
|
||||
import { tableHasColumn } from './helpers.js';
|
||||
import type { SchemaMigrationDefinition } from './types.js';
|
||||
|
||||
export const ARBITER_INTERVENTION_COUNT_MIGRATION: SchemaMigrationDefinition = {
|
||||
version: 20,
|
||||
name: 'arbiter_intervention_count',
|
||||
apply(database: Database) {
|
||||
if (
|
||||
!tableHasColumn(database, 'paired_tasks', 'arbiter_intervention_count')
|
||||
) {
|
||||
database.exec(`
|
||||
ALTER TABLE paired_tasks
|
||||
ADD COLUMN arbiter_intervention_count INTEGER NOT NULL DEFAULT 0
|
||||
`);
|
||||
}
|
||||
|
||||
database.exec(`
|
||||
UPDATE paired_tasks
|
||||
SET arbiter_intervention_count = 0
|
||||
WHERE arbiter_intervention_count IS NULL
|
||||
`);
|
||||
},
|
||||
};
|
||||
@@ -4,7 +4,7 @@ import { tableHasColumn } from './helpers.js';
|
||||
import type { SchemaMigrationDefinition } from './types.js';
|
||||
|
||||
export const REVIEWER_FAILURE_COUNT_MIGRATION: SchemaMigrationDefinition = {
|
||||
version: 19,
|
||||
version: 21,
|
||||
name: 'reviewer_failure_count',
|
||||
apply(database: Database) {
|
||||
if (!tableHasColumn(database, 'paired_tasks', 'reviewer_failure_count')) {
|
||||
@@ -17,7 +17,7 @@ import type { SchemaMigrationDefinition } from './types.js';
|
||||
* `tableHasColumn` guards make this a no-op.
|
||||
*/
|
||||
export const TURN_PROGRESS_TEXT_COMPAT_MIGRATION: SchemaMigrationDefinition = {
|
||||
version: 20,
|
||||
version: 22,
|
||||
name: 'turn_progress_text_compat',
|
||||
apply(database: Database) {
|
||||
if (!tableHasColumn(database, 'paired_turns', 'progress_text')) {
|
||||
@@ -18,8 +18,10 @@ import { TURN_PROGRESS_TEXT_MIGRATION } from './015_turn-progress-text.js';
|
||||
import { ROOM_SKILL_OVERRIDES_MIGRATION } from './016_room-skill-overrides.js';
|
||||
import { SCHEDULED_TASK_ROOM_ROLE_MIGRATION } from './017_scheduled-task-room-role.js';
|
||||
import { PAIRED_TURN_OUTPUT_ATTACHMENTS_MIGRATION } from './018_paired-turn-output-attachments.js';
|
||||
import { REVIEWER_FAILURE_COUNT_MIGRATION } from './019_reviewer-failure-count.js';
|
||||
import { TURN_PROGRESS_TEXT_COMPAT_MIGRATION } from './020_turn-progress-text-compat.js';
|
||||
import { TURN_PROGRESS_TEXT_RECOVERY_MIGRATION } from './019_turn-progress-text-recovery.js';
|
||||
import { ARBITER_INTERVENTION_COUNT_MIGRATION } from './020_arbiter-intervention-count.js';
|
||||
import { REVIEWER_FAILURE_COUNT_MIGRATION } from './021_reviewer-failure-count.js';
|
||||
import { TURN_PROGRESS_TEXT_COMPAT_MIGRATION } from './022_turn-progress-text-compat.js';
|
||||
import type {
|
||||
SchemaMigrationArgs,
|
||||
SchemaMigrationDefinition,
|
||||
@@ -46,6 +48,8 @@ const ORDERED_SCHEMA_MIGRATIONS: readonly SchemaMigrationDefinition[] = [
|
||||
ROOM_SKILL_OVERRIDES_MIGRATION,
|
||||
SCHEDULED_TASK_ROOM_ROLE_MIGRATION,
|
||||
PAIRED_TURN_OUTPUT_ATTACHMENTS_MIGRATION,
|
||||
TURN_PROGRESS_TEXT_RECOVERY_MIGRATION,
|
||||
ARBITER_INTERVENTION_COUNT_MIGRATION,
|
||||
REVIEWER_FAILURE_COUNT_MIGRATION,
|
||||
TURN_PROGRESS_TEXT_COMPAT_MIGRATION,
|
||||
];
|
||||
|
||||
@@ -55,6 +55,7 @@ export type PairedTaskUpdates = Partial<
|
||||
| 'finalize_step_done_count'
|
||||
| 'task_done_then_user_reopen_count'
|
||||
| 'empty_step_done_streak'
|
||||
| 'arbiter_intervention_count'
|
||||
| 'status'
|
||||
| 'arbiter_verdict'
|
||||
| 'arbiter_requested_at'
|
||||
@@ -182,6 +183,7 @@ export function createPairedTaskInDatabase(
|
||||
finalize_step_done_count,
|
||||
task_done_then_user_reopen_count,
|
||||
empty_step_done_streak,
|
||||
arbiter_intervention_count,
|
||||
status,
|
||||
arbiter_verdict,
|
||||
arbiter_requested_at,
|
||||
@@ -189,7 +191,7 @@ export function createPairedTaskInDatabase(
|
||||
created_at,
|
||||
updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
)
|
||||
.run(
|
||||
@@ -212,6 +214,7 @@ export function createPairedTaskInDatabase(
|
||||
task.finalize_step_done_count ?? 0,
|
||||
task.task_done_then_user_reopen_count ?? 0,
|
||||
task.empty_step_done_streak ?? 0,
|
||||
task.arbiter_intervention_count ?? 0,
|
||||
task.status,
|
||||
task.arbiter_verdict,
|
||||
task.arbiter_requested_at,
|
||||
@@ -362,6 +365,10 @@ export function updatePairedTaskInDatabase(
|
||||
fields.push('empty_step_done_streak = ?');
|
||||
values.push(updates.empty_step_done_streak);
|
||||
}
|
||||
if (updates.arbiter_intervention_count !== undefined) {
|
||||
fields.push('arbiter_intervention_count = ?');
|
||||
values.push(updates.arbiter_intervention_count);
|
||||
}
|
||||
if (updates.status !== undefined) {
|
||||
fields.push('status = ?');
|
||||
values.push(updates.status);
|
||||
@@ -445,6 +452,10 @@ export function updatePairedTaskIfUnchangedInDatabase(
|
||||
fields.push('empty_step_done_streak = ?');
|
||||
values.push(updates.empty_step_done_streak);
|
||||
}
|
||||
if (updates.arbiter_intervention_count !== undefined) {
|
||||
fields.push('arbiter_intervention_count = ?');
|
||||
values.push(updates.arbiter_intervention_count);
|
||||
}
|
||||
if (updates.status !== undefined) {
|
||||
fields.push('status = ?');
|
||||
values.push(updates.status);
|
||||
|
||||
@@ -462,6 +462,24 @@ export function recoverInterruptedPairedTurnAttemptsForServiceInDatabase(
|
||||
const error =
|
||||
args.error ?? 'Interrupted by service restart before completion.';
|
||||
return database.transaction(() => {
|
||||
database
|
||||
.prepare(
|
||||
`
|
||||
UPDATE paired_turn_attempts
|
||||
SET state = 'failed',
|
||||
active_run_id = NULL,
|
||||
updated_at = ?,
|
||||
completed_at = ?,
|
||||
last_error = COALESCE(last_error, ?)
|
||||
WHERE state = 'running'
|
||||
AND executor_service_id IN (${servicePlaceholders})
|
||||
AND task_id IN (
|
||||
SELECT id FROM paired_tasks WHERE status = 'completed'
|
||||
)
|
||||
`,
|
||||
)
|
||||
.run(now, now, error, ...serviceIds);
|
||||
|
||||
const rows = database
|
||||
.prepare(
|
||||
`
|
||||
|
||||
@@ -211,6 +211,7 @@ function parseLastAgentSeqState(
|
||||
`Invalid last_agent_seq JSON for ${serviceId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -273,6 +273,34 @@ export function getOpenWorkItemFromDatabase(
|
||||
return row ? hydrateWorkItemRow(row) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recent delivered owner-side work items for a chat, newest last.
|
||||
* Used by single-mode prompt building to surface the bot's own prior
|
||||
* answers (which only live in work_items.result_payload) so a new turn
|
||||
* can reference what it previously said.
|
||||
*/
|
||||
export function getRecentDeliveredOwnerWorkItemsForChatFromDatabase(
|
||||
database: Database,
|
||||
chatJid: string,
|
||||
limit: number,
|
||||
): WorkItem[] {
|
||||
if (limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
const rows = database
|
||||
.prepare(
|
||||
`SELECT *
|
||||
FROM work_items
|
||||
WHERE chat_jid = ?
|
||||
AND status = 'delivered'
|
||||
AND (delivery_role = 'owner' OR delivery_role IS NULL)
|
||||
ORDER BY id DESC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(chatJid, limit) as StoredWorkItemRow[];
|
||||
return rows.map(hydrateWorkItemRow).reverse();
|
||||
}
|
||||
|
||||
export function getOpenWorkItemForChatFromDatabase(
|
||||
database: Database,
|
||||
chatJid: string,
|
||||
|
||||
@@ -53,8 +53,8 @@ import {
|
||||
import { createMessageRuntime } from './message-runtime.js';
|
||||
import { nudgeSchedulerLoop, startSchedulerLoop } from './task-scheduler.js';
|
||||
import { startUnifiedDashboard } from './unified-dashboard.js';
|
||||
import { startWebDashboardServer } from './web-dashboard-server.js';
|
||||
import { startUsagePrimer } from './usage-primer.js';
|
||||
import { startWebDashboardServer } from './web-dashboard-server.js';
|
||||
import { Channel, NewMessage, RegisteredGroup } from './types.js';
|
||||
import { logger } from './logger.js';
|
||||
import { initCodexTokenRotation } from './codex-token-rotation.js';
|
||||
|
||||
71
src/message-agent-executor-attempt-runner.test.ts
Normal file
71
src/message-agent-executor-attempt-runner.test.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mockRunAgentProcess = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock('./agent-runner.js', () => ({
|
||||
runAgentProcess: mockRunAgentProcess,
|
||||
}));
|
||||
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
getCodexAccountCount: vi.fn(() => 2),
|
||||
}));
|
||||
|
||||
import { runMessageAgentAttempt } from './message-agent-executor-attempt-runner.js';
|
||||
import type { AgentOutput } from './agent-runner.js';
|
||||
|
||||
describe('runMessageAgentAttempt', () => {
|
||||
it('stores pre-stream Codex launch errors in paired summary', async () => {
|
||||
const error = new Error(
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||
);
|
||||
mockRunAgentProcess.mockRejectedValueOnce(error);
|
||||
const updateSummary = vi.fn();
|
||||
|
||||
const attempt = await runMessageAgentAttempt({
|
||||
provider: 'codex',
|
||||
currentSessionId: undefined,
|
||||
isClaudeCodeAgent: false,
|
||||
canRetryClaudeCredentials: false,
|
||||
shouldPersistSession: false,
|
||||
effectiveGroup: {
|
||||
name: 'Test',
|
||||
folder: 'test',
|
||||
trigger: '@test',
|
||||
added_at: new Date().toISOString(),
|
||||
agentType: 'codex',
|
||||
},
|
||||
agentInput: {
|
||||
prompt: 'arbiter prompt',
|
||||
groupFolder: 'test',
|
||||
chatJid: 'dc:test',
|
||||
runId: 'run-test',
|
||||
isMain: false,
|
||||
assistantName: 'Andy',
|
||||
},
|
||||
activeRole: 'arbiter',
|
||||
effectiveServiceId: 'codex-review',
|
||||
effectiveAgentType: 'codex',
|
||||
sessionFolder: 'test:arbiter',
|
||||
onPersistSession: vi.fn(),
|
||||
registerProcess: vi.fn(),
|
||||
onOutput: vi.fn(async (_output: AgentOutput) => undefined),
|
||||
pairedExecutionLifecycle: {
|
||||
updateSummary,
|
||||
recordFinalOutputBeforeDelivery: vi.fn(() => false),
|
||||
},
|
||||
log: {
|
||||
child: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
} as any,
|
||||
});
|
||||
|
||||
expect(attempt.error).toBe(error);
|
||||
expect(attempt.sawOutput).toBe(false);
|
||||
expect(updateSummary).toHaveBeenCalledWith({
|
||||
errorText:
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
shouldResetCodexSessionOnAgentFailure,
|
||||
shouldResetSessionOnAgentFailure,
|
||||
} from './session-recovery.js';
|
||||
import { getErrorMessage } from './utils.js';
|
||||
import type {
|
||||
AgentType,
|
||||
OutboundAttachment,
|
||||
@@ -162,6 +163,9 @@ class MessageAgentAttemptRunner {
|
||||
);
|
||||
return this.buildAttempt({ output });
|
||||
} catch (error) {
|
||||
this.args.pairedExecutionLifecycle.updateSummary({
|
||||
errorText: getErrorMessage(error),
|
||||
});
|
||||
return this.buildAttempt({ error });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,11 @@ vi.mock('./message-runtime-follow-up.js', () => ({
|
||||
import type { AgentOutput } from './agent-runner.js';
|
||||
import * as db from './db.js';
|
||||
import * as pairedExecutionContextModule from './paired-execution-context.js';
|
||||
import { createPairedExecutionLifecycle } from './message-agent-executor-paired.js';
|
||||
import {
|
||||
createPairedExecutionLifecycle,
|
||||
resolveOwnerNextStepNotice,
|
||||
} from './message-agent-executor-paired.js';
|
||||
import type { PairedTask } from './types.js';
|
||||
|
||||
const log = {
|
||||
info: vi.fn(),
|
||||
@@ -325,3 +329,42 @@ describe('createPairedExecutionLifecycle completion handling', () => {
|
||||
expect(enqueueMessageCheck).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveOwnerNextStepNotice routing indicator', () => {
|
||||
const task = (over: Partial<PairedTask>): PairedTask =>
|
||||
({
|
||||
status: 'active',
|
||||
reviewer_agent_type: 'claude-code',
|
||||
completion_reason: null,
|
||||
...over,
|
||||
}) as PairedTask;
|
||||
|
||||
it('announces reviewer request while waiting', () => {
|
||||
expect(resolveOwnerNextStepNotice(task({ status: 'review_ready' }))).toBe(
|
||||
'🔁 리뷰어 검토 요청됨 — 리뷰어 응답을 기다리는 중입니다.',
|
||||
);
|
||||
});
|
||||
|
||||
it('announces arbiter call when arbiter is requested', () => {
|
||||
expect(
|
||||
resolveOwnerNextStepNotice(task({ status: 'arbiter_requested' })),
|
||||
).toBe('⚖️ 의견 차이로 중재자 호출됨 — 중재 판정을 기다리는 중입니다.');
|
||||
});
|
||||
|
||||
it('stays silent when the task finishes (no redundant done line)', () => {
|
||||
expect(
|
||||
resolveOwnerNextStepNotice(
|
||||
task({ status: 'completed', completion_reason: 'done' }),
|
||||
),
|
||||
).toBeNull();
|
||||
expect(
|
||||
resolveOwnerNextStepNotice(
|
||||
task({ status: 'completed', completion_reason: 'escalated' }),
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('stays silent for in-progress owner work', () => {
|
||||
expect(resolveOwnerNextStepNotice(task({ status: 'active' }))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,25 +64,63 @@ function completeStoredExecution(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Routing indicator shown to the user after an owner turn when the turn does
|
||||
* NOT end the task — i.e. when more is still pending — so a same-looking status
|
||||
* line (e.g. TASK_DONE) no longer reads as inconsistent: the user can always
|
||||
* tell whether the reviewer was requested or the arbiter was called, vs. the
|
||||
* task simply finishing (no line — the owner's final message is the ending).
|
||||
* Purely additive — it reflects the routing the state machine already chose; it
|
||||
* does not change any routing, and it never duplicates the completion message.
|
||||
*/
|
||||
export function resolveOwnerNextStepNotice(
|
||||
task: PairedTaskRecord,
|
||||
): string | null {
|
||||
switch (task.status) {
|
||||
case 'review_ready':
|
||||
case 'in_review':
|
||||
return '🔁 리뷰어 검토 요청됨 — 리뷰어 응답을 기다리는 중입니다.';
|
||||
case 'arbiter_requested':
|
||||
case 'in_arbitration':
|
||||
return '⚖️ 의견 차이로 중재자 호출됨 — 중재 판정을 기다리는 중입니다.';
|
||||
default:
|
||||
// 'completed' included: the owner's final message is itself the ending,
|
||||
// so no extra done/escalation line is appended here (escalation has its
|
||||
// own dedicated notice above).
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function notifyPairedCompletionIfNeeded(args: {
|
||||
task: PairedTaskRecord | null | undefined;
|
||||
chatJid: string;
|
||||
completedRole: PairedRoomRole;
|
||||
onOutput?: (output: AgentOutput) => Promise<void>;
|
||||
}): Promise<void> {
|
||||
if (args.task?.status !== 'completed' || !args.task.completion_reason) return;
|
||||
const sender = getLastHumanMessageSender(args.chatJid);
|
||||
const mention = sender ? `<@${sender}>` : '';
|
||||
const notifications: Record<string, string> = {
|
||||
escalated: `${mention} ⚠️ 자동 해결 불가 — 확인이 필요합니다.`,
|
||||
};
|
||||
const message = notifications[args.task.completion_reason];
|
||||
if (!message) return;
|
||||
await args.onOutput?.({
|
||||
status: 'success',
|
||||
result: message,
|
||||
output: { visibility: 'public', text: message },
|
||||
phase: 'final',
|
||||
});
|
||||
const task = args.task;
|
||||
if (!task) return;
|
||||
|
||||
const emit = (text: string): Promise<void> | undefined =>
|
||||
args.onOutput?.({
|
||||
status: 'success',
|
||||
result: text,
|
||||
output: { visibility: 'public', text },
|
||||
phase: 'final',
|
||||
});
|
||||
|
||||
// Escalation notice fires for any role's completion that escalated to the user.
|
||||
if (task.status === 'completed' && task.completion_reason === 'escalated') {
|
||||
const sender = getLastHumanMessageSender(args.chatJid);
|
||||
const mention = sender ? `<@${sender}> ` : '';
|
||||
await emit(`${mention}⚠️ 자동 해결 불가 — 확인이 필요합니다.`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Owner turn: surface the next routing step so the ending is unambiguous.
|
||||
if (args.completedRole !== 'owner') return;
|
||||
const notice = resolveOwnerNextStepNotice(task);
|
||||
if (!notice) return;
|
||||
await emit(notice);
|
||||
}
|
||||
|
||||
export interface PairedExecutionLifecycle {
|
||||
@@ -505,8 +543,11 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
if (!missingVisibleVerdict) {
|
||||
return false;
|
||||
}
|
||||
this.pairedExecutionSummary =
|
||||
const noVerdictSummary =
|
||||
'Execution completed without a visible terminal verdict.';
|
||||
this.pairedExecutionSummary = this.pairedExecutionSummary
|
||||
? `${this.pairedExecutionSummary}\n${noVerdictSummary}`
|
||||
: noVerdictSummary;
|
||||
log.warn(
|
||||
{
|
||||
pairedTaskId: pairedExecutionContext?.task.id ?? null,
|
||||
@@ -578,7 +619,8 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
private async notifyCompletionAndQueueFollowUp(
|
||||
state: FinalizeState,
|
||||
): Promise<void> {
|
||||
const { chatJid, onOutput, pairedExecutionContext } = this.args;
|
||||
const { chatJid, completedRole, onOutput, pairedExecutionContext } =
|
||||
this.args;
|
||||
if (!pairedExecutionContext || state.interruptedByHumanMessage) {
|
||||
return;
|
||||
}
|
||||
@@ -586,6 +628,7 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
await notifyPairedCompletionIfNeeded({
|
||||
task: finishedTask,
|
||||
chatJid,
|
||||
completedRole,
|
||||
onOutput,
|
||||
});
|
||||
this.queueFollowUpIfNeeded(state, finishedTask);
|
||||
@@ -620,6 +663,7 @@ class PairedExecutionLifecycleController implements PairedExecutionLifecycle {
|
||||
sawOutput: state.sawOutputForFollowUp,
|
||||
taskStatus: finishedTask?.status ?? null,
|
||||
outputSummary: this.pairedExecutionSummary,
|
||||
ownerFailureCount: finishedTask?.owner_failure_count ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -172,6 +172,19 @@ describe('message-agent-executor-rules', () => {
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('requeues owner follow-up after a silent arbiter Codex account failure returns the task active', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
completedRole: 'arbiter',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
taskStatus: 'active',
|
||||
outputSummary:
|
||||
'Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.\nExecution completed without a visible terminal verdict.',
|
||||
}),
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('resolves pending arbiter follow-up requeue after repeated owner execution failures', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
@@ -194,6 +207,19 @@ describe('message-agent-executor-rules', () => {
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('requeues a silent owner failure caused by Codex pool unavailability', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
completedRole: 'owner',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
taskStatus: 'active',
|
||||
outputSummary:
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||
}),
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('does not request an executor-side follow-up after successful owner output moved the task to review_ready', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
resolveNextTurnAction,
|
||||
shouldRetrySilentOwnerExecution,
|
||||
} from './message-runtime-rules.js';
|
||||
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||
import { parseVisibleVerdict } from './paired-verdict.js';
|
||||
import type { PairedRoomRole, PairedTaskStatus } from './types.js';
|
||||
export {
|
||||
@@ -17,13 +18,31 @@ export {
|
||||
|
||||
export type PairedFollowUpQueueAction = 'pending' | 'none';
|
||||
|
||||
function isSilentCodexAccountFailure(summary?: string | null): boolean {
|
||||
return isTerminalCodexAccountFailure(summary);
|
||||
}
|
||||
|
||||
export function resolvePairedFollowUpQueueAction(args: {
|
||||
completedRole: PairedRoomRole;
|
||||
executionStatus: 'succeeded' | 'failed';
|
||||
sawOutput: boolean;
|
||||
taskStatus: PairedTaskStatus | null;
|
||||
outputSummary?: string | null;
|
||||
ownerFailureCount?: number | null;
|
||||
}): PairedFollowUpQueueAction {
|
||||
if (
|
||||
args.executionStatus === 'failed' &&
|
||||
args.sawOutput === false &&
|
||||
isSilentCodexAccountFailure(args.outputSummary)
|
||||
) {
|
||||
if (args.completedRole === 'arbiter' && args.taskStatus === 'active') {
|
||||
return 'pending';
|
||||
}
|
||||
if (args.completedRole !== 'owner') {
|
||||
return 'none';
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
shouldRetrySilentOwnerExecution({
|
||||
completedRole: args.completedRole,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import * as config from './config.js';
|
||||
|
||||
const NO_VISIBLE_VERDICT_SUMMARY = `검토 중입니다.\nExecution completed without a visible terminal verdict.`;
|
||||
vi.mock('./agent-runner.js', () => ({
|
||||
runAgentProcess: vi.fn(),
|
||||
writeGroupsSnapshot: vi.fn(),
|
||||
@@ -1190,7 +1191,7 @@ describe('runAgentForGroup room memory', () => {
|
||||
taskId: 'paired-task-review-no-verdict',
|
||||
role: 'reviewer',
|
||||
status: 'failed',
|
||||
summary: 'Execution completed without a visible terminal verdict.',
|
||||
summary: NO_VISIBLE_VERDICT_SUMMARY,
|
||||
}),
|
||||
);
|
||||
expect(db.failPairedTurn).toHaveBeenCalledWith({
|
||||
@@ -1202,7 +1203,7 @@ describe('runAgentForGroup room memory', () => {
|
||||
intentKind: 'reviewer-turn',
|
||||
role: 'reviewer',
|
||||
},
|
||||
error: 'Execution completed without a visible terminal verdict.',
|
||||
error: NO_VISIBLE_VERDICT_SUMMARY,
|
||||
});
|
||||
expect(db.completePairedTurn).not.toHaveBeenCalled();
|
||||
expect(db.insertPairedTurnOutput).not.toHaveBeenCalled();
|
||||
|
||||
@@ -21,6 +21,50 @@ export type PairedFollowUpSource = Parameters<
|
||||
typeof resolveFollowUpDispatch
|
||||
>[0]['source'];
|
||||
|
||||
type PairedTurnOutputContext = ReturnType<typeof getPairedTurnOutputs>[number];
|
||||
|
||||
type PairedFollowUpTaskContext = Pick<
|
||||
PairedTask,
|
||||
'id' | 'status' | 'round_trip_count' | 'updated_at' | 'owner_failure_count'
|
||||
>;
|
||||
|
||||
function timestampMs(value: string | null | undefined): number | null {
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function isPersistedOutputOlderThanTaskRevision(args: {
|
||||
output: PairedTurnOutputContext;
|
||||
task: Partial<Pick<PairedTask, 'updated_at'>> | null | undefined;
|
||||
}): boolean {
|
||||
const outputCreatedAt = timestampMs(args.output.created_at);
|
||||
const taskUpdatedAt = timestampMs(args.task?.updated_at);
|
||||
if (outputCreatedAt == null || taskUpdatedAt == null) {
|
||||
return false;
|
||||
}
|
||||
return outputCreatedAt < taskUpdatedAt;
|
||||
}
|
||||
|
||||
function shouldPreferFallbackTurnOutputContext(args: {
|
||||
output: PairedTurnOutputContext | null | undefined;
|
||||
task: Partial<Pick<PairedTask, 'updated_at'>> | null | undefined;
|
||||
fallbackRole: PairedRoomRole | null | undefined;
|
||||
}): boolean {
|
||||
if (!args.output || !args.fallbackRole) {
|
||||
return false;
|
||||
}
|
||||
if (args.output.role === args.fallbackRole) {
|
||||
return false;
|
||||
}
|
||||
return isPersistedOutputOlderThanTaskRevision({
|
||||
output: args.output,
|
||||
task: args.task,
|
||||
});
|
||||
}
|
||||
|
||||
export interface PairedFollowUpDecision {
|
||||
taskId: string | null;
|
||||
taskStatus: PairedTaskStatus | null;
|
||||
@@ -44,7 +88,10 @@ export type PairedFollowUpDispatchResult =
|
||||
});
|
||||
|
||||
export function resolveLatestPairedTurnOutputContext(args: {
|
||||
task: Pick<PairedTask, 'id'> | null | undefined;
|
||||
task:
|
||||
| (Pick<PairedTask, 'id'> & Partial<Pick<PairedTask, 'updated_at'>>)
|
||||
| null
|
||||
| undefined;
|
||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||
fallbackLastTurnOutputVerdict?: VisibleVerdict | null;
|
||||
}): {
|
||||
@@ -54,12 +101,19 @@ export function resolveLatestPairedTurnOutputContext(args: {
|
||||
const latestOutput = args.task
|
||||
? getPairedTurnOutputs(args.task.id).at(-1)
|
||||
: null;
|
||||
const output = shouldPreferFallbackTurnOutputContext({
|
||||
output: latestOutput,
|
||||
task: args.task,
|
||||
fallbackRole: args.fallbackLastTurnOutputRole,
|
||||
})
|
||||
? null
|
||||
: latestOutput;
|
||||
return {
|
||||
role: latestOutput?.role ?? args.fallbackLastTurnOutputRole ?? null,
|
||||
role: output?.role ?? args.fallbackLastTurnOutputRole ?? null,
|
||||
verdict:
|
||||
resolveStoredVisibleVerdict({
|
||||
verdict: latestOutput?.verdict ?? null,
|
||||
outputText: latestOutput?.output_text ?? null,
|
||||
verdict: output?.verdict ?? null,
|
||||
outputText: output?.output_text ?? null,
|
||||
}) ??
|
||||
args.fallbackLastTurnOutputVerdict ??
|
||||
null,
|
||||
@@ -67,7 +121,13 @@ export function resolveLatestPairedTurnOutputContext(args: {
|
||||
}
|
||||
|
||||
export function resolvePairedFollowUpDecision(args: {
|
||||
task: Pick<PairedTask, 'id' | 'status'> | null | undefined;
|
||||
task:
|
||||
| Pick<
|
||||
PairedFollowUpTaskContext,
|
||||
'id' | 'status' | 'updated_at' | 'owner_failure_count'
|
||||
>
|
||||
| null
|
||||
| undefined;
|
||||
source: PairedFollowUpSource;
|
||||
completedRole?: PairedRoomRole;
|
||||
executionStatus?: 'succeeded' | 'failed';
|
||||
@@ -93,6 +153,7 @@ export function resolvePairedFollowUpDecision(args: {
|
||||
taskStatus: args.task?.status ?? null,
|
||||
lastTurnOutputRole,
|
||||
lastTurnOutputVerdict,
|
||||
ownerFailureCount: args.task?.owner_failure_count ?? null,
|
||||
});
|
||||
const dispatch = resolveFollowUpDispatch({
|
||||
source: args.source,
|
||||
@@ -115,10 +176,7 @@ export function resolvePairedFollowUpDecision(args: {
|
||||
export function schedulePairedFollowUpIntent(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
task:
|
||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
||||
| null
|
||||
| undefined;
|
||||
task: PairedFollowUpTaskContext | null | undefined;
|
||||
intentKind: ScheduledPairedFollowUpIntentKind;
|
||||
enqueue: () => void;
|
||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||
@@ -148,6 +206,7 @@ export function schedulePairedFollowUpIntent(args: {
|
||||
taskStatus: args.task.status,
|
||||
lastTurnOutputRole: latestOutputContext.role,
|
||||
lastTurnOutputVerdict: latestOutputContext.verdict,
|
||||
ownerFailureCount: args.task.owner_failure_count ?? null,
|
||||
intentKind: args.intentKind,
|
||||
}) &&
|
||||
!(
|
||||
@@ -171,10 +230,7 @@ export function schedulePairedFollowUpIntent(args: {
|
||||
export function schedulePairedFollowUpWithMessageCheck(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
task:
|
||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
||||
| null
|
||||
| undefined;
|
||||
task: PairedFollowUpTaskContext | null | undefined;
|
||||
intentKind: ScheduledPairedFollowUpIntentKind;
|
||||
enqueueMessageCheck: () => void;
|
||||
fallbackLastTurnOutputRole?: PairedRoomRole | null;
|
||||
@@ -198,10 +254,7 @@ export function schedulePairedFollowUpWithMessageCheck(args: {
|
||||
export function dispatchPairedFollowUpForEvent(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
task:
|
||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
||||
| null
|
||||
| undefined;
|
||||
task: PairedFollowUpTaskContext | null | undefined;
|
||||
source: PairedFollowUpSource;
|
||||
completedRole?: PairedRoomRole;
|
||||
executionStatus?: 'succeeded' | 'failed';
|
||||
@@ -270,10 +323,7 @@ export function dispatchPairedFollowUpForEvent(args: {
|
||||
export function enqueuePairedFollowUpAfterEvent(args: {
|
||||
chatJid: string;
|
||||
runId: string;
|
||||
task:
|
||||
| Pick<PairedTask, 'id' | 'status' | 'round_trip_count' | 'updated_at'>
|
||||
| null
|
||||
| undefined;
|
||||
task: PairedFollowUpTaskContext | null | undefined;
|
||||
source: PairedFollowUpSource;
|
||||
completedRole?: PairedRoomRole;
|
||||
executionStatus?: 'succeeded' | 'failed';
|
||||
|
||||
@@ -11,9 +11,12 @@ vi.mock('./session-commands.js', () => ({
|
||||
|
||||
import { handleQueuedRunGates } from './message-runtime-gating.js';
|
||||
|
||||
describe('message-runtime-gating', () => {
|
||||
const baseArgs = {
|
||||
chatJid: 'room-1',
|
||||
function makeArgs(overrides: {
|
||||
chatJid?: string;
|
||||
sendMessage?: () => Promise<void>;
|
||||
}) {
|
||||
return {
|
||||
chatJid: overrides.chatJid ?? 'room-1',
|
||||
group: {
|
||||
folder: 'room-folder',
|
||||
name: 'room',
|
||||
@@ -26,9 +29,13 @@ describe('message-runtime-gating', () => {
|
||||
triggerPattern: /^코덱스/,
|
||||
timezone: 'Asia/Seoul',
|
||||
hasImplicitContinuationWindow: () => false,
|
||||
sessionCommandDeps: {} as never,
|
||||
sessionCommandDeps: {
|
||||
sendMessage: overrides.sendMessage ?? (async () => {}),
|
||||
} as never,
|
||||
};
|
||||
}
|
||||
|
||||
describe('message-runtime-gating', () => {
|
||||
beforeEach(() => {
|
||||
handleSessionCommandMock.mockReset();
|
||||
});
|
||||
@@ -39,15 +46,15 @@ describe('message-runtime-gating', () => {
|
||||
success: false,
|
||||
});
|
||||
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
const result = await handleQueuedRunGates(makeArgs({}));
|
||||
|
||||
expect(result).toEqual({ handled: true, success: false });
|
||||
});
|
||||
|
||||
it('falls through when no session command is handled', async () => {
|
||||
it('falls through when no session command matched', async () => {
|
||||
handleSessionCommandMock.mockResolvedValue({ handled: false });
|
||||
|
||||
const result = await handleQueuedRunGates(baseArgs);
|
||||
const result = await handleQueuedRunGates(makeArgs({}));
|
||||
|
||||
expect(result).toEqual({ handled: false });
|
||||
});
|
||||
|
||||
125
src/message-runtime-queue-owner-retry.test.ts
Normal file
125
src/message-runtime-queue-owner-retry.test.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { _initTestDatabase, createPairedTask } from './db.js';
|
||||
import { runQueuedGroupTurn } from './message-runtime-queue.js';
|
||||
import { resetPairedFollowUpScheduleState } from './paired-follow-up-scheduler.js';
|
||||
import type { Channel, PairedTask, RegisteredGroup } from './types.js';
|
||||
|
||||
function makeGroup(): RegisteredGroup {
|
||||
return {
|
||||
name: 'Test Group',
|
||||
folder: 'test-group',
|
||||
trigger: '@Andy',
|
||||
added_at: '2026-03-30T00:00:00.000Z',
|
||||
requiresTrigger: false,
|
||||
agentType: 'codex',
|
||||
workDir: '/repo',
|
||||
};
|
||||
}
|
||||
|
||||
function makeTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||
return {
|
||||
id: 'task-owner-retry',
|
||||
chat_jid: 'group@test',
|
||||
group_folder: 'test-group',
|
||||
owner_service_id: 'codex-main',
|
||||
reviewer_service_id: 'claude',
|
||||
owner_agent_type: 'codex',
|
||||
reviewer_agent_type: 'claude-code',
|
||||
arbiter_agent_type: null,
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: null,
|
||||
round_trip_count: 0,
|
||||
owner_failure_count: 1,
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
task_done_then_user_reopen_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-30T00:00:00.000Z',
|
||||
updated_at: '2026-03-30T00:00:10.000Z',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeChannel(): Channel {
|
||||
return {
|
||||
name: 'discord-owner',
|
||||
connect: vi.fn(),
|
||||
sendMessage: vi.fn(),
|
||||
isConnected: vi.fn(() => true),
|
||||
ownsJid: vi.fn(() => true),
|
||||
disconnect: vi.fn(),
|
||||
} as unknown as Channel;
|
||||
}
|
||||
|
||||
describe('message-runtime-queue owner retry turns', () => {
|
||||
beforeEach(() => {
|
||||
_initTestDatabase();
|
||||
resetPairedFollowUpScheduleState();
|
||||
});
|
||||
|
||||
it('keeps stale human messages in failed owner retries as owner follow-up', async () => {
|
||||
const task = makeTask();
|
||||
createPairedTask(task);
|
||||
const executeTurn = vi.fn(async () => ({
|
||||
outputStatus: 'error' as const,
|
||||
deliverySucceeded: false,
|
||||
visiblePhase: 'silent',
|
||||
}));
|
||||
|
||||
const outcome = await runQueuedGroupTurn({
|
||||
chatJid: task.chat_jid,
|
||||
group: makeGroup(),
|
||||
runId: 'run-owner-retry-stale-human',
|
||||
log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as any,
|
||||
timezone: 'UTC',
|
||||
missedMessages: [
|
||||
{
|
||||
id: 'human-owner-stale-1',
|
||||
chat_jid: task.chat_jid,
|
||||
sender: 'user@test',
|
||||
sender_name: 'User',
|
||||
content: '원 요청입니다',
|
||||
timestamp: '2026-03-30T00:00:03.000Z',
|
||||
seq: 46,
|
||||
is_bot_message: false,
|
||||
},
|
||||
],
|
||||
task,
|
||||
roleToChannel: {
|
||||
owner: null,
|
||||
reviewer: makeChannel(),
|
||||
arbiter: null,
|
||||
},
|
||||
ownerChannel: makeChannel(),
|
||||
lastAgentTimestamps: {},
|
||||
saveState: vi.fn(),
|
||||
executeTurn,
|
||||
getFixedRoleChannelName: () => 'discord-review',
|
||||
labelPairedSenders: (_chatJid, messages) => messages,
|
||||
formatMessages: () => 'formatted prompt',
|
||||
});
|
||||
|
||||
expect(outcome).toBe(false);
|
||||
expect(executeTurn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
hasHumanMessage: false,
|
||||
deliveryRole: 'owner',
|
||||
forcedRole: 'owner',
|
||||
pairedTurnIdentity: {
|
||||
turnId: 'task-owner-retry:2026-03-30T00:00:10.000Z:owner-follow-up',
|
||||
taskId: 'task-owner-retry',
|
||||
taskUpdatedAt: '2026-03-30T00:00:10.000Z',
|
||||
intentKind: 'owner-follow-up',
|
||||
role: 'owner',
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -56,6 +56,33 @@ function resolveQueuedTurnReservationIntent(args: {
|
||||
return 'owner-follow-up';
|
||||
}
|
||||
|
||||
function timestampMs(value: string | null | undefined): number | null {
|
||||
if (!value) return null;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function isHumanMessageFreshForTask(
|
||||
task: PairedTask | null | undefined,
|
||||
message: NewMessage,
|
||||
): boolean {
|
||||
if (!isExternalHumanMessage(message)) {
|
||||
return false;
|
||||
}
|
||||
if (!task) {
|
||||
return true;
|
||||
}
|
||||
if (task.status !== 'active' || (task.owner_failure_count ?? 0) <= 0) {
|
||||
return true;
|
||||
}
|
||||
const messageTime = timestampMs(message.timestamp);
|
||||
const taskUpdatedAt = timestampMs(task.updated_at);
|
||||
if (messageTime == null || taskUpdatedAt == null) {
|
||||
return true;
|
||||
}
|
||||
return messageTime > taskUpdatedAt;
|
||||
}
|
||||
|
||||
function buildQueuedGroupTurnPrompt(args: {
|
||||
turnRole: 'owner' | 'reviewer' | 'arbiter';
|
||||
currentTask: PairedTask | null | undefined;
|
||||
@@ -248,7 +275,9 @@ export async function runQueuedGroupTurn(args: {
|
||||
args;
|
||||
let currentTask = task;
|
||||
const hasHumanMsg = task
|
||||
? missedMessages.some(isExternalHumanMessage)
|
||||
? missedMessages.some((message) =>
|
||||
isHumanMessageFreshForTask(task, message),
|
||||
)
|
||||
: !isBotOnlyPairedRoomTurn(chatJid, missedMessages);
|
||||
let fallbackMessages = missedMessages;
|
||||
if (currentTask !== undefined && hasHumanMsg) {
|
||||
|
||||
@@ -2727,9 +2727,20 @@ describe('createMessageRuntime', () => {
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockImplementation(
|
||||
() => pairedTask,
|
||||
);
|
||||
vi.mocked(db.getPairedTurnOutputs).mockReturnValue([
|
||||
{
|
||||
id: 1,
|
||||
task_id: pairedTask.id,
|
||||
turn_number: 1,
|
||||
role: 'owner',
|
||||
output_text: 'STEP_DONE\nowner work ready for review',
|
||||
created_at: '2026-03-30T00:00:00.500Z',
|
||||
},
|
||||
] as any);
|
||||
vi.mocked(agentRunner.runAgentProcess).mockImplementation(
|
||||
async (_group, _input, _onProcess, onOutput) => {
|
||||
pairedTask.status = 'active';
|
||||
pairedTask.updated_at = '2026-03-30T00:00:01.000Z';
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
phase: 'final',
|
||||
|
||||
@@ -802,6 +802,32 @@ describe('MessageTurnController outbound audit logging', () => {
|
||||
replaceMessageId: 'progress-1',
|
||||
});
|
||||
});
|
||||
|
||||
it('publishes a failure final when an error finishes before any visible output', async () => {
|
||||
const channel = makeChannel();
|
||||
const deliverFinalText = vi.fn().mockResolvedValue(true);
|
||||
const controller = new MessageTurnController({
|
||||
chatJid: 'dc:test-room',
|
||||
group: makeGroup(),
|
||||
runId: 'run-silent-error-final',
|
||||
channel,
|
||||
idleTimeout: 1_000,
|
||||
failureFinalText: '실패',
|
||||
isClaudeCodeAgent: true,
|
||||
clearSession: vi.fn(),
|
||||
requestClose: vi.fn(),
|
||||
deliverFinalText,
|
||||
deliveryRole: 'owner',
|
||||
});
|
||||
|
||||
await controller.start();
|
||||
const finishResult = await controller.finish('error');
|
||||
|
||||
expect(finishResult.visiblePhase).toBe('final');
|
||||
expect(deliverFinalText).toHaveBeenCalledWith('실패', {
|
||||
replaceMessageId: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('MessageTurnController human interruptions', () => {
|
||||
|
||||
@@ -385,6 +385,12 @@ export class MessageTurnController {
|
||||
this.hadError
|
||||
) {
|
||||
await this.publishFailureFinal();
|
||||
} else if (
|
||||
outputStatus === 'error' &&
|
||||
this.visiblePhase === 'silent' &&
|
||||
!this.terminalObserved()
|
||||
) {
|
||||
await this.publishFailureFinal();
|
||||
}
|
||||
|
||||
this.clearProgressTicker();
|
||||
@@ -615,15 +621,19 @@ export class MessageTurnController {
|
||||
return;
|
||||
}
|
||||
|
||||
// Capture the value before the await: a concurrent resetProgressState()
|
||||
// can null `latestProgressText` while editMessage is in flight, which used
|
||||
// to throw "null is not an object (evaluating 'this.latestProgressText.length')".
|
||||
const progressText = this.latestProgressText;
|
||||
if (
|
||||
!this.progressMessageId ||
|
||||
!this.options.channel.editMessage ||
|
||||
!this.latestProgressText
|
||||
!progressText
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const rendered = this.renderProgressMessage(this.latestProgressText);
|
||||
const rendered = this.renderProgressMessage(progressText);
|
||||
|
||||
try {
|
||||
await this.options.channel.editMessage(
|
||||
@@ -635,7 +645,7 @@ export class MessageTurnController {
|
||||
this.progressEditFailCount = 0;
|
||||
this.logOutboundAudit('progress-edit', {
|
||||
messageId: this.progressMessageId,
|
||||
textLength: this.latestProgressText.length,
|
||||
textLength: progressText.length,
|
||||
renderedLength: rendered.length,
|
||||
});
|
||||
} catch (err) {
|
||||
|
||||
@@ -4,7 +4,11 @@ import path from 'path';
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { validateOutboundAttachments } from './outbound-attachments.js';
|
||||
import {
|
||||
appendRejectionNotice,
|
||||
describeRejectedAttachments,
|
||||
validateOutboundAttachments,
|
||||
} from './outbound-attachments.js';
|
||||
|
||||
const ONE_PIXEL_PNG = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
|
||||
@@ -38,6 +42,7 @@ function writeFile(
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
for (const dir of cleanupDirs.splice(0)) {
|
||||
fs.rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
@@ -176,7 +181,13 @@ describe('validateOutboundAttachments', () => {
|
||||
});
|
||||
|
||||
describe('validateOutboundAttachments policy checks', () => {
|
||||
function useIsolatedDefaultTempDir(): void {
|
||||
const tempDir = makeTempDir(os.tmpdir(), 'ejclaw-policy-temp-');
|
||||
vi.spyOn(os, 'tmpdir').mockReturnValue(tempDir);
|
||||
}
|
||||
|
||||
it('requires workspace paths to be explicitly allowlisted', () => {
|
||||
useIsolatedDefaultTempDir();
|
||||
const dir = makeTempDir(process.cwd(), '.ejclaw-attachment-');
|
||||
const imagePath = writeFile(dir, 'workspace-shot.png', ONE_PIXEL_PNG);
|
||||
|
||||
@@ -223,6 +234,7 @@ describe('validateOutboundAttachments policy checks', () => {
|
||||
});
|
||||
|
||||
it('rejects symlink attempts that escape the allowed directory', () => {
|
||||
useIsolatedDefaultTempDir();
|
||||
const workspaceDir = makeTempDir(process.cwd(), '.ejclaw-attachment-');
|
||||
const targetPath = writeFile(
|
||||
workspaceDir,
|
||||
@@ -242,6 +254,7 @@ describe('validateOutboundAttachments policy checks', () => {
|
||||
});
|
||||
|
||||
it('rejects symlink attempts that escape a user-configured directory', () => {
|
||||
useIsolatedDefaultTempDir();
|
||||
const allowedDir = makeTempDir(process.cwd(), '.ejclaw-user-images-');
|
||||
const secretDir = makeTempDir(process.cwd(), '.ejclaw-secret-images-');
|
||||
const targetPath = writeFile(secretDir, 'secret-shot.png', ONE_PIXEL_PNG);
|
||||
@@ -316,3 +329,59 @@ describe('validateOutboundAttachments policy checks', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('describeRejectedAttachments', () => {
|
||||
it('returns an empty string when nothing was rejected', () => {
|
||||
expect(describeRejectedAttachments([])).toBe('');
|
||||
});
|
||||
|
||||
it('summarizes rejected attachments by basename and reason', () => {
|
||||
const note = describeRejectedAttachments([
|
||||
{
|
||||
path: '/home/claude/jarvis-tts/sample.wav',
|
||||
reason: 'outside-allowed-dirs',
|
||||
},
|
||||
{ path: '/tmp/missing.png', reason: 'not-found' },
|
||||
]);
|
||||
|
||||
expect(note).toContain('첨부 2건을 전송하지 못했습니다');
|
||||
expect(note).toContain('sample.wav (허용된 폴더 밖의 경로)');
|
||||
expect(note).toContain('missing.png (파일을 찾을 수 없음)');
|
||||
});
|
||||
|
||||
it('falls back to the raw reason code when no label is mapped', () => {
|
||||
const note = describeRejectedAttachments([
|
||||
{ path: '/tmp/x.bin', reason: 'some-new-reason' },
|
||||
]);
|
||||
|
||||
expect(note).toContain('x.bin (some-new-reason)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('appendRejectionNotice', () => {
|
||||
it('leaves the body untouched when nothing was rejected', () => {
|
||||
expect(appendRejectionNotice('보고서입니다', [])).toBe('보고서입니다');
|
||||
});
|
||||
|
||||
it('appends the failure notice to the visible body', () => {
|
||||
const body = appendRejectionNotice('샘플을 첨부합니다', [
|
||||
{ path: '/home/claude/jarvis-tts/a.wav', reason: 'outside-allowed-dirs' },
|
||||
]);
|
||||
|
||||
expect(body).toContain('샘플을 첨부합니다');
|
||||
expect(body).toContain('첨부 1건을 전송하지 못했습니다');
|
||||
expect(body).toContain('a.wav (허용된 폴더 밖의 경로)');
|
||||
});
|
||||
|
||||
it('returns the notice alone when the body would otherwise be empty', () => {
|
||||
const body = appendRejectionNotice('', [
|
||||
{ path: '/tmp/missing.png', reason: 'not-found' },
|
||||
]);
|
||||
|
||||
expect(body).toBe(
|
||||
describeRejectedAttachments([
|
||||
{ path: '/tmp/missing.png', reason: 'not-found' },
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -92,6 +92,13 @@ export function getDefaultAttachmentBaseDirs(): string[] {
|
||||
.filter((dir): dir is string => Boolean(dir));
|
||||
}
|
||||
|
||||
function resolveAllowedBaseDirs(baseDirs?: string[]): string[] {
|
||||
return unique([
|
||||
...getDefaultAttachmentBaseDirs(),
|
||||
...(baseDirs ?? []).map(resolveExistingDir),
|
||||
]).filter((dir): dir is string => Boolean(dir));
|
||||
}
|
||||
|
||||
function isWithinBaseDir(realPath: string, baseDir: string): boolean {
|
||||
const relative = path.relative(baseDir, realPath);
|
||||
return (
|
||||
@@ -235,14 +242,56 @@ function normalizeAttachmentName(
|
||||
return candidate || 'attachment';
|
||||
}
|
||||
|
||||
const REJECTION_REASON_LABELS: Record<string, string> = {
|
||||
'outside-allowed-dirs': '허용된 폴더 밖의 경로',
|
||||
'not-found': '파일을 찾을 수 없음',
|
||||
'not-absolute': '절대경로가 아님',
|
||||
'not-file': '파일이 아님',
|
||||
'too-large': '8MB 용량 초과',
|
||||
'unsupported-extension': '지원하지 않는 형식',
|
||||
'invalid-image-signature': '이미지 형식 손상 또는 불일치',
|
||||
'invalid-file-signature': '파일 형식 손상 또는 불일치',
|
||||
'mime-mismatch': '형식 불일치',
|
||||
'validation-error': '검증 오류',
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds a concise, user-facing note describing attachments that failed to
|
||||
* send. Returning a non-empty string lets the delivery layer surface the
|
||||
* failure in the visible message instead of dropping it silently — otherwise a
|
||||
* stripped MEDIA: directive leaves the user with a message that claims a file
|
||||
* was attached when none was delivered.
|
||||
*/
|
||||
export function describeRejectedAttachments(
|
||||
rejected: Array<{ path: string; reason: string }>,
|
||||
): string {
|
||||
if (rejected.length === 0) return '';
|
||||
const lines = rejected.map(({ path: rejectedPath, reason }) => {
|
||||
const label = REJECTION_REASON_LABELS[reason] ?? reason;
|
||||
return `• ${path.basename(rejectedPath)} (${label})`;
|
||||
});
|
||||
return `⚠️ 첨부 ${rejected.length}건을 전송하지 못했습니다:\n${lines.join('\n')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Folds a rejected-attachment notice into the visible outbound text so the
|
||||
* delivery layer never sends a message that silently dropped its attachments.
|
||||
* Returns the notice on its own when the body would otherwise be empty.
|
||||
*/
|
||||
export function appendRejectionNotice(
|
||||
cleanText: string,
|
||||
rejected: Array<{ path: string; reason: string }>,
|
||||
): string {
|
||||
const note = describeRejectedAttachments(rejected);
|
||||
if (!note) return cleanText;
|
||||
return cleanText ? `${cleanText}\n\n${note}` : note;
|
||||
}
|
||||
|
||||
export function validateOutboundAttachments(
|
||||
attachments: OutboundAttachment[] | undefined,
|
||||
options: { baseDirs?: string[] } = {},
|
||||
): ValidateOutboundAttachmentsResult {
|
||||
const baseDirs = unique([
|
||||
...getDefaultAttachmentBaseDirs(),
|
||||
...(options.baseDirs ?? []).map(resolveExistingDir),
|
||||
]).filter((dir): dir is string => Boolean(dir));
|
||||
const baseDirs = resolveAllowedBaseDirs(options.baseDirs);
|
||||
const defaultTempDir = resolveExistingDir(os.tmpdir());
|
||||
const files: ValidatedOutboundAttachment[] = [];
|
||||
const rejected: Array<{ path: string; reason: string }> = [];
|
||||
|
||||
207
src/owner-final-codex-unavailable.test.ts
Normal file
207
src/owner-final-codex-unavailable.test.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./db.js', () => {
|
||||
const updatePairedTask = vi.fn();
|
||||
return {
|
||||
getPairedTaskById: vi.fn(),
|
||||
getPairedWorkspace: vi.fn(),
|
||||
updatePairedTask,
|
||||
updatePairedTaskIfUnchanged: vi.fn((id, _expectedUpdatedAt, updates) => {
|
||||
updatePairedTask(id, updates);
|
||||
return true;
|
||||
}),
|
||||
releasePairedTaskExecutionLease: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('./paired-workspace-manager.js', () => ({
|
||||
isOwnerWorkspaceRepairNeededError: vi.fn(() => false),
|
||||
markPairedTaskReviewReady: vi.fn(),
|
||||
prepareReviewerWorkspaceForExecution: vi.fn(),
|
||||
provisionOwnerWorkspaceForPairedTask: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./logger.js', () => ({
|
||||
logger: {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
import * as config from './config.js';
|
||||
import * as db from './db.js';
|
||||
import { completePairedExecutionContext } from './paired-execution-context.js';
|
||||
import { resolvePairedFollowUpQueueAction } from './message-agent-executor-rules.js';
|
||||
import type { PairedTask } from './types.js';
|
||||
|
||||
const CODEX_UNAVAILABLE_SUMMARY =
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.';
|
||||
|
||||
function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||
return {
|
||||
id: 'task-1',
|
||||
chat_jid: 'dc:test',
|
||||
group_folder: 'ejclaw',
|
||||
owner_service_id: config.CODEX_MAIN_SERVICE_ID,
|
||||
reviewer_service_id: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: null,
|
||||
round_trip_count: 0,
|
||||
owner_failure_count: 0,
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
task_done_then_user_reopen_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
status: 'merge_ready',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-28T00:00:00.000Z',
|
||||
updated_at: '2026-03-28T00:00:00.000Z',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('owner final Codex unavailable handling', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(db.getPairedWorkspace).mockReturnValue(undefined);
|
||||
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(true);
|
||||
});
|
||||
|
||||
it('preserves active tasks when the first owner turn cannot start Codex', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'active',
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'failed',
|
||||
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||
});
|
||||
|
||||
const updates = vi.mocked(db.updatePairedTask).mock.calls[0]?.[1];
|
||||
expect(updates).toEqual(
|
||||
expect.objectContaining({
|
||||
owner_failure_count: 1,
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
}),
|
||||
);
|
||||
expect(updates?.status).not.toBe('completed');
|
||||
});
|
||||
|
||||
it('preserves merge_ready when owner finalization cannot start Codex', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'failed',
|
||||
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
owner_failure_count: 1,
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('requests arbiter after repeated owner Codex account failures', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'active',
|
||||
owner_failure_count: 1,
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'failed',
|
||||
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'arbiter_requested',
|
||||
owner_failure_count: 2,
|
||||
arbiter_requested_at: expect.any(String),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('escalates to the user after persistent owner and arbiter Codex account failures', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'active',
|
||||
owner_failure_count: 3,
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'failed',
|
||||
summary: CODEX_UNAVAILABLE_SUMMARY,
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
owner_failure_count: 4,
|
||||
arbiter_verdict: 'escalate',
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: 'escalated',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('requeues owner finalization once after preserving merge_ready', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
completedRole: 'owner',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
taskStatus: 'merge_ready',
|
||||
ownerFailureCount: 1,
|
||||
outputSummary: CODEX_UNAVAILABLE_SUMMARY,
|
||||
}),
|
||||
).toBe('pending');
|
||||
});
|
||||
|
||||
it('queues arbiter after repeated owner finalization failures', () => {
|
||||
expect(
|
||||
resolvePairedFollowUpQueueAction({
|
||||
completedRole: 'owner',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
taskStatus: 'arbiter_requested',
|
||||
ownerFailureCount: 2,
|
||||
outputSummary: CODEX_UNAVAILABLE_SUMMARY,
|
||||
}),
|
||||
).toBe('pending');
|
||||
});
|
||||
});
|
||||
81
src/paired-arbiter-codex-owner-wake.test.ts
Normal file
81
src/paired-arbiter-codex-owner-wake.test.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./db.js', () => ({
|
||||
getPairedTurnOutputs: vi.fn(() => []),
|
||||
reservePairedTurnReservation: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
import { getPairedTurnOutputs } from './db.js';
|
||||
import { dispatchPairedFollowUpForEvent } from './message-runtime-follow-up.js';
|
||||
import {
|
||||
matchesExpectedPairedFollowUpIntent,
|
||||
resolveNextTurnAction,
|
||||
} from './message-runtime-rules.js';
|
||||
|
||||
describe('arbiter Codex failure owner wake', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(getPairedTurnOutputs).mockReturnValue([]);
|
||||
});
|
||||
|
||||
it('keeps owner follow-ups schedulable when active task has owner failure count', () => {
|
||||
const followUpContext = {
|
||||
taskStatus: 'active' as const,
|
||||
lastTurnOutputRole: 'owner' as const,
|
||||
lastTurnOutputVerdict: 'step_done' as const,
|
||||
ownerFailureCount: 1,
|
||||
};
|
||||
|
||||
expect(resolveNextTurnAction(followUpContext)).toEqual({
|
||||
kind: 'owner-follow-up',
|
||||
});
|
||||
expect(
|
||||
matchesExpectedPairedFollowUpIntent({
|
||||
...followUpContext,
|
||||
intentKind: 'owner-follow-up',
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('requeues owner follow-up when arbiter Codex failure returns the task active', () => {
|
||||
const enqueue = vi.fn();
|
||||
vi.mocked(getPairedTurnOutputs).mockReturnValue([
|
||||
{
|
||||
id: 1,
|
||||
task_id: 'task-arbiter-codex-unavailable',
|
||||
turn_number: 1,
|
||||
role: 'owner',
|
||||
output_text: 'STEP_DONE\nwaiting for CI',
|
||||
verdict: 'step_done',
|
||||
created_at: '2026-03-30T00:00:01.000Z',
|
||||
},
|
||||
] as any);
|
||||
|
||||
const result = dispatchPairedFollowUpForEvent({
|
||||
chatJid: 'group@test',
|
||||
runId: 'run-arbiter-codex-unavailable',
|
||||
task: {
|
||||
id: 'task-arbiter-codex-unavailable',
|
||||
status: 'active',
|
||||
round_trip_count: 1,
|
||||
owner_failure_count: 1,
|
||||
updated_at: '2026-03-30T00:00:02.000Z',
|
||||
},
|
||||
source: 'executor-recovery',
|
||||
completedRole: 'arbiter',
|
||||
executionStatus: 'failed',
|
||||
sawOutput: false,
|
||||
enqueue,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
kind: 'paired-follow-up',
|
||||
intentKind: 'owner-follow-up',
|
||||
scheduled: true,
|
||||
taskStatus: 'active',
|
||||
lastTurnOutputRole: 'owner',
|
||||
nextTurnAction: { kind: 'owner-follow-up' },
|
||||
});
|
||||
expect(enqueue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isArbiterEnabled } from './config.js';
|
||||
import { ARBITER_MAX_INTERVENTIONS, isArbiterEnabled } from './config.js';
|
||||
import { logger } from './logger.js';
|
||||
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
||||
import type { PairedTaskStatus } from './types.js';
|
||||
@@ -11,6 +11,13 @@ export function requestArbiterOrEscalate(args: {
|
||||
arbiterLogMessage: string;
|
||||
escalateLogMessage: string;
|
||||
logContext?: Record<string, unknown>;
|
||||
/**
|
||||
* How many times the arbiter has already intervened on this task. When this
|
||||
* reaches {@link maxArbiterInterventions} the task is escalated to the user
|
||||
* instead of re-invoking the arbiter, bounding the owner↔reviewer↔arbiter loop.
|
||||
*/
|
||||
arbiterInterventionCount?: number;
|
||||
maxArbiterInterventions?: number;
|
||||
patch?: {
|
||||
title?: string | null;
|
||||
source_ref?: string | null;
|
||||
@@ -23,6 +30,7 @@ export function requestArbiterOrEscalate(args: {
|
||||
finalize_step_done_count?: number;
|
||||
task_done_then_user_reopen_count?: number;
|
||||
empty_step_done_streak?: number;
|
||||
arbiter_intervention_count?: number;
|
||||
arbiter_verdict?: string | null;
|
||||
arbiter_requested_at?: string | null;
|
||||
completion_reason?: string | null;
|
||||
@@ -37,7 +45,10 @@ export function requestArbiterOrEscalate(args: {
|
||||
escalateLogMessage,
|
||||
logContext,
|
||||
} = args;
|
||||
if (isArbiterEnabled()) {
|
||||
const interventionCount = args.arbiterInterventionCount ?? 0;
|
||||
const maxInterventions =
|
||||
args.maxArbiterInterventions ?? ARBITER_MAX_INTERVENTIONS;
|
||||
if (isArbiterEnabled() && interventionCount < maxInterventions) {
|
||||
const transitioned = transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus,
|
||||
@@ -55,6 +66,33 @@ export function requestArbiterOrEscalate(args: {
|
||||
return transitioned;
|
||||
}
|
||||
|
||||
if (isArbiterEnabled() && interventionCount >= maxInterventions) {
|
||||
const transitioned = transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
...args.patch,
|
||||
arbiter_verdict: 'escalate',
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: 'escalated',
|
||||
},
|
||||
});
|
||||
if (transitioned) {
|
||||
logger.info(
|
||||
{
|
||||
...(logContext ?? { taskId }),
|
||||
arbiterInterventionCount: interventionCount,
|
||||
maxArbiterInterventions: maxInterventions,
|
||||
},
|
||||
'Arbiter intervention budget exhausted — escalating to user instead of re-invoking arbiter',
|
||||
);
|
||||
}
|
||||
return transitioned;
|
||||
}
|
||||
|
||||
const transitioned = transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus,
|
||||
|
||||
@@ -135,6 +135,16 @@ describe('paired completion signals', () => {
|
||||
visibleVerdict: 'done',
|
||||
}),
|
||||
).toEqual({ kind: 'request_owner_finalize' });
|
||||
expect(
|
||||
resolveReviewerFailureSignal({
|
||||
visibleVerdict: 'step_done',
|
||||
}),
|
||||
).toEqual({ kind: 'request_owner_changes' });
|
||||
expect(
|
||||
resolveReviewerFailureSignal({
|
||||
visibleVerdict: 'done_with_concerns',
|
||||
}),
|
||||
).toEqual({ kind: 'request_owner_changes' });
|
||||
expect(
|
||||
resolveReviewerFailureSignal({
|
||||
visibleVerdict: 'needs_context',
|
||||
|
||||
@@ -98,13 +98,15 @@ export function resolveReviewerFailureSignal(args: {
|
||||
case 'task_done':
|
||||
case 'done':
|
||||
return { kind: 'request_owner_finalize' };
|
||||
case 'step_done':
|
||||
case 'done_with_concerns':
|
||||
return { kind: 'request_owner_changes' };
|
||||
case 'blocked':
|
||||
case 'needs_context':
|
||||
return {
|
||||
kind: 'complete',
|
||||
completionReason: 'escalated',
|
||||
};
|
||||
case 'done_with_concerns':
|
||||
case 'continue':
|
||||
default:
|
||||
return { kind: 'preserve_review_ready' };
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { logger } from './logger.js';
|
||||
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||
import { transitionPairedTaskStatus } from './paired-task-status.js';
|
||||
import { classifyArbiterVerdict } from './paired-verdict.js';
|
||||
import type { PairedTask } from './types.js';
|
||||
@@ -8,8 +9,42 @@ const ARBITER_RESOLUTION_ROUND_TRIP_COUNT = 0;
|
||||
export function handleFailedArbiterExecution(args: {
|
||||
task: PairedTask;
|
||||
taskId: string;
|
||||
summary?: string | null;
|
||||
}): void {
|
||||
const { task, taskId } = args;
|
||||
const { task, taskId, summary } = args;
|
||||
if (isTerminalCodexAccountFailure(summary)) {
|
||||
const now = new Date().toISOString();
|
||||
const nextOwnerFailureCount = Math.max(
|
||||
(task.owner_failure_count ?? 0) + 1,
|
||||
1,
|
||||
);
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'active',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
owner_failure_count: nextOwnerFailureCount,
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
},
|
||||
});
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role: 'arbiter',
|
||||
status: task.status,
|
||||
nextOwnerFailureCount,
|
||||
summary: summary?.slice(0, 200),
|
||||
},
|
||||
'Returned arbiter task to owner after terminal Codex account failure',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const fallbackStatus =
|
||||
task.status === 'in_arbitration' || task.status === 'arbiter_requested'
|
||||
@@ -43,9 +78,19 @@ export function handleArbiterCompletion(args: {
|
||||
const { task, taskId, summary } = args;
|
||||
const now = new Date().toISOString();
|
||||
const arbiterVerdict = classifyArbiterVerdict(summary);
|
||||
// Persist a running count of arbiter interventions so the loop can be bounded.
|
||||
// This must NOT be reset by the round_trip_count reset below — otherwise the
|
||||
// owner↔reviewer loop could re-trigger the arbiter without limit.
|
||||
const nextArbiterInterventionCount =
|
||||
(task.arbiter_intervention_count ?? 0) + 1;
|
||||
|
||||
logger.info(
|
||||
{ taskId, arbiterVerdict, summary: summary?.slice(0, 200) },
|
||||
{
|
||||
taskId,
|
||||
arbiterVerdict,
|
||||
arbiterInterventionCount: nextArbiterInterventionCount,
|
||||
summary: summary?.slice(0, 200),
|
||||
},
|
||||
'Arbiter verdict rendered',
|
||||
);
|
||||
|
||||
@@ -63,6 +108,7 @@ export function handleArbiterCompletion(args: {
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_intervention_count: nextArbiterInterventionCount,
|
||||
arbiter_verdict: arbiterVerdict,
|
||||
arbiter_requested_at: null,
|
||||
},
|
||||
@@ -86,12 +132,17 @@ export function handleArbiterCompletion(args: {
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_intervention_count: nextArbiterInterventionCount,
|
||||
arbiter_verdict: arbiterVerdict,
|
||||
arbiter_requested_at: null,
|
||||
},
|
||||
});
|
||||
logger.info(
|
||||
{ taskId, arbiterVerdict },
|
||||
{
|
||||
taskId,
|
||||
arbiterVerdict,
|
||||
arbiterInterventionCount: nextArbiterInterventionCount,
|
||||
},
|
||||
'Arbiter requested owner changes — resuming owner flow',
|
||||
);
|
||||
return;
|
||||
|
||||
138
src/paired-execution-context-owner-follow-up.test.ts
Normal file
138
src/paired-execution-context-owner-follow-up.test.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./db.js', () => {
|
||||
const updatePairedTask = vi.fn();
|
||||
return {
|
||||
createPairedTask: vi.fn(),
|
||||
getLatestPairedTaskForChat: vi.fn(),
|
||||
getLatestOpenPairedTaskForChat: vi.fn(),
|
||||
getPairedTaskById: vi.fn(),
|
||||
getPairedTurnOutputs: vi.fn(() => []),
|
||||
insertPairedTurnOutput: vi.fn(),
|
||||
updatePairedTask,
|
||||
updatePairedTaskIfUnchanged: vi.fn((id, _expectedUpdatedAt, updates) => {
|
||||
updatePairedTask(id, updates);
|
||||
return true;
|
||||
}),
|
||||
upsertPairedProject: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('./paired-workspace-manager.js', () => ({
|
||||
isOwnerWorkspaceRepairNeededError: vi.fn(() => false),
|
||||
markPairedTaskReviewReady: vi.fn(),
|
||||
prepareReviewerWorkspaceForExecution: vi.fn(),
|
||||
provisionOwnerWorkspaceForPairedTask: vi.fn(() => ({
|
||||
id: 'task-1:owner',
|
||||
task_id: 'task-1',
|
||||
role: 'owner',
|
||||
workspace_dir: '/tmp/paired/task-1/owner',
|
||||
snapshot_source_dir: null,
|
||||
snapshot_ref: null,
|
||||
status: 'ready',
|
||||
snapshot_refreshed_at: null,
|
||||
created_at: '2026-03-28T00:00:00.000Z',
|
||||
updated_at: '2026-03-28T00:00:00.000Z',
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock('./logger.js', () => ({
|
||||
logger: {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
import * as config from './config.js';
|
||||
import * as db from './db.js';
|
||||
import { preparePairedExecutionContext } from './paired-execution-context.js';
|
||||
import type { PairedTask, RegisteredGroup, RoomRoleContext } from './types.js';
|
||||
|
||||
const group: RegisteredGroup = {
|
||||
name: 'Paired Room',
|
||||
folder: 'paired-room',
|
||||
trigger: '@codex',
|
||||
added_at: '2026-03-28T00:00:00.000Z',
|
||||
agentType: 'codex',
|
||||
workDir: '/repo/canonical',
|
||||
};
|
||||
|
||||
const ownerContext: RoomRoleContext = {
|
||||
serviceId: config.CODEX_MAIN_SERVICE_ID,
|
||||
role: 'owner',
|
||||
ownerServiceId: config.CODEX_MAIN_SERVICE_ID,
|
||||
reviewerServiceId: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||
failoverOwner: false,
|
||||
};
|
||||
|
||||
function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||
return {
|
||||
id: 'task-1',
|
||||
chat_jid: 'dc:test',
|
||||
group_folder: group.folder,
|
||||
owner_service_id: config.CODEX_MAIN_SERVICE_ID,
|
||||
reviewer_service_id: config.REVIEWER_SERVICE_ID_FOR_TYPE,
|
||||
title: null,
|
||||
source_ref: 'HEAD',
|
||||
plan_notes: null,
|
||||
review_requested_at: null,
|
||||
round_trip_count: 0,
|
||||
owner_failure_count: 0,
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
task_done_then_user_reopen_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
created_at: '2026-03-28T00:00:00.000Z',
|
||||
updated_at: '2026-03-28T00:00:00.000Z',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('paired owner follow-up preparation', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(undefined);
|
||||
});
|
||||
|
||||
it('does not reset owner failure counters for scheduled owner follow-up turns', () => {
|
||||
vi.mocked(db.getLatestOpenPairedTaskForChat).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'active',
|
||||
owner_failure_count: 1,
|
||||
owner_step_done_streak: 2,
|
||||
empty_step_done_streak: 2,
|
||||
}),
|
||||
);
|
||||
|
||||
preparePairedExecutionContext({
|
||||
group,
|
||||
chatJid: 'dc:test',
|
||||
runId: 'run-owner-follow-up-no-reset',
|
||||
roomRoleContext: ownerContext,
|
||||
hasHumanMessage: true,
|
||||
pairedTurnIdentity: {
|
||||
turnId: 'task-1:2026-03-28T00:00:00.000Z:owner-follow-up',
|
||||
taskId: 'task-1',
|
||||
taskUpdatedAt: '2026-03-28T00:00:00.000Z',
|
||||
intentKind: 'owner-follow-up',
|
||||
role: 'owner',
|
||||
},
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
round_trip_count: 0,
|
||||
owner_failure_count: 0,
|
||||
owner_step_done_streak: 0,
|
||||
empty_step_done_streak: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
getPairedWorkspace,
|
||||
hasActiveCiWatcherForChat,
|
||||
} from './db.js';
|
||||
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||
import { logger } from './logger.js';
|
||||
import { markPairedTaskReviewReady } from './paired-workspace-manager.js';
|
||||
import { requestArbiterOrEscalate } from './paired-arbiter-request.js';
|
||||
@@ -21,6 +22,7 @@ import type { PairedTask } from './types.js';
|
||||
|
||||
type OwnerFinalizeOutcome = 'stop' | 're_review';
|
||||
const OWNER_FAILURE_ESCALATION_THRESHOLD = 2;
|
||||
const OWNER_CODEX_UNAVAILABLE_USER_ESCALATION_THRESHOLD = 4;
|
||||
const EMPTY_STEP_DONE_THRESHOLD = 2;
|
||||
|
||||
export function handleFailedOwnerExecution(args: {
|
||||
@@ -31,6 +33,107 @@ export function handleFailedOwnerExecution(args: {
|
||||
const { task, taskId, summary } = args;
|
||||
const now = new Date().toISOString();
|
||||
const nextFailureCount = (task.owner_failure_count ?? 0) + 1;
|
||||
if (isTerminalCodexAccountFailure(summary)) {
|
||||
if (nextFailureCount >= OWNER_CODEX_UNAVAILABLE_USER_ESCALATION_THRESHOLD) {
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
owner_failure_count: nextFailureCount,
|
||||
arbiter_verdict: 'escalate',
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: 'escalated',
|
||||
},
|
||||
});
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role: 'owner',
|
||||
previousStatus: task.status,
|
||||
ownerFailureCount: nextFailureCount,
|
||||
summary: summary?.slice(0, 160),
|
||||
},
|
||||
'Escalated owner task after persistent Codex account failures',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
||||
requestArbiterOrEscalate({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
now,
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
arbiterLogMessage:
|
||||
'Owner Codex unavailable repeatedly — requesting arbiter',
|
||||
escalateLogMessage:
|
||||
'Owner Codex unavailable repeatedly — escalating to user',
|
||||
logContext: {
|
||||
taskId,
|
||||
role: 'owner',
|
||||
previousStatus: task.status,
|
||||
ownerFailureCount: nextFailureCount,
|
||||
summary: summary?.slice(0, 160),
|
||||
},
|
||||
patch: {
|
||||
owner_failure_count: nextFailureCount,
|
||||
arbiter_verdict: null,
|
||||
completion_reason: null,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const patch = {
|
||||
owner_failure_count: nextFailureCount,
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
};
|
||||
if (task.status === 'active' || task.status === 'merge_ready') {
|
||||
applyPairedTaskPatch({
|
||||
taskId,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch,
|
||||
});
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role: 'owner',
|
||||
status: task.status,
|
||||
ownerFailureCount: nextFailureCount,
|
||||
summary: summary?.slice(0, 200),
|
||||
},
|
||||
'Preserved owner task after terminal Codex account failure',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'active',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch,
|
||||
});
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role: 'owner',
|
||||
status: task.status,
|
||||
ownerFailureCount: nextFailureCount,
|
||||
summary: summary?.slice(0, 200),
|
||||
},
|
||||
'Reset owner task to active after terminal Codex account failure',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (nextFailureCount >= OWNER_FAILURE_ESCALATION_THRESHOLD) {
|
||||
requestArbiterOrEscalate({
|
||||
@@ -38,6 +141,7 @@ export function handleFailedOwnerExecution(args: {
|
||||
currentStatus: task.status,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
now,
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
arbiterLogMessage:
|
||||
'Owner failed repeatedly without a visible verdict — requesting arbiter',
|
||||
escalateLogMessage:
|
||||
@@ -135,6 +239,7 @@ function handleOwnerFinalizeCompletion(args: {
|
||||
currentStatus: task.status,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
now,
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
arbiterLogMessage,
|
||||
escalateLogMessage,
|
||||
logContext: {
|
||||
@@ -164,6 +269,7 @@ function handleOwnerFinalizeCompletion(args: {
|
||||
currentStatus: task.status,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
now,
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
arbiterLogMessage:
|
||||
'Owner repeated STEP_DONE during finalize without code changes — requesting arbiter',
|
||||
escalateLogMessage:
|
||||
@@ -363,6 +469,7 @@ export function handleOwnerCompletion(args: {
|
||||
currentStatus: task.status,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
now,
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
arbiterLogMessage: 'Owner blocked/needs_context — requesting arbiter',
|
||||
escalateLogMessage: 'Owner blocked/needs_context — escalating to user',
|
||||
logContext: {
|
||||
@@ -389,6 +496,7 @@ export function handleOwnerCompletion(args: {
|
||||
currentStatus: task.status,
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
now,
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
arbiterLogMessage:
|
||||
'Owner repeated STEP_DONE without code changes — requesting arbiter',
|
||||
escalateLogMessage:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { ARBITER_DEADLOCK_THRESHOLD } from './config.js';
|
||||
import { getPairedWorkspace } from './db.js';
|
||||
import { isTerminalCodexAccountFailure } from './agent-error-detection.js';
|
||||
import { logger } from './logger.js';
|
||||
import { requestArbiterOrEscalate } from './paired-arbiter-request.js';
|
||||
import {
|
||||
@@ -33,6 +34,31 @@ export function handleFailedReviewerExecution(args: {
|
||||
const { task, taskId, summary } = args;
|
||||
const now = new Date().toISOString();
|
||||
|
||||
if (isTerminalCodexAccountFailure(summary)) {
|
||||
transitionPairedTaskStatus({
|
||||
taskId,
|
||||
currentStatus: task.status,
|
||||
nextStatus: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
arbiter_verdict: 'escalate',
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: 'reviewer_codex_unavailable',
|
||||
},
|
||||
});
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role: 'reviewer',
|
||||
status: task.status,
|
||||
summary: summary?.slice(0, 200),
|
||||
},
|
||||
'Completed reviewer task after terminal Codex account failure instead of preserving review loop',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (summary) {
|
||||
const verdict = parseVisibleVerdict(summary);
|
||||
const signal = resolveReviewerFailureSignal({
|
||||
@@ -40,7 +66,8 @@ export function handleFailedReviewerExecution(args: {
|
||||
});
|
||||
if (
|
||||
signal.kind === 'request_owner_finalize' ||
|
||||
signal.kind === 'complete'
|
||||
signal.kind === 'complete' ||
|
||||
signal.kind === 'request_owner_changes'
|
||||
) {
|
||||
const ownerWs =
|
||||
signal.kind === 'request_owner_finalize'
|
||||
@@ -56,7 +83,9 @@ export function handleFailedReviewerExecution(args: {
|
||||
nextStatus:
|
||||
signal.kind === 'request_owner_finalize'
|
||||
? 'merge_ready'
|
||||
: 'completed',
|
||||
: signal.kind === 'request_owner_changes'
|
||||
? 'active'
|
||||
: 'completed',
|
||||
expectedUpdatedAt: task.updated_at,
|
||||
updatedAt: now,
|
||||
patch: {
|
||||
@@ -221,6 +250,7 @@ export function handleReviewerCompletion(args: {
|
||||
arbiterLogMessage,
|
||||
escalateLogMessage,
|
||||
logContext: { taskId, verdict, summary: summary?.slice(0, 100) },
|
||||
arbiterInterventionCount: task.arbiter_intervention_count ?? 0,
|
||||
patch: { reviewer_failure_count: 0 },
|
||||
});
|
||||
return;
|
||||
|
||||
@@ -197,6 +197,96 @@ describe('paired execution routing loop guards', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('returns arbiter terminal Codex account failures to owner without re-arming arbiter', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_arbitration',
|
||||
owner_failure_count: 1,
|
||||
owner_step_done_streak: 3,
|
||||
finalize_step_done_count: 1,
|
||||
empty_step_done_streak: 2,
|
||||
arbiter_verdict: 'revise',
|
||||
arbiter_requested_at: '2026-03-28T00:00:04.000Z',
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'arbiter',
|
||||
status: 'failed',
|
||||
summary:
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'active',
|
||||
owner_failure_count: 2,
|
||||
owner_step_done_streak: 0,
|
||||
finalize_step_done_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('completes reviewer task after terminal Codex account failure instead of preserving review_ready loop', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_review',
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'reviewer',
|
||||
status: 'failed',
|
||||
summary:
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
arbiter_verdict: 'escalate',
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: 'reviewer_codex_unavailable',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves owner task after terminal Codex account failure instead of completing silently', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'active',
|
||||
updated_at: '2026-03-28T00:00:05.000Z',
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'failed',
|
||||
summary:
|
||||
'auth-expired: All Codex rotation accounts unavailable; re-auth required before launching Codex\nExecution completed without a visible terminal verdict.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
owner_failure_count: 1,
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
completion_reason: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps arbiter REVISE on owner flow while clearing stale loop counters', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
|
||||
@@ -148,6 +148,7 @@ function buildPairedTask(overrides: Partial<PairedTask> = {}): PairedTask {
|
||||
finalize_step_done_count: 0,
|
||||
task_done_then_user_reopen_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_intervention_count: 0,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
@@ -1225,6 +1226,81 @@ describe('paired execution context', () => {
|
||||
).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('escalates to the user instead of re-invoking the arbiter once the intervention budget is exhausted', () => {
|
||||
vi.spyOn(config, 'isArbiterEnabled').mockReturnValue(true);
|
||||
|
||||
const repoDir = createCanonicalRepoWithCommit('reviewed');
|
||||
const approvedSourceRef = resolveTreeRef(repoDir);
|
||||
fs.writeFileSync(path.join(repoDir, 'README.md'), 'changed again\n');
|
||||
execFileSync('git', ['add', 'README.md'], {
|
||||
cwd: repoDir,
|
||||
stdio: 'ignore',
|
||||
});
|
||||
execFileSync('git', ['commit', '-m', 'code change'], {
|
||||
cwd: repoDir,
|
||||
stdio: 'ignore',
|
||||
});
|
||||
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'merge_ready',
|
||||
source_ref: approvedSourceRef,
|
||||
round_trip_count: config.ARBITER_DEADLOCK_THRESHOLD,
|
||||
// Arbiter has already intervened the maximum allowed number of times.
|
||||
arbiter_intervention_count: config.ARBITER_MAX_INTERVENTIONS,
|
||||
}),
|
||||
);
|
||||
vi.mocked(db.getPairedWorkspace).mockImplementation((_taskId, role) =>
|
||||
role === 'owner' ? buildWorkspace('owner', repoDir) : undefined,
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'succeeded',
|
||||
summary: 'DONE',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'completed',
|
||||
completion_reason: 'escalated',
|
||||
}),
|
||||
);
|
||||
expect(db.updatePairedTask).not.toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({ status: 'arbiter_requested' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('increments the arbiter intervention count and keeps it across the round-trip reset on REVISE', () => {
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'in_arbitration',
|
||||
round_trip_count: config.ARBITER_DEADLOCK_THRESHOLD,
|
||||
arbiter_intervention_count: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'arbiter',
|
||||
status: 'succeeded',
|
||||
summary: 'VERDICT: REVISE\nOwner should address the reviewer feedback.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
'task-1',
|
||||
expect.objectContaining({
|
||||
status: 'active',
|
||||
round_trip_count: 0,
|
||||
arbiter_intervention_count: 1,
|
||||
arbiter_verdict: 'revise',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(['BLOCKED', 'NEEDS_CONTEXT'])(
|
||||
'escalates immediately when owner reports %s during finalize without arbiter',
|
||||
(summary) => {
|
||||
@@ -1473,3 +1549,64 @@ describe('paired execution context', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('paired execution context completion lease cleanup', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('propagates completion handler errors when no execution lease was reserved', () => {
|
||||
const transitionError = new Error('transition failed without lease');
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(
|
||||
buildPairedTask({
|
||||
status: 'merge_ready',
|
||||
}),
|
||||
);
|
||||
vi.mocked(db.updatePairedTaskIfUnchanged).mockImplementationOnce(() => {
|
||||
throw transitionError;
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
completePairedExecutionContext({
|
||||
taskId: 'task-1',
|
||||
role: 'owner',
|
||||
status: 'failed',
|
||||
summary: 'push failed',
|
||||
}),
|
||||
).toThrow('transition failed without lease');
|
||||
expect(db.releasePairedTaskExecutionLease).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('paired execution context reviewer failures', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('moves failed reviewer STEP_DONE output back to active for owner follow-up', () => {
|
||||
const task = buildPairedTask({
|
||||
status: 'in_review',
|
||||
updated_at: '2026-03-28T00:05:00.000Z',
|
||||
});
|
||||
vi.mocked(db.getPairedTaskById).mockReturnValue(task);
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: task.id,
|
||||
role: 'reviewer',
|
||||
status: 'failed',
|
||||
runId: 'run-failed-reviewer-step-done',
|
||||
summary: 'STEP_DONE\n오너 수정이 더 필요합니다.',
|
||||
});
|
||||
|
||||
expect(db.updatePairedTask).toHaveBeenCalledWith(
|
||||
task.id,
|
||||
expect.objectContaining({
|
||||
status: 'active',
|
||||
}),
|
||||
);
|
||||
expect(db.releasePairedTaskExecutionLease).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
runId: 'run-failed-reviewer-step-done',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -146,6 +146,7 @@ function createActiveTaskForRoom(args: {
|
||||
finalize_step_done_count: 0,
|
||||
task_done_then_user_reopen_count: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_intervention_count: 0,
|
||||
status: 'active',
|
||||
arbiter_verdict: null,
|
||||
arbiter_requested_at: null,
|
||||
@@ -413,6 +414,24 @@ export interface PairedExecutionRecoveryPlan {
|
||||
prompt: string;
|
||||
}
|
||||
|
||||
function isOwnerContinuationTurn(
|
||||
turnIdentity: PairedTurnIdentity | undefined,
|
||||
): boolean {
|
||||
return (
|
||||
turnIdentity?.role === 'owner' && turnIdentity.intentKind !== 'owner-turn'
|
||||
);
|
||||
}
|
||||
|
||||
function shouldResetOwnerLoopCounters(args: {
|
||||
hasHumanMessage?: boolean;
|
||||
pairedTurnIdentity?: PairedTurnIdentity;
|
||||
}): boolean {
|
||||
if (args.hasHumanMessage !== true) {
|
||||
return false;
|
||||
}
|
||||
return !isOwnerContinuationTurn(args.pairedTurnIdentity);
|
||||
}
|
||||
|
||||
export function preparePairedExecutionContext(args: {
|
||||
group: RegisteredGroup;
|
||||
chatJid: string;
|
||||
@@ -459,7 +478,7 @@ export function preparePairedExecutionContext(args: {
|
||||
// merge_ready is split into a fresh task before this function runs.
|
||||
// Only reset round_trip_count when a human message is present —
|
||||
// bot-only ping-pong must accumulate the counter for loop detection.
|
||||
const hasHuman = args.hasHumanMessage === true;
|
||||
const hasHuman = shouldResetOwnerLoopCounters(args);
|
||||
const needsStatusReset =
|
||||
latestTask.status === 'review_ready' || latestTask.status === 'in_review';
|
||||
if (hasHuman || needsStatusReset) {
|
||||
@@ -478,6 +497,7 @@ export function preparePairedExecutionContext(args: {
|
||||
reviewer_failure_count: 0,
|
||||
owner_step_done_streak: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_intervention_count: 0,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
@@ -495,6 +515,7 @@ export function preparePairedExecutionContext(args: {
|
||||
reviewer_failure_count: 0,
|
||||
owner_step_done_streak: 0,
|
||||
empty_step_done_streak: 0,
|
||||
arbiter_intervention_count: 0,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
@@ -622,15 +643,18 @@ export function preparePairedExecutionContext(args: {
|
||||
};
|
||||
}
|
||||
|
||||
export function completePairedExecutionContext(args: {
|
||||
type CompletePairedExecutionContextArgs = {
|
||||
taskId: string;
|
||||
role: PairedRoomRole;
|
||||
status: 'succeeded' | 'failed';
|
||||
runId?: string;
|
||||
summary?: string | null;
|
||||
}): void {
|
||||
};
|
||||
|
||||
export function completePairedExecutionContext(
|
||||
args: CompletePairedExecutionContextArgs,
|
||||
): void {
|
||||
const { taskId, role, status } = args;
|
||||
let completionError: unknown;
|
||||
logger.info(
|
||||
{
|
||||
taskId,
|
||||
@@ -642,96 +666,126 @@ export function completePairedExecutionContext(args: {
|
||||
);
|
||||
|
||||
try {
|
||||
const task = getPairedTaskById(taskId);
|
||||
if (!task) {
|
||||
return;
|
||||
}
|
||||
if (task.status === 'completed') {
|
||||
logger.info(
|
||||
{
|
||||
taskId,
|
||||
role,
|
||||
status,
|
||||
completionReason: task.completion_reason ?? null,
|
||||
},
|
||||
'Ignoring late paired execution completion for an already completed task',
|
||||
);
|
||||
return;
|
||||
}
|
||||
completePairedExecutionContextBody(args);
|
||||
} catch (error) {
|
||||
releaseExecutionLeaseAfterCompletion({
|
||||
taskId,
|
||||
role,
|
||||
runId: args.runId,
|
||||
completionError: error,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (status !== 'succeeded') {
|
||||
if (role === 'reviewer') {
|
||||
handleFailedReviewerExecution({
|
||||
releaseExecutionLeaseAfterCompletion({
|
||||
taskId,
|
||||
role,
|
||||
runId: args.runId,
|
||||
});
|
||||
}
|
||||
|
||||
function completePairedExecutionContextBody(
|
||||
args: CompletePairedExecutionContextArgs,
|
||||
): void {
|
||||
const { taskId, role, status } = args;
|
||||
const task = getPairedTaskById(taskId);
|
||||
if (!task) {
|
||||
return;
|
||||
}
|
||||
if (task.status === 'completed') {
|
||||
logger.info(
|
||||
{
|
||||
taskId,
|
||||
role,
|
||||
status,
|
||||
completionReason: task.completion_reason ?? null,
|
||||
},
|
||||
'Ignoring late paired execution completion for an already completed task',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (status !== 'succeeded') {
|
||||
if (role === 'reviewer') {
|
||||
handleFailedReviewerExecution({
|
||||
task,
|
||||
taskId,
|
||||
summary: args.summary,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (role === 'arbiter') {
|
||||
handleFailedArbiterExecution({
|
||||
task,
|
||||
taskId,
|
||||
summary: args.summary,
|
||||
});
|
||||
return;
|
||||
}
|
||||
handleFailedOwnerExecution({ task, taskId, summary: args.summary });
|
||||
return;
|
||||
}
|
||||
|
||||
if (role === 'owner') {
|
||||
try {
|
||||
provisionOwnerWorkspaceForPairedTask(taskId);
|
||||
} catch (error) {
|
||||
if (isOwnerWorkspaceRepairNeededError(error)) {
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role,
|
||||
repairMessage: error.blockMessage || error.message,
|
||||
},
|
||||
'Owner workspace post-run guard blocked completion handling',
|
||||
);
|
||||
handleFailedOwnerExecution({
|
||||
task,
|
||||
taskId,
|
||||
summary: args.summary,
|
||||
summary: error.blockMessage || error.message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (role === 'arbiter') {
|
||||
handleFailedArbiterExecution({ task, taskId });
|
||||
return;
|
||||
}
|
||||
handleFailedOwnerExecution({ task, taskId, summary: args.summary });
|
||||
return;
|
||||
throw error;
|
||||
}
|
||||
handleOwnerCompletion({ task, taskId, summary: args.summary });
|
||||
return;
|
||||
}
|
||||
|
||||
if (role === 'owner') {
|
||||
try {
|
||||
provisionOwnerWorkspaceForPairedTask(taskId);
|
||||
} catch (error) {
|
||||
if (isOwnerWorkspaceRepairNeededError(error)) {
|
||||
logger.warn(
|
||||
{
|
||||
taskId,
|
||||
role,
|
||||
repairMessage: error.blockMessage || error.message,
|
||||
},
|
||||
'Owner workspace post-run guard blocked completion handling',
|
||||
);
|
||||
handleFailedOwnerExecution({
|
||||
task,
|
||||
taskId,
|
||||
summary: error.blockMessage || error.message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
handleOwnerCompletion({ task, taskId, summary: args.summary });
|
||||
return;
|
||||
}
|
||||
if (role === 'reviewer') {
|
||||
handleReviewerCompletion({ task, taskId, summary: args.summary });
|
||||
return;
|
||||
}
|
||||
|
||||
if (role === 'reviewer') {
|
||||
handleReviewerCompletion({ task, taskId, summary: args.summary });
|
||||
return;
|
||||
}
|
||||
if (role === 'arbiter') {
|
||||
handleArbiterCompletion({ task, taskId, summary: args.summary });
|
||||
}
|
||||
}
|
||||
|
||||
if (role === 'arbiter') {
|
||||
handleArbiterCompletion({ task, taskId, summary: args.summary });
|
||||
}
|
||||
} catch (error) {
|
||||
completionError = error;
|
||||
throw error;
|
||||
} finally {
|
||||
if (!args.runId) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
releasePairedTaskExecutionLease({ taskId, runId: args.runId });
|
||||
} catch (releaseError) {
|
||||
logger.error(
|
||||
{
|
||||
taskId,
|
||||
role,
|
||||
runId: args.runId,
|
||||
releaseError,
|
||||
},
|
||||
'Failed to release paired task execution lease after completion',
|
||||
);
|
||||
if (!completionError) {
|
||||
throw releaseError;
|
||||
}
|
||||
function releaseExecutionLeaseAfterCompletion(args: {
|
||||
taskId: string;
|
||||
role: PairedRoomRole;
|
||||
runId?: string;
|
||||
completionError?: unknown;
|
||||
}): void {
|
||||
if (!args.runId) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
releasePairedTaskExecutionLease({ taskId: args.taskId, runId: args.runId });
|
||||
} catch (releaseError) {
|
||||
logger.error(
|
||||
{
|
||||
taskId: args.taskId,
|
||||
role: args.role,
|
||||
runId: args.runId,
|
||||
releaseError,
|
||||
},
|
||||
'Failed to release paired task execution lease after completion',
|
||||
);
|
||||
if (!args.completionError) {
|
||||
throw releaseError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ export function transitionPairedTaskStatus(args: {
|
||||
finalize_step_done_count?: number;
|
||||
task_done_then_user_reopen_count?: number;
|
||||
empty_step_done_streak?: number;
|
||||
arbiter_intervention_count?: number;
|
||||
arbiter_verdict?: string | null;
|
||||
arbiter_requested_at?: string | null;
|
||||
completion_reason?: string | null;
|
||||
@@ -116,6 +117,7 @@ export function applyPairedTaskPatch(args: {
|
||||
finalize_step_done_count?: number;
|
||||
task_done_then_user_reopen_count?: number;
|
||||
empty_step_done_streak?: number;
|
||||
arbiter_intervention_count?: number;
|
||||
status?: PairedTaskStatus;
|
||||
arbiter_verdict?: string | null;
|
||||
arbiter_requested_at?: string | null;
|
||||
|
||||
@@ -224,6 +224,7 @@ describe('runCodexRotationLoop', () => {
|
||||
|
||||
expect(outcome).toEqual({ type: 'success' });
|
||||
expect(rotateCodexToken).toHaveBeenCalledTimes(1);
|
||||
expect(rotateCodexToken).toHaveBeenCalledWith('auth-expired');
|
||||
expect(markCodexTokenHealthy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ import { clearGlobalFailover } from './service-routing.js';
|
||||
export interface TriggerInfo {
|
||||
reason: AgentTriggerReason;
|
||||
retryAfterMs?: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface RotationAttemptResult {
|
||||
@@ -206,6 +207,34 @@ function evaluateCodexAttempt(
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
if (
|
||||
!attempt.sawOutput &&
|
||||
typeof output.error === 'string' &&
|
||||
output.error.length > 0
|
||||
) {
|
||||
const retryTrigger =
|
||||
attempt.streamedTriggerReason &&
|
||||
isCodexRotationReason(attempt.streamedTriggerReason.reason)
|
||||
? {
|
||||
shouldRotate: true as const,
|
||||
reason: attempt.streamedTriggerReason.reason,
|
||||
}
|
||||
: detectCodexRotationTrigger(output.error);
|
||||
if (retryTrigger.shouldRotate) {
|
||||
return {
|
||||
type: 'continue',
|
||||
trigger: { reason: retryTrigger.reason },
|
||||
rotationMessage: output.error,
|
||||
};
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{ ...logContext, provider: 'codex', error: output.error },
|
||||
'Rotated Codex account returned an error field without visible output',
|
||||
);
|
||||
return { type: 'error' };
|
||||
}
|
||||
|
||||
if (
|
||||
!attempt.sawOutput &&
|
||||
attempt.streamedTriggerReason &&
|
||||
@@ -350,7 +379,7 @@ export async function runClaudeRotationLoop(
|
||||
}
|
||||
|
||||
export async function runCodexRotationLoop(
|
||||
initialTrigger: { reason: CodexRotationReason },
|
||||
initialTrigger: { reason: CodexRotationReason; message?: string },
|
||||
runAttempt: () => Promise<CodexRotationAttemptResult>,
|
||||
logContext: Record<string, unknown>,
|
||||
rotationMessage?: string,
|
||||
@@ -358,7 +387,13 @@ export async function runCodexRotationLoop(
|
||||
let trigger = initialTrigger;
|
||||
let lastRotationMessage = rotationMessage;
|
||||
|
||||
while (getCodexAccountCount() > 1 && rotateCodexToken(lastRotationMessage)) {
|
||||
while (getCodexAccountCount() > 1) {
|
||||
const rotationReasonMessage =
|
||||
lastRotationMessage ?? trigger.message ?? trigger.reason;
|
||||
if (!rotateCodexToken(rotationReasonMessage)) {
|
||||
break;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{ ...logContext, reason: trigger.reason },
|
||||
'Codex account unhealthy, retrying with rotated account',
|
||||
|
||||
@@ -227,13 +227,7 @@ export function listClaudeAccounts(): ClaudeAccountSummary[] {
|
||||
if (def) out.push(def);
|
||||
const dir = path.join(settingsHomeDir(), '.claude-accounts');
|
||||
if (fs.existsSync(dir)) {
|
||||
const entries = fs.readdirSync(dir);
|
||||
const indices = entries
|
||||
.filter((e) => /^\d+$/.test(e))
|
||||
.map((e) => Number.parseInt(e, 10))
|
||||
.filter((n) => Number.isFinite(n) && n >= 1)
|
||||
.sort((a, b) => a - b);
|
||||
for (const i of indices) {
|
||||
for (const i of readAccountIndices(dir)) {
|
||||
const acc = readClaudeAccount(i);
|
||||
if (acc) out.push(acc);
|
||||
}
|
||||
@@ -247,13 +241,7 @@ export function listCodexAccounts(): CodexAccountSummary[] {
|
||||
if (def) out.push(def);
|
||||
const dir = path.join(settingsHomeDir(), '.codex-accounts');
|
||||
if (fs.existsSync(dir)) {
|
||||
const entries = fs.readdirSync(dir);
|
||||
const indices = entries
|
||||
.filter((e) => /^\d+$/.test(e))
|
||||
.map((e) => Number.parseInt(e, 10))
|
||||
.filter((n) => Number.isFinite(n) && n >= 1)
|
||||
.sort((a, b) => a - b);
|
||||
for (const i of indices) {
|
||||
for (const i of readAccountIndices(dir)) {
|
||||
const acc = readCodexAccount(i);
|
||||
if (acc) out.push(acc);
|
||||
}
|
||||
@@ -261,6 +249,16 @@ export function listCodexAccounts(): CodexAccountSummary[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
function readAccountIndices(dir: string): number[] {
|
||||
const indices: number[] = [];
|
||||
for (const entry of fs.readdirSync(dir)) {
|
||||
if (!/^\d+$/.test(entry)) continue;
|
||||
const index = Number.parseInt(entry, 10);
|
||||
if (Number.isFinite(index) && index >= 1) indices.push(index);
|
||||
}
|
||||
return indices.sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function pickEnvValue(content: string, key: string): string | undefined {
|
||||
const re = new RegExp(`^${key}=(.*)$`, 'm');
|
||||
const match = content.match(re);
|
||||
@@ -526,13 +524,7 @@ export function getActiveCodexSettingsIndex(): number | null {
|
||||
}
|
||||
const dir = path.join(settingsHomeDir(), '.codex-accounts');
|
||||
if (fs.existsSync(dir)) {
|
||||
const indices = fs
|
||||
.readdirSync(dir)
|
||||
.filter((e) => /^\d+$/.test(e))
|
||||
.map((e) => Number.parseInt(e, 10))
|
||||
.filter((n) => Number.isFinite(n) && n >= 1)
|
||||
.sort((a, b) => a - b);
|
||||
for (const i of indices) {
|
||||
for (const i of readAccountIndices(dir)) {
|
||||
if (codexAuthPath(i) === activePath) return i;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -438,7 +438,9 @@ describe('evaluateStreamedOutput', () => {
|
||||
expect(result.state.sawSuccessNullResultWithoutOutput).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('evaluateStreamedOutput error handling', () => {
|
||||
describe('error → Claude rotation trigger', () => {
|
||||
it('returns rotation trigger with retryAfterMs', () => {
|
||||
vi.mocked(classifyRotationTrigger).mockReturnValue({
|
||||
@@ -513,18 +515,26 @@ describe('evaluateStreamedOutput', () => {
|
||||
|
||||
describe('error → Codex rotation trigger', () => {
|
||||
it('returns rotation trigger for Codex primary', () => {
|
||||
const errorMessage =
|
||||
'unexpected status 401 Unauthorized: Missing bearer or basic authentication in header';
|
||||
vi.mocked(detectCodexRotationTrigger).mockReturnValue({
|
||||
shouldRotate: true,
|
||||
reason: '429',
|
||||
});
|
||||
|
||||
const result = evaluateStreamedOutput(
|
||||
errorOutput('429 rate limit'),
|
||||
errorOutput(errorMessage),
|
||||
freshState(),
|
||||
codexOpts,
|
||||
);
|
||||
expect(result.newTrigger).toEqual({ reason: '429' });
|
||||
expect(result.state.streamedTriggerReason).toEqual({ reason: '429' });
|
||||
expect(result.newTrigger).toEqual({
|
||||
reason: '429',
|
||||
message: errorMessage,
|
||||
});
|
||||
expect(result.state.streamedTriggerReason).toEqual({
|
||||
reason: '429',
|
||||
message: errorMessage,
|
||||
});
|
||||
});
|
||||
|
||||
it('does not check Codex rotation for Claude agent type', () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user