import { ChildProcess, spawn } from 'child_process'; import fs from 'fs'; import os from 'os'; import path from 'path'; import { getAllCodexAccounts, getCodexAuthPath, updateCodexAccountUsage, } from './codex-token-rotation.js'; import { formatResetRemaining, type UsageRow } from './dashboard-usage-rows.js'; import { logger } from './logger.js'; export interface CodexRateLimit { limitId?: string; limitName: string | null; primary: { usedPercent: number; resetsAt: string | number; windowDurationMins?: number }; secondary: { usedPercent: number; resetsAt: string | number; windowDurationMins?: number } | null; } /** * Result returned by the refresh functions. * Caller is responsible for persisting into module-level cache. */ export interface CodexUsageRefreshResult { rows: UsageRow[]; /** Non-null only when at least one account was successfully fetched. */ fetchedAt: string | null; } /** Full scan interval — exported so the orchestrator can schedule it. */ export const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour /** * Threshold above which a Codex account is considered "out of budget" for * the purpose of admitting new turns. A turn is blocked only when *every* * Codex account is either at/over this threshold on either window or * already in a rate-limit cooldown. */ export const CODEX_EXHAUSTION_THRESHOLD_PCT = 95; export interface UsageExhaustionInfo { exhausted: boolean; /** ISO timestamp of the soonest reset that would relieve exhaustion. */ nextResetAt: string | null; } function parseResetMs(s: string | undefined | null): number | null { if (!s) return null; const t = Date.parse(s); return Number.isFinite(t) ? t : null; } /** Coerce a string-or-number resetsAt (numeric epoch in seconds) to ISO. */ function toIsoMaybe(value: string | number | undefined): string | undefined { if (value == null) return undefined; if (typeof value === 'number') { return new Date(value * 1000).toISOString(); } // Already a string — pass through if it parses, else drop. const ms = Date.parse(value); return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined; } /** * Reports whether every configured Codex account is either ≥ 95% on a * window or in a rate-limit cooldown, and — when so — the soonest moment * at which any one account is expected to recover. * * Reads the in-process rotation snapshot; performs no I/O. Returns * `exhausted: false` when there are no configured accounts, or when usage * data is missing for any account (err on letting work through). */ export function getCodexUsageExhaustion(): UsageExhaustionInfo { const accounts = getAllCodexAccounts(); if (accounts.length === 0) return { exhausted: false, nextResetAt: null }; let earliestRecoveryMs = Infinity; for (const a of accounts) { if (a.isRateLimited) continue; // counts as exhausted, recovery unknown here const h5 = a.cachedUsagePct; const d7 = a.cachedUsageD7Pct; // -1 / undefined means "unknown" → treat as headroom if (h5 == null || h5 < 0 || d7 == null || d7 < 0) { return { exhausted: false, nextResetAt: null }; } if (Math.max(h5, d7) < CODEX_EXHAUSTION_THRESHOLD_PCT) { return { exhausted: false, nextResetAt: null }; } const h5ResetMs = h5 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetAt) : null; const d7ResetMs = d7 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetD7At) : null; const candidates = [h5ResetMs, d7ResetMs].filter( (v): v is number => v != null, ); if (candidates.length > 0) { const accountRecovery = Math.max(...candidates); if (accountRecovery < earliestRecoveryMs) { earliestRecoveryMs = accountRecovery; } } } const nextResetAt = Number.isFinite(earliestRecoveryMs) ? new Date(earliestRecoveryMs).toISOString() : null; return { exhausted: true, nextResetAt }; } /** Backwards-compat boolean wrapper. */ export function isCodexUsageExhausted(): boolean { return getCodexUsageExhaustion().exhausted; } function getFnmCodexBinDirs(): string[] { const fnmRoot = path.join(os.homedir(), '.local', 'share', 'fnm'); const dirs: string[] = []; // Prefer the alias `default` first if it exists. const defaultBin = path.join(fnmRoot, 'aliases', 'default', 'bin'); if (fs.existsSync(defaultBin)) dirs.push(defaultBin); // Then fall back to scanning every installed node version. const versionsRoot = path.join(fnmRoot, 'node-versions'); try { if (fs.existsSync(versionsRoot)) { for (const entry of fs.readdirSync(versionsRoot)) { const bin = path.join(versionsRoot, entry, 'installation', 'bin'); if (fs.existsSync(bin)) dirs.push(bin); } } } catch { /* ignore */ } return dirs; } function getPreferredCodexPathEntries(): string[] { const entries = [ path.dirname(process.execPath), path.join(os.homedir(), '.npm-global', 'bin'), // fnm-managed node installs (where `npm i -g @openai/codex` actually // lands when the host uses fnm). Without these, ejclaw running under // bun/systemd cannot find the `codex` binary even though the user has // it installed via fnm's default node. ...getFnmCodexBinDirs(), ]; if (process.versions.bun || path.basename(process.execPath) === 'bun') { entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin')); } return [...new Set(entries)]; } function findCodexBinary(): string { const candidates = [ path.join(os.homedir(), '.npm-global', 'bin', 'codex'), ...getFnmCodexBinDirs().map((dir) => path.join(dir, 'codex')), ]; for (const candidate of candidates) { if (fs.existsSync(candidate)) return candidate; } // Last resort: rely on PATH (we extend it via getPreferredCodexPathEntries). return 'codex'; } function getCodexHomeForAccount(accountIndex?: number): string | null { const authPath = getCodexAuthPath(accountIndex); if (!authPath || !fs.existsSync(authPath)) return null; return path.dirname(authPath); } export async function fetchCodexUsage( codexHomeOverride?: string, ): Promise { const codexBin = findCodexBinary(); return new Promise((resolve) => { let done = false; let proc: ChildProcess | null = null; const finish = (value: CodexRateLimit[] | null) => { if (done) return; done = true; clearTimeout(timer); if (proc) { try { proc.kill(); } catch { /* ignore */ } } resolve(value); }; const timer = setTimeout(() => finish(null), 20_000); const spawnEnv: Record = { ...(process.env as Record), PATH: [...getPreferredCodexPathEntries(), process.env.PATH || ''] .filter(Boolean) .join(path.delimiter), }; if (codexHomeOverride) { spawnEnv.CODEX_HOME = codexHomeOverride; } try { proc = spawn(codexBin, ['app-server'], { stdio: ['pipe', 'pipe', 'pipe'], env: spawnEnv, }); } catch { resolve(null); return; } if (!proc.stdout || !proc.stdin) { finish(null); return; } proc.on('error', () => finish(null)); proc.on('close', () => finish(null)); let buffer = ''; proc.stdout.on('data', (chunk: Buffer) => { buffer += chunk.toString(); const lines = buffer.split('\n'); buffer = lines.pop() || ''; for (const line of lines) { if (!line.trim()) continue; try { const message = JSON.parse(line); if (message.id === 1) { proc!.stdin!.write( JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'account/rateLimits/read', params: {}, }) + '\n', ); } else if (message.id === 2 && message.result) { const byId = message.result.rateLimitsByLimitId; finish( byId && typeof byId === 'object' ? Object.entries(byId).map(([id, val]) => ({ ...(val as CodexRateLimit), limitId: id, })) : null, ); } } catch { /* ignore */ } } }); proc.stdin.write( JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { clientInfo: { name: 'usage-monitor', version: '1.0' } }, }) + '\n', ); }); } /** * Extract usage percentages from the primary 'codex' rate-limit bucket * and update the rotation state for a given account. * * Bucket selection: * 1. limitId === 'codex' → use it * 2. No 'codex' bucket + single bucket → use it * 3. No 'codex' bucket + multiple buckets → unknown (show —) * * All buckets are logged at info level for observability. */ export function applyCodexUsageToAccount( usage: CodexRateLimit[], accountIndex: number, ): void { if (usage.length === 0) return; // Log all buckets for observability logger.info( { account: accountIndex + 1, buckets: usage.map((l) => ({ id: l.limitId, h5: l.primary?.usedPercent ?? null, d7: l.secondary?.usedPercent ?? null, })), }, `Codex account #${accountIndex + 1}: ${usage.length} rate-limit bucket(s)`, ); // Select the effective bucket const primaryBucket = usage.find((l) => l.limitId === 'codex'); const effective = primaryBucket ?? (usage.length === 1 ? usage[0] : null); if (!effective) { // Multiple unknown buckets — cannot determine which is authoritative logger.warn( { account: accountIndex + 1 }, `Codex account #${accountIndex + 1}: no 'codex' bucket found among ${usage.length} buckets, showing unknown`, ); updateCodexAccountUsage(-1, undefined, accountIndex, -1, undefined); return; } // Historical shape: primary = 5h window, secondary = 7d window. Some plans // (e.g. Plus) return only ONE window in `primary` (a weekly/7d window) with // `secondary: null`. Guard the null so we don't throw, and route a lone // weekly window into the 7d slot so usage still renders. // Store raw ISO timestamps (not pre-formatted strings) so the exhaustion // gate can compute "minutes until reset" later. The dashboard render path // formats these at display time via `formatResetRemaining`. let pct = Math.round(effective.primary.usedPercent); let resetIso = toIsoMaybe(effective.primary.resetsAt); let d7Pct = effective.secondary ? Math.round(effective.secondary.usedPercent) : -1; let resetD7Iso = effective.secondary ? toIsoMaybe(effective.secondary.resetsAt) : undefined; if ( !effective.secondary && (effective.primary.windowDurationMins ?? 0) > 1440 ) { // Lone window is weekly (> 1 day) — show it as 7d, leave 5h unknown. d7Pct = pct; resetD7Iso = resetIso; pct = -1; resetIso = undefined; } updateCodexAccountUsage(pct, resetIso, accountIndex, d7Pct, resetD7Iso); logger.info( { account: accountIndex + 1, bucket: effective.limitId, h5: pct, d7: d7Pct, reset: resetIso, }, `Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`, ); } /** * Build display-ready usage rows from Codex rotation state. * Called after refreshing usage data. */ export function buildCodexUsageRowsFromState(): UsageRow[] { const codexAccounts = getAllCodexAccounts(); if (codexAccounts.length === 0) return []; const isMulti = codexAccounts.length > 1; return codexAccounts.map((acct) => { const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' '; const label = isMulti ? `Codex${acct.index + 1}${icon} ${acct.planType}` : 'Codex'; return { name: label, h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1, h5reset: acct.resetAt ? formatResetRemaining(acct.resetAt) : '', d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1, d7reset: acct.resetD7At ? formatResetRemaining(acct.resetD7At) : '', }; }); } /** * Scan ALL Codex accounts by spawning app-server with each auth. * Returns refresh result — caller owns cache state. */ export async function refreshAllCodexAccountUsage(): Promise { const codexAccounts = getAllCodexAccounts(); if (codexAccounts.length <= 1) { return { rows: buildCodexUsageRowsFromState(), fetchedAt: null }; } logger.info( { accountCount: codexAccounts.length }, 'Scanning all Codex accounts for usage data', ); let anySuccess = false; for (const acct of codexAccounts) { const accountDir = getCodexHomeForAccount(acct.index); if (!accountDir) continue; try { const usage = await fetchCodexUsage(accountDir); if (usage && Array.isArray(usage) && usage.length > 0) { applyCodexUsageToAccount(usage, acct.index); anySuccess = true; } } catch (err) { logger.debug( { err, account: acct.index + 1 }, 'Failed to fetch usage for Codex account', ); } } return { rows: buildCodexUsageRowsFromState(), fetchedAt: anySuccess ? new Date().toISOString() : null, }; } /** * Quick-refresh the active Codex account's usage. * Returns refresh result — caller owns cache state. */ export async function refreshActiveCodexUsage(): Promise { const codexAccounts = getAllCodexAccounts(); if (codexAccounts.length === 0) { return { rows: [], fetchedAt: null }; } const active = codexAccounts.find((a) => a.isActive); if (!active) { return { rows: buildCodexUsageRowsFromState(), fetchedAt: null }; } const accountDir = getCodexHomeForAccount(active.index); if (!accountDir) { return { rows: buildCodexUsageRowsFromState(), fetchedAt: null }; } let fetchedAt: string | null = null; try { const usage = await fetchCodexUsage(accountDir); if (usage && Array.isArray(usage) && usage.length > 0) { applyCodexUsageToAccount(usage, active.index); fetchedAt = new Date().toISOString(); } } catch (err) { logger.debug({ err }, 'Failed to fetch active Codex account usage'); } return { rows: buildCodexUsageRowsFromState(), fetchedAt }; }