6ab38ca461
Some Codex plans (e.g. Plus) return only a weekly window in `primary` with `secondary: null`. applyCodexUsageToAccount dereferenced `secondary.usedPercent` unconditionally, throwing a TypeError that refreshActiveCodexUsage swallowed at debug level, so usage never applied and the dashboard row stayed blank (-1). Guard the null secondary, and when a lone window is weekly (windowDurationMins > 1440) route it into the 7d slot leaving 5h unknown. Accounts that report both windows are unchanged. Adds a regression test for the secondary:null case.
450 lines
14 KiB
TypeScript
450 lines
14 KiB
TypeScript
import { ChildProcess, spawn } from 'child_process';
|
|
import fs from 'fs';
|
|
import os from 'os';
|
|
import path from 'path';
|
|
|
|
import {
|
|
getAllCodexAccounts,
|
|
getCodexAuthPath,
|
|
updateCodexAccountUsage,
|
|
} from './codex-token-rotation.js';
|
|
import { formatResetRemaining, type UsageRow } from './dashboard-usage-rows.js';
|
|
import { logger } from './logger.js';
|
|
|
|
export interface CodexRateLimit {
|
|
limitId?: string;
|
|
limitName: string | null;
|
|
primary: { usedPercent: number; resetsAt: string | number; windowDurationMins?: number };
|
|
secondary: { usedPercent: number; resetsAt: string | number; windowDurationMins?: number } | null;
|
|
}
|
|
|
|
/**
|
|
* Result returned by the refresh functions.
|
|
* Caller is responsible for persisting into module-level cache.
|
|
*/
|
|
export interface CodexUsageRefreshResult {
|
|
rows: UsageRow[];
|
|
/** Non-null only when at least one account was successfully fetched. */
|
|
fetchedAt: string | null;
|
|
}
|
|
|
|
/** Full scan interval — exported so the orchestrator can schedule it. */
|
|
export const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
|
|
|
/**
|
|
* Threshold above which a Codex account is considered "out of budget" for
|
|
* the purpose of admitting new turns. A turn is blocked only when *every*
|
|
* Codex account is either at/over this threshold on either window or
|
|
* already in a rate-limit cooldown.
|
|
*/
|
|
export const CODEX_EXHAUSTION_THRESHOLD_PCT = 95;
|
|
|
|
export interface UsageExhaustionInfo {
|
|
exhausted: boolean;
|
|
/** ISO timestamp of the soonest reset that would relieve exhaustion. */
|
|
nextResetAt: string | null;
|
|
}
|
|
|
|
function parseResetMs(s: string | undefined | null): number | null {
|
|
if (!s) return null;
|
|
const t = Date.parse(s);
|
|
return Number.isFinite(t) ? t : null;
|
|
}
|
|
|
|
/** Coerce a string-or-number resetsAt (numeric epoch in seconds) to ISO. */
|
|
function toIsoMaybe(value: string | number | undefined): string | undefined {
|
|
if (value == null) return undefined;
|
|
if (typeof value === 'number') {
|
|
return new Date(value * 1000).toISOString();
|
|
}
|
|
// Already a string — pass through if it parses, else drop.
|
|
const ms = Date.parse(value);
|
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined;
|
|
}
|
|
|
|
/**
|
|
* Reports whether every configured Codex account is either ≥ 95% on a
|
|
* window or in a rate-limit cooldown, and — when so — the soonest moment
|
|
* at which any one account is expected to recover.
|
|
*
|
|
* Reads the in-process rotation snapshot; performs no I/O. Returns
|
|
* `exhausted: false` when there are no configured accounts, or when usage
|
|
* data is missing for any account (err on letting work through).
|
|
*/
|
|
export function getCodexUsageExhaustion(): UsageExhaustionInfo {
|
|
const accounts = getAllCodexAccounts();
|
|
if (accounts.length === 0) return { exhausted: false, nextResetAt: null };
|
|
|
|
let earliestRecoveryMs = Infinity;
|
|
|
|
for (const a of accounts) {
|
|
if (a.isRateLimited) continue; // counts as exhausted, recovery unknown here
|
|
const h5 = a.cachedUsagePct;
|
|
const d7 = a.cachedUsageD7Pct;
|
|
// -1 / undefined means "unknown" → treat as headroom
|
|
if (h5 == null || h5 < 0 || d7 == null || d7 < 0) {
|
|
return { exhausted: false, nextResetAt: null };
|
|
}
|
|
if (Math.max(h5, d7) < CODEX_EXHAUSTION_THRESHOLD_PCT) {
|
|
return { exhausted: false, nextResetAt: null };
|
|
}
|
|
|
|
const h5ResetMs =
|
|
h5 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetAt) : null;
|
|
const d7ResetMs =
|
|
d7 >= CODEX_EXHAUSTION_THRESHOLD_PCT ? parseResetMs(a.resetD7At) : null;
|
|
const candidates = [h5ResetMs, d7ResetMs].filter(
|
|
(v): v is number => v != null,
|
|
);
|
|
if (candidates.length > 0) {
|
|
const accountRecovery = Math.max(...candidates);
|
|
if (accountRecovery < earliestRecoveryMs) {
|
|
earliestRecoveryMs = accountRecovery;
|
|
}
|
|
}
|
|
}
|
|
|
|
const nextResetAt = Number.isFinite(earliestRecoveryMs)
|
|
? new Date(earliestRecoveryMs).toISOString()
|
|
: null;
|
|
return { exhausted: true, nextResetAt };
|
|
}
|
|
|
|
/** Backwards-compat boolean wrapper. */
|
|
export function isCodexUsageExhausted(): boolean {
|
|
return getCodexUsageExhaustion().exhausted;
|
|
}
|
|
|
|
function getFnmCodexBinDirs(): string[] {
|
|
const fnmRoot = path.join(os.homedir(), '.local', 'share', 'fnm');
|
|
const dirs: string[] = [];
|
|
// Prefer the alias `default` first if it exists.
|
|
const defaultBin = path.join(fnmRoot, 'aliases', 'default', 'bin');
|
|
if (fs.existsSync(defaultBin)) dirs.push(defaultBin);
|
|
// Then fall back to scanning every installed node version.
|
|
const versionsRoot = path.join(fnmRoot, 'node-versions');
|
|
try {
|
|
if (fs.existsSync(versionsRoot)) {
|
|
for (const entry of fs.readdirSync(versionsRoot)) {
|
|
const bin = path.join(versionsRoot, entry, 'installation', 'bin');
|
|
if (fs.existsSync(bin)) dirs.push(bin);
|
|
}
|
|
}
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
return dirs;
|
|
}
|
|
|
|
function getPreferredCodexPathEntries(): string[] {
|
|
const entries = [
|
|
path.dirname(process.execPath),
|
|
path.join(os.homedir(), '.npm-global', 'bin'),
|
|
// fnm-managed node installs (where `npm i -g @openai/codex` actually
|
|
// lands when the host uses fnm). Without these, ejclaw running under
|
|
// bun/systemd cannot find the `codex` binary even though the user has
|
|
// it installed via fnm's default node.
|
|
...getFnmCodexBinDirs(),
|
|
];
|
|
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
|
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
|
}
|
|
return [...new Set(entries)];
|
|
}
|
|
|
|
function findCodexBinary(): string {
|
|
const candidates = [
|
|
path.join(os.homedir(), '.npm-global', 'bin', 'codex'),
|
|
...getFnmCodexBinDirs().map((dir) => path.join(dir, 'codex')),
|
|
];
|
|
for (const candidate of candidates) {
|
|
if (fs.existsSync(candidate)) return candidate;
|
|
}
|
|
// Last resort: rely on PATH (we extend it via getPreferredCodexPathEntries).
|
|
return 'codex';
|
|
}
|
|
|
|
function getCodexHomeForAccount(accountIndex?: number): string | null {
|
|
const authPath = getCodexAuthPath(accountIndex);
|
|
if (!authPath || !fs.existsSync(authPath)) return null;
|
|
return path.dirname(authPath);
|
|
}
|
|
|
|
export async function fetchCodexUsage(
|
|
codexHomeOverride?: string,
|
|
): Promise<CodexRateLimit[] | null> {
|
|
const codexBin = findCodexBinary();
|
|
|
|
return new Promise((resolve) => {
|
|
let done = false;
|
|
let proc: ChildProcess | null = null;
|
|
const finish = (value: CodexRateLimit[] | null) => {
|
|
if (done) return;
|
|
done = true;
|
|
clearTimeout(timer);
|
|
if (proc) {
|
|
try {
|
|
proc.kill();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
resolve(value);
|
|
};
|
|
|
|
const timer = setTimeout(() => finish(null), 20_000);
|
|
|
|
const spawnEnv: Record<string, string> = {
|
|
...(process.env as Record<string, string>),
|
|
PATH: [...getPreferredCodexPathEntries(), process.env.PATH || '']
|
|
.filter(Boolean)
|
|
.join(path.delimiter),
|
|
};
|
|
if (codexHomeOverride) {
|
|
spawnEnv.CODEX_HOME = codexHomeOverride;
|
|
}
|
|
|
|
try {
|
|
proc = spawn(codexBin, ['app-server'], {
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
env: spawnEnv,
|
|
});
|
|
} catch {
|
|
resolve(null);
|
|
return;
|
|
}
|
|
|
|
if (!proc.stdout || !proc.stdin) {
|
|
finish(null);
|
|
return;
|
|
}
|
|
|
|
proc.on('error', () => finish(null));
|
|
proc.on('close', () => finish(null));
|
|
|
|
let buffer = '';
|
|
proc.stdout.on('data', (chunk: Buffer) => {
|
|
buffer += chunk.toString();
|
|
const lines = buffer.split('\n');
|
|
buffer = lines.pop() || '';
|
|
for (const line of lines) {
|
|
if (!line.trim()) continue;
|
|
try {
|
|
const message = JSON.parse(line);
|
|
if (message.id === 1) {
|
|
proc!.stdin!.write(
|
|
JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
method: 'account/rateLimits/read',
|
|
params: {},
|
|
}) + '\n',
|
|
);
|
|
} else if (message.id === 2 && message.result) {
|
|
const byId = message.result.rateLimitsByLimitId;
|
|
finish(
|
|
byId && typeof byId === 'object'
|
|
? Object.entries(byId).map(([id, val]) => ({
|
|
...(val as CodexRateLimit),
|
|
limitId: id,
|
|
}))
|
|
: null,
|
|
);
|
|
}
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
});
|
|
|
|
proc.stdin.write(
|
|
JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
method: 'initialize',
|
|
params: { clientInfo: { name: 'usage-monitor', version: '1.0' } },
|
|
}) + '\n',
|
|
);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Extract usage percentages from the primary 'codex' rate-limit bucket
|
|
* and update the rotation state for a given account.
|
|
*
|
|
* Bucket selection:
|
|
* 1. limitId === 'codex' → use it
|
|
* 2. No 'codex' bucket + single bucket → use it
|
|
* 3. No 'codex' bucket + multiple buckets → unknown (show —)
|
|
*
|
|
* All buckets are logged at info level for observability.
|
|
*/
|
|
export function applyCodexUsageToAccount(
|
|
usage: CodexRateLimit[],
|
|
accountIndex: number,
|
|
): void {
|
|
if (usage.length === 0) return;
|
|
|
|
// Log all buckets for observability
|
|
logger.info(
|
|
{
|
|
account: accountIndex + 1,
|
|
buckets: usage.map((l) => ({
|
|
id: l.limitId,
|
|
h5: l.primary?.usedPercent ?? null,
|
|
d7: l.secondary?.usedPercent ?? null,
|
|
})),
|
|
},
|
|
`Codex account #${accountIndex + 1}: ${usage.length} rate-limit bucket(s)`,
|
|
);
|
|
|
|
// Select the effective bucket
|
|
const primaryBucket = usage.find((l) => l.limitId === 'codex');
|
|
const effective = primaryBucket ?? (usage.length === 1 ? usage[0] : null);
|
|
|
|
if (!effective) {
|
|
// Multiple unknown buckets — cannot determine which is authoritative
|
|
logger.warn(
|
|
{ account: accountIndex + 1 },
|
|
`Codex account #${accountIndex + 1}: no 'codex' bucket found among ${usage.length} buckets, showing unknown`,
|
|
);
|
|
updateCodexAccountUsage(-1, undefined, accountIndex, -1, undefined);
|
|
return;
|
|
}
|
|
|
|
// Historical shape: primary = 5h window, secondary = 7d window. Some plans
|
|
// (e.g. Plus) return only ONE window in `primary` (a weekly/7d window) with
|
|
// `secondary: null`. Guard the null so we don't throw, and route a lone
|
|
// weekly window into the 7d slot so usage still renders.
|
|
// Store raw ISO timestamps (not pre-formatted strings) so the exhaustion
|
|
// gate can compute "minutes until reset" later. The dashboard render path
|
|
// formats these at display time via `formatResetRemaining`.
|
|
let pct = Math.round(effective.primary.usedPercent);
|
|
let resetIso = toIsoMaybe(effective.primary.resetsAt);
|
|
let d7Pct = effective.secondary
|
|
? Math.round(effective.secondary.usedPercent)
|
|
: -1;
|
|
let resetD7Iso = effective.secondary
|
|
? toIsoMaybe(effective.secondary.resetsAt)
|
|
: undefined;
|
|
if (
|
|
!effective.secondary &&
|
|
(effective.primary.windowDurationMins ?? 0) > 1440
|
|
) {
|
|
// Lone window is weekly (> 1 day) — show it as 7d, leave 5h unknown.
|
|
d7Pct = pct;
|
|
resetD7Iso = resetIso;
|
|
pct = -1;
|
|
resetIso = undefined;
|
|
}
|
|
updateCodexAccountUsage(pct, resetIso, accountIndex, d7Pct, resetD7Iso);
|
|
logger.info(
|
|
{
|
|
account: accountIndex + 1,
|
|
bucket: effective.limitId,
|
|
h5: pct,
|
|
d7: d7Pct,
|
|
reset: resetIso,
|
|
},
|
|
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Build display-ready usage rows from Codex rotation state.
|
|
* Called after refreshing usage data.
|
|
*/
|
|
export function buildCodexUsageRowsFromState(): UsageRow[] {
|
|
const codexAccounts = getAllCodexAccounts();
|
|
if (codexAccounts.length === 0) return [];
|
|
|
|
const isMulti = codexAccounts.length > 1;
|
|
return codexAccounts.map((acct) => {
|
|
const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' ';
|
|
const label = isMulti
|
|
? `Codex${acct.index + 1}${icon} ${acct.planType}`
|
|
: 'Codex';
|
|
return {
|
|
name: label,
|
|
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
|
h5reset: acct.resetAt ? formatResetRemaining(acct.resetAt) : '',
|
|
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
|
d7reset: acct.resetD7At ? formatResetRemaining(acct.resetD7At) : '',
|
|
};
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Scan ALL Codex accounts by spawning app-server with each auth.
|
|
* Returns refresh result — caller owns cache state.
|
|
*/
|
|
export async function refreshAllCodexAccountUsage(): Promise<CodexUsageRefreshResult> {
|
|
const codexAccounts = getAllCodexAccounts();
|
|
if (codexAccounts.length <= 1) {
|
|
return { rows: buildCodexUsageRowsFromState(), fetchedAt: null };
|
|
}
|
|
|
|
logger.info(
|
|
{ accountCount: codexAccounts.length },
|
|
'Scanning all Codex accounts for usage data',
|
|
);
|
|
|
|
let anySuccess = false;
|
|
for (const acct of codexAccounts) {
|
|
const accountDir = getCodexHomeForAccount(acct.index);
|
|
if (!accountDir) continue;
|
|
|
|
try {
|
|
const usage = await fetchCodexUsage(accountDir);
|
|
if (usage && Array.isArray(usage) && usage.length > 0) {
|
|
applyCodexUsageToAccount(usage, acct.index);
|
|
anySuccess = true;
|
|
}
|
|
} catch (err) {
|
|
logger.debug(
|
|
{ err, account: acct.index + 1 },
|
|
'Failed to fetch usage for Codex account',
|
|
);
|
|
}
|
|
}
|
|
|
|
return {
|
|
rows: buildCodexUsageRowsFromState(),
|
|
fetchedAt: anySuccess ? new Date().toISOString() : null,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Quick-refresh the active Codex account's usage.
|
|
* Returns refresh result — caller owns cache state.
|
|
*/
|
|
export async function refreshActiveCodexUsage(): Promise<CodexUsageRefreshResult> {
|
|
const codexAccounts = getAllCodexAccounts();
|
|
if (codexAccounts.length === 0) {
|
|
return { rows: [], fetchedAt: null };
|
|
}
|
|
|
|
const active = codexAccounts.find((a) => a.isActive);
|
|
if (!active) {
|
|
return { rows: buildCodexUsageRowsFromState(), fetchedAt: null };
|
|
}
|
|
|
|
const accountDir = getCodexHomeForAccount(active.index);
|
|
if (!accountDir) {
|
|
return { rows: buildCodexUsageRowsFromState(), fetchedAt: null };
|
|
}
|
|
|
|
let fetchedAt: string | null = null;
|
|
try {
|
|
const usage = await fetchCodexUsage(accountDir);
|
|
if (usage && Array.isArray(usage) && usage.length > 0) {
|
|
applyCodexUsageToAccount(usage, active.index);
|
|
fetchedAt = new Date().toISOString();
|
|
}
|
|
} catch (err) {
|
|
logger.debug({ err }, 'Failed to fetch active Codex account usage');
|
|
}
|
|
|
|
return { rows: buildCodexUsageRowsFromState(), fetchedAt };
|
|
}
|