fix: remove Codex API key auth path, add secret redaction and d7 auto-rotation
- Remove OPENAI_API_KEY from .env and Codex child process env to prevent API billing when subscription quota is exhausted - Remove writeCodexApiKeyAuth entirely — Codex now uses OAuth only - Add 9-pattern secret redaction in formatOutbound() to prevent key leaks - Fix Codex usage bucket aggregation: use 'codex' bucket only instead of max across all buckets (bengalfox = Codex Spark, not needed) - Add d7≥100% auto-rotation in updateCodexAccountUsage to skip exhausted accounts and prevent API billing fallback - Add findNextCodexAvailable that checks both rate-limits and d7 usage - Clean stale apikey auth.json files from all session directories - Update tests: OAuth-only auth, d7 auto-skip (3 new tests), dashboard Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -112,12 +112,13 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes API-key auth when OPENAI_API_KEY is available', () => {
|
||||
it('ignores OPENAI_API_KEY and always uses OAuth auth', () => {
|
||||
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||
fs.writeFileSync(
|
||||
rotatedAuthPath,
|
||||
JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: 'x' } }),
|
||||
);
|
||||
const rotatedAuth = {
|
||||
auth_mode: 'chatgpt',
|
||||
tokens: { access_token: 'x' },
|
||||
};
|
||||
fs.writeFileSync(rotatedAuthPath, JSON.stringify(rotatedAuth));
|
||||
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
|
||||
mockReadEnvFile.mockReturnValue({
|
||||
OPENAI_API_KEY: 'sk-test-api-key',
|
||||
@@ -139,9 +140,10 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
||||
tokens?: unknown;
|
||||
};
|
||||
|
||||
expect(auth.auth_mode).toBe('apikey');
|
||||
expect(auth.OPENAI_API_KEY).toBe('sk-test-api-key');
|
||||
expect(auth.tokens).toBeUndefined();
|
||||
// API key auth is never used — always OAuth
|
||||
expect(auth.auth_mode).toBe('chatgpt');
|
||||
expect(auth.OPENAI_API_KEY).toBeUndefined();
|
||||
expect(auth.tokens).toEqual({ access_token: 'x' });
|
||||
});
|
||||
|
||||
it('falls back to rotated OAuth auth when no API key is configured', () => {
|
||||
|
||||
@@ -20,19 +20,8 @@ import {
|
||||
} from './platform-prompts.js';
|
||||
import type { AgentType, RegisteredGroup } from './types.js';
|
||||
|
||||
function writeCodexApiKeyAuth(authPath: string, openaiKey: string): void {
|
||||
fs.writeFileSync(
|
||||
authPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
auth_mode: 'apikey',
|
||||
OPENAI_API_KEY: openaiKey,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + '\n',
|
||||
);
|
||||
}
|
||||
// writeCodexApiKeyAuth removed — Codex uses OAuth only.
|
||||
// API key auth caused unintended billing.
|
||||
|
||||
function syncDirectoryEntries(sources: string[], destination: string): void {
|
||||
for (const source of sources) {
|
||||
@@ -184,12 +173,10 @@ function prepareCodexSessionEnvironment(args: {
|
||||
isPairedRoom: boolean;
|
||||
memoryBriefing?: string;
|
||||
}): void {
|
||||
const openaiKey =
|
||||
args.envVars.CODEX_OPENAI_API_KEY ||
|
||||
process.env.CODEX_OPENAI_API_KEY ||
|
||||
args.envVars.OPENAI_API_KEY ||
|
||||
process.env.OPENAI_API_KEY;
|
||||
if (openaiKey) args.env.OPENAI_API_KEY = openaiKey;
|
||||
// API key auth intentionally removed — Codex uses OAuth only.
|
||||
// Never pass any API key to Codex child process to prevent API billing.
|
||||
delete args.env.OPENAI_API_KEY;
|
||||
delete args.env.CODEX_OPENAI_API_KEY;
|
||||
|
||||
const codexModel =
|
||||
args.group.agentConfig?.codexModel ||
|
||||
@@ -208,9 +195,8 @@ function prepareCodexSessionEnvironment(args: {
|
||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||
|
||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||
if (openaiKey) {
|
||||
writeCodexApiKeyAuth(authDst, openaiKey);
|
||||
} else {
|
||||
// Always use OAuth auth from rotated accounts (API key auth removed)
|
||||
{
|
||||
const rotatedAuthSrc = getActiveCodexAuthPath();
|
||||
const authSrc =
|
||||
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||
@@ -417,8 +403,6 @@ export function prepareGroupEnvironment(
|
||||
'CLAUDE_THINKING',
|
||||
'CLAUDE_THINKING_BUDGET',
|
||||
'CLAUDE_EFFORT',
|
||||
'OPENAI_API_KEY',
|
||||
'CODEX_OPENAI_API_KEY',
|
||||
'CODEX_MODEL',
|
||||
'CODEX_EFFORT',
|
||||
'MEMENTO_MCP_SSE_URL',
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
import { DATA_DIR } from './config.js';
|
||||
@@ -46,7 +47,10 @@ function mapWindow(w?: {
|
||||
|
||||
interface UsageCacheEntry {
|
||||
usage: ClaudeUsageData;
|
||||
/** Timestamp of last *successful* API fetch (use for stale detection). */
|
||||
fetchedAt: number;
|
||||
/** Timestamp of last API *attempt* including failures (use for throttling). */
|
||||
lastAttemptAt?: number;
|
||||
}
|
||||
|
||||
let usageDiskCache: Record<string, UsageCacheEntry> = {};
|
||||
@@ -77,13 +81,15 @@ const MIN_FETCH_INTERVAL_MS = 300_000;
|
||||
|
||||
async function fetchUsageForToken(
|
||||
token: string,
|
||||
accountIndex?: number,
|
||||
): Promise<ClaudeUsageData | null> {
|
||||
loadUsageDiskCache();
|
||||
|
||||
// Return cached data if fetched recently (avoid API rate-limit)
|
||||
// Return cached data if attempted recently (avoid API rate-limit)
|
||||
const key = cacheKey(token);
|
||||
const cached = usageDiskCache[key];
|
||||
if (cached && Date.now() - cached.fetchedAt < MIN_FETCH_INTERVAL_MS) {
|
||||
const lastAttempt = cached?.lastAttemptAt ?? cached?.fetchedAt ?? 0;
|
||||
if (cached && Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS) {
|
||||
return cached.usage;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
@@ -101,21 +107,35 @@ async function fetchUsageForToken(
|
||||
});
|
||||
|
||||
if (res.status === 401) {
|
||||
logger.warn('Claude usage API: token expired or invalid (401)');
|
||||
logger.warn(
|
||||
{ account: accountIndex != null ? accountIndex + 1 : '?', tokenKey: key },
|
||||
'Claude usage API: token expired or invalid (401)',
|
||||
);
|
||||
return null;
|
||||
}
|
||||
if (res.status === 429) {
|
||||
const staleMs = cached ? Date.now() - cached.fetchedAt : 0;
|
||||
logger.warn(
|
||||
{ account: accountIndex != null ? accountIndex + 1 : '?', tokenKey: key, staleMinutes: Math.round(staleMs / 60_000) },
|
||||
'Claude usage API: rate limited (429), returning cached data',
|
||||
);
|
||||
return usageDiskCache[cacheKey(token)]?.usage ?? null;
|
||||
// Record attempt time so we don't retry for MIN_FETCH_INTERVAL_MS
|
||||
if (cached) {
|
||||
cached.lastAttemptAt = Date.now();
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
return cached?.usage ?? null;
|
||||
}
|
||||
if (!res.ok) {
|
||||
logger.warn(
|
||||
{ status: res.status },
|
||||
{ account: accountIndex != null ? accountIndex + 1 : '?', status: res.status, tokenKey: key },
|
||||
`Claude usage API: unexpected status ${res.status}`,
|
||||
);
|
||||
return usageDiskCache[cacheKey(token)]?.usage ?? null;
|
||||
if (cached) {
|
||||
cached.lastAttemptAt = Date.now();
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
return cached?.usage ?? null;
|
||||
}
|
||||
|
||||
const data = (await res.json()) as UsageApiResponse;
|
||||
@@ -127,18 +147,29 @@ async function fetchUsageForToken(
|
||||
seven_day_opus: mapWindow(data.seven_day_opus),
|
||||
};
|
||||
|
||||
// Persist to disk cache
|
||||
usageDiskCache[cacheKey(token)] = { usage: result, fetchedAt: Date.now() };
|
||||
// Persist to disk cache — success: update both fetchedAt and lastAttemptAt
|
||||
const now = Date.now();
|
||||
usageDiskCache[key] = { usage: result, fetchedAt: now, lastAttemptAt: now };
|
||||
saveUsageDiskCache();
|
||||
|
||||
logger.debug(
|
||||
{ tokenKey: key, h5: result.five_hour?.utilization, d7: result.seven_day?.utilization },
|
||||
'Claude usage API: fetched successfully',
|
||||
);
|
||||
|
||||
return result;
|
||||
} catch (err) {
|
||||
if ((err as Error).name === 'AbortError') {
|
||||
logger.warn('Claude usage API: request timed out');
|
||||
logger.warn({ tokenKey: key }, 'Claude usage API: request timed out');
|
||||
} else {
|
||||
logger.warn({ err }, 'Claude usage API: fetch failed');
|
||||
logger.warn({ err, tokenKey: key }, 'Claude usage API: fetch failed');
|
||||
}
|
||||
return usageDiskCache[cacheKey(token)]?.usage ?? null;
|
||||
// Record attempt time so we back off on persistent failures
|
||||
if (cached) {
|
||||
cached.lastAttemptAt = Date.now();
|
||||
saveUsageDiskCache();
|
||||
}
|
||||
return cached?.usage ?? null;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
@@ -154,7 +185,7 @@ export async function fetchClaudeUsage(): Promise<ClaudeUsageData | null> {
|
||||
logger.debug('No Claude OAuth token available for usage check');
|
||||
return null;
|
||||
}
|
||||
return fetchUsageForToken(token);
|
||||
return fetchUsageForToken(token, undefined);
|
||||
}
|
||||
|
||||
export interface ClaudeAccountProfile {
|
||||
@@ -164,6 +195,32 @@ export interface ClaudeAccountProfile {
|
||||
|
||||
const profileCache = new Map<number, ClaudeAccountProfile>();
|
||||
|
||||
/**
|
||||
* Read planType from credentials file as fallback when profile API fails.
|
||||
* Account 0: ~/.claude/.credentials.json
|
||||
* Account 1+: ~/.claude-accounts/{index}/.credentials.json
|
||||
*/
|
||||
function readCredentialsPlanType(accountIndex: number): string | null {
|
||||
try {
|
||||
const credsPath =
|
||||
accountIndex === 0
|
||||
? path.join(os.homedir(), '.claude', '.credentials.json')
|
||||
: path.join(
|
||||
os.homedir(),
|
||||
'.claude-accounts',
|
||||
String(accountIndex),
|
||||
'.credentials.json',
|
||||
);
|
||||
if (!fs.existsSync(credsPath)) return null;
|
||||
const data = readJsonFile<{
|
||||
claudeAiOauth?: { subscriptionType?: string };
|
||||
}>(credsPath);
|
||||
return data?.claudeAiOauth?.subscriptionType || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchProfileForToken(
|
||||
token: string,
|
||||
): Promise<ClaudeAccountProfile | null> {
|
||||
@@ -192,7 +249,7 @@ async function fetchProfileForToken(
|
||||
? 'max'
|
||||
: data.account?.has_claude_pro
|
||||
? 'pro'
|
||||
: orgType.replace('claude_', '') || 'free';
|
||||
: orgType.replace('claude_', '') || '?';
|
||||
return {
|
||||
email: data.account?.email || '?',
|
||||
planType,
|
||||
@@ -210,7 +267,23 @@ async function fetchProfileForToken(
|
||||
export async function fetchAllClaudeProfiles(): Promise<void> {
|
||||
const allTokens = getAllTokens();
|
||||
for (const t of allTokens) {
|
||||
const profile = await fetchProfileForToken(t.token);
|
||||
let profile = await fetchProfileForToken(t.token);
|
||||
|
||||
// Fallback: if profile API failed or returned unknown plan, use credentials file
|
||||
if (!profile || profile.planType === '?') {
|
||||
const credsPlan = readCredentialsPlanType(t.index);
|
||||
if (credsPlan) {
|
||||
profile = {
|
||||
email: profile?.email || '?',
|
||||
planType: credsPlan,
|
||||
};
|
||||
logger.info(
|
||||
{ account: t.index + 1, plan: credsPlan, source: 'credentials' },
|
||||
`Claude account #${t.index + 1}: ${credsPlan} (from credentials)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (profile) {
|
||||
profileCache.set(t.index, profile);
|
||||
logger.info(
|
||||
@@ -246,7 +319,7 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
// Single token fallback
|
||||
const token = process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
||||
if (!token) return [];
|
||||
const usage = await fetchUsageForToken(token);
|
||||
const usage = await fetchUsageForToken(token, 0);
|
||||
return [
|
||||
{
|
||||
index: 0,
|
||||
@@ -260,7 +333,7 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
||||
|
||||
const results: ClaudeAccountUsage[] = [];
|
||||
for (const t of allTokens) {
|
||||
const usage = await fetchUsageForToken(t.token);
|
||||
const usage = await fetchUsageForToken(t.token, t.index);
|
||||
results.push({
|
||||
index: t.index,
|
||||
masked: t.masked,
|
||||
|
||||
124
src/codex-token-rotation.test.ts
Normal file
124
src/codex-token-rotation.test.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('./agent-error-detection.js', () => ({
|
||||
classifyAgentError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||
classifyCodexAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||
}));
|
||||
|
||||
vi.mock('./config.js', () => ({
|
||||
DATA_DIR: '/tmp/ejclaw-codex-rot-data',
|
||||
}));
|
||||
|
||||
vi.mock('./logger.js', () => ({
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('./utils.js', async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import('./utils.js')>('./utils.js');
|
||||
return {
|
||||
...actual,
|
||||
writeJsonFile: vi.fn(), // no-op to prevent state file writes
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('os', async () => {
|
||||
const actual = await vi.importActual<typeof import('os')>('os');
|
||||
return {
|
||||
...actual,
|
||||
default: {
|
||||
...actual,
|
||||
homedir: () => process.env.CODEX_ROT_TEST_HOME || '/tmp',
|
||||
},
|
||||
homedir: () => process.env.CODEX_ROT_TEST_HOME || '/tmp',
|
||||
};
|
||||
});
|
||||
|
||||
function createFakeAccounts(homeDir: string, count: number): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
const dir = path.join(homeDir, '.codex-accounts', String(i));
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'auth.json'),
|
||||
JSON.stringify({
|
||||
auth_mode: 'chatgpt',
|
||||
tokens: { account_id: `acct-${i}`, access_token: `token-${i}` },
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
||||
let tempHome: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-rot-'));
|
||||
process.env.CODEX_ROT_TEST_HOME = tempHome;
|
||||
createFakeAccounts(tempHome, 4);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.CODEX_ROT_TEST_HOME;
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('advanceCodexAccount skips accounts with d7 ≥ 100%', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
expect(mod.getCodexAccountCount()).toBe(4);
|
||||
|
||||
// Mark account #1 (next after #0) as 7d-exhausted
|
||||
mod.updateCodexAccountUsage(80, undefined, 1, 100, undefined);
|
||||
|
||||
// Current is #0, advance should skip #1 (d7=100%) → land on #2
|
||||
mod.advanceCodexAccount();
|
||||
|
||||
const accounts = mod.getAllCodexAccounts();
|
||||
expect(accounts[0].isActive).toBe(false);
|
||||
expect(accounts[1].isActive).toBe(false);
|
||||
expect(accounts[2].isActive).toBe(true);
|
||||
});
|
||||
|
||||
it('updateCodexAccountUsage auto-rotates when current account hits d7 ≥ 100%', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
expect(mod.getCodexAccountCount()).toBe(4);
|
||||
|
||||
// Current is #0 — report d7=100% for the current account
|
||||
mod.updateCodexAccountUsage(80, undefined, 0, 100, undefined);
|
||||
|
||||
// Should have auto-rotated away from #0 to #1
|
||||
const accounts = mod.getAllCodexAccounts();
|
||||
expect(accounts[0].isActive).toBe(false);
|
||||
expect(accounts[1].isActive).toBe(true);
|
||||
});
|
||||
|
||||
it('advanceCodexAccount falls back when all accounts are d7-exhausted', async () => {
|
||||
const mod = await import('./codex-token-rotation.js');
|
||||
mod.initCodexTokenRotation();
|
||||
expect(mod.getCodexAccountCount()).toBe(4);
|
||||
|
||||
// Exhaust d7 on all accounts except current (#0)
|
||||
mod.updateCodexAccountUsage(80, undefined, 1, 100, undefined);
|
||||
mod.updateCodexAccountUsage(80, undefined, 2, 100, undefined);
|
||||
mod.updateCodexAccountUsage(80, undefined, 3, 100, undefined);
|
||||
|
||||
// Advance — all others d7-exhausted, falls back to rate-limit-only check
|
||||
// findNextAvailable (base) should still find #1 since it's not rate-limited
|
||||
mod.advanceCodexAccount();
|
||||
|
||||
const accounts = mod.getAllCodexAccounts();
|
||||
const active = accounts.find((a) => a.isActive);
|
||||
expect(active).toBeDefined();
|
||||
expect(active!.index).toBe(1); // fallback picks next non-rate-limited
|
||||
});
|
||||
});
|
||||
@@ -137,7 +137,7 @@ function saveCodexState(): void {
|
||||
}
|
||||
}
|
||||
|
||||
function loadCodexState(): void {
|
||||
function loadCodexState(quiet = false): void {
|
||||
const state = readJsonFile<{
|
||||
currentIndex?: number;
|
||||
rateLimits?: (number | null)[];
|
||||
@@ -164,6 +164,8 @@ function loadCodexState(): void {
|
||||
const until = state.rateLimits[i];
|
||||
if (typeof until === 'number' && until > now) {
|
||||
accounts[i].rateLimitedUntil = until;
|
||||
} else {
|
||||
accounts[i].rateLimitedUntil = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -173,8 +175,10 @@ function loadCodexState(): void {
|
||||
i < Math.min(state.usagePcts.length, accounts.length);
|
||||
i++
|
||||
) {
|
||||
if (typeof state.usagePcts[i] === 'number')
|
||||
accounts[i].lastUsagePct = state.usagePcts[i]!;
|
||||
accounts[i].lastUsagePct =
|
||||
typeof state.usagePcts[i] === 'number'
|
||||
? state.usagePcts[i]!
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
if (Array.isArray(state.usageD7Pcts)) {
|
||||
@@ -183,13 +187,15 @@ function loadCodexState(): void {
|
||||
i < Math.min(state.usageD7Pcts.length, accounts.length);
|
||||
i++
|
||||
) {
|
||||
if (typeof state.usageD7Pcts[i] === 'number')
|
||||
accounts[i].lastUsageD7Pct = state.usageD7Pcts[i]!;
|
||||
accounts[i].lastUsageD7Pct =
|
||||
typeof state.usageD7Pcts[i] === 'number'
|
||||
? state.usageD7Pcts[i]!
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
if (Array.isArray(state.resetAts)) {
|
||||
for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) {
|
||||
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]!;
|
||||
accounts[i].resetAt = state.resetAts[i] ?? undefined;
|
||||
}
|
||||
}
|
||||
if (Array.isArray(state.resetD7Ats)) {
|
||||
@@ -198,13 +204,25 @@ function loadCodexState(): void {
|
||||
i < Math.min(state.resetD7Ats.length, accounts.length);
|
||||
i++
|
||||
) {
|
||||
if (state.resetD7Ats[i]) accounts[i].resetD7At = state.resetD7Ats[i]!;
|
||||
accounts[i].resetD7At = state.resetD7Ats[i] ?? undefined;
|
||||
}
|
||||
}
|
||||
logger.info(
|
||||
{ currentIndex, accountCount: accounts.length },
|
||||
'Codex rotation state restored',
|
||||
);
|
||||
if (!quiet) {
|
||||
logger.info(
|
||||
{ currentIndex, accountCount: accounts.length },
|
||||
'Codex rotation state restored',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-read the on-disk rotation state (written by any service).
|
||||
* Call before dashboard renders so the renderer picks up rotations
|
||||
* performed by the Codex service process.
|
||||
*/
|
||||
export function reloadCodexStateFromDisk(): void {
|
||||
if (accounts.length <= 1) return;
|
||||
loadCodexState(true);
|
||||
}
|
||||
|
||||
/** Get the auth.json path for the current active account. */
|
||||
@@ -272,19 +290,37 @@ export function rotateCodexToken(
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
||||
*/
|
||||
function findNextCodexAvailable(fromIndex?: number): number | null {
|
||||
const now = Date.now();
|
||||
const start = fromIndex ?? currentIndex;
|
||||
for (let i = 1; i < accounts.length; i++) {
|
||||
const idx = (start + i) % accounts.length;
|
||||
const acct = accounts[idx];
|
||||
const rlOk = !acct.rateLimitedUntil || acct.rateLimitedUntil <= now;
|
||||
const usageOk = acct.lastUsageD7Pct == null || acct.lastUsageD7Pct < 100;
|
||||
if (rlOk && usageOk) return idx;
|
||||
}
|
||||
// All exhausted — fall back to rate-limit-only check
|
||||
return findNextAvailable(accounts, start);
|
||||
}
|
||||
|
||||
/**
|
||||
* Advance to the next healthy account (round-robin).
|
||||
* Called after each successful request to spread load evenly
|
||||
* and keep usage data fresh for all accounts.
|
||||
* Skips accounts with 7d usage ≥ 100% to avoid API billing.
|
||||
*/
|
||||
export function advanceCodexAccount(): void {
|
||||
if (accounts.length <= 1) return;
|
||||
const nextIdx = findNextAvailable(accounts, currentIndex);
|
||||
const nextIdx = findNextCodexAvailable();
|
||||
if (nextIdx !== null) {
|
||||
currentIndex = nextIdx;
|
||||
saveCodexState();
|
||||
}
|
||||
// All others rate-limited, stay on current
|
||||
// All others rate-limited/exhausted, stay on current
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -306,6 +342,19 @@ export function updateCodexAccountUsage(
|
||||
if (resetAt) acct.resetAt = resetAt;
|
||||
if (resetD7At) acct.resetD7At = resetD7At;
|
||||
saveCodexState();
|
||||
|
||||
// Auto-rotate away from 7d-exhausted current account to avoid API billing
|
||||
if (idx === currentIndex && d7Pct != null && d7Pct >= 100 && accounts.length > 1) {
|
||||
const nextIdx = findNextCodexAvailable(idx);
|
||||
if (nextIdx !== null && nextIdx !== idx) {
|
||||
logger.info(
|
||||
{ from: idx + 1, to: nextIdx + 1, d7Pct },
|
||||
`Codex auto-rotating: account #${idx + 1} at ${d7Pct}% 7d → #${nextIdx + 1}`,
|
||||
);
|
||||
currentIndex = nextIdx;
|
||||
saveCodexState();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
import fs from 'fs';
|
||||
|
||||
import { GroupQueue } from './group-queue.js';
|
||||
import { GroupQueue, type GroupRunContext } from './group-queue.js';
|
||||
|
||||
// Mock config to control concurrency limit
|
||||
vi.mock('./config.js', () => ({
|
||||
@@ -121,6 +121,100 @@ describe('GroupQueue', () => {
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
});
|
||||
|
||||
it('force-terminates a lingering process after output was delivered', async () => {
|
||||
class StubbornProcess extends EventEmitter {
|
||||
exitCode: number | null = null;
|
||||
signalCode: NodeJS.Signals | null = null;
|
||||
kill = vi.fn(() => true);
|
||||
}
|
||||
|
||||
const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder';
|
||||
let releaseRun!: (value: boolean) => void;
|
||||
let runId: string | undefined;
|
||||
const blocker = new Promise<boolean>((resolve) => {
|
||||
releaseRun = resolve;
|
||||
});
|
||||
|
||||
const processMessages = vi.fn(
|
||||
async (_groupJid: string, context: GroupRunContext) => {
|
||||
runId = context.runId;
|
||||
return await blocker;
|
||||
},
|
||||
);
|
||||
|
||||
queue.setProcessMessagesFn(processMessages);
|
||||
queue.enqueueMessageCheck('group1@g.us', ipcDir);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
expect(runId).toEqual(expect.any(String));
|
||||
|
||||
const proc = new StubbornProcess();
|
||||
queue.registerProcess(
|
||||
'group1@g.us',
|
||||
proc as unknown as import('child_process').ChildProcess,
|
||||
'proc-1',
|
||||
ipcDir,
|
||||
);
|
||||
|
||||
queue.closeStdin('group1@g.us', {
|
||||
runId,
|
||||
reason: 'output-delivered-close',
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(proc.kill).toHaveBeenCalledWith('SIGTERM');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
expect(proc.kill).toHaveBeenCalledWith('SIGKILL');
|
||||
|
||||
releaseRun(true);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
});
|
||||
|
||||
it('clears post-close termination timers once the run exits', async () => {
|
||||
class FakeProcess extends EventEmitter {
|
||||
exitCode: number | null = null;
|
||||
signalCode: NodeJS.Signals | null = null;
|
||||
kill = vi.fn(() => true);
|
||||
}
|
||||
|
||||
const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder';
|
||||
let releaseRun!: (value: boolean) => void;
|
||||
const blocker = new Promise<boolean>((resolve) => {
|
||||
releaseRun = resolve;
|
||||
});
|
||||
|
||||
const proc = new FakeProcess();
|
||||
const processMessages = vi.fn(
|
||||
async (_groupJid: string, context: GroupRunContext) => {
|
||||
queue.registerProcess(
|
||||
'group1@g.us',
|
||||
proc as unknown as import('child_process').ChildProcess,
|
||||
'proc-1',
|
||||
ipcDir,
|
||||
);
|
||||
|
||||
queue.closeStdin('group1@g.us', {
|
||||
runId: context.runId,
|
||||
reason: 'output-delivered-close',
|
||||
});
|
||||
|
||||
await blocker;
|
||||
return true;
|
||||
},
|
||||
);
|
||||
|
||||
queue.setProcessMessagesFn(processMessages);
|
||||
queue.enqueueMessageCheck('group1@g.us', ipcDir);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
releaseRun(true);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(75_000);
|
||||
expect(proc.kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// --- Global concurrency limit ---
|
||||
|
||||
it('respects global concurrency limit', async () => {
|
||||
|
||||
@@ -23,6 +23,8 @@ const MAX_RETRIES = 5;
|
||||
const BASE_RETRY_MS = 5000;
|
||||
const MAX_CONCURRENT_TASKS =
|
||||
MAX_CONCURRENT_AGENTS > 1 ? MAX_CONCURRENT_AGENTS - 1 : 1;
|
||||
const POST_CLOSE_SIGTERM_DELAY_MS = 60_000;
|
||||
const POST_CLOSE_SIGKILL_DELAY_MS = 75_000;
|
||||
|
||||
interface GroupState {
|
||||
active: boolean;
|
||||
@@ -38,6 +40,8 @@ interface GroupState {
|
||||
retryCount: number;
|
||||
retryTimer: ReturnType<typeof setTimeout> | null;
|
||||
retryScheduledAt: number | null;
|
||||
postCloseTermTimer: ReturnType<typeof setTimeout> | null;
|
||||
postCloseKillTimer: ReturnType<typeof setTimeout> | null;
|
||||
startedAt: number | null;
|
||||
}
|
||||
|
||||
@@ -78,6 +82,8 @@ export class GroupQueue {
|
||||
retryCount: 0,
|
||||
retryTimer: null,
|
||||
retryScheduledAt: null,
|
||||
postCloseTermTimer: null,
|
||||
postCloseKillTimer: null,
|
||||
startedAt: null,
|
||||
};
|
||||
this.groups.set(groupJid, state);
|
||||
@@ -293,6 +299,91 @@ export class GroupQueue {
|
||||
}
|
||||
}
|
||||
|
||||
private clearPostCloseTimers(state: GroupState): void {
|
||||
if (state.postCloseTermTimer) {
|
||||
clearTimeout(state.postCloseTermTimer);
|
||||
state.postCloseTermTimer = null;
|
||||
}
|
||||
if (state.postCloseKillTimer) {
|
||||
clearTimeout(state.postCloseKillTimer);
|
||||
state.postCloseKillTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private schedulePostCloseTermination(
|
||||
groupJid: string,
|
||||
state: GroupState,
|
||||
runId: string | null,
|
||||
reason: string,
|
||||
): void {
|
||||
const proc = state.process;
|
||||
if (!proc || !runId || state.isTaskProcess) {
|
||||
return;
|
||||
}
|
||||
|
||||
const processName = state.processName;
|
||||
const isSameActiveProcess = () =>
|
||||
state.process === proc &&
|
||||
state.currentRunId === runId &&
|
||||
this.isProcessAlive(proc);
|
||||
|
||||
this.clearPostCloseTimers(state);
|
||||
|
||||
state.postCloseTermTimer = setTimeout(() => {
|
||||
state.postCloseTermTimer = null;
|
||||
if (!isSameActiveProcess()) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
{
|
||||
groupJid,
|
||||
runId,
|
||||
processName,
|
||||
reason,
|
||||
delayMs: POST_CLOSE_SIGTERM_DELAY_MS,
|
||||
},
|
||||
'Force-terminating lingering agent after stdin close request',
|
||||
);
|
||||
|
||||
try {
|
||||
proc.kill('SIGTERM');
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ groupJid, runId, processName, err },
|
||||
'Failed to SIGTERM lingering agent after stdin close request',
|
||||
);
|
||||
}
|
||||
}, POST_CLOSE_SIGTERM_DELAY_MS);
|
||||
|
||||
state.postCloseKillTimer = setTimeout(() => {
|
||||
state.postCloseKillTimer = null;
|
||||
if (!isSameActiveProcess()) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error(
|
||||
{
|
||||
groupJid,
|
||||
runId,
|
||||
processName,
|
||||
reason,
|
||||
delayMs: POST_CLOSE_SIGKILL_DELAY_MS,
|
||||
},
|
||||
'Force-killing stubborn agent after stdin close request',
|
||||
);
|
||||
|
||||
try {
|
||||
proc.kill('SIGKILL');
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ groupJid, runId, processName, err },
|
||||
'Failed to SIGKILL stubborn agent after stdin close request',
|
||||
);
|
||||
}
|
||||
}, POST_CLOSE_SIGKILL_DELAY_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Signal the active agent process to wind down by writing a close sentinel.
|
||||
*/
|
||||
@@ -327,6 +418,15 @@ export class GroupQueue {
|
||||
'Failed to signal active agent to close stdin',
|
||||
);
|
||||
}
|
||||
|
||||
if (metadata?.reason === 'output-delivered-close') {
|
||||
this.schedulePostCloseTermination(
|
||||
groupJid,
|
||||
state,
|
||||
metadata.runId ?? state.currentRunId,
|
||||
metadata.reason,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async runForGroup(
|
||||
@@ -371,6 +471,7 @@ export class GroupQueue {
|
||||
);
|
||||
this.scheduleRetry(groupJid, state, runId);
|
||||
} finally {
|
||||
this.clearPostCloseTimers(state);
|
||||
const durationMs = state.startedAt ? Date.now() - state.startedAt : null;
|
||||
logger.info(
|
||||
{
|
||||
@@ -416,6 +517,7 @@ export class GroupQueue {
|
||||
} catch (err) {
|
||||
logger.error({ groupJid, taskId: task.id, err }, 'Error running task');
|
||||
} finally {
|
||||
this.clearPostCloseTimers(state);
|
||||
state.active = false;
|
||||
state.isTaskProcess = false;
|
||||
state.runningTaskId = null;
|
||||
@@ -621,6 +723,7 @@ export class GroupQueue {
|
||||
}> = [];
|
||||
|
||||
for (const [groupJid, state] of this.groups) {
|
||||
this.clearPostCloseTimers(state);
|
||||
if (state.retryTimer) {
|
||||
clearTimeout(state.retryTimer);
|
||||
state.retryTimer = null;
|
||||
|
||||
@@ -28,10 +28,35 @@ export function stripInternalTags(text: string): string {
|
||||
return text.replace(/<internal>[\s\S]*?<\/internal>/g, '').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Patterns that match common API keys / tokens.
|
||||
* Matched strings are replaced with `[REDACTED]`.
|
||||
*/
|
||||
const SECRET_PATTERNS: RegExp[] = [
|
||||
/sk-ant-[A-Za-z0-9_-]{20,}/g, // Anthropic
|
||||
/sk-[A-Za-z0-9_-]{20,}/g, // OpenAI
|
||||
/gsk_[A-Za-z0-9_-]{20,}/g, // Groq
|
||||
/xai-[A-Za-z0-9_-]{20,}/g, // xAI
|
||||
/ghp_[A-Za-z0-9_]{36,}/g, // GitHub PAT classic
|
||||
/github_pat_[A-Za-z0-9_]{20,}/g, // GitHub PAT fine-grained
|
||||
/glpat-[A-Za-z0-9_-]{20,}/g, // GitLab PAT
|
||||
/AKIA[A-Z0-9]{16}/g, // AWS Access Key
|
||||
/Bearer\s+eyJ[A-Za-z0-9_-]{40,}/g, // Bearer JWT
|
||||
];
|
||||
|
||||
function redactSecrets(text: string): string {
|
||||
let result = text;
|
||||
for (const pattern of SECRET_PATTERNS) {
|
||||
pattern.lastIndex = 0;
|
||||
result = result.replace(pattern, '[REDACTED]');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function formatOutbound(rawText: string): string {
|
||||
const text = stripInternalTags(rawText);
|
||||
if (!text) return '';
|
||||
return text;
|
||||
return redactSecrets(text);
|
||||
}
|
||||
|
||||
export function findChannel(
|
||||
|
||||
@@ -17,11 +17,22 @@ export interface StatusSnapshotEntry {
|
||||
pendingTasks: number;
|
||||
}
|
||||
|
||||
export interface UsageRowSnapshot {
|
||||
name: string;
|
||||
h5pct: number;
|
||||
h5reset: string;
|
||||
d7pct: number;
|
||||
d7reset: string;
|
||||
}
|
||||
|
||||
export interface StatusSnapshot {
|
||||
agentType: AgentType;
|
||||
assistantName: string;
|
||||
updatedAt: string;
|
||||
entries: StatusSnapshotEntry[];
|
||||
usageRows?: UsageRowSnapshot[];
|
||||
/** ISO timestamp of the last successful usage data fetch (separate from updatedAt heartbeat). */
|
||||
usageRowsFetchedAt?: string;
|
||||
}
|
||||
|
||||
const STATUS_SNAPSHOT_DIR = path.join(CACHE_DIR, 'status-dashboard');
|
||||
|
||||
@@ -18,8 +18,82 @@ vi.mock('./codex-token-rotation.js', () => ({
|
||||
|
||||
import {
|
||||
buildClaudeUsageRows,
|
||||
extractCodexUsageRows,
|
||||
mergeClaudeDashboardAccounts,
|
||||
} from './unified-dashboard.js';
|
||||
import type { StatusSnapshot } from './status-dashboard.js';
|
||||
|
||||
describe('extractCodexUsageRows staleness', () => {
|
||||
const freshRows = [
|
||||
{ name: 'Codex1* pro', h5pct: 42, h5reset: '2h', d7pct: 65, d7reset: '3d' },
|
||||
];
|
||||
const TEN_MIN = 600_000;
|
||||
|
||||
it('returns real rows when usageRowsFetchedAt is within maxAgeMs', () => {
|
||||
const now = Date.now();
|
||||
const snapshot: StatusSnapshot = {
|
||||
agentType: 'codex',
|
||||
assistantName: 'test',
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
entries: [],
|
||||
usageRows: freshRows,
|
||||
usageRowsFetchedAt: new Date(now - TEN_MIN + 1000).toISOString(), // 9min59s ago
|
||||
};
|
||||
|
||||
const result = extractCodexUsageRows(snapshot, TEN_MIN, now);
|
||||
expect(result).toEqual(freshRows);
|
||||
});
|
||||
|
||||
it('returns degraded row when usageRowsFetchedAt exceeds maxAgeMs', () => {
|
||||
const now = Date.now();
|
||||
const snapshot: StatusSnapshot = {
|
||||
agentType: 'codex',
|
||||
assistantName: 'test',
|
||||
updatedAt: new Date(now).toISOString(), // heartbeat is fresh
|
||||
entries: [],
|
||||
usageRows: freshRows,
|
||||
usageRowsFetchedAt: new Date(now - TEN_MIN - 1000).toISOString(), // 10min1s ago
|
||||
};
|
||||
|
||||
const result = extractCodexUsageRows(snapshot, TEN_MIN, now);
|
||||
expect(result).toEqual([
|
||||
{ name: 'Codex', h5pct: -1, h5reset: '', d7pct: -1, d7reset: '' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns degraded row when usageRowsFetchedAt is missing', () => {
|
||||
const now = Date.now();
|
||||
const snapshot: StatusSnapshot = {
|
||||
agentType: 'codex',
|
||||
assistantName: 'test',
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
entries: [],
|
||||
usageRows: freshRows,
|
||||
// usageRowsFetchedAt intentionally omitted
|
||||
};
|
||||
|
||||
const result = extractCodexUsageRows(snapshot, TEN_MIN, now);
|
||||
expect(result).toEqual([
|
||||
{ name: 'Codex', h5pct: -1, h5reset: '', d7pct: -1, d7reset: '' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns empty array when snapshot is undefined', () => {
|
||||
expect(extractCodexUsageRows(undefined, TEN_MIN)).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array when usageRows is empty', () => {
|
||||
const snapshot: StatusSnapshot = {
|
||||
agentType: 'codex',
|
||||
assistantName: 'test',
|
||||
updatedAt: new Date().toISOString(),
|
||||
entries: [],
|
||||
usageRows: [],
|
||||
usageRowsFetchedAt: new Date().toISOString(),
|
||||
};
|
||||
expect(extractCodexUsageRows(snapshot, TEN_MIN)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('unified dashboard Claude usage rows', () => {
|
||||
it('keeps both Claude accounts visible when one account usage is unavailable', () => {
|
||||
|
||||
@@ -27,6 +27,7 @@ import { logger } from './logger.js';
|
||||
import {
|
||||
readStatusSnapshots,
|
||||
writeStatusSnapshot,
|
||||
type StatusSnapshot,
|
||||
} from './status-dashboard.js';
|
||||
import type {
|
||||
AgentType,
|
||||
@@ -63,6 +64,14 @@ const STATUS_ICONS: Record<string, string> = {
|
||||
|
||||
const CHANNEL_META_REFRESH_MS = 300000;
|
||||
const STATUS_SNAPSHOT_MAX_AGE_MS = 60000;
|
||||
/** Usage data can be up to 10 min old before considered stale. */
|
||||
const USAGE_SNAPSHOT_MAX_AGE_MS = 600_000;
|
||||
/**
|
||||
* Renderer refreshes usage cache every 30s (not 5min).
|
||||
* Claude API calls are internally rate-limited to 5min per token,
|
||||
* so this only affects how quickly Codex snapshot data is picked up.
|
||||
*/
|
||||
const RENDERER_USAGE_REFRESH_MS = 30_000;
|
||||
|
||||
let statusMessageId: string | null = null;
|
||||
let cachedUsageContent = '';
|
||||
@@ -70,27 +79,18 @@ let cachedClaudeAccounts: ClaudeAccountUsage[] = [];
|
||||
let usageUpdateInProgress = false;
|
||||
let channelMetaCache = new Map<string, ChannelMeta>();
|
||||
let channelMetaLastRefresh = 0;
|
||||
|
||||
function formatResetKST(value: string | number): string {
|
||||
try {
|
||||
const date =
|
||||
typeof value === 'number' ? new Date(value * 1000) : new Date(value);
|
||||
return date.toLocaleString('ko-KR', {
|
||||
timeZone: 'Asia/Seoul',
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
});
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
let dashboardUpdateLogged = false;
|
||||
/** Codex service only: cached usage rows written into the status snapshot. */
|
||||
let cachedCodexUsageRows: UsageRow[] = [];
|
||||
/** Codex service only: ISO timestamp of last successful usage fetch. */
|
||||
let codexUsageFetchedAt: string | null = null;
|
||||
|
||||
function formatResetRemaining(value: string | number): string {
|
||||
if (value === '' || value == null) return '';
|
||||
try {
|
||||
const date =
|
||||
typeof value === 'number' ? new Date(value * 1000) : new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return '';
|
||||
const diffMs = date.getTime() - Date.now();
|
||||
if (diffMs <= 0) return ' reset';
|
||||
const hours = Math.floor(diffMs / 3_600_000);
|
||||
@@ -228,6 +228,8 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
|
||||
pendingMessages: boolean;
|
||||
pendingTasks: number;
|
||||
}>,
|
||||
...(cachedCodexUsageRows.length > 0 && { usageRows: cachedCodexUsageRows }),
|
||||
...(codexUsageFetchedAt && { usageRowsFetchedAt: codexUsageFetchedAt }),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -421,7 +423,10 @@ async function fetchCodexUsage(
|
||||
const byId = message.result.rateLimitsByLimitId;
|
||||
finish(
|
||||
byId && typeof byId === 'object'
|
||||
? (Object.values(byId) as CodexRateLimit[])
|
||||
? Object.entries(byId).map(([id, val]) => ({
|
||||
...(val as CodexRateLimit),
|
||||
limitId: id,
|
||||
}))
|
||||
: null,
|
||||
);
|
||||
}
|
||||
@@ -499,11 +504,34 @@ export function buildClaudeUsageRows(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract Codex usage rows from a snapshot, applying staleness check.
|
||||
* Returns real rows if usageRowsFetchedAt is within maxAgeMs, otherwise a
|
||||
* single degraded row. Returns empty array if no usage data present.
|
||||
* Exported for testing.
|
||||
*/
|
||||
export function extractCodexUsageRows(
|
||||
snapshot: StatusSnapshot | undefined,
|
||||
maxAgeMs: number,
|
||||
now: number = Date.now(),
|
||||
): UsageRow[] {
|
||||
if (!snapshot?.usageRows || snapshot.usageRows.length === 0) return [];
|
||||
|
||||
const fetchedAt = snapshot.usageRowsFetchedAt
|
||||
? new Date(snapshot.usageRowsFetchedAt).getTime()
|
||||
: 0;
|
||||
const usageAge = now - fetchedAt;
|
||||
if (usageAge <= maxAgeMs) {
|
||||
return [...snapshot.usageRows];
|
||||
}
|
||||
// Usage data is stale — return degraded indicator
|
||||
return [{ name: 'Codex', h5pct: -1, h5reset: '', d7pct: -1, d7reset: '' }];
|
||||
}
|
||||
|
||||
async function buildUsageContent(): Promise<string> {
|
||||
const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED;
|
||||
let liveClaudeAccounts: ClaudeAccountUsage[] | null = null;
|
||||
|
||||
const codexUsagePromise = fetchCodexUsage();
|
||||
if (shouldFetchClaudeUsage) {
|
||||
try {
|
||||
liveClaudeAccounts = await fetchAllClaudeUsage();
|
||||
@@ -511,12 +539,11 @@ async function buildUsageContent(): Promise<string> {
|
||||
logger.warn({ err }, 'Failed to fetch Claude usage for dashboard');
|
||||
}
|
||||
}
|
||||
const codexUsage = await codexUsagePromise;
|
||||
|
||||
const lines: string[] = ['📊 *사용량*'];
|
||||
const bar = (pct: number) => {
|
||||
const filled = Math.max(0, Math.min(10, Math.round(pct / 10)));
|
||||
return '█'.repeat(filled) + '░'.repeat(10 - filled);
|
||||
const filled = Math.max(0, Math.min(5, Math.round(pct / 20)));
|
||||
return '█'.repeat(filled) + '░'.repeat(5 - filled);
|
||||
};
|
||||
|
||||
const rows: UsageRow[] = [];
|
||||
@@ -529,60 +556,11 @@ async function buildUsageContent(): Promise<string> {
|
||||
rows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
|
||||
}
|
||||
|
||||
const codexAccounts = getAllCodexAccounts();
|
||||
if (codexAccounts.length > 1) {
|
||||
// Multi-account: show each account with plan + status
|
||||
for (const acct of codexAccounts) {
|
||||
const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' ';
|
||||
const label = `Codex${acct.index + 1}${icon}`;
|
||||
if (acct.isActive && codexUsage && Array.isArray(codexUsage)) {
|
||||
const relevant = codexUsage.filter(
|
||||
(limit) =>
|
||||
limit.primary.usedPercent > 0 || limit.secondary.usedPercent > 0,
|
||||
);
|
||||
const display = relevant.length > 0 ? relevant : codexUsage.slice(0, 1);
|
||||
for (const limit of display) {
|
||||
rows.push({
|
||||
name: `${label} ${acct.planType}`,
|
||||
h5pct: Math.round(limit.primary.usedPercent),
|
||||
h5reset: formatResetRemaining(limit.primary.resetsAt),
|
||||
d7pct: Math.round(limit.secondary.usedPercent),
|
||||
d7reset: formatResetRemaining(limit.secondary.resetsAt),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Show cached usage from last scan
|
||||
const pct = acct.cachedUsagePct != null ? acct.cachedUsagePct : -1;
|
||||
const d7pct =
|
||||
acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1;
|
||||
const reset = acct.resetAt || '';
|
||||
const d7reset = acct.resetD7At || '';
|
||||
rows.push({
|
||||
name: `${label} ${acct.planType}`,
|
||||
h5pct: pct,
|
||||
h5reset: reset,
|
||||
d7pct,
|
||||
d7reset,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (codexUsage && Array.isArray(codexUsage)) {
|
||||
// Single account: existing behavior
|
||||
const relevant = codexUsage.filter(
|
||||
(limit) =>
|
||||
limit.primary.usedPercent > 0 || limit.secondary.usedPercent > 0,
|
||||
);
|
||||
const display = relevant.length > 0 ? relevant : codexUsage.slice(0, 1);
|
||||
for (const limit of display) {
|
||||
rows.push({
|
||||
name: 'Codex',
|
||||
h5pct: Math.round(limit.primary.usedPercent),
|
||||
h5reset: formatResetKST(limit.primary.resetsAt),
|
||||
d7pct: Math.round(limit.secondary.usedPercent),
|
||||
d7reset: formatResetKST(limit.secondary.resetsAt),
|
||||
});
|
||||
}
|
||||
}
|
||||
// Codex usage: read from Codex service's own status snapshot.
|
||||
// Each service owns its usage data — no cross-service auth access needed.
|
||||
const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS)
|
||||
.find(s => s.agentType === 'codex');
|
||||
rows.push(...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS));
|
||||
|
||||
if (rows.length > 0) {
|
||||
// Emoji characters take 2 columns in monospace — count visual width
|
||||
@@ -592,22 +570,34 @@ async function buildUsageContent(): Promise<string> {
|
||||
Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1;
|
||||
const padName = (s: string) =>
|
||||
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
|
||||
// Strip whitespace and trailing 'm' from reset strings for compact display
|
||||
const compactReset = (s: string) =>
|
||||
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
|
||||
|
||||
lines.push('```');
|
||||
lines.push(`${' '.repeat(maxNameWidth)}5-Hour 7-Day`);
|
||||
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
|
||||
for (const row of rows) {
|
||||
const h5 =
|
||||
row.h5pct >= 0
|
||||
? `${bar(row.h5pct)} ${String(row.h5pct).padStart(3)}%`
|
||||
: ' — ';
|
||||
? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%`
|
||||
: ' — ';
|
||||
const d7 =
|
||||
row.d7pct >= 0
|
||||
? `${bar(row.d7pct)} ${String(row.d7pct).padStart(3)}%`
|
||||
: ' — ';
|
||||
const reset =
|
||||
row.h5reset || row.d7reset
|
||||
? ` ${row.h5reset || ''}${row.d7reset ? ` / ${row.d7reset}` : ''}`
|
||||
: '';
|
||||
lines.push(`${padName(row.name)}${h5} ${d7}${reset}`);
|
||||
? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%`
|
||||
: ' — ';
|
||||
lines.push(`${padName(row.name)}${h5} ${d7}`);
|
||||
const r5 = compactReset(row.h5reset);
|
||||
const r7 = compactReset(row.d7reset);
|
||||
if (r5 || r7) {
|
||||
// Align reset values under 5h / 7d columns
|
||||
// h5 column starts at maxNameWidth; d7 column starts at maxNameWidth + 9 + 1
|
||||
const d7ColStart = maxNameWidth + 10;
|
||||
let resetLine = ' '.repeat(maxNameWidth);
|
||||
if (r5) resetLine += r5;
|
||||
resetLine = resetLine.padEnd(d7ColStart);
|
||||
if (r7) resetLine += r7;
|
||||
lines.push(resetLine);
|
||||
}
|
||||
}
|
||||
lines.push('```');
|
||||
} else {
|
||||
@@ -672,9 +662,96 @@ function buildUnifiedDashboardContent(): string {
|
||||
const CODEX_ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||
const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
||||
|
||||
/**
|
||||
* Extract usage percentages from the primary 'codex' rate-limit bucket
|
||||
* and update the rotation state for a given account.
|
||||
*
|
||||
* Bucket selection:
|
||||
* 1. limitId === 'codex' → use it (user confirmed bengalfox = Codex Spark, not needed)
|
||||
* 2. No 'codex' bucket + single bucket → use it
|
||||
* 3. No 'codex' bucket + multiple buckets → unknown (show —)
|
||||
*
|
||||
* All buckets are logged at info level for observability.
|
||||
*/
|
||||
function applyCodexUsageToAccount(
|
||||
usage: CodexRateLimit[],
|
||||
accountIndex: number,
|
||||
): void {
|
||||
if (usage.length === 0) return;
|
||||
|
||||
// Log all buckets for observability
|
||||
logger.info(
|
||||
{
|
||||
account: accountIndex + 1,
|
||||
buckets: usage.map((l) => ({
|
||||
id: l.limitId,
|
||||
h5: l.primary.usedPercent,
|
||||
d7: l.secondary.usedPercent,
|
||||
})),
|
||||
},
|
||||
`Codex account #${accountIndex + 1}: ${usage.length} rate-limit bucket(s)`,
|
||||
);
|
||||
|
||||
// Select the effective bucket
|
||||
const primaryBucket = usage.find((l) => l.limitId === 'codex');
|
||||
let effective: CodexRateLimit | null = null;
|
||||
|
||||
if (primaryBucket) {
|
||||
effective = primaryBucket;
|
||||
} else if (usage.length === 1) {
|
||||
effective = usage[0];
|
||||
} else {
|
||||
// Multiple unknown buckets — cannot determine which is authoritative
|
||||
logger.warn(
|
||||
{ account: accountIndex + 1 },
|
||||
`Codex account #${accountIndex + 1}: no 'codex' bucket found among ${usage.length} buckets, showing unknown`,
|
||||
);
|
||||
updateCodexAccountUsage(-1, undefined, accountIndex, -1, undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
const pct = Math.round(effective.primary.usedPercent);
|
||||
const d7Pct = Math.round(effective.secondary.usedPercent);
|
||||
const resetStr = effective.primary.resetsAt
|
||||
? formatResetRemaining(effective.primary.resetsAt)
|
||||
: undefined;
|
||||
const resetD7Str = effective.secondary.resetsAt
|
||||
? formatResetRemaining(effective.secondary.resetsAt)
|
||||
: undefined;
|
||||
updateCodexAccountUsage(pct, resetStr, accountIndex, d7Pct, resetD7Str);
|
||||
logger.info(
|
||||
{ account: accountIndex + 1, bucket: effective.limitId, h5: pct, d7: d7Pct, reset: resetStr },
|
||||
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build display-ready usage rows from Codex rotation state.
|
||||
* Called by the Codex service after refreshing usage data.
|
||||
*/
|
||||
function buildCodexUsageRowsFromState(): UsageRow[] {
|
||||
const codexAccounts = getAllCodexAccounts();
|
||||
if (codexAccounts.length === 0) return [];
|
||||
|
||||
const isMulti = codexAccounts.length > 1;
|
||||
return codexAccounts.map((acct) => {
|
||||
const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' ';
|
||||
const label = isMulti
|
||||
? `Codex${acct.index + 1}${icon} ${acct.planType}`
|
||||
: 'Codex';
|
||||
return {
|
||||
name: label,
|
||||
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
||||
h5reset: acct.resetAt || '',
|
||||
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
||||
d7reset: acct.resetD7At || '',
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan ALL Codex accounts by spawning app-server with each auth.
|
||||
* Called on startup and every hour to keep cached usage fresh.
|
||||
* Codex service only — called on startup and every hour.
|
||||
*/
|
||||
async function refreshAllCodexAccountUsage(): Promise<void> {
|
||||
const codexAccounts = getAllCodexAccounts();
|
||||
@@ -685,47 +762,16 @@ async function refreshAllCodexAccountUsage(): Promise<void> {
|
||||
'Scanning all Codex accounts for usage data',
|
||||
);
|
||||
|
||||
let anySuccess = false;
|
||||
for (const acct of codexAccounts) {
|
||||
const accountDir = path.join(CODEX_ACCOUNTS_DIR, String(acct.index + 1));
|
||||
if (!fs.existsSync(accountDir)) continue;
|
||||
|
||||
try {
|
||||
const usage = await fetchCodexUsage(accountDir);
|
||||
if (usage && Array.isArray(usage)) {
|
||||
const relevant = usage.filter(
|
||||
(l) => l.primary.usedPercent > 0 || l.secondary.usedPercent > 0,
|
||||
);
|
||||
const display = relevant.length > 0 ? relevant : usage.slice(0, 1);
|
||||
if (usage.length > 0) {
|
||||
// Find max primary (5h) and secondary (7d) across all limits
|
||||
// Always capture reset times from first limit as baseline
|
||||
let maxH5 = 0;
|
||||
let maxD7 = 0;
|
||||
let h5Reset: string | number | undefined = usage[0].primary.resetsAt;
|
||||
let d7Reset: string | number | undefined =
|
||||
usage[0].secondary.resetsAt;
|
||||
for (const limit of usage) {
|
||||
if (limit.primary.usedPercent >= maxH5) {
|
||||
maxH5 = limit.primary.usedPercent;
|
||||
h5Reset = limit.primary.resetsAt;
|
||||
}
|
||||
if (limit.secondary.usedPercent >= maxD7) {
|
||||
maxD7 = limit.secondary.usedPercent;
|
||||
d7Reset = limit.secondary.resetsAt;
|
||||
}
|
||||
}
|
||||
const pct = Math.round(maxH5);
|
||||
const d7Pct = Math.round(maxD7);
|
||||
const resetStr = h5Reset ? formatResetRemaining(h5Reset) : undefined;
|
||||
const resetD7Str = d7Reset
|
||||
? formatResetRemaining(d7Reset)
|
||||
: undefined;
|
||||
updateCodexAccountUsage(pct, resetStr, acct.index, d7Pct, resetD7Str);
|
||||
logger.info(
|
||||
{ account: acct.index + 1, h5: pct, d7: d7Pct, reset: resetStr },
|
||||
`Codex account #${acct.index + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
||||
);
|
||||
}
|
||||
if (usage && Array.isArray(usage) && usage.length > 0) {
|
||||
applyCodexUsageToAccount(usage, acct.index);
|
||||
anySuccess = true;
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
@@ -734,6 +780,38 @@ async function refreshAllCodexAccountUsage(): Promise<void> {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (anySuccess) {
|
||||
codexUsageFetchedAt = new Date().toISOString();
|
||||
}
|
||||
cachedCodexUsageRows = buildCodexUsageRowsFromState();
|
||||
}
|
||||
|
||||
/**
|
||||
* Quick-refresh the active Codex account's usage (5-min interval).
|
||||
* Codex service only — fetches the active account and rebuilds cached rows.
|
||||
*/
|
||||
async function refreshActiveCodexUsage(): Promise<void> {
|
||||
const codexAccounts = getAllCodexAccounts();
|
||||
if (codexAccounts.length === 0) return;
|
||||
|
||||
const active = codexAccounts.find((a) => a.isActive);
|
||||
if (!active) return;
|
||||
|
||||
const accountDir = path.join(CODEX_ACCOUNTS_DIR, String(active.index + 1));
|
||||
if (!fs.existsSync(accountDir)) return;
|
||||
|
||||
try {
|
||||
const usage = await fetchCodexUsage(accountDir);
|
||||
if (usage && Array.isArray(usage) && usage.length > 0) {
|
||||
applyCodexUsageToAccount(usage, active.index);
|
||||
codexUsageFetchedAt = new Date().toISOString();
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug({ err }, 'Failed to fetch active Codex account usage');
|
||||
}
|
||||
|
||||
cachedCodexUsageRows = buildCodexUsageRowsFromState();
|
||||
}
|
||||
|
||||
async function refreshUsageCache(): Promise<void> {
|
||||
@@ -793,6 +871,13 @@ export async function startUnifiedDashboard(
|
||||
const id = await channel.sendAndTrack(statusJid, content);
|
||||
if (id) statusMessageId = id;
|
||||
}
|
||||
if (!dashboardUpdateLogged) {
|
||||
logger.info(
|
||||
{ messageId: statusMessageId, contentLength: content.length },
|
||||
'Dashboard updated successfully (first)',
|
||||
);
|
||||
dashboardUpdateLogged = true;
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Dashboard update failed');
|
||||
statusMessageId = null;
|
||||
@@ -803,15 +888,17 @@ export async function startUnifiedDashboard(
|
||||
await updateStatus();
|
||||
|
||||
if (isRenderer) {
|
||||
setInterval(refreshUsageCache, opts.usageUpdateInterval);
|
||||
// Full scan of all Codex accounts on startup + hourly
|
||||
// After scan, refresh dashboard so cached data is visible immediately
|
||||
void refreshAllCodexAccountUsage().then(() => {
|
||||
void refreshUsageCache().then(() => void updateStatus());
|
||||
});
|
||||
// Renderer: refresh usage cache at shorter interval so Codex snapshot
|
||||
// data is picked up quickly. Claude API calls are internally rate-limited
|
||||
// to 5min per token, so this only affects local reads.
|
||||
setInterval(refreshUsageCache, RENDERER_USAGE_REFRESH_MS);
|
||||
} else {
|
||||
// Codex service: fetch own usage and expose via status snapshot.
|
||||
// Active account every usageUpdateInterval; full scan on startup + hourly.
|
||||
void refreshAllCodexAccountUsage().then(() => void refreshActiveCodexUsage());
|
||||
setInterval(() => void refreshActiveCodexUsage(), opts.usageUpdateInterval);
|
||||
setInterval(
|
||||
() =>
|
||||
void refreshAllCodexAccountUsage().then(() => void refreshUsageCache()),
|
||||
() => void refreshAllCodexAccountUsage(),
|
||||
CODEX_FULL_SCAN_INTERVAL,
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user