fix: remove Codex API key auth path, add secret redaction and d7 auto-rotation
- Remove OPENAI_API_KEY from .env and Codex child process env to prevent API billing when subscription quota is exhausted - Remove writeCodexApiKeyAuth entirely — Codex now uses OAuth only - Add 9-pattern secret redaction in formatOutbound() to prevent key leaks - Fix Codex usage bucket aggregation: use 'codex' bucket only instead of max across all buckets (bengalfox = Codex Spark, not needed) - Add d7≥100% auto-rotation in updateCodexAccountUsage to skip exhausted accounts and prevent API billing fallback - Add findNextCodexAvailable that checks both rate-limits and d7 usage - Clean stale apikey auth.json files from all session directories - Update tests: OAuth-only auth, d7 auto-skip (3 new tests), dashboard Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -112,12 +112,13 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('writes API-key auth when OPENAI_API_KEY is available', () => {
|
it('ignores OPENAI_API_KEY and always uses OAuth auth', () => {
|
||||||
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
const rotatedAuthPath = path.join(tempRoot, 'rotated-auth.json');
|
||||||
fs.writeFileSync(
|
const rotatedAuth = {
|
||||||
rotatedAuthPath,
|
auth_mode: 'chatgpt',
|
||||||
JSON.stringify({ auth_mode: 'chatgpt', tokens: { access_token: 'x' } }),
|
tokens: { access_token: 'x' },
|
||||||
);
|
};
|
||||||
|
fs.writeFileSync(rotatedAuthPath, JSON.stringify(rotatedAuth));
|
||||||
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
|
mockGetActiveCodexAuthPath.mockReturnValue(rotatedAuthPath);
|
||||||
mockReadEnvFile.mockReturnValue({
|
mockReadEnvFile.mockReturnValue({
|
||||||
OPENAI_API_KEY: 'sk-test-api-key',
|
OPENAI_API_KEY: 'sk-test-api-key',
|
||||||
@@ -139,9 +140,10 @@ describe('prepareGroupEnvironment codex auth handling', () => {
|
|||||||
tokens?: unknown;
|
tokens?: unknown;
|
||||||
};
|
};
|
||||||
|
|
||||||
expect(auth.auth_mode).toBe('apikey');
|
// API key auth is never used — always OAuth
|
||||||
expect(auth.OPENAI_API_KEY).toBe('sk-test-api-key');
|
expect(auth.auth_mode).toBe('chatgpt');
|
||||||
expect(auth.tokens).toBeUndefined();
|
expect(auth.OPENAI_API_KEY).toBeUndefined();
|
||||||
|
expect(auth.tokens).toEqual({ access_token: 'x' });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('falls back to rotated OAuth auth when no API key is configured', () => {
|
it('falls back to rotated OAuth auth when no API key is configured', () => {
|
||||||
|
|||||||
@@ -20,19 +20,8 @@ import {
|
|||||||
} from './platform-prompts.js';
|
} from './platform-prompts.js';
|
||||||
import type { AgentType, RegisteredGroup } from './types.js';
|
import type { AgentType, RegisteredGroup } from './types.js';
|
||||||
|
|
||||||
function writeCodexApiKeyAuth(authPath: string, openaiKey: string): void {
|
// writeCodexApiKeyAuth removed — Codex uses OAuth only.
|
||||||
fs.writeFileSync(
|
// API key auth caused unintended billing.
|
||||||
authPath,
|
|
||||||
JSON.stringify(
|
|
||||||
{
|
|
||||||
auth_mode: 'apikey',
|
|
||||||
OPENAI_API_KEY: openaiKey,
|
|
||||||
},
|
|
||||||
null,
|
|
||||||
2,
|
|
||||||
) + '\n',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function syncDirectoryEntries(sources: string[], destination: string): void {
|
function syncDirectoryEntries(sources: string[], destination: string): void {
|
||||||
for (const source of sources) {
|
for (const source of sources) {
|
||||||
@@ -184,12 +173,10 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
isPairedRoom: boolean;
|
isPairedRoom: boolean;
|
||||||
memoryBriefing?: string;
|
memoryBriefing?: string;
|
||||||
}): void {
|
}): void {
|
||||||
const openaiKey =
|
// API key auth intentionally removed — Codex uses OAuth only.
|
||||||
args.envVars.CODEX_OPENAI_API_KEY ||
|
// Never pass any API key to Codex child process to prevent API billing.
|
||||||
process.env.CODEX_OPENAI_API_KEY ||
|
delete args.env.OPENAI_API_KEY;
|
||||||
args.envVars.OPENAI_API_KEY ||
|
delete args.env.CODEX_OPENAI_API_KEY;
|
||||||
process.env.OPENAI_API_KEY;
|
|
||||||
if (openaiKey) args.env.OPENAI_API_KEY = openaiKey;
|
|
||||||
|
|
||||||
const codexModel =
|
const codexModel =
|
||||||
args.group.agentConfig?.codexModel ||
|
args.group.agentConfig?.codexModel ||
|
||||||
@@ -208,9 +195,8 @@ function prepareCodexSessionEnvironment(args: {
|
|||||||
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
fs.mkdirSync(sessionCodexDir, { recursive: true });
|
||||||
|
|
||||||
const authDst = path.join(sessionCodexDir, 'auth.json');
|
const authDst = path.join(sessionCodexDir, 'auth.json');
|
||||||
if (openaiKey) {
|
// Always use OAuth auth from rotated accounts (API key auth removed)
|
||||||
writeCodexApiKeyAuth(authDst, openaiKey);
|
{
|
||||||
} else {
|
|
||||||
const rotatedAuthSrc = getActiveCodexAuthPath();
|
const rotatedAuthSrc = getActiveCodexAuthPath();
|
||||||
const authSrc =
|
const authSrc =
|
||||||
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
rotatedAuthSrc && fs.existsSync(rotatedAuthSrc)
|
||||||
@@ -417,8 +403,6 @@ export function prepareGroupEnvironment(
|
|||||||
'CLAUDE_THINKING',
|
'CLAUDE_THINKING',
|
||||||
'CLAUDE_THINKING_BUDGET',
|
'CLAUDE_THINKING_BUDGET',
|
||||||
'CLAUDE_EFFORT',
|
'CLAUDE_EFFORT',
|
||||||
'OPENAI_API_KEY',
|
|
||||||
'CODEX_OPENAI_API_KEY',
|
|
||||||
'CODEX_MODEL',
|
'CODEX_MODEL',
|
||||||
'CODEX_EFFORT',
|
'CODEX_EFFORT',
|
||||||
'MEMENTO_MCP_SSE_URL',
|
'MEMENTO_MCP_SSE_URL',
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
import os from 'os';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
import { DATA_DIR } from './config.js';
|
import { DATA_DIR } from './config.js';
|
||||||
@@ -46,7 +47,10 @@ function mapWindow(w?: {
|
|||||||
|
|
||||||
interface UsageCacheEntry {
|
interface UsageCacheEntry {
|
||||||
usage: ClaudeUsageData;
|
usage: ClaudeUsageData;
|
||||||
|
/** Timestamp of last *successful* API fetch (use for stale detection). */
|
||||||
fetchedAt: number;
|
fetchedAt: number;
|
||||||
|
/** Timestamp of last API *attempt* including failures (use for throttling). */
|
||||||
|
lastAttemptAt?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
let usageDiskCache: Record<string, UsageCacheEntry> = {};
|
let usageDiskCache: Record<string, UsageCacheEntry> = {};
|
||||||
@@ -77,13 +81,15 @@ const MIN_FETCH_INTERVAL_MS = 300_000;
|
|||||||
|
|
||||||
async function fetchUsageForToken(
|
async function fetchUsageForToken(
|
||||||
token: string,
|
token: string,
|
||||||
|
accountIndex?: number,
|
||||||
): Promise<ClaudeUsageData | null> {
|
): Promise<ClaudeUsageData | null> {
|
||||||
loadUsageDiskCache();
|
loadUsageDiskCache();
|
||||||
|
|
||||||
// Return cached data if fetched recently (avoid API rate-limit)
|
// Return cached data if attempted recently (avoid API rate-limit)
|
||||||
const key = cacheKey(token);
|
const key = cacheKey(token);
|
||||||
const cached = usageDiskCache[key];
|
const cached = usageDiskCache[key];
|
||||||
if (cached && Date.now() - cached.fetchedAt < MIN_FETCH_INTERVAL_MS) {
|
const lastAttempt = cached?.lastAttemptAt ?? cached?.fetchedAt ?? 0;
|
||||||
|
if (cached && Date.now() - lastAttempt < MIN_FETCH_INTERVAL_MS) {
|
||||||
return cached.usage;
|
return cached.usage;
|
||||||
}
|
}
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
@@ -101,21 +107,35 @@ async function fetchUsageForToken(
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (res.status === 401) {
|
if (res.status === 401) {
|
||||||
logger.warn('Claude usage API: token expired or invalid (401)');
|
logger.warn(
|
||||||
|
{ account: accountIndex != null ? accountIndex + 1 : '?', tokenKey: key },
|
||||||
|
'Claude usage API: token expired or invalid (401)',
|
||||||
|
);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
|
const staleMs = cached ? Date.now() - cached.fetchedAt : 0;
|
||||||
logger.warn(
|
logger.warn(
|
||||||
|
{ account: accountIndex != null ? accountIndex + 1 : '?', tokenKey: key, staleMinutes: Math.round(staleMs / 60_000) },
|
||||||
'Claude usage API: rate limited (429), returning cached data',
|
'Claude usage API: rate limited (429), returning cached data',
|
||||||
);
|
);
|
||||||
return usageDiskCache[cacheKey(token)]?.usage ?? null;
|
// Record attempt time so we don't retry for MIN_FETCH_INTERVAL_MS
|
||||||
|
if (cached) {
|
||||||
|
cached.lastAttemptAt = Date.now();
|
||||||
|
saveUsageDiskCache();
|
||||||
|
}
|
||||||
|
return cached?.usage ?? null;
|
||||||
}
|
}
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{ status: res.status },
|
{ account: accountIndex != null ? accountIndex + 1 : '?', status: res.status, tokenKey: key },
|
||||||
`Claude usage API: unexpected status ${res.status}`,
|
`Claude usage API: unexpected status ${res.status}`,
|
||||||
);
|
);
|
||||||
return usageDiskCache[cacheKey(token)]?.usage ?? null;
|
if (cached) {
|
||||||
|
cached.lastAttemptAt = Date.now();
|
||||||
|
saveUsageDiskCache();
|
||||||
|
}
|
||||||
|
return cached?.usage ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = (await res.json()) as UsageApiResponse;
|
const data = (await res.json()) as UsageApiResponse;
|
||||||
@@ -127,18 +147,29 @@ async function fetchUsageForToken(
|
|||||||
seven_day_opus: mapWindow(data.seven_day_opus),
|
seven_day_opus: mapWindow(data.seven_day_opus),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Persist to disk cache
|
// Persist to disk cache — success: update both fetchedAt and lastAttemptAt
|
||||||
usageDiskCache[cacheKey(token)] = { usage: result, fetchedAt: Date.now() };
|
const now = Date.now();
|
||||||
|
usageDiskCache[key] = { usage: result, fetchedAt: now, lastAttemptAt: now };
|
||||||
saveUsageDiskCache();
|
saveUsageDiskCache();
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
{ tokenKey: key, h5: result.five_hour?.utilization, d7: result.seven_day?.utilization },
|
||||||
|
'Claude usage API: fetched successfully',
|
||||||
|
);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if ((err as Error).name === 'AbortError') {
|
if ((err as Error).name === 'AbortError') {
|
||||||
logger.warn('Claude usage API: request timed out');
|
logger.warn({ tokenKey: key }, 'Claude usage API: request timed out');
|
||||||
} else {
|
} else {
|
||||||
logger.warn({ err }, 'Claude usage API: fetch failed');
|
logger.warn({ err, tokenKey: key }, 'Claude usage API: fetch failed');
|
||||||
}
|
}
|
||||||
return usageDiskCache[cacheKey(token)]?.usage ?? null;
|
// Record attempt time so we back off on persistent failures
|
||||||
|
if (cached) {
|
||||||
|
cached.lastAttemptAt = Date.now();
|
||||||
|
saveUsageDiskCache();
|
||||||
|
}
|
||||||
|
return cached?.usage ?? null;
|
||||||
} finally {
|
} finally {
|
||||||
clearTimeout(timer);
|
clearTimeout(timer);
|
||||||
}
|
}
|
||||||
@@ -154,7 +185,7 @@ export async function fetchClaudeUsage(): Promise<ClaudeUsageData | null> {
|
|||||||
logger.debug('No Claude OAuth token available for usage check');
|
logger.debug('No Claude OAuth token available for usage check');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return fetchUsageForToken(token);
|
return fetchUsageForToken(token, undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClaudeAccountProfile {
|
export interface ClaudeAccountProfile {
|
||||||
@@ -164,6 +195,32 @@ export interface ClaudeAccountProfile {
|
|||||||
|
|
||||||
const profileCache = new Map<number, ClaudeAccountProfile>();
|
const profileCache = new Map<number, ClaudeAccountProfile>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read planType from credentials file as fallback when profile API fails.
|
||||||
|
* Account 0: ~/.claude/.credentials.json
|
||||||
|
* Account 1+: ~/.claude-accounts/{index}/.credentials.json
|
||||||
|
*/
|
||||||
|
function readCredentialsPlanType(accountIndex: number): string | null {
|
||||||
|
try {
|
||||||
|
const credsPath =
|
||||||
|
accountIndex === 0
|
||||||
|
? path.join(os.homedir(), '.claude', '.credentials.json')
|
||||||
|
: path.join(
|
||||||
|
os.homedir(),
|
||||||
|
'.claude-accounts',
|
||||||
|
String(accountIndex),
|
||||||
|
'.credentials.json',
|
||||||
|
);
|
||||||
|
if (!fs.existsSync(credsPath)) return null;
|
||||||
|
const data = readJsonFile<{
|
||||||
|
claudeAiOauth?: { subscriptionType?: string };
|
||||||
|
}>(credsPath);
|
||||||
|
return data?.claudeAiOauth?.subscriptionType || null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchProfileForToken(
|
async function fetchProfileForToken(
|
||||||
token: string,
|
token: string,
|
||||||
): Promise<ClaudeAccountProfile | null> {
|
): Promise<ClaudeAccountProfile | null> {
|
||||||
@@ -192,7 +249,7 @@ async function fetchProfileForToken(
|
|||||||
? 'max'
|
? 'max'
|
||||||
: data.account?.has_claude_pro
|
: data.account?.has_claude_pro
|
||||||
? 'pro'
|
? 'pro'
|
||||||
: orgType.replace('claude_', '') || 'free';
|
: orgType.replace('claude_', '') || '?';
|
||||||
return {
|
return {
|
||||||
email: data.account?.email || '?',
|
email: data.account?.email || '?',
|
||||||
planType,
|
planType,
|
||||||
@@ -210,7 +267,23 @@ async function fetchProfileForToken(
|
|||||||
export async function fetchAllClaudeProfiles(): Promise<void> {
|
export async function fetchAllClaudeProfiles(): Promise<void> {
|
||||||
const allTokens = getAllTokens();
|
const allTokens = getAllTokens();
|
||||||
for (const t of allTokens) {
|
for (const t of allTokens) {
|
||||||
const profile = await fetchProfileForToken(t.token);
|
let profile = await fetchProfileForToken(t.token);
|
||||||
|
|
||||||
|
// Fallback: if profile API failed or returned unknown plan, use credentials file
|
||||||
|
if (!profile || profile.planType === '?') {
|
||||||
|
const credsPlan = readCredentialsPlanType(t.index);
|
||||||
|
if (credsPlan) {
|
||||||
|
profile = {
|
||||||
|
email: profile?.email || '?',
|
||||||
|
planType: credsPlan,
|
||||||
|
};
|
||||||
|
logger.info(
|
||||||
|
{ account: t.index + 1, plan: credsPlan, source: 'credentials' },
|
||||||
|
`Claude account #${t.index + 1}: ${credsPlan} (from credentials)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (profile) {
|
if (profile) {
|
||||||
profileCache.set(t.index, profile);
|
profileCache.set(t.index, profile);
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -246,7 +319,7 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
|||||||
// Single token fallback
|
// Single token fallback
|
||||||
const token = process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
const token = process.env.CLAUDE_CODE_OAUTH_TOKEN;
|
||||||
if (!token) return [];
|
if (!token) return [];
|
||||||
const usage = await fetchUsageForToken(token);
|
const usage = await fetchUsageForToken(token, 0);
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
index: 0,
|
index: 0,
|
||||||
@@ -260,7 +333,7 @@ export async function fetchAllClaudeUsage(): Promise<ClaudeAccountUsage[]> {
|
|||||||
|
|
||||||
const results: ClaudeAccountUsage[] = [];
|
const results: ClaudeAccountUsage[] = [];
|
||||||
for (const t of allTokens) {
|
for (const t of allTokens) {
|
||||||
const usage = await fetchUsageForToken(t.token);
|
const usage = await fetchUsageForToken(t.token, t.index);
|
||||||
results.push({
|
results.push({
|
||||||
index: t.index,
|
index: t.index,
|
||||||
masked: t.masked,
|
masked: t.masked,
|
||||||
|
|||||||
124
src/codex-token-rotation.test.ts
Normal file
124
src/codex-token-rotation.test.ts
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('./agent-error-detection.js', () => ({
|
||||||
|
classifyAgentError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||||
|
classifyCodexAuthError: vi.fn(() => ({ category: 'none', reason: '' })),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./config.js', () => ({
|
||||||
|
DATA_DIR: '/tmp/ejclaw-codex-rot-data',
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./logger.js', () => ({
|
||||||
|
logger: {
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
debug: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./utils.js', async () => {
|
||||||
|
const actual =
|
||||||
|
await vi.importActual<typeof import('./utils.js')>('./utils.js');
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
writeJsonFile: vi.fn(), // no-op to prevent state file writes
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock('os', async () => {
|
||||||
|
const actual = await vi.importActual<typeof import('os')>('os');
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
default: {
|
||||||
|
...actual,
|
||||||
|
homedir: () => process.env.CODEX_ROT_TEST_HOME || '/tmp',
|
||||||
|
},
|
||||||
|
homedir: () => process.env.CODEX_ROT_TEST_HOME || '/tmp',
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
function createFakeAccounts(homeDir: string, count: number): void {
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const dir = path.join(homeDir, '.codex-accounts', String(i));
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(dir, 'auth.json'),
|
||||||
|
JSON.stringify({
|
||||||
|
auth_mode: 'chatgpt',
|
||||||
|
tokens: { account_id: `acct-${i}`, access_token: `token-${i}` },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('codex-token-rotation d7 ≥ 100% auto-skip', () => {
|
||||||
|
let tempHome: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-rot-'));
|
||||||
|
process.env.CODEX_ROT_TEST_HOME = tempHome;
|
||||||
|
createFakeAccounts(tempHome, 4);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
delete process.env.CODEX_ROT_TEST_HOME;
|
||||||
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('advanceCodexAccount skips accounts with d7 ≥ 100%', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
expect(mod.getCodexAccountCount()).toBe(4);
|
||||||
|
|
||||||
|
// Mark account #1 (next after #0) as 7d-exhausted
|
||||||
|
mod.updateCodexAccountUsage(80, undefined, 1, 100, undefined);
|
||||||
|
|
||||||
|
// Current is #0, advance should skip #1 (d7=100%) → land on #2
|
||||||
|
mod.advanceCodexAccount();
|
||||||
|
|
||||||
|
const accounts = mod.getAllCodexAccounts();
|
||||||
|
expect(accounts[0].isActive).toBe(false);
|
||||||
|
expect(accounts[1].isActive).toBe(false);
|
||||||
|
expect(accounts[2].isActive).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('updateCodexAccountUsage auto-rotates when current account hits d7 ≥ 100%', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
expect(mod.getCodexAccountCount()).toBe(4);
|
||||||
|
|
||||||
|
// Current is #0 — report d7=100% for the current account
|
||||||
|
mod.updateCodexAccountUsage(80, undefined, 0, 100, undefined);
|
||||||
|
|
||||||
|
// Should have auto-rotated away from #0 to #1
|
||||||
|
const accounts = mod.getAllCodexAccounts();
|
||||||
|
expect(accounts[0].isActive).toBe(false);
|
||||||
|
expect(accounts[1].isActive).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('advanceCodexAccount falls back when all accounts are d7-exhausted', async () => {
|
||||||
|
const mod = await import('./codex-token-rotation.js');
|
||||||
|
mod.initCodexTokenRotation();
|
||||||
|
expect(mod.getCodexAccountCount()).toBe(4);
|
||||||
|
|
||||||
|
// Exhaust d7 on all accounts except current (#0)
|
||||||
|
mod.updateCodexAccountUsage(80, undefined, 1, 100, undefined);
|
||||||
|
mod.updateCodexAccountUsage(80, undefined, 2, 100, undefined);
|
||||||
|
mod.updateCodexAccountUsage(80, undefined, 3, 100, undefined);
|
||||||
|
|
||||||
|
// Advance — all others d7-exhausted, falls back to rate-limit-only check
|
||||||
|
// findNextAvailable (base) should still find #1 since it's not rate-limited
|
||||||
|
mod.advanceCodexAccount();
|
||||||
|
|
||||||
|
const accounts = mod.getAllCodexAccounts();
|
||||||
|
const active = accounts.find((a) => a.isActive);
|
||||||
|
expect(active).toBeDefined();
|
||||||
|
expect(active!.index).toBe(1); // fallback picks next non-rate-limited
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -137,7 +137,7 @@ function saveCodexState(): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadCodexState(): void {
|
function loadCodexState(quiet = false): void {
|
||||||
const state = readJsonFile<{
|
const state = readJsonFile<{
|
||||||
currentIndex?: number;
|
currentIndex?: number;
|
||||||
rateLimits?: (number | null)[];
|
rateLimits?: (number | null)[];
|
||||||
@@ -164,6 +164,8 @@ function loadCodexState(): void {
|
|||||||
const until = state.rateLimits[i];
|
const until = state.rateLimits[i];
|
||||||
if (typeof until === 'number' && until > now) {
|
if (typeof until === 'number' && until > now) {
|
||||||
accounts[i].rateLimitedUntil = until;
|
accounts[i].rateLimitedUntil = until;
|
||||||
|
} else {
|
||||||
|
accounts[i].rateLimitedUntil = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -173,8 +175,10 @@ function loadCodexState(): void {
|
|||||||
i < Math.min(state.usagePcts.length, accounts.length);
|
i < Math.min(state.usagePcts.length, accounts.length);
|
||||||
i++
|
i++
|
||||||
) {
|
) {
|
||||||
if (typeof state.usagePcts[i] === 'number')
|
accounts[i].lastUsagePct =
|
||||||
accounts[i].lastUsagePct = state.usagePcts[i]!;
|
typeof state.usagePcts[i] === 'number'
|
||||||
|
? state.usagePcts[i]!
|
||||||
|
: undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (Array.isArray(state.usageD7Pcts)) {
|
if (Array.isArray(state.usageD7Pcts)) {
|
||||||
@@ -183,13 +187,15 @@ function loadCodexState(): void {
|
|||||||
i < Math.min(state.usageD7Pcts.length, accounts.length);
|
i < Math.min(state.usageD7Pcts.length, accounts.length);
|
||||||
i++
|
i++
|
||||||
) {
|
) {
|
||||||
if (typeof state.usageD7Pcts[i] === 'number')
|
accounts[i].lastUsageD7Pct =
|
||||||
accounts[i].lastUsageD7Pct = state.usageD7Pcts[i]!;
|
typeof state.usageD7Pcts[i] === 'number'
|
||||||
|
? state.usageD7Pcts[i]!
|
||||||
|
: undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (Array.isArray(state.resetAts)) {
|
if (Array.isArray(state.resetAts)) {
|
||||||
for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) {
|
for (let i = 0; i < Math.min(state.resetAts.length, accounts.length); i++) {
|
||||||
if (state.resetAts[i]) accounts[i].resetAt = state.resetAts[i]!;
|
accounts[i].resetAt = state.resetAts[i] ?? undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (Array.isArray(state.resetD7Ats)) {
|
if (Array.isArray(state.resetD7Ats)) {
|
||||||
@@ -198,13 +204,25 @@ function loadCodexState(): void {
|
|||||||
i < Math.min(state.resetD7Ats.length, accounts.length);
|
i < Math.min(state.resetD7Ats.length, accounts.length);
|
||||||
i++
|
i++
|
||||||
) {
|
) {
|
||||||
if (state.resetD7Ats[i]) accounts[i].resetD7At = state.resetD7Ats[i]!;
|
accounts[i].resetD7At = state.resetD7Ats[i] ?? undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.info(
|
if (!quiet) {
|
||||||
{ currentIndex, accountCount: accounts.length },
|
logger.info(
|
||||||
'Codex rotation state restored',
|
{ currentIndex, accountCount: accounts.length },
|
||||||
);
|
'Codex rotation state restored',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Re-read the on-disk rotation state (written by any service).
|
||||||
|
* Call before dashboard renders so the renderer picks up rotations
|
||||||
|
* performed by the Codex service process.
|
||||||
|
*/
|
||||||
|
export function reloadCodexStateFromDisk(): void {
|
||||||
|
if (accounts.length <= 1) return;
|
||||||
|
loadCodexState(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Get the auth.json path for the current active account. */
|
/** Get the auth.json path for the current active account. */
|
||||||
@@ -272,19 +290,37 @@ export function rotateCodexToken(
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the next Codex account that is neither rate-limited nor 7d-exhausted.
|
||||||
|
*/
|
||||||
|
function findNextCodexAvailable(fromIndex?: number): number | null {
|
||||||
|
const now = Date.now();
|
||||||
|
const start = fromIndex ?? currentIndex;
|
||||||
|
for (let i = 1; i < accounts.length; i++) {
|
||||||
|
const idx = (start + i) % accounts.length;
|
||||||
|
const acct = accounts[idx];
|
||||||
|
const rlOk = !acct.rateLimitedUntil || acct.rateLimitedUntil <= now;
|
||||||
|
const usageOk = acct.lastUsageD7Pct == null || acct.lastUsageD7Pct < 100;
|
||||||
|
if (rlOk && usageOk) return idx;
|
||||||
|
}
|
||||||
|
// All exhausted — fall back to rate-limit-only check
|
||||||
|
return findNextAvailable(accounts, start);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Advance to the next healthy account (round-robin).
|
* Advance to the next healthy account (round-robin).
|
||||||
* Called after each successful request to spread load evenly
|
* Called after each successful request to spread load evenly
|
||||||
* and keep usage data fresh for all accounts.
|
* and keep usage data fresh for all accounts.
|
||||||
|
* Skips accounts with 7d usage ≥ 100% to avoid API billing.
|
||||||
*/
|
*/
|
||||||
export function advanceCodexAccount(): void {
|
export function advanceCodexAccount(): void {
|
||||||
if (accounts.length <= 1) return;
|
if (accounts.length <= 1) return;
|
||||||
const nextIdx = findNextAvailable(accounts, currentIndex);
|
const nextIdx = findNextCodexAvailable();
|
||||||
if (nextIdx !== null) {
|
if (nextIdx !== null) {
|
||||||
currentIndex = nextIdx;
|
currentIndex = nextIdx;
|
||||||
saveCodexState();
|
saveCodexState();
|
||||||
}
|
}
|
||||||
// All others rate-limited, stay on current
|
// All others rate-limited/exhausted, stay on current
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -306,6 +342,19 @@ export function updateCodexAccountUsage(
|
|||||||
if (resetAt) acct.resetAt = resetAt;
|
if (resetAt) acct.resetAt = resetAt;
|
||||||
if (resetD7At) acct.resetD7At = resetD7At;
|
if (resetD7At) acct.resetD7At = resetD7At;
|
||||||
saveCodexState();
|
saveCodexState();
|
||||||
|
|
||||||
|
// Auto-rotate away from 7d-exhausted current account to avoid API billing
|
||||||
|
if (idx === currentIndex && d7Pct != null && d7Pct >= 100 && accounts.length > 1) {
|
||||||
|
const nextIdx = findNextCodexAvailable(idx);
|
||||||
|
if (nextIdx !== null && nextIdx !== idx) {
|
||||||
|
logger.info(
|
||||||
|
{ from: idx + 1, to: nextIdx + 1, d7Pct },
|
||||||
|
`Codex auto-rotating: account #${idx + 1} at ${d7Pct}% 7d → #${nextIdx + 1}`,
|
||||||
|
);
|
||||||
|
currentIndex = nextIdx;
|
||||||
|
saveCodexState();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
|
|||||||
import { EventEmitter } from 'events';
|
import { EventEmitter } from 'events';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
|
||||||
import { GroupQueue } from './group-queue.js';
|
import { GroupQueue, type GroupRunContext } from './group-queue.js';
|
||||||
|
|
||||||
// Mock config to control concurrency limit
|
// Mock config to control concurrency limit
|
||||||
vi.mock('./config.js', () => ({
|
vi.mock('./config.js', () => ({
|
||||||
@@ -121,6 +121,100 @@ describe('GroupQueue', () => {
|
|||||||
await vi.advanceTimersByTimeAsync(10);
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('force-terminates a lingering process after output was delivered', async () => {
|
||||||
|
class StubbornProcess extends EventEmitter {
|
||||||
|
exitCode: number | null = null;
|
||||||
|
signalCode: NodeJS.Signals | null = null;
|
||||||
|
kill = vi.fn(() => true);
|
||||||
|
}
|
||||||
|
|
||||||
|
const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder';
|
||||||
|
let releaseRun!: (value: boolean) => void;
|
||||||
|
let runId: string | undefined;
|
||||||
|
const blocker = new Promise<boolean>((resolve) => {
|
||||||
|
releaseRun = resolve;
|
||||||
|
});
|
||||||
|
|
||||||
|
const processMessages = vi.fn(
|
||||||
|
async (_groupJid: string, context: GroupRunContext) => {
|
||||||
|
runId = context.runId;
|
||||||
|
return await blocker;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
queue.setProcessMessagesFn(processMessages);
|
||||||
|
queue.enqueueMessageCheck('group1@g.us', ipcDir);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
|
expect(runId).toEqual(expect.any(String));
|
||||||
|
|
||||||
|
const proc = new StubbornProcess();
|
||||||
|
queue.registerProcess(
|
||||||
|
'group1@g.us',
|
||||||
|
proc as unknown as import('child_process').ChildProcess,
|
||||||
|
'proc-1',
|
||||||
|
ipcDir,
|
||||||
|
);
|
||||||
|
|
||||||
|
queue.closeStdin('group1@g.us', {
|
||||||
|
runId,
|
||||||
|
reason: 'output-delivered-close',
|
||||||
|
});
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(60_000);
|
||||||
|
expect(proc.kill).toHaveBeenCalledWith('SIGTERM');
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(15_000);
|
||||||
|
expect(proc.kill).toHaveBeenCalledWith('SIGKILL');
|
||||||
|
|
||||||
|
releaseRun(true);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears post-close termination timers once the run exits', async () => {
|
||||||
|
class FakeProcess extends EventEmitter {
|
||||||
|
exitCode: number | null = null;
|
||||||
|
signalCode: NodeJS.Signals | null = null;
|
||||||
|
kill = vi.fn(() => true);
|
||||||
|
}
|
||||||
|
|
||||||
|
const ipcDir = '/tmp/ejclaw-test-data/ipc/group-folder';
|
||||||
|
let releaseRun!: (value: boolean) => void;
|
||||||
|
const blocker = new Promise<boolean>((resolve) => {
|
||||||
|
releaseRun = resolve;
|
||||||
|
});
|
||||||
|
|
||||||
|
const proc = new FakeProcess();
|
||||||
|
const processMessages = vi.fn(
|
||||||
|
async (_groupJid: string, context: GroupRunContext) => {
|
||||||
|
queue.registerProcess(
|
||||||
|
'group1@g.us',
|
||||||
|
proc as unknown as import('child_process').ChildProcess,
|
||||||
|
'proc-1',
|
||||||
|
ipcDir,
|
||||||
|
);
|
||||||
|
|
||||||
|
queue.closeStdin('group1@g.us', {
|
||||||
|
runId: context.runId,
|
||||||
|
reason: 'output-delivered-close',
|
||||||
|
});
|
||||||
|
|
||||||
|
await blocker;
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
queue.setProcessMessagesFn(processMessages);
|
||||||
|
queue.enqueueMessageCheck('group1@g.us', ipcDir);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
|
releaseRun(true);
|
||||||
|
await vi.advanceTimersByTimeAsync(10);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(75_000);
|
||||||
|
expect(proc.kill).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
// --- Global concurrency limit ---
|
// --- Global concurrency limit ---
|
||||||
|
|
||||||
it('respects global concurrency limit', async () => {
|
it('respects global concurrency limit', async () => {
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ const MAX_RETRIES = 5;
|
|||||||
const BASE_RETRY_MS = 5000;
|
const BASE_RETRY_MS = 5000;
|
||||||
const MAX_CONCURRENT_TASKS =
|
const MAX_CONCURRENT_TASKS =
|
||||||
MAX_CONCURRENT_AGENTS > 1 ? MAX_CONCURRENT_AGENTS - 1 : 1;
|
MAX_CONCURRENT_AGENTS > 1 ? MAX_CONCURRENT_AGENTS - 1 : 1;
|
||||||
|
const POST_CLOSE_SIGTERM_DELAY_MS = 60_000;
|
||||||
|
const POST_CLOSE_SIGKILL_DELAY_MS = 75_000;
|
||||||
|
|
||||||
interface GroupState {
|
interface GroupState {
|
||||||
active: boolean;
|
active: boolean;
|
||||||
@@ -38,6 +40,8 @@ interface GroupState {
|
|||||||
retryCount: number;
|
retryCount: number;
|
||||||
retryTimer: ReturnType<typeof setTimeout> | null;
|
retryTimer: ReturnType<typeof setTimeout> | null;
|
||||||
retryScheduledAt: number | null;
|
retryScheduledAt: number | null;
|
||||||
|
postCloseTermTimer: ReturnType<typeof setTimeout> | null;
|
||||||
|
postCloseKillTimer: ReturnType<typeof setTimeout> | null;
|
||||||
startedAt: number | null;
|
startedAt: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,6 +82,8 @@ export class GroupQueue {
|
|||||||
retryCount: 0,
|
retryCount: 0,
|
||||||
retryTimer: null,
|
retryTimer: null,
|
||||||
retryScheduledAt: null,
|
retryScheduledAt: null,
|
||||||
|
postCloseTermTimer: null,
|
||||||
|
postCloseKillTimer: null,
|
||||||
startedAt: null,
|
startedAt: null,
|
||||||
};
|
};
|
||||||
this.groups.set(groupJid, state);
|
this.groups.set(groupJid, state);
|
||||||
@@ -293,6 +299,91 @@ export class GroupQueue {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private clearPostCloseTimers(state: GroupState): void {
|
||||||
|
if (state.postCloseTermTimer) {
|
||||||
|
clearTimeout(state.postCloseTermTimer);
|
||||||
|
state.postCloseTermTimer = null;
|
||||||
|
}
|
||||||
|
if (state.postCloseKillTimer) {
|
||||||
|
clearTimeout(state.postCloseKillTimer);
|
||||||
|
state.postCloseKillTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private schedulePostCloseTermination(
|
||||||
|
groupJid: string,
|
||||||
|
state: GroupState,
|
||||||
|
runId: string | null,
|
||||||
|
reason: string,
|
||||||
|
): void {
|
||||||
|
const proc = state.process;
|
||||||
|
if (!proc || !runId || state.isTaskProcess) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const processName = state.processName;
|
||||||
|
const isSameActiveProcess = () =>
|
||||||
|
state.process === proc &&
|
||||||
|
state.currentRunId === runId &&
|
||||||
|
this.isProcessAlive(proc);
|
||||||
|
|
||||||
|
this.clearPostCloseTimers(state);
|
||||||
|
|
||||||
|
state.postCloseTermTimer = setTimeout(() => {
|
||||||
|
state.postCloseTermTimer = null;
|
||||||
|
if (!isSameActiveProcess()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
groupJid,
|
||||||
|
runId,
|
||||||
|
processName,
|
||||||
|
reason,
|
||||||
|
delayMs: POST_CLOSE_SIGTERM_DELAY_MS,
|
||||||
|
},
|
||||||
|
'Force-terminating lingering agent after stdin close request',
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
proc.kill('SIGTERM');
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ groupJid, runId, processName, err },
|
||||||
|
'Failed to SIGTERM lingering agent after stdin close request',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, POST_CLOSE_SIGTERM_DELAY_MS);
|
||||||
|
|
||||||
|
state.postCloseKillTimer = setTimeout(() => {
|
||||||
|
state.postCloseKillTimer = null;
|
||||||
|
if (!isSameActiveProcess()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.error(
|
||||||
|
{
|
||||||
|
groupJid,
|
||||||
|
runId,
|
||||||
|
processName,
|
||||||
|
reason,
|
||||||
|
delayMs: POST_CLOSE_SIGKILL_DELAY_MS,
|
||||||
|
},
|
||||||
|
'Force-killing stubborn agent after stdin close request',
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
proc.kill('SIGKILL');
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ groupJid, runId, processName, err },
|
||||||
|
'Failed to SIGKILL stubborn agent after stdin close request',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, POST_CLOSE_SIGKILL_DELAY_MS);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Signal the active agent process to wind down by writing a close sentinel.
|
* Signal the active agent process to wind down by writing a close sentinel.
|
||||||
*/
|
*/
|
||||||
@@ -327,6 +418,15 @@ export class GroupQueue {
|
|||||||
'Failed to signal active agent to close stdin',
|
'Failed to signal active agent to close stdin',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (metadata?.reason === 'output-delivered-close') {
|
||||||
|
this.schedulePostCloseTermination(
|
||||||
|
groupJid,
|
||||||
|
state,
|
||||||
|
metadata.runId ?? state.currentRunId,
|
||||||
|
metadata.reason,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runForGroup(
|
private async runForGroup(
|
||||||
@@ -371,6 +471,7 @@ export class GroupQueue {
|
|||||||
);
|
);
|
||||||
this.scheduleRetry(groupJid, state, runId);
|
this.scheduleRetry(groupJid, state, runId);
|
||||||
} finally {
|
} finally {
|
||||||
|
this.clearPostCloseTimers(state);
|
||||||
const durationMs = state.startedAt ? Date.now() - state.startedAt : null;
|
const durationMs = state.startedAt ? Date.now() - state.startedAt : null;
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
@@ -416,6 +517,7 @@ export class GroupQueue {
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error({ groupJid, taskId: task.id, err }, 'Error running task');
|
logger.error({ groupJid, taskId: task.id, err }, 'Error running task');
|
||||||
} finally {
|
} finally {
|
||||||
|
this.clearPostCloseTimers(state);
|
||||||
state.active = false;
|
state.active = false;
|
||||||
state.isTaskProcess = false;
|
state.isTaskProcess = false;
|
||||||
state.runningTaskId = null;
|
state.runningTaskId = null;
|
||||||
@@ -621,6 +723,7 @@ export class GroupQueue {
|
|||||||
}> = [];
|
}> = [];
|
||||||
|
|
||||||
for (const [groupJid, state] of this.groups) {
|
for (const [groupJid, state] of this.groups) {
|
||||||
|
this.clearPostCloseTimers(state);
|
||||||
if (state.retryTimer) {
|
if (state.retryTimer) {
|
||||||
clearTimeout(state.retryTimer);
|
clearTimeout(state.retryTimer);
|
||||||
state.retryTimer = null;
|
state.retryTimer = null;
|
||||||
|
|||||||
@@ -28,10 +28,35 @@ export function stripInternalTags(text: string): string {
|
|||||||
return text.replace(/<internal>[\s\S]*?<\/internal>/g, '').trim();
|
return text.replace(/<internal>[\s\S]*?<\/internal>/g, '').trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Patterns that match common API keys / tokens.
|
||||||
|
* Matched strings are replaced with `[REDACTED]`.
|
||||||
|
*/
|
||||||
|
const SECRET_PATTERNS: RegExp[] = [
|
||||||
|
/sk-ant-[A-Za-z0-9_-]{20,}/g, // Anthropic
|
||||||
|
/sk-[A-Za-z0-9_-]{20,}/g, // OpenAI
|
||||||
|
/gsk_[A-Za-z0-9_-]{20,}/g, // Groq
|
||||||
|
/xai-[A-Za-z0-9_-]{20,}/g, // xAI
|
||||||
|
/ghp_[A-Za-z0-9_]{36,}/g, // GitHub PAT classic
|
||||||
|
/github_pat_[A-Za-z0-9_]{20,}/g, // GitHub PAT fine-grained
|
||||||
|
/glpat-[A-Za-z0-9_-]{20,}/g, // GitLab PAT
|
||||||
|
/AKIA[A-Z0-9]{16}/g, // AWS Access Key
|
||||||
|
/Bearer\s+eyJ[A-Za-z0-9_-]{40,}/g, // Bearer JWT
|
||||||
|
];
|
||||||
|
|
||||||
|
function redactSecrets(text: string): string {
|
||||||
|
let result = text;
|
||||||
|
for (const pattern of SECRET_PATTERNS) {
|
||||||
|
pattern.lastIndex = 0;
|
||||||
|
result = result.replace(pattern, '[REDACTED]');
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
export function formatOutbound(rawText: string): string {
|
export function formatOutbound(rawText: string): string {
|
||||||
const text = stripInternalTags(rawText);
|
const text = stripInternalTags(rawText);
|
||||||
if (!text) return '';
|
if (!text) return '';
|
||||||
return text;
|
return redactSecrets(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function findChannel(
|
export function findChannel(
|
||||||
|
|||||||
@@ -17,11 +17,22 @@ export interface StatusSnapshotEntry {
|
|||||||
pendingTasks: number;
|
pendingTasks: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface UsageRowSnapshot {
|
||||||
|
name: string;
|
||||||
|
h5pct: number;
|
||||||
|
h5reset: string;
|
||||||
|
d7pct: number;
|
||||||
|
d7reset: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface StatusSnapshot {
|
export interface StatusSnapshot {
|
||||||
agentType: AgentType;
|
agentType: AgentType;
|
||||||
assistantName: string;
|
assistantName: string;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
entries: StatusSnapshotEntry[];
|
entries: StatusSnapshotEntry[];
|
||||||
|
usageRows?: UsageRowSnapshot[];
|
||||||
|
/** ISO timestamp of the last successful usage data fetch (separate from updatedAt heartbeat). */
|
||||||
|
usageRowsFetchedAt?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const STATUS_SNAPSHOT_DIR = path.join(CACHE_DIR, 'status-dashboard');
|
const STATUS_SNAPSHOT_DIR = path.join(CACHE_DIR, 'status-dashboard');
|
||||||
|
|||||||
@@ -18,8 +18,82 @@ vi.mock('./codex-token-rotation.js', () => ({
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
buildClaudeUsageRows,
|
buildClaudeUsageRows,
|
||||||
|
extractCodexUsageRows,
|
||||||
mergeClaudeDashboardAccounts,
|
mergeClaudeDashboardAccounts,
|
||||||
} from './unified-dashboard.js';
|
} from './unified-dashboard.js';
|
||||||
|
import type { StatusSnapshot } from './status-dashboard.js';
|
||||||
|
|
||||||
|
describe('extractCodexUsageRows staleness', () => {
|
||||||
|
const freshRows = [
|
||||||
|
{ name: 'Codex1* pro', h5pct: 42, h5reset: '2h', d7pct: 65, d7reset: '3d' },
|
||||||
|
];
|
||||||
|
const TEN_MIN = 600_000;
|
||||||
|
|
||||||
|
it('returns real rows when usageRowsFetchedAt is within maxAgeMs', () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const snapshot: StatusSnapshot = {
|
||||||
|
agentType: 'codex',
|
||||||
|
assistantName: 'test',
|
||||||
|
updatedAt: new Date(now).toISOString(),
|
||||||
|
entries: [],
|
||||||
|
usageRows: freshRows,
|
||||||
|
usageRowsFetchedAt: new Date(now - TEN_MIN + 1000).toISOString(), // 9min59s ago
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = extractCodexUsageRows(snapshot, TEN_MIN, now);
|
||||||
|
expect(result).toEqual(freshRows);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns degraded row when usageRowsFetchedAt exceeds maxAgeMs', () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const snapshot: StatusSnapshot = {
|
||||||
|
agentType: 'codex',
|
||||||
|
assistantName: 'test',
|
||||||
|
updatedAt: new Date(now).toISOString(), // heartbeat is fresh
|
||||||
|
entries: [],
|
||||||
|
usageRows: freshRows,
|
||||||
|
usageRowsFetchedAt: new Date(now - TEN_MIN - 1000).toISOString(), // 10min1s ago
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = extractCodexUsageRows(snapshot, TEN_MIN, now);
|
||||||
|
expect(result).toEqual([
|
||||||
|
{ name: 'Codex', h5pct: -1, h5reset: '', d7pct: -1, d7reset: '' },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns degraded row when usageRowsFetchedAt is missing', () => {
|
||||||
|
const now = Date.now();
|
||||||
|
const snapshot: StatusSnapshot = {
|
||||||
|
agentType: 'codex',
|
||||||
|
assistantName: 'test',
|
||||||
|
updatedAt: new Date(now).toISOString(),
|
||||||
|
entries: [],
|
||||||
|
usageRows: freshRows,
|
||||||
|
// usageRowsFetchedAt intentionally omitted
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = extractCodexUsageRows(snapshot, TEN_MIN, now);
|
||||||
|
expect(result).toEqual([
|
||||||
|
{ name: 'Codex', h5pct: -1, h5reset: '', d7pct: -1, d7reset: '' },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty array when snapshot is undefined', () => {
|
||||||
|
expect(extractCodexUsageRows(undefined, TEN_MIN)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty array when usageRows is empty', () => {
|
||||||
|
const snapshot: StatusSnapshot = {
|
||||||
|
agentType: 'codex',
|
||||||
|
assistantName: 'test',
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
entries: [],
|
||||||
|
usageRows: [],
|
||||||
|
usageRowsFetchedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
expect(extractCodexUsageRows(snapshot, TEN_MIN)).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('unified dashboard Claude usage rows', () => {
|
describe('unified dashboard Claude usage rows', () => {
|
||||||
it('keeps both Claude accounts visible when one account usage is unavailable', () => {
|
it('keeps both Claude accounts visible when one account usage is unavailable', () => {
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import { logger } from './logger.js';
|
|||||||
import {
|
import {
|
||||||
readStatusSnapshots,
|
readStatusSnapshots,
|
||||||
writeStatusSnapshot,
|
writeStatusSnapshot,
|
||||||
|
type StatusSnapshot,
|
||||||
} from './status-dashboard.js';
|
} from './status-dashboard.js';
|
||||||
import type {
|
import type {
|
||||||
AgentType,
|
AgentType,
|
||||||
@@ -63,6 +64,14 @@ const STATUS_ICONS: Record<string, string> = {
|
|||||||
|
|
||||||
const CHANNEL_META_REFRESH_MS = 300000;
|
const CHANNEL_META_REFRESH_MS = 300000;
|
||||||
const STATUS_SNAPSHOT_MAX_AGE_MS = 60000;
|
const STATUS_SNAPSHOT_MAX_AGE_MS = 60000;
|
||||||
|
/** Usage data can be up to 10 min old before considered stale. */
|
||||||
|
const USAGE_SNAPSHOT_MAX_AGE_MS = 600_000;
|
||||||
|
/**
|
||||||
|
* Renderer refreshes usage cache every 30s (not 5min).
|
||||||
|
* Claude API calls are internally rate-limited to 5min per token,
|
||||||
|
* so this only affects how quickly Codex snapshot data is picked up.
|
||||||
|
*/
|
||||||
|
const RENDERER_USAGE_REFRESH_MS = 30_000;
|
||||||
|
|
||||||
let statusMessageId: string | null = null;
|
let statusMessageId: string | null = null;
|
||||||
let cachedUsageContent = '';
|
let cachedUsageContent = '';
|
||||||
@@ -70,27 +79,18 @@ let cachedClaudeAccounts: ClaudeAccountUsage[] = [];
|
|||||||
let usageUpdateInProgress = false;
|
let usageUpdateInProgress = false;
|
||||||
let channelMetaCache = new Map<string, ChannelMeta>();
|
let channelMetaCache = new Map<string, ChannelMeta>();
|
||||||
let channelMetaLastRefresh = 0;
|
let channelMetaLastRefresh = 0;
|
||||||
|
let dashboardUpdateLogged = false;
|
||||||
function formatResetKST(value: string | number): string {
|
/** Codex service only: cached usage rows written into the status snapshot. */
|
||||||
try {
|
let cachedCodexUsageRows: UsageRow[] = [];
|
||||||
const date =
|
/** Codex service only: ISO timestamp of last successful usage fetch. */
|
||||||
typeof value === 'number' ? new Date(value * 1000) : new Date(value);
|
let codexUsageFetchedAt: string | null = null;
|
||||||
return date.toLocaleString('ko-KR', {
|
|
||||||
timeZone: 'Asia/Seoul',
|
|
||||||
month: 'short',
|
|
||||||
day: 'numeric',
|
|
||||||
hour: '2-digit',
|
|
||||||
minute: '2-digit',
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
return String(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatResetRemaining(value: string | number): string {
|
function formatResetRemaining(value: string | number): string {
|
||||||
|
if (value === '' || value == null) return '';
|
||||||
try {
|
try {
|
||||||
const date =
|
const date =
|
||||||
typeof value === 'number' ? new Date(value * 1000) : new Date(value);
|
typeof value === 'number' ? new Date(value * 1000) : new Date(value);
|
||||||
|
if (Number.isNaN(date.getTime())) return '';
|
||||||
const diffMs = date.getTime() - Date.now();
|
const diffMs = date.getTime() - Date.now();
|
||||||
if (diffMs <= 0) return ' reset';
|
if (diffMs <= 0) return ' reset';
|
||||||
const hours = Math.floor(diffMs / 3_600_000);
|
const hours = Math.floor(diffMs / 3_600_000);
|
||||||
@@ -228,6 +228,8 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
|
|||||||
pendingMessages: boolean;
|
pendingMessages: boolean;
|
||||||
pendingTasks: number;
|
pendingTasks: number;
|
||||||
}>,
|
}>,
|
||||||
|
...(cachedCodexUsageRows.length > 0 && { usageRows: cachedCodexUsageRows }),
|
||||||
|
...(codexUsageFetchedAt && { usageRowsFetchedAt: codexUsageFetchedAt }),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -421,7 +423,10 @@ async function fetchCodexUsage(
|
|||||||
const byId = message.result.rateLimitsByLimitId;
|
const byId = message.result.rateLimitsByLimitId;
|
||||||
finish(
|
finish(
|
||||||
byId && typeof byId === 'object'
|
byId && typeof byId === 'object'
|
||||||
? (Object.values(byId) as CodexRateLimit[])
|
? Object.entries(byId).map(([id, val]) => ({
|
||||||
|
...(val as CodexRateLimit),
|
||||||
|
limitId: id,
|
||||||
|
}))
|
||||||
: null,
|
: null,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -499,11 +504,34 @@ export function buildClaudeUsageRows(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract Codex usage rows from a snapshot, applying staleness check.
|
||||||
|
* Returns real rows if usageRowsFetchedAt is within maxAgeMs, otherwise a
|
||||||
|
* single degraded row. Returns empty array if no usage data present.
|
||||||
|
* Exported for testing.
|
||||||
|
*/
|
||||||
|
export function extractCodexUsageRows(
|
||||||
|
snapshot: StatusSnapshot | undefined,
|
||||||
|
maxAgeMs: number,
|
||||||
|
now: number = Date.now(),
|
||||||
|
): UsageRow[] {
|
||||||
|
if (!snapshot?.usageRows || snapshot.usageRows.length === 0) return [];
|
||||||
|
|
||||||
|
const fetchedAt = snapshot.usageRowsFetchedAt
|
||||||
|
? new Date(snapshot.usageRowsFetchedAt).getTime()
|
||||||
|
: 0;
|
||||||
|
const usageAge = now - fetchedAt;
|
||||||
|
if (usageAge <= maxAgeMs) {
|
||||||
|
return [...snapshot.usageRows];
|
||||||
|
}
|
||||||
|
// Usage data is stale — return degraded indicator
|
||||||
|
return [{ name: 'Codex', h5pct: -1, h5reset: '', d7pct: -1, d7reset: '' }];
|
||||||
|
}
|
||||||
|
|
||||||
async function buildUsageContent(): Promise<string> {
|
async function buildUsageContent(): Promise<string> {
|
||||||
const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED;
|
const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED;
|
||||||
let liveClaudeAccounts: ClaudeAccountUsage[] | null = null;
|
let liveClaudeAccounts: ClaudeAccountUsage[] | null = null;
|
||||||
|
|
||||||
const codexUsagePromise = fetchCodexUsage();
|
|
||||||
if (shouldFetchClaudeUsage) {
|
if (shouldFetchClaudeUsage) {
|
||||||
try {
|
try {
|
||||||
liveClaudeAccounts = await fetchAllClaudeUsage();
|
liveClaudeAccounts = await fetchAllClaudeUsage();
|
||||||
@@ -511,12 +539,11 @@ async function buildUsageContent(): Promise<string> {
|
|||||||
logger.warn({ err }, 'Failed to fetch Claude usage for dashboard');
|
logger.warn({ err }, 'Failed to fetch Claude usage for dashboard');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const codexUsage = await codexUsagePromise;
|
|
||||||
|
|
||||||
const lines: string[] = ['📊 *사용량*'];
|
const lines: string[] = ['📊 *사용량*'];
|
||||||
const bar = (pct: number) => {
|
const bar = (pct: number) => {
|
||||||
const filled = Math.max(0, Math.min(10, Math.round(pct / 10)));
|
const filled = Math.max(0, Math.min(5, Math.round(pct / 20)));
|
||||||
return '█'.repeat(filled) + '░'.repeat(10 - filled);
|
return '█'.repeat(filled) + '░'.repeat(5 - filled);
|
||||||
};
|
};
|
||||||
|
|
||||||
const rows: UsageRow[] = [];
|
const rows: UsageRow[] = [];
|
||||||
@@ -529,60 +556,11 @@ async function buildUsageContent(): Promise<string> {
|
|||||||
rows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
|
rows.push(...buildClaudeUsageRows(cachedClaudeAccounts));
|
||||||
}
|
}
|
||||||
|
|
||||||
const codexAccounts = getAllCodexAccounts();
|
// Codex usage: read from Codex service's own status snapshot.
|
||||||
if (codexAccounts.length > 1) {
|
// Each service owns its usage data — no cross-service auth access needed.
|
||||||
// Multi-account: show each account with plan + status
|
const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS)
|
||||||
for (const acct of codexAccounts) {
|
.find(s => s.agentType === 'codex');
|
||||||
const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' ';
|
rows.push(...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS));
|
||||||
const label = `Codex${acct.index + 1}${icon}`;
|
|
||||||
if (acct.isActive && codexUsage && Array.isArray(codexUsage)) {
|
|
||||||
const relevant = codexUsage.filter(
|
|
||||||
(limit) =>
|
|
||||||
limit.primary.usedPercent > 0 || limit.secondary.usedPercent > 0,
|
|
||||||
);
|
|
||||||
const display = relevant.length > 0 ? relevant : codexUsage.slice(0, 1);
|
|
||||||
for (const limit of display) {
|
|
||||||
rows.push({
|
|
||||||
name: `${label} ${acct.planType}`,
|
|
||||||
h5pct: Math.round(limit.primary.usedPercent),
|
|
||||||
h5reset: formatResetRemaining(limit.primary.resetsAt),
|
|
||||||
d7pct: Math.round(limit.secondary.usedPercent),
|
|
||||||
d7reset: formatResetRemaining(limit.secondary.resetsAt),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Show cached usage from last scan
|
|
||||||
const pct = acct.cachedUsagePct != null ? acct.cachedUsagePct : -1;
|
|
||||||
const d7pct =
|
|
||||||
acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1;
|
|
||||||
const reset = acct.resetAt || '';
|
|
||||||
const d7reset = acct.resetD7At || '';
|
|
||||||
rows.push({
|
|
||||||
name: `${label} ${acct.planType}`,
|
|
||||||
h5pct: pct,
|
|
||||||
h5reset: reset,
|
|
||||||
d7pct,
|
|
||||||
d7reset,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (codexUsage && Array.isArray(codexUsage)) {
|
|
||||||
// Single account: existing behavior
|
|
||||||
const relevant = codexUsage.filter(
|
|
||||||
(limit) =>
|
|
||||||
limit.primary.usedPercent > 0 || limit.secondary.usedPercent > 0,
|
|
||||||
);
|
|
||||||
const display = relevant.length > 0 ? relevant : codexUsage.slice(0, 1);
|
|
||||||
for (const limit of display) {
|
|
||||||
rows.push({
|
|
||||||
name: 'Codex',
|
|
||||||
h5pct: Math.round(limit.primary.usedPercent),
|
|
||||||
h5reset: formatResetKST(limit.primary.resetsAt),
|
|
||||||
d7pct: Math.round(limit.secondary.usedPercent),
|
|
||||||
d7reset: formatResetKST(limit.secondary.resetsAt),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (rows.length > 0) {
|
if (rows.length > 0) {
|
||||||
// Emoji characters take 2 columns in monospace — count visual width
|
// Emoji characters take 2 columns in monospace — count visual width
|
||||||
@@ -592,22 +570,34 @@ async function buildUsageContent(): Promise<string> {
|
|||||||
Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1;
|
Math.max(8, ...rows.map((r) => visualWidth(r.name))) + 1;
|
||||||
const padName = (s: string) =>
|
const padName = (s: string) =>
|
||||||
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
|
s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s)));
|
||||||
|
// Strip whitespace and trailing 'm' from reset strings for compact display
|
||||||
|
const compactReset = (s: string) =>
|
||||||
|
s ? s.replace(/\s+/g, '').replace(/m$/, '') : '';
|
||||||
|
|
||||||
lines.push('```');
|
lines.push('```');
|
||||||
lines.push(`${' '.repeat(maxNameWidth)}5-Hour 7-Day`);
|
lines.push(`${' '.repeat(maxNameWidth)}5h 7d`);
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
const h5 =
|
const h5 =
|
||||||
row.h5pct >= 0
|
row.h5pct >= 0
|
||||||
? `${bar(row.h5pct)} ${String(row.h5pct).padStart(3)}%`
|
? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%`
|
||||||
: ' — ';
|
: ' — ';
|
||||||
const d7 =
|
const d7 =
|
||||||
row.d7pct >= 0
|
row.d7pct >= 0
|
||||||
? `${bar(row.d7pct)} ${String(row.d7pct).padStart(3)}%`
|
? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%`
|
||||||
: ' — ';
|
: ' — ';
|
||||||
const reset =
|
lines.push(`${padName(row.name)}${h5} ${d7}`);
|
||||||
row.h5reset || row.d7reset
|
const r5 = compactReset(row.h5reset);
|
||||||
? ` ${row.h5reset || ''}${row.d7reset ? ` / ${row.d7reset}` : ''}`
|
const r7 = compactReset(row.d7reset);
|
||||||
: '';
|
if (r5 || r7) {
|
||||||
lines.push(`${padName(row.name)}${h5} ${d7}${reset}`);
|
// Align reset values under 5h / 7d columns
|
||||||
|
// h5 column starts at maxNameWidth; d7 column starts at maxNameWidth + 9 + 1
|
||||||
|
const d7ColStart = maxNameWidth + 10;
|
||||||
|
let resetLine = ' '.repeat(maxNameWidth);
|
||||||
|
if (r5) resetLine += r5;
|
||||||
|
resetLine = resetLine.padEnd(d7ColStart);
|
||||||
|
if (r7) resetLine += r7;
|
||||||
|
lines.push(resetLine);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
lines.push('```');
|
lines.push('```');
|
||||||
} else {
|
} else {
|
||||||
@@ -672,9 +662,96 @@ function buildUnifiedDashboardContent(): string {
|
|||||||
const CODEX_ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
const CODEX_ACCOUNTS_DIR = path.join(os.homedir(), '.codex-accounts');
|
||||||
const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
const CODEX_FULL_SCAN_INTERVAL = 3_600_000; // 1 hour
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract usage percentages from the primary 'codex' rate-limit bucket
|
||||||
|
* and update the rotation state for a given account.
|
||||||
|
*
|
||||||
|
* Bucket selection:
|
||||||
|
* 1. limitId === 'codex' → use it (user confirmed bengalfox = Codex Spark, not needed)
|
||||||
|
* 2. No 'codex' bucket + single bucket → use it
|
||||||
|
* 3. No 'codex' bucket + multiple buckets → unknown (show —)
|
||||||
|
*
|
||||||
|
* All buckets are logged at info level for observability.
|
||||||
|
*/
|
||||||
|
function applyCodexUsageToAccount(
|
||||||
|
usage: CodexRateLimit[],
|
||||||
|
accountIndex: number,
|
||||||
|
): void {
|
||||||
|
if (usage.length === 0) return;
|
||||||
|
|
||||||
|
// Log all buckets for observability
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
account: accountIndex + 1,
|
||||||
|
buckets: usage.map((l) => ({
|
||||||
|
id: l.limitId,
|
||||||
|
h5: l.primary.usedPercent,
|
||||||
|
d7: l.secondary.usedPercent,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
`Codex account #${accountIndex + 1}: ${usage.length} rate-limit bucket(s)`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Select the effective bucket
|
||||||
|
const primaryBucket = usage.find((l) => l.limitId === 'codex');
|
||||||
|
let effective: CodexRateLimit | null = null;
|
||||||
|
|
||||||
|
if (primaryBucket) {
|
||||||
|
effective = primaryBucket;
|
||||||
|
} else if (usage.length === 1) {
|
||||||
|
effective = usage[0];
|
||||||
|
} else {
|
||||||
|
// Multiple unknown buckets — cannot determine which is authoritative
|
||||||
|
logger.warn(
|
||||||
|
{ account: accountIndex + 1 },
|
||||||
|
`Codex account #${accountIndex + 1}: no 'codex' bucket found among ${usage.length} buckets, showing unknown`,
|
||||||
|
);
|
||||||
|
updateCodexAccountUsage(-1, undefined, accountIndex, -1, undefined);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pct = Math.round(effective.primary.usedPercent);
|
||||||
|
const d7Pct = Math.round(effective.secondary.usedPercent);
|
||||||
|
const resetStr = effective.primary.resetsAt
|
||||||
|
? formatResetRemaining(effective.primary.resetsAt)
|
||||||
|
: undefined;
|
||||||
|
const resetD7Str = effective.secondary.resetsAt
|
||||||
|
? formatResetRemaining(effective.secondary.resetsAt)
|
||||||
|
: undefined;
|
||||||
|
updateCodexAccountUsage(pct, resetStr, accountIndex, d7Pct, resetD7Str);
|
||||||
|
logger.info(
|
||||||
|
{ account: accountIndex + 1, bucket: effective.limitId, h5: pct, d7: d7Pct, reset: resetStr },
|
||||||
|
`Codex account #${accountIndex + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build display-ready usage rows from Codex rotation state.
|
||||||
|
* Called by the Codex service after refreshing usage data.
|
||||||
|
*/
|
||||||
|
function buildCodexUsageRowsFromState(): UsageRow[] {
|
||||||
|
const codexAccounts = getAllCodexAccounts();
|
||||||
|
if (codexAccounts.length === 0) return [];
|
||||||
|
|
||||||
|
const isMulti = codexAccounts.length > 1;
|
||||||
|
return codexAccounts.map((acct) => {
|
||||||
|
const icon = acct.isActive ? '*' : acct.isRateLimited ? '!' : ' ';
|
||||||
|
const label = isMulti
|
||||||
|
? `Codex${acct.index + 1}${icon} ${acct.planType}`
|
||||||
|
: 'Codex';
|
||||||
|
return {
|
||||||
|
name: label,
|
||||||
|
h5pct: acct.cachedUsagePct != null ? acct.cachedUsagePct : -1,
|
||||||
|
h5reset: acct.resetAt || '',
|
||||||
|
d7pct: acct.cachedUsageD7Pct != null ? acct.cachedUsageD7Pct : -1,
|
||||||
|
d7reset: acct.resetD7At || '',
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Scan ALL Codex accounts by spawning app-server with each auth.
|
* Scan ALL Codex accounts by spawning app-server with each auth.
|
||||||
* Called on startup and every hour to keep cached usage fresh.
|
* Codex service only — called on startup and every hour.
|
||||||
*/
|
*/
|
||||||
async function refreshAllCodexAccountUsage(): Promise<void> {
|
async function refreshAllCodexAccountUsage(): Promise<void> {
|
||||||
const codexAccounts = getAllCodexAccounts();
|
const codexAccounts = getAllCodexAccounts();
|
||||||
@@ -685,47 +762,16 @@ async function refreshAllCodexAccountUsage(): Promise<void> {
|
|||||||
'Scanning all Codex accounts for usage data',
|
'Scanning all Codex accounts for usage data',
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let anySuccess = false;
|
||||||
for (const acct of codexAccounts) {
|
for (const acct of codexAccounts) {
|
||||||
const accountDir = path.join(CODEX_ACCOUNTS_DIR, String(acct.index + 1));
|
const accountDir = path.join(CODEX_ACCOUNTS_DIR, String(acct.index + 1));
|
||||||
if (!fs.existsSync(accountDir)) continue;
|
if (!fs.existsSync(accountDir)) continue;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const usage = await fetchCodexUsage(accountDir);
|
const usage = await fetchCodexUsage(accountDir);
|
||||||
if (usage && Array.isArray(usage)) {
|
if (usage && Array.isArray(usage) && usage.length > 0) {
|
||||||
const relevant = usage.filter(
|
applyCodexUsageToAccount(usage, acct.index);
|
||||||
(l) => l.primary.usedPercent > 0 || l.secondary.usedPercent > 0,
|
anySuccess = true;
|
||||||
);
|
|
||||||
const display = relevant.length > 0 ? relevant : usage.slice(0, 1);
|
|
||||||
if (usage.length > 0) {
|
|
||||||
// Find max primary (5h) and secondary (7d) across all limits
|
|
||||||
// Always capture reset times from first limit as baseline
|
|
||||||
let maxH5 = 0;
|
|
||||||
let maxD7 = 0;
|
|
||||||
let h5Reset: string | number | undefined = usage[0].primary.resetsAt;
|
|
||||||
let d7Reset: string | number | undefined =
|
|
||||||
usage[0].secondary.resetsAt;
|
|
||||||
for (const limit of usage) {
|
|
||||||
if (limit.primary.usedPercent >= maxH5) {
|
|
||||||
maxH5 = limit.primary.usedPercent;
|
|
||||||
h5Reset = limit.primary.resetsAt;
|
|
||||||
}
|
|
||||||
if (limit.secondary.usedPercent >= maxD7) {
|
|
||||||
maxD7 = limit.secondary.usedPercent;
|
|
||||||
d7Reset = limit.secondary.resetsAt;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const pct = Math.round(maxH5);
|
|
||||||
const d7Pct = Math.round(maxD7);
|
|
||||||
const resetStr = h5Reset ? formatResetRemaining(h5Reset) : undefined;
|
|
||||||
const resetD7Str = d7Reset
|
|
||||||
? formatResetRemaining(d7Reset)
|
|
||||||
: undefined;
|
|
||||||
updateCodexAccountUsage(pct, resetStr, acct.index, d7Pct, resetD7Str);
|
|
||||||
logger.info(
|
|
||||||
{ account: acct.index + 1, h5: pct, d7: d7Pct, reset: resetStr },
|
|
||||||
`Codex account #${acct.index + 1} usage: 5h=${pct}% 7d=${d7Pct}%`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -734,6 +780,38 @@ async function refreshAllCodexAccountUsage(): Promise<void> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (anySuccess) {
|
||||||
|
codexUsageFetchedAt = new Date().toISOString();
|
||||||
|
}
|
||||||
|
cachedCodexUsageRows = buildCodexUsageRowsFromState();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Quick-refresh the active Codex account's usage (5-min interval).
|
||||||
|
* Codex service only — fetches the active account and rebuilds cached rows.
|
||||||
|
*/
|
||||||
|
async function refreshActiveCodexUsage(): Promise<void> {
|
||||||
|
const codexAccounts = getAllCodexAccounts();
|
||||||
|
if (codexAccounts.length === 0) return;
|
||||||
|
|
||||||
|
const active = codexAccounts.find((a) => a.isActive);
|
||||||
|
if (!active) return;
|
||||||
|
|
||||||
|
const accountDir = path.join(CODEX_ACCOUNTS_DIR, String(active.index + 1));
|
||||||
|
if (!fs.existsSync(accountDir)) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const usage = await fetchCodexUsage(accountDir);
|
||||||
|
if (usage && Array.isArray(usage) && usage.length > 0) {
|
||||||
|
applyCodexUsageToAccount(usage, active.index);
|
||||||
|
codexUsageFetchedAt = new Date().toISOString();
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.debug({ err }, 'Failed to fetch active Codex account usage');
|
||||||
|
}
|
||||||
|
|
||||||
|
cachedCodexUsageRows = buildCodexUsageRowsFromState();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refreshUsageCache(): Promise<void> {
|
async function refreshUsageCache(): Promise<void> {
|
||||||
@@ -793,6 +871,13 @@ export async function startUnifiedDashboard(
|
|||||||
const id = await channel.sendAndTrack(statusJid, content);
|
const id = await channel.sendAndTrack(statusJid, content);
|
||||||
if (id) statusMessageId = id;
|
if (id) statusMessageId = id;
|
||||||
}
|
}
|
||||||
|
if (!dashboardUpdateLogged) {
|
||||||
|
logger.info(
|
||||||
|
{ messageId: statusMessageId, contentLength: content.length },
|
||||||
|
'Dashboard updated successfully (first)',
|
||||||
|
);
|
||||||
|
dashboardUpdateLogged = true;
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn({ err }, 'Dashboard update failed');
|
logger.warn({ err }, 'Dashboard update failed');
|
||||||
statusMessageId = null;
|
statusMessageId = null;
|
||||||
@@ -803,15 +888,17 @@ export async function startUnifiedDashboard(
|
|||||||
await updateStatus();
|
await updateStatus();
|
||||||
|
|
||||||
if (isRenderer) {
|
if (isRenderer) {
|
||||||
setInterval(refreshUsageCache, opts.usageUpdateInterval);
|
// Renderer: refresh usage cache at shorter interval so Codex snapshot
|
||||||
// Full scan of all Codex accounts on startup + hourly
|
// data is picked up quickly. Claude API calls are internally rate-limited
|
||||||
// After scan, refresh dashboard so cached data is visible immediately
|
// to 5min per token, so this only affects local reads.
|
||||||
void refreshAllCodexAccountUsage().then(() => {
|
setInterval(refreshUsageCache, RENDERER_USAGE_REFRESH_MS);
|
||||||
void refreshUsageCache().then(() => void updateStatus());
|
} else {
|
||||||
});
|
// Codex service: fetch own usage and expose via status snapshot.
|
||||||
|
// Active account every usageUpdateInterval; full scan on startup + hourly.
|
||||||
|
void refreshAllCodexAccountUsage().then(() => void refreshActiveCodexUsage());
|
||||||
|
setInterval(() => void refreshActiveCodexUsage(), opts.usageUpdateInterval);
|
||||||
setInterval(
|
setInterval(
|
||||||
() =>
|
() => void refreshAllCodexAccountUsage(),
|
||||||
void refreshAllCodexAccountUsage().then(() => void refreshUsageCache()),
|
|
||||||
CODEX_FULL_SCAN_INTERVAL,
|
CODEX_FULL_SCAN_INTERVAL,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user