feat: add optional Codex warm-up scheduler
This commit is contained in:
15
.env.example
15
.env.example
@@ -36,6 +36,21 @@ STATUS_CHANNEL_ID= # Discord channel ID for live status updat
|
|||||||
# STATUS_SHOW_ROOMS=true # Show room status in dashboard
|
# STATUS_SHOW_ROOMS=true # Show room status in dashboard
|
||||||
# STATUS_SHOW_ROOM_DETAILS=false # Show detailed room info
|
# STATUS_SHOW_ROOM_DETAILS=false # Show detailed room info
|
||||||
|
|
||||||
|
# --- Codex warm-up (optional, disabled by default) ---
|
||||||
|
# Sends a tiny real Codex prompt to near-zero-usage accounts so their reset
|
||||||
|
# countdown starts after a fresh quota window. Keep disabled if OpenAI changes
|
||||||
|
# policy or starts blocking automated warm-up prompts.
|
||||||
|
# CODEX_WARMUP_ENABLED=false
|
||||||
|
# CODEX_WARMUP_PROMPT=Reply exactly OK. Do not run tools.
|
||||||
|
# CODEX_WARMUP_MODEL= # Defaults to CODEX_MODEL
|
||||||
|
# CODEX_WARMUP_INTERVAL_MS=300000 # Check every 5min
|
||||||
|
# CODEX_WARMUP_MIN_INTERVAL_MS=18300000 # Per-account minimum gap: 5h5m
|
||||||
|
# CODEX_WARMUP_STAGGER_MS=1800000 # Global gap between accounts: 30min
|
||||||
|
# CODEX_WARMUP_MAX_USAGE_PCT=0 # Only warm accounts at/below this 5h usage
|
||||||
|
# CODEX_WARMUP_MAX_D7_USAGE_PCT=99 # Skip weekly-exhausted accounts
|
||||||
|
# CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES=2
|
||||||
|
# CODEX_WARMUP_FAILURE_COOLDOWN_MS=21600000 # Back off 6h after repeated failure
|
||||||
|
|
||||||
# --- Fallback provider (when Claude/Codex both unavailable) ---
|
# --- Fallback provider (when Claude/Codex both unavailable) ---
|
||||||
# FALLBACK_ENABLED=true # Enable fallback provider
|
# FALLBACK_ENABLED=true # Enable fallback provider
|
||||||
# FALLBACK_PROVIDER_NAME=kimi # Provider name for logging
|
# FALLBACK_PROVIDER_NAME=kimi # Provider name for logging
|
||||||
|
|||||||
268
src/codex-warmup.test.ts
Normal file
268
src/codex-warmup.test.ts
Normal file
@@ -0,0 +1,268 @@
|
|||||||
|
import { EventEmitter } from 'events';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('child_process', () => ({
|
||||||
|
spawn: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./codex-token-rotation.js', () => ({
|
||||||
|
getAllCodexAccounts: vi.fn(),
|
||||||
|
getCodexAuthPath: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('./logger.js', () => ({
|
||||||
|
logger: {
|
||||||
|
info: vi.fn(),
|
||||||
|
warn: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
debug: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
type FakeProcess = EventEmitter & {
|
||||||
|
stdout: EventEmitter;
|
||||||
|
stderr: EventEmitter;
|
||||||
|
stdin: { write: ReturnType<typeof vi.fn>; end: ReturnType<typeof vi.fn> };
|
||||||
|
kill: ReturnType<typeof vi.fn>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function createFakeCodexProcess(exitCode = 0): FakeProcess {
|
||||||
|
const proc = new EventEmitter() as FakeProcess;
|
||||||
|
proc.stdout = new EventEmitter();
|
||||||
|
proc.stderr = new EventEmitter();
|
||||||
|
proc.stdin = { write: vi.fn(), end: vi.fn() };
|
||||||
|
proc.kill = vi.fn();
|
||||||
|
setImmediate(() => {
|
||||||
|
proc.stdout.emit('data', Buffer.from('OK\n'));
|
||||||
|
proc.emit('close', exitCode, null);
|
||||||
|
});
|
||||||
|
return proc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function authPathFor(tempHome: string, accountIndex: number): string {
|
||||||
|
return path.join(
|
||||||
|
tempHome,
|
||||||
|
'.codex-accounts',
|
||||||
|
String(accountIndex + 1),
|
||||||
|
'auth.json',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('Codex warm-up scheduler', () => {
|
||||||
|
let tempHome: string;
|
||||||
|
let statePath: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-warmup-'));
|
||||||
|
statePath = path.join(tempHome, 'codex-warmup-state.json');
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
const p = authPathFor(tempHome, i);
|
||||||
|
fs.mkdirSync(path.dirname(p), { recursive: true });
|
||||||
|
fs.writeFileSync(p, '{}');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('warms one eligible zero-usage account with a real codex exec and persists account cooldown state', async () => {
|
||||||
|
const childProcess = await import('child_process');
|
||||||
|
const rotation = await import('./codex-token-rotation.js');
|
||||||
|
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||||
|
const now = new Date('2026-04-24T09:00:00Z').getTime();
|
||||||
|
let capturedEnv: Record<string, string> | undefined;
|
||||||
|
let capturedArgs: readonly string[] | undefined;
|
||||||
|
|
||||||
|
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
accountId: 'busy-account',
|
||||||
|
planType: 'pro',
|
||||||
|
isActive: true,
|
||||||
|
isRateLimited: false,
|
||||||
|
cachedUsagePct: 14,
|
||||||
|
cachedUsageD7Pct: 30,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
index: 1,
|
||||||
|
accountId: 'rate-limited-account',
|
||||||
|
planType: 'pro',
|
||||||
|
isActive: false,
|
||||||
|
isRateLimited: true,
|
||||||
|
cachedUsagePct: 0,
|
||||||
|
cachedUsageD7Pct: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
index: 2,
|
||||||
|
accountId: 'fresh-account',
|
||||||
|
planType: 'team',
|
||||||
|
isActive: false,
|
||||||
|
isRateLimited: false,
|
||||||
|
cachedUsagePct: 0,
|
||||||
|
cachedUsageD7Pct: 0,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||||
|
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||||
|
);
|
||||||
|
vi.mocked(childProcess.spawn).mockImplementation(((
|
||||||
|
_cmd: string,
|
||||||
|
args?: readonly string[],
|
||||||
|
opts?: { env?: Record<string, string> },
|
||||||
|
) => {
|
||||||
|
capturedArgs = args;
|
||||||
|
capturedEnv = opts?.env;
|
||||||
|
return createFakeCodexProcess(0) as never;
|
||||||
|
}) as unknown as typeof childProcess.spawn);
|
||||||
|
|
||||||
|
const result = await runCodexWarmupCycle(
|
||||||
|
{
|
||||||
|
enabled: true,
|
||||||
|
prompt: 'Reply exactly OK. Do not run tools.',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
intervalMs: 300_000,
|
||||||
|
minIntervalMs: 18_300_000,
|
||||||
|
staggerMs: 1_800_000,
|
||||||
|
maxUsagePct: 0,
|
||||||
|
maxD7UsagePct: 99,
|
||||||
|
commandTimeoutMs: 120_000,
|
||||||
|
failureCooldownMs: 21_600_000,
|
||||||
|
maxConsecutiveFailures: 2,
|
||||||
|
},
|
||||||
|
{ nowMs: now, statePath },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({ status: 'warmed', accountIndex: 2 });
|
||||||
|
expect(childProcess.spawn).toHaveBeenCalledTimes(1);
|
||||||
|
expect(capturedArgs).toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
'exec',
|
||||||
|
'--ephemeral',
|
||||||
|
'--ignore-rules',
|
||||||
|
'--skip-git-repo-check',
|
||||||
|
'--sandbox',
|
||||||
|
'read-only',
|
||||||
|
'-m',
|
||||||
|
'gpt-5.5',
|
||||||
|
'Reply exactly OK. Do not run tools.',
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
expect(capturedEnv?.CODEX_HOME).toBe(
|
||||||
|
path.dirname(authPathFor(tempHome, 2)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||||
|
expect(state.lastWarmupAt).toBe('2026-04-24T09:00:00.000Z');
|
||||||
|
expect(state.accounts['2'].lastWarmupAt).toBe('2026-04-24T09:00:00.000Z');
|
||||||
|
expect(state.consecutiveFailures).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('auto-backs off after repeated codex exec failures so OpenAI-side blocking does not hammer accounts', async () => {
|
||||||
|
const childProcess = await import('child_process');
|
||||||
|
const rotation = await import('./codex-token-rotation.js');
|
||||||
|
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||||
|
const now = new Date('2026-04-24T09:00:00Z').getTime();
|
||||||
|
|
||||||
|
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
accountId: 'fresh-account',
|
||||||
|
planType: 'pro',
|
||||||
|
isActive: false,
|
||||||
|
isRateLimited: false,
|
||||||
|
cachedUsagePct: 0,
|
||||||
|
cachedUsageD7Pct: 0,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||||
|
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||||
|
);
|
||||||
|
vi.mocked(childProcess.spawn).mockImplementation(
|
||||||
|
() => createFakeCodexProcess(1) as never,
|
||||||
|
);
|
||||||
|
|
||||||
|
const config = {
|
||||||
|
enabled: true,
|
||||||
|
prompt: 'Reply exactly OK. Do not run tools.',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
intervalMs: 300_000,
|
||||||
|
minIntervalMs: 18_300_000,
|
||||||
|
staggerMs: 0,
|
||||||
|
maxUsagePct: 0,
|
||||||
|
maxD7UsagePct: 99,
|
||||||
|
commandTimeoutMs: 120_000,
|
||||||
|
failureCooldownMs: 21_600_000,
|
||||||
|
maxConsecutiveFailures: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const failed = await runCodexWarmupCycle(config, { nowMs: now, statePath });
|
||||||
|
expect(failed.status).toBe('failed');
|
||||||
|
|
||||||
|
const backedOff = await runCodexWarmupCycle(config, {
|
||||||
|
nowMs: now + 60_000,
|
||||||
|
statePath,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(backedOff).toEqual({
|
||||||
|
status: 'skipped',
|
||||||
|
reason: 'disabled_cooldown',
|
||||||
|
});
|
||||||
|
expect(childProcess.spawn).toHaveBeenCalledTimes(1);
|
||||||
|
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||||
|
expect(state.disabledUntil).toBe('2026-04-24T15:00:00.000Z');
|
||||||
|
expect(state.consecutiveFailures).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('respects global stagger and does not warm another account too soon', async () => {
|
||||||
|
const childProcess = await import('child_process');
|
||||||
|
const rotation = await import('./codex-token-rotation.js');
|
||||||
|
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||||
|
const now = new Date('2026-04-24T09:20:00Z').getTime();
|
||||||
|
|
||||||
|
fs.writeFileSync(
|
||||||
|
statePath,
|
||||||
|
JSON.stringify({
|
||||||
|
lastWarmupAt: '2026-04-24T09:00:00.000Z',
|
||||||
|
consecutiveFailures: 0,
|
||||||
|
accounts: { '0': { lastWarmupAt: '2026-04-24T09:00:00.000Z' } },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||||
|
{
|
||||||
|
index: 1,
|
||||||
|
accountId: 'another-fresh-account',
|
||||||
|
planType: 'pro',
|
||||||
|
isActive: false,
|
||||||
|
isRateLimited: false,
|
||||||
|
cachedUsagePct: 0,
|
||||||
|
cachedUsageD7Pct: 0,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await runCodexWarmupCycle(
|
||||||
|
{
|
||||||
|
enabled: true,
|
||||||
|
prompt: '.',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
intervalMs: 300_000,
|
||||||
|
minIntervalMs: 18_300_000,
|
||||||
|
staggerMs: 1_800_000,
|
||||||
|
maxUsagePct: 0,
|
||||||
|
maxD7UsagePct: 99,
|
||||||
|
commandTimeoutMs: 120_000,
|
||||||
|
failureCooldownMs: 21_600_000,
|
||||||
|
maxConsecutiveFailures: 2,
|
||||||
|
},
|
||||||
|
{ nowMs: now, statePath },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({ status: 'skipped', reason: 'stagger_wait' });
|
||||||
|
expect(childProcess.spawn).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
294
src/codex-warmup.ts
Normal file
294
src/codex-warmup.ts
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
import { ChildProcess, spawn } from 'child_process';
|
||||||
|
import fs from 'fs';
|
||||||
|
import os from 'os';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
import { DATA_DIR } from './config.js';
|
||||||
|
import type { AppConfig } from './config/schema.js';
|
||||||
|
import {
|
||||||
|
getAllCodexAccounts,
|
||||||
|
getCodexAuthPath,
|
||||||
|
} from './codex-token-rotation.js';
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
|
||||||
|
export type CodexWarmupConfig = AppConfig['codexWarmup'];
|
||||||
|
|
||||||
|
interface CodexWarmupAccountState {
|
||||||
|
lastWarmupAt?: string;
|
||||||
|
lastAttemptAt?: string;
|
||||||
|
lastErrorAt?: string;
|
||||||
|
failures?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CodexWarmupState {
|
||||||
|
lastWarmupAt?: string;
|
||||||
|
lastAttemptAt?: string;
|
||||||
|
disabledUntil?: string;
|
||||||
|
consecutiveFailures?: number;
|
||||||
|
accounts?: Record<string, CodexWarmupAccountState>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CodexWarmupRuntimeOptions {
|
||||||
|
nowMs?: number;
|
||||||
|
statePath?: string;
|
||||||
|
shouldSkip?: () => boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CodexWarmupCycleResult =
|
||||||
|
| { status: 'disabled' }
|
||||||
|
| { status: 'skipped'; reason: string }
|
||||||
|
| { status: 'warmed'; accountIndex: number }
|
||||||
|
| { status: 'failed'; accountIndex: number; reason: string };
|
||||||
|
|
||||||
|
const DEFAULT_STATE_FILE = path.join(DATA_DIR, 'codex-warmup-state.json');
|
||||||
|
|
||||||
|
function parseTimestamp(value?: string): number | null {
|
||||||
|
if (!value) return null;
|
||||||
|
const parsed = Date.parse(value);
|
||||||
|
return Number.isFinite(parsed) ? parsed : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readWarmupState(statePath: string): CodexWarmupState {
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(statePath)) return { accounts: {} };
|
||||||
|
const parsed = JSON.parse(
|
||||||
|
fs.readFileSync(statePath, 'utf8'),
|
||||||
|
) as CodexWarmupState;
|
||||||
|
if (!parsed.accounts || typeof parsed.accounts !== 'object') {
|
||||||
|
parsed.accounts = {};
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
{ err, statePath },
|
||||||
|
'Failed to read Codex warm-up state; starting fresh',
|
||||||
|
);
|
||||||
|
return { accounts: {} };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeWarmupState(statePath: string, state: CodexWarmupState): void {
|
||||||
|
fs.mkdirSync(path.dirname(statePath), { recursive: true });
|
||||||
|
fs.writeFileSync(`${statePath}.tmp`, `${JSON.stringify(state, null, 2)}\n`);
|
||||||
|
fs.renameSync(`${statePath}.tmp`, statePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getPreferredCodexPathEntries(): string[] {
|
||||||
|
const entries = [
|
||||||
|
path.dirname(process.execPath),
|
||||||
|
path.join(os.homedir(), '.npm-global', 'bin'),
|
||||||
|
];
|
||||||
|
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
||||||
|
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
||||||
|
}
|
||||||
|
return [...new Set(entries)];
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveCodexBinary(): string {
|
||||||
|
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
||||||
|
return fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureAccountState(
|
||||||
|
state: CodexWarmupState,
|
||||||
|
accountIndex: number,
|
||||||
|
): CodexWarmupAccountState {
|
||||||
|
state.accounts ??= {};
|
||||||
|
const key = String(accountIndex);
|
||||||
|
state.accounts[key] ??= {};
|
||||||
|
return state.accounts[key];
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectWarmupCandidate(
|
||||||
|
config: CodexWarmupConfig,
|
||||||
|
state: CodexWarmupState,
|
||||||
|
nowMs: number,
|
||||||
|
): { accountIndex: number } | { reason: string } {
|
||||||
|
const disabledUntilMs = parseTimestamp(state.disabledUntil);
|
||||||
|
if (disabledUntilMs != null && disabledUntilMs > nowMs) {
|
||||||
|
return { reason: 'disabled_cooldown' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastGlobalWarmupMs = parseTimestamp(state.lastWarmupAt);
|
||||||
|
if (
|
||||||
|
lastGlobalWarmupMs != null &&
|
||||||
|
config.staggerMs > 0 &&
|
||||||
|
nowMs - lastGlobalWarmupMs < config.staggerMs
|
||||||
|
) {
|
||||||
|
return { reason: 'stagger_wait' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const accounts = getAllCodexAccounts();
|
||||||
|
if (accounts.length === 0) return { reason: 'no_accounts' };
|
||||||
|
|
||||||
|
for (const account of accounts) {
|
||||||
|
if (account.isRateLimited) continue;
|
||||||
|
if (typeof account.cachedUsagePct !== 'number') continue;
|
||||||
|
if (typeof account.cachedUsageD7Pct !== 'number') continue;
|
||||||
|
if (account.cachedUsagePct > config.maxUsagePct) continue;
|
||||||
|
if (account.cachedUsageD7Pct > config.maxD7UsagePct) continue;
|
||||||
|
|
||||||
|
const accountState = state.accounts?.[String(account.index)];
|
||||||
|
const lastWarmupMs = parseTimestamp(accountState?.lastWarmupAt);
|
||||||
|
if (lastWarmupMs != null && nowMs - lastWarmupMs < config.minIntervalMs) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { accountIndex: account.index };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { reason: 'no_eligible_accounts' };
|
||||||
|
}
|
||||||
|
|
||||||
|
function runCodexWarmupCommand(
|
||||||
|
accountDir: string,
|
||||||
|
config: CodexWarmupConfig,
|
||||||
|
): Promise<{ ok: boolean; reason: string }> {
|
||||||
|
const args = [
|
||||||
|
'exec',
|
||||||
|
'--ephemeral',
|
||||||
|
'--ignore-rules',
|
||||||
|
'--skip-git-repo-check',
|
||||||
|
'--sandbox',
|
||||||
|
'read-only',
|
||||||
|
'-C',
|
||||||
|
os.tmpdir(),
|
||||||
|
'-m',
|
||||||
|
config.model,
|
||||||
|
config.prompt,
|
||||||
|
];
|
||||||
|
|
||||||
|
const spawnEnv: Record<string, string> = {
|
||||||
|
...(process.env as Record<string, string>),
|
||||||
|
CODEX_HOME: accountDir,
|
||||||
|
PATH: [...getPreferredCodexPathEntries(), process.env.PATH || '']
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(path.delimiter),
|
||||||
|
};
|
||||||
|
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
let done = false;
|
||||||
|
let proc: ChildProcess | null = null;
|
||||||
|
let stderr = '';
|
||||||
|
const finish = (result: { ok: boolean; reason: string }) => {
|
||||||
|
if (done) return;
|
||||||
|
done = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
if (proc && result.reason === 'timeout') {
|
||||||
|
try {
|
||||||
|
proc.kill();
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolve(result);
|
||||||
|
};
|
||||||
|
|
||||||
|
const timer = setTimeout(
|
||||||
|
() => finish({ ok: false, reason: 'timeout' }),
|
||||||
|
config.commandTimeoutMs,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
proc = spawn(resolveCodexBinary(), args, {
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
env: spawnEnv,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err }, 'Failed to spawn Codex warm-up command');
|
||||||
|
finish({ ok: false, reason: 'spawn_error' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
proc.stderr?.on('data', (chunk: Buffer) => {
|
||||||
|
stderr += chunk.toString();
|
||||||
|
if (stderr.length > 2000) stderr = stderr.slice(-2000);
|
||||||
|
});
|
||||||
|
proc.on('error', (err) => {
|
||||||
|
logger.warn({ err }, 'Codex warm-up process error');
|
||||||
|
finish({ ok: false, reason: 'process_error' });
|
||||||
|
});
|
||||||
|
proc.on('close', (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
finish({ ok: true, reason: 'ok' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.warn(
|
||||||
|
{ exitCode: code, stderr: stderr.trim() || undefined },
|
||||||
|
'Codex warm-up command failed',
|
||||||
|
);
|
||||||
|
finish({ ok: false, reason: `exit_${code ?? 'unknown'}` });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runCodexWarmupCycle(
|
||||||
|
config: CodexWarmupConfig,
|
||||||
|
runtime: CodexWarmupRuntimeOptions = {},
|
||||||
|
): Promise<CodexWarmupCycleResult> {
|
||||||
|
if (!config.enabled) return { status: 'disabled' };
|
||||||
|
if (runtime.shouldSkip?.())
|
||||||
|
return { status: 'skipped', reason: 'runtime_busy' };
|
||||||
|
|
||||||
|
const nowMs = runtime.nowMs ?? Date.now();
|
||||||
|
const nowIso = new Date(nowMs).toISOString();
|
||||||
|
const statePath = runtime.statePath ?? DEFAULT_STATE_FILE;
|
||||||
|
const state = readWarmupState(statePath);
|
||||||
|
const selected = selectWarmupCandidate(config, state, nowMs);
|
||||||
|
if ('reason' in selected) {
|
||||||
|
return { status: 'skipped', reason: selected.reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
const authPath = getCodexAuthPath(selected.accountIndex);
|
||||||
|
if (!authPath || !fs.existsSync(authPath)) {
|
||||||
|
return { status: 'skipped', reason: 'missing_auth' };
|
||||||
|
}
|
||||||
|
const accountDir = path.dirname(authPath);
|
||||||
|
const accountState = ensureAccountState(state, selected.accountIndex);
|
||||||
|
state.lastAttemptAt = nowIso;
|
||||||
|
accountState.lastAttemptAt = nowIso;
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{ account: selected.accountIndex + 1, model: config.model },
|
||||||
|
'Starting Codex warm-up prompt',
|
||||||
|
);
|
||||||
|
const result = await runCodexWarmupCommand(accountDir, config);
|
||||||
|
|
||||||
|
if (result.ok) {
|
||||||
|
state.lastWarmupAt = nowIso;
|
||||||
|
state.consecutiveFailures = 0;
|
||||||
|
delete state.disabledUntil;
|
||||||
|
accountState.lastWarmupAt = nowIso;
|
||||||
|
accountState.failures = 0;
|
||||||
|
writeWarmupState(statePath, state);
|
||||||
|
logger.info(
|
||||||
|
{ account: selected.accountIndex + 1 },
|
||||||
|
'Codex warm-up prompt completed',
|
||||||
|
);
|
||||||
|
return { status: 'warmed', accountIndex: selected.accountIndex };
|
||||||
|
}
|
||||||
|
|
||||||
|
state.consecutiveFailures = (state.consecutiveFailures ?? 0) + 1;
|
||||||
|
accountState.lastErrorAt = nowIso;
|
||||||
|
accountState.failures = (accountState.failures ?? 0) + 1;
|
||||||
|
if (state.consecutiveFailures >= config.maxConsecutiveFailures) {
|
||||||
|
state.disabledUntil = new Date(
|
||||||
|
nowMs + config.failureCooldownMs,
|
||||||
|
).toISOString();
|
||||||
|
}
|
||||||
|
writeWarmupState(statePath, state);
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
account: selected.accountIndex + 1,
|
||||||
|
reason: result.reason,
|
||||||
|
consecutiveFailures: state.consecutiveFailures,
|
||||||
|
disabledUntil: state.disabledUntil,
|
||||||
|
},
|
||||||
|
'Codex warm-up prompt failed',
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
status: 'failed',
|
||||||
|
accountIndex: selected.accountIndex,
|
||||||
|
reason: result.reason,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -31,6 +31,17 @@ describe('config/env loading', () => {
|
|||||||
delete process.env.DISCORD_CODEX_REVIEW_BOT_TOKEN;
|
delete process.env.DISCORD_CODEX_REVIEW_BOT_TOKEN;
|
||||||
delete process.env.PAIRED_CARRY_FORWARD_LATEST_OWNER_FINAL;
|
delete process.env.PAIRED_CARRY_FORWARD_LATEST_OWNER_FINAL;
|
||||||
delete process.env.PAIRED_FORCE_FRESH_CLAUDE_REVIEWER_SESSION;
|
delete process.env.PAIRED_FORCE_FRESH_CLAUDE_REVIEWER_SESSION;
|
||||||
|
delete process.env.CODEX_WARMUP_ENABLED;
|
||||||
|
delete process.env.CODEX_WARMUP_PROMPT;
|
||||||
|
delete process.env.CODEX_WARMUP_MODEL;
|
||||||
|
delete process.env.CODEX_WARMUP_INTERVAL_MS;
|
||||||
|
delete process.env.CODEX_WARMUP_MIN_INTERVAL_MS;
|
||||||
|
delete process.env.CODEX_WARMUP_STAGGER_MS;
|
||||||
|
delete process.env.CODEX_WARMUP_MAX_USAGE_PCT;
|
||||||
|
delete process.env.CODEX_WARMUP_MAX_D7_USAGE_PCT;
|
||||||
|
delete process.env.CODEX_WARMUP_COMMAND_TIMEOUT_MS;
|
||||||
|
delete process.env.CODEX_WARMUP_FAILURE_COOLDOWN_MS;
|
||||||
|
delete process.env.CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES;
|
||||||
delete process.env.SESSION_COMMAND_USER_IDS;
|
delete process.env.SESSION_COMMAND_USER_IDS;
|
||||||
vi.resetModules();
|
vi.resetModules();
|
||||||
});
|
});
|
||||||
@@ -124,6 +135,33 @@ describe('config/env loading', () => {
|
|||||||
expect(config.PAIRED_FORCE_FRESH_CLAUDE_REVIEWER_SESSION).toBe(true);
|
expect(config.PAIRED_FORCE_FRESH_CLAUDE_REVIEWER_SESSION).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps Codex warm-up disabled by default and exposes conservative opt-in env config', async () => {
|
||||||
|
let config = await import('./config.js');
|
||||||
|
expect(config.CODEX_WARMUP_CONFIG.enabled).toBe(false);
|
||||||
|
expect(config.CODEX_WARMUP_CONFIG.maxUsagePct).toBe(0);
|
||||||
|
expect(
|
||||||
|
config.CODEX_WARMUP_CONFIG.maxConsecutiveFailures,
|
||||||
|
).toBeGreaterThanOrEqual(1);
|
||||||
|
|
||||||
|
vi.resetModules();
|
||||||
|
process.env.CODEX_MODEL = 'gpt-5.5';
|
||||||
|
process.env.CODEX_WARMUP_ENABLED = 'true';
|
||||||
|
process.env.CODEX_WARMUP_PROMPT = '.';
|
||||||
|
process.env.CODEX_WARMUP_STAGGER_MS = '600000';
|
||||||
|
process.env.CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES = '1';
|
||||||
|
config = await import('./config.js');
|
||||||
|
|
||||||
|
expect(config.CODEX_WARMUP_CONFIG).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
enabled: true,
|
||||||
|
prompt: '.',
|
||||||
|
model: 'gpt-5.5',
|
||||||
|
staggerMs: 600000,
|
||||||
|
maxConsecutiveFailures: 1,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('fails fast when a legacy Discord token alias is configured', async () => {
|
it('fails fast when a legacy Discord token alias is configured', async () => {
|
||||||
process.env.DISCORD_BOT_TOKEN = 'legacy-owner-token';
|
process.env.DISCORD_BOT_TOKEN = 'legacy-owner-token';
|
||||||
|
|
||||||
|
|||||||
@@ -140,6 +140,7 @@ export const USAGE_UPDATE_INTERVAL = CONFIG.status.usageUpdateInterval;
|
|||||||
export const STATUS_SHOW_ROOMS = CONFIG.status.showRooms;
|
export const STATUS_SHOW_ROOMS = CONFIG.status.showRooms;
|
||||||
export const STATUS_SHOW_ROOM_DETAILS = CONFIG.status.showRoomDetails;
|
export const STATUS_SHOW_ROOM_DETAILS = CONFIG.status.showRoomDetails;
|
||||||
export const USAGE_DASHBOARD_ENABLED = CONFIG.status.usageDashboardEnabled;
|
export const USAGE_DASHBOARD_ENABLED = CONFIG.status.usageDashboardEnabled;
|
||||||
|
export const CODEX_WARMUP_CONFIG = CONFIG.codexWarmup;
|
||||||
|
|
||||||
// Timezone for scheduled tasks (cron expressions, etc.)
|
// Timezone for scheduled tasks (cron expressions, etc.)
|
||||||
// Uses system timezone by default
|
// Uses system timezone by default
|
||||||
|
|||||||
@@ -53,6 +53,11 @@ function readIntegerAtLeast(
|
|||||||
return Math.max(minimum, readInteger(key, fallback) || fallback);
|
return Math.max(minimum, readInteger(key, fallback) || fallback);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readPercent(key: string, fallback: number): number {
|
||||||
|
const value = readInteger(key, fallback);
|
||||||
|
return Math.min(100, Math.max(0, value));
|
||||||
|
}
|
||||||
|
|
||||||
function readAgentType(
|
function readAgentType(
|
||||||
key: string,
|
key: string,
|
||||||
fallback?: AgentType,
|
fallback?: AgentType,
|
||||||
@@ -255,6 +260,40 @@ export function loadConfig(): AppConfig {
|
|||||||
timezone:
|
timezone:
|
||||||
readText('TZ') ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
|
readText('TZ') ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||||
},
|
},
|
||||||
|
codexWarmup: {
|
||||||
|
enabled: readBoolean('CODEX_WARMUP_ENABLED', false),
|
||||||
|
prompt:
|
||||||
|
readNonEmptyText('CODEX_WARMUP_PROMPT') ??
|
||||||
|
'Reply exactly OK. Do not run tools.',
|
||||||
|
model:
|
||||||
|
readNonEmptyText('CODEX_WARMUP_MODEL') ??
|
||||||
|
readNonEmptyText('CODEX_MODEL') ??
|
||||||
|
'codex',
|
||||||
|
intervalMs: readIntegerAtLeast('CODEX_WARMUP_INTERVAL_MS', 300000, 60000),
|
||||||
|
minIntervalMs: readIntegerAtLeast(
|
||||||
|
'CODEX_WARMUP_MIN_INTERVAL_MS',
|
||||||
|
18300000,
|
||||||
|
60000,
|
||||||
|
),
|
||||||
|
staggerMs: Math.max(0, readInteger('CODEX_WARMUP_STAGGER_MS', 1800000)),
|
||||||
|
maxUsagePct: readPercent('CODEX_WARMUP_MAX_USAGE_PCT', 0),
|
||||||
|
maxD7UsagePct: readPercent('CODEX_WARMUP_MAX_D7_USAGE_PCT', 99),
|
||||||
|
commandTimeoutMs: readIntegerAtLeast(
|
||||||
|
'CODEX_WARMUP_COMMAND_TIMEOUT_MS',
|
||||||
|
120000,
|
||||||
|
10000,
|
||||||
|
),
|
||||||
|
failureCooldownMs: readIntegerAtLeast(
|
||||||
|
'CODEX_WARMUP_FAILURE_COOLDOWN_MS',
|
||||||
|
21600000,
|
||||||
|
60000,
|
||||||
|
),
|
||||||
|
maxConsecutiveFailures: readIntegerAtLeast(
|
||||||
|
'CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES',
|
||||||
|
2,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
},
|
||||||
sessionCommands: {
|
sessionCommands: {
|
||||||
allowedSenders: new Set(
|
allowedSenders: new Set(
|
||||||
(readText('SESSION_COMMAND_ALLOWED_SENDERS') ?? '')
|
(readText('SESSION_COMMAND_ALLOWED_SENDERS') ?? '')
|
||||||
|
|||||||
@@ -78,6 +78,19 @@ export interface AppConfig {
|
|||||||
usageDashboardEnabled: boolean;
|
usageDashboardEnabled: boolean;
|
||||||
timezone: string;
|
timezone: string;
|
||||||
};
|
};
|
||||||
|
codexWarmup: {
|
||||||
|
enabled: boolean;
|
||||||
|
prompt: string;
|
||||||
|
model: string;
|
||||||
|
intervalMs: number;
|
||||||
|
minIntervalMs: number;
|
||||||
|
staggerMs: number;
|
||||||
|
maxUsagePct: number;
|
||||||
|
maxD7UsagePct: number;
|
||||||
|
commandTimeoutMs: number;
|
||||||
|
failureCooldownMs: number;
|
||||||
|
maxConsecutiveFailures: number;
|
||||||
|
};
|
||||||
sessionCommands: {
|
sessionCommands: {
|
||||||
allowedSenders: Set<string>;
|
allowedSenders: Set<string>;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import {
|
|||||||
STATUS_SHOW_ROOM_DETAILS,
|
STATUS_SHOW_ROOM_DETAILS,
|
||||||
STATUS_SHOW_ROOMS,
|
STATUS_SHOW_ROOMS,
|
||||||
USAGE_DASHBOARD_ENABLED,
|
USAGE_DASHBOARD_ENABLED,
|
||||||
|
CODEX_WARMUP_CONFIG,
|
||||||
getMoaConfig,
|
getMoaConfig,
|
||||||
} from './config.js';
|
} from './config.js';
|
||||||
import { fetchKimiUsage, buildKimiUsageRows } from './kimi-usage.js';
|
import { fetchKimiUsage, buildKimiUsageRows } from './kimi-usage.js';
|
||||||
@@ -28,6 +29,7 @@ import {
|
|||||||
refreshActiveCodexUsage,
|
refreshActiveCodexUsage,
|
||||||
refreshAllCodexAccountUsage,
|
refreshAllCodexAccountUsage,
|
||||||
} from './codex-usage-collector.js';
|
} from './codex-usage-collector.js';
|
||||||
|
import { runCodexWarmupCycle } from './codex-warmup.js';
|
||||||
import {
|
import {
|
||||||
composeDashboardContent,
|
composeDashboardContent,
|
||||||
formatElapsed,
|
formatElapsed,
|
||||||
@@ -735,18 +737,49 @@ export async function startUnifiedDashboard(
|
|||||||
cachedCodexUsageRows = result.rows;
|
cachedCodexUsageRows = result.rows;
|
||||||
if (result.fetchedAt) codexUsageFetchedAt = result.fetchedAt;
|
if (result.fetchedAt) codexUsageFetchedAt = result.fetchedAt;
|
||||||
};
|
};
|
||||||
void refreshAllCodexAccountUsage().then((r) => {
|
const isWarmupRuntimeBusy = () => {
|
||||||
applyCodexRefresh(r);
|
const groups = opts.roomBindings();
|
||||||
return refreshActiveCodexUsage().then(applyCodexRefresh);
|
return opts.queue
|
||||||
});
|
.getStatuses(Object.keys(groups))
|
||||||
|
.some((status) => status.status === 'processing');
|
||||||
|
};
|
||||||
|
let codexWarmupInFlight = false;
|
||||||
|
const runCodexWarmup = async () => {
|
||||||
|
if (!CODEX_WARMUP_CONFIG.enabled || codexWarmupInFlight) return;
|
||||||
|
codexWarmupInFlight = true;
|
||||||
|
try {
|
||||||
|
const result = await runCodexWarmupCycle(CODEX_WARMUP_CONFIG, {
|
||||||
|
shouldSkip: isWarmupRuntimeBusy,
|
||||||
|
});
|
||||||
|
if (result.status === 'warmed') {
|
||||||
|
applyCodexRefresh(await refreshAllCodexAccountUsage());
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err }, 'Codex warm-up cycle failed unexpectedly');
|
||||||
|
} finally {
|
||||||
|
codexWarmupInFlight = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
void refreshAllCodexAccountUsage()
|
||||||
|
.then((r) => {
|
||||||
|
applyCodexRefresh(r);
|
||||||
|
return refreshActiveCodexUsage().then(applyCodexRefresh);
|
||||||
|
})
|
||||||
|
.then(() => runCodexWarmup());
|
||||||
setInterval(
|
setInterval(
|
||||||
() => void refreshActiveCodexUsage().then(applyCodexRefresh),
|
() => void refreshActiveCodexUsage().then(applyCodexRefresh),
|
||||||
opts.usageUpdateInterval,
|
opts.usageUpdateInterval,
|
||||||
);
|
);
|
||||||
setInterval(
|
setInterval(
|
||||||
() => void refreshAllCodexAccountUsage().then(applyCodexRefresh),
|
() =>
|
||||||
|
void refreshAllCodexAccountUsage()
|
||||||
|
.then(applyCodexRefresh)
|
||||||
|
.then(() => runCodexWarmup()),
|
||||||
CODEX_FULL_SCAN_INTERVAL,
|
CODEX_FULL_SCAN_INTERVAL,
|
||||||
);
|
);
|
||||||
|
if (CODEX_WARMUP_CONFIG.enabled) {
|
||||||
|
setInterval(() => void runCodexWarmup(), CODEX_WARMUP_CONFIG.intervalMs);
|
||||||
|
}
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user