feat: add optional Codex warm-up scheduler
This commit is contained in:
ejclaw
15
.env.example
15
.env.example
@@ -36,6 +36,21 @@ STATUS_CHANNEL_ID= # Discord channel ID for live status updat
|
||||
# STATUS_SHOW_ROOMS=true # Show room status in dashboard
|
||||
# STATUS_SHOW_ROOM_DETAILS=false # Show detailed room info
|
||||
|
||||
# --- Codex warm-up (optional, disabled by default) ---
|
||||
# Sends a tiny real Codex prompt to near-zero-usage accounts so their reset
|
||||
# countdown starts after a fresh quota window. Keep disabled if OpenAI changes
|
||||
# policy or starts blocking automated warm-up prompts.
|
||||
# CODEX_WARMUP_ENABLED=false
|
||||
# CODEX_WARMUP_PROMPT=Reply exactly OK. Do not run tools.
|
||||
# CODEX_WARMUP_MODEL= # Defaults to CODEX_MODEL
|
||||
# CODEX_WARMUP_INTERVAL_MS=300000 # Check every 5min
|
||||
# CODEX_WARMUP_MIN_INTERVAL_MS=18300000 # Per-account minimum gap: 5h5m
|
||||
# CODEX_WARMUP_STAGGER_MS=1800000 # Global gap between accounts: 30min
|
||||
# CODEX_WARMUP_MAX_USAGE_PCT=0 # Only warm accounts at/below this 5h usage
|
||||
# CODEX_WARMUP_MAX_D7_USAGE_PCT=99 # Skip weekly-exhausted accounts
|
||||
# CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES=2
|
||||
# CODEX_WARMUP_FAILURE_COOLDOWN_MS=21600000 # Back off 6h after repeated failure
|
||||
|
||||
# --- Fallback provider (when Claude/Codex both unavailable) ---
|
||||
# FALLBACK_ENABLED=true # Enable fallback provider
|
||||
# FALLBACK_PROVIDER_NAME=kimi # Provider name for logging
|
||||
|
||||
268
src/codex-warmup.test.ts
Normal file
268
src/codex-warmup.test.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('child_process', () => ({
|
||||
spawn: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./codex-token-rotation.js', () => ({
|
||||
getAllCodexAccounts: vi.fn(),
|
||||
getCodexAuthPath: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./logger.js', () => ({
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
type FakeProcess = EventEmitter & {
|
||||
stdout: EventEmitter;
|
||||
stderr: EventEmitter;
|
||||
stdin: { write: ReturnType<typeof vi.fn>; end: ReturnType<typeof vi.fn> };
|
||||
kill: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
function createFakeCodexProcess(exitCode = 0): FakeProcess {
|
||||
const proc = new EventEmitter() as FakeProcess;
|
||||
proc.stdout = new EventEmitter();
|
||||
proc.stderr = new EventEmitter();
|
||||
proc.stdin = { write: vi.fn(), end: vi.fn() };
|
||||
proc.kill = vi.fn();
|
||||
setImmediate(() => {
|
||||
proc.stdout.emit('data', Buffer.from('OK\n'));
|
||||
proc.emit('close', exitCode, null);
|
||||
});
|
||||
return proc;
|
||||
}
|
||||
|
||||
function authPathFor(tempHome: string, accountIndex: number): string {
|
||||
return path.join(
|
||||
tempHome,
|
||||
'.codex-accounts',
|
||||
String(accountIndex + 1),
|
||||
'auth.json',
|
||||
);
|
||||
}
|
||||
|
||||
describe('Codex warm-up scheduler', () => {
|
||||
let tempHome: string;
|
||||
let statePath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
tempHome = fs.mkdtempSync(path.join('/tmp', 'ejclaw-codex-warmup-'));
|
||||
statePath = path.join(tempHome, 'codex-warmup-state.json');
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const p = authPathFor(tempHome, i);
|
||||
fs.mkdirSync(path.dirname(p), { recursive: true });
|
||||
fs.writeFileSync(p, '{}');
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('warms one eligible zero-usage account with a real codex exec and persists account cooldown state', async () => {
|
||||
const childProcess = await import('child_process');
|
||||
const rotation = await import('./codex-token-rotation.js');
|
||||
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||
const now = new Date('2026-04-24T09:00:00Z').getTime();
|
||||
let capturedEnv: Record<string, string> | undefined;
|
||||
let capturedArgs: readonly string[] | undefined;
|
||||
|
||||
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'busy-account',
|
||||
planType: 'pro',
|
||||
isActive: true,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 14,
|
||||
cachedUsageD7Pct: 30,
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
accountId: 'rate-limited-account',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: true,
|
||||
cachedUsagePct: 0,
|
||||
cachedUsageD7Pct: 0,
|
||||
},
|
||||
{
|
||||
index: 2,
|
||||
accountId: 'fresh-account',
|
||||
planType: 'team',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 0,
|
||||
cachedUsageD7Pct: 0,
|
||||
},
|
||||
]);
|
||||
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||
);
|
||||
vi.mocked(childProcess.spawn).mockImplementation(((
|
||||
_cmd: string,
|
||||
args?: readonly string[],
|
||||
opts?: { env?: Record<string, string> },
|
||||
) => {
|
||||
capturedArgs = args;
|
||||
capturedEnv = opts?.env;
|
||||
return createFakeCodexProcess(0) as never;
|
||||
}) as unknown as typeof childProcess.spawn);
|
||||
|
||||
const result = await runCodexWarmupCycle(
|
||||
{
|
||||
enabled: true,
|
||||
prompt: 'Reply exactly OK. Do not run tools.',
|
||||
model: 'gpt-5.5',
|
||||
intervalMs: 300_000,
|
||||
minIntervalMs: 18_300_000,
|
||||
staggerMs: 1_800_000,
|
||||
maxUsagePct: 0,
|
||||
maxD7UsagePct: 99,
|
||||
commandTimeoutMs: 120_000,
|
||||
failureCooldownMs: 21_600_000,
|
||||
maxConsecutiveFailures: 2,
|
||||
},
|
||||
{ nowMs: now, statePath },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ status: 'warmed', accountIndex: 2 });
|
||||
expect(childProcess.spawn).toHaveBeenCalledTimes(1);
|
||||
expect(capturedArgs).toEqual(
|
||||
expect.arrayContaining([
|
||||
'exec',
|
||||
'--ephemeral',
|
||||
'--ignore-rules',
|
||||
'--skip-git-repo-check',
|
||||
'--sandbox',
|
||||
'read-only',
|
||||
'-m',
|
||||
'gpt-5.5',
|
||||
'Reply exactly OK. Do not run tools.',
|
||||
]),
|
||||
);
|
||||
expect(capturedEnv?.CODEX_HOME).toBe(
|
||||
path.dirname(authPathFor(tempHome, 2)),
|
||||
);
|
||||
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||
expect(state.lastWarmupAt).toBe('2026-04-24T09:00:00.000Z');
|
||||
expect(state.accounts['2'].lastWarmupAt).toBe('2026-04-24T09:00:00.000Z');
|
||||
expect(state.consecutiveFailures).toBe(0);
|
||||
});
|
||||
|
||||
it('auto-backs off after repeated codex exec failures so OpenAI-side blocking does not hammer accounts', async () => {
|
||||
const childProcess = await import('child_process');
|
||||
const rotation = await import('./codex-token-rotation.js');
|
||||
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||
const now = new Date('2026-04-24T09:00:00Z').getTime();
|
||||
|
||||
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||
{
|
||||
index: 0,
|
||||
accountId: 'fresh-account',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 0,
|
||||
cachedUsageD7Pct: 0,
|
||||
},
|
||||
]);
|
||||
vi.mocked(rotation.getCodexAuthPath).mockImplementation(
|
||||
(accountIndex = 0) => authPathFor(tempHome, accountIndex),
|
||||
);
|
||||
vi.mocked(childProcess.spawn).mockImplementation(
|
||||
() => createFakeCodexProcess(1) as never,
|
||||
);
|
||||
|
||||
const config = {
|
||||
enabled: true,
|
||||
prompt: 'Reply exactly OK. Do not run tools.',
|
||||
model: 'gpt-5.5',
|
||||
intervalMs: 300_000,
|
||||
minIntervalMs: 18_300_000,
|
||||
staggerMs: 0,
|
||||
maxUsagePct: 0,
|
||||
maxD7UsagePct: 99,
|
||||
commandTimeoutMs: 120_000,
|
||||
failureCooldownMs: 21_600_000,
|
||||
maxConsecutiveFailures: 1,
|
||||
};
|
||||
|
||||
const failed = await runCodexWarmupCycle(config, { nowMs: now, statePath });
|
||||
expect(failed.status).toBe('failed');
|
||||
|
||||
const backedOff = await runCodexWarmupCycle(config, {
|
||||
nowMs: now + 60_000,
|
||||
statePath,
|
||||
});
|
||||
|
||||
expect(backedOff).toEqual({
|
||||
status: 'skipped',
|
||||
reason: 'disabled_cooldown',
|
||||
});
|
||||
expect(childProcess.spawn).toHaveBeenCalledTimes(1);
|
||||
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||
expect(state.disabledUntil).toBe('2026-04-24T15:00:00.000Z');
|
||||
expect(state.consecutiveFailures).toBe(1);
|
||||
});
|
||||
|
||||
it('respects global stagger and does not warm another account too soon', async () => {
|
||||
const childProcess = await import('child_process');
|
||||
const rotation = await import('./codex-token-rotation.js');
|
||||
const { runCodexWarmupCycle } = await import('./codex-warmup.js');
|
||||
const now = new Date('2026-04-24T09:20:00Z').getTime();
|
||||
|
||||
fs.writeFileSync(
|
||||
statePath,
|
||||
JSON.stringify({
|
||||
lastWarmupAt: '2026-04-24T09:00:00.000Z',
|
||||
consecutiveFailures: 0,
|
||||
accounts: { '0': { lastWarmupAt: '2026-04-24T09:00:00.000Z' } },
|
||||
}),
|
||||
);
|
||||
vi.mocked(rotation.getAllCodexAccounts).mockReturnValue([
|
||||
{
|
||||
index: 1,
|
||||
accountId: 'another-fresh-account',
|
||||
planType: 'pro',
|
||||
isActive: false,
|
||||
isRateLimited: false,
|
||||
cachedUsagePct: 0,
|
||||
cachedUsageD7Pct: 0,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await runCodexWarmupCycle(
|
||||
{
|
||||
enabled: true,
|
||||
prompt: '.',
|
||||
model: 'gpt-5.5',
|
||||
intervalMs: 300_000,
|
||||
minIntervalMs: 18_300_000,
|
||||
staggerMs: 1_800_000,
|
||||
maxUsagePct: 0,
|
||||
maxD7UsagePct: 99,
|
||||
commandTimeoutMs: 120_000,
|
||||
failureCooldownMs: 21_600_000,
|
||||
maxConsecutiveFailures: 2,
|
||||
},
|
||||
{ nowMs: now, statePath },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ status: 'skipped', reason: 'stagger_wait' });
|
||||
expect(childProcess.spawn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
294
src/codex-warmup.ts
Normal file
294
src/codex-warmup.ts
Normal file
@@ -0,0 +1,294 @@
|
||||
import { ChildProcess, spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
import { DATA_DIR } from './config.js';
|
||||
import type { AppConfig } from './config/schema.js';
|
||||
import {
|
||||
getAllCodexAccounts,
|
||||
getCodexAuthPath,
|
||||
} from './codex-token-rotation.js';
|
||||
import { logger } from './logger.js';
|
||||
|
||||
export type CodexWarmupConfig = AppConfig['codexWarmup'];
|
||||
|
||||
interface CodexWarmupAccountState {
|
||||
lastWarmupAt?: string;
|
||||
lastAttemptAt?: string;
|
||||
lastErrorAt?: string;
|
||||
failures?: number;
|
||||
}
|
||||
|
||||
interface CodexWarmupState {
|
||||
lastWarmupAt?: string;
|
||||
lastAttemptAt?: string;
|
||||
disabledUntil?: string;
|
||||
consecutiveFailures?: number;
|
||||
accounts?: Record<string, CodexWarmupAccountState>;
|
||||
}
|
||||
|
||||
interface CodexWarmupRuntimeOptions {
|
||||
nowMs?: number;
|
||||
statePath?: string;
|
||||
shouldSkip?: () => boolean;
|
||||
}
|
||||
|
||||
export type CodexWarmupCycleResult =
|
||||
| { status: 'disabled' }
|
||||
| { status: 'skipped'; reason: string }
|
||||
| { status: 'warmed'; accountIndex: number }
|
||||
| { status: 'failed'; accountIndex: number; reason: string };
|
||||
|
||||
const DEFAULT_STATE_FILE = path.join(DATA_DIR, 'codex-warmup-state.json');
|
||||
|
||||
function parseTimestamp(value?: string): number | null {
|
||||
if (!value) return null;
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function readWarmupState(statePath: string): CodexWarmupState {
|
||||
try {
|
||||
if (!fs.existsSync(statePath)) return { accounts: {} };
|
||||
const parsed = JSON.parse(
|
||||
fs.readFileSync(statePath, 'utf8'),
|
||||
) as CodexWarmupState;
|
||||
if (!parsed.accounts || typeof parsed.accounts !== 'object') {
|
||||
parsed.accounts = {};
|
||||
}
|
||||
return parsed;
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, statePath },
|
||||
'Failed to read Codex warm-up state; starting fresh',
|
||||
);
|
||||
return { accounts: {} };
|
||||
}
|
||||
}
|
||||
|
||||
function writeWarmupState(statePath: string, state: CodexWarmupState): void {
|
||||
fs.mkdirSync(path.dirname(statePath), { recursive: true });
|
||||
fs.writeFileSync(`${statePath}.tmp`, `${JSON.stringify(state, null, 2)}\n`);
|
||||
fs.renameSync(`${statePath}.tmp`, statePath);
|
||||
}
|
||||
|
||||
function getPreferredCodexPathEntries(): string[] {
|
||||
const entries = [
|
||||
path.dirname(process.execPath),
|
||||
path.join(os.homedir(), '.npm-global', 'bin'),
|
||||
];
|
||||
if (process.versions.bun || path.basename(process.execPath) === 'bun') {
|
||||
entries.push(path.join(os.homedir(), '.hermes', 'node', 'bin'));
|
||||
}
|
||||
return [...new Set(entries)];
|
||||
}
|
||||
|
||||
function resolveCodexBinary(): string {
|
||||
const npmGlobalBin = path.join(os.homedir(), '.npm-global', 'bin', 'codex');
|
||||
return fs.existsSync(npmGlobalBin) ? npmGlobalBin : 'codex';
|
||||
}
|
||||
|
||||
function ensureAccountState(
|
||||
state: CodexWarmupState,
|
||||
accountIndex: number,
|
||||
): CodexWarmupAccountState {
|
||||
state.accounts ??= {};
|
||||
const key = String(accountIndex);
|
||||
state.accounts[key] ??= {};
|
||||
return state.accounts[key];
|
||||
}
|
||||
|
||||
function selectWarmupCandidate(
|
||||
config: CodexWarmupConfig,
|
||||
state: CodexWarmupState,
|
||||
nowMs: number,
|
||||
): { accountIndex: number } | { reason: string } {
|
||||
const disabledUntilMs = parseTimestamp(state.disabledUntil);
|
||||
if (disabledUntilMs != null && disabledUntilMs > nowMs) {
|
||||
return { reason: 'disabled_cooldown' };
|
||||
}
|
||||
|
||||
const lastGlobalWarmupMs = parseTimestamp(state.lastWarmupAt);
|
||||
if (
|
||||
lastGlobalWarmupMs != null &&
|
||||
config.staggerMs > 0 &&
|
||||
nowMs - lastGlobalWarmupMs < config.staggerMs
|
||||
) {
|
||||
return { reason: 'stagger_wait' };
|
||||
}
|
||||
|
||||
const accounts = getAllCodexAccounts();
|
||||
if (accounts.length === 0) return { reason: 'no_accounts' };
|
||||
|
||||
for (const account of accounts) {
|
||||
if (account.isRateLimited) continue;
|
||||
if (typeof account.cachedUsagePct !== 'number') continue;
|
||||
if (typeof account.cachedUsageD7Pct !== 'number') continue;
|
||||
if (account.cachedUsagePct > config.maxUsagePct) continue;
|
||||
if (account.cachedUsageD7Pct > config.maxD7UsagePct) continue;
|
||||
|
||||
const accountState = state.accounts?.[String(account.index)];
|
||||
const lastWarmupMs = parseTimestamp(accountState?.lastWarmupAt);
|
||||
if (lastWarmupMs != null && nowMs - lastWarmupMs < config.minIntervalMs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return { accountIndex: account.index };
|
||||
}
|
||||
|
||||
return { reason: 'no_eligible_accounts' };
|
||||
}
|
||||
|
||||
function runCodexWarmupCommand(
|
||||
accountDir: string,
|
||||
config: CodexWarmupConfig,
|
||||
): Promise<{ ok: boolean; reason: string }> {
|
||||
const args = [
|
||||
'exec',
|
||||
'--ephemeral',
|
||||
'--ignore-rules',
|
||||
'--skip-git-repo-check',
|
||||
'--sandbox',
|
||||
'read-only',
|
||||
'-C',
|
||||
os.tmpdir(),
|
||||
'-m',
|
||||
config.model,
|
||||
config.prompt,
|
||||
];
|
||||
|
||||
const spawnEnv: Record<string, string> = {
|
||||
...(process.env as Record<string, string>),
|
||||
CODEX_HOME: accountDir,
|
||||
PATH: [...getPreferredCodexPathEntries(), process.env.PATH || '']
|
||||
.filter(Boolean)
|
||||
.join(path.delimiter),
|
||||
};
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let done = false;
|
||||
let proc: ChildProcess | null = null;
|
||||
let stderr = '';
|
||||
const finish = (result: { ok: boolean; reason: string }) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
if (proc && result.reason === 'timeout') {
|
||||
try {
|
||||
proc.kill();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
const timer = setTimeout(
|
||||
() => finish({ ok: false, reason: 'timeout' }),
|
||||
config.commandTimeoutMs,
|
||||
);
|
||||
|
||||
try {
|
||||
proc = spawn(resolveCodexBinary(), args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: spawnEnv,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Failed to spawn Codex warm-up command');
|
||||
finish({ ok: false, reason: 'spawn_error' });
|
||||
return;
|
||||
}
|
||||
|
||||
proc.stderr?.on('data', (chunk: Buffer) => {
|
||||
stderr += chunk.toString();
|
||||
if (stderr.length > 2000) stderr = stderr.slice(-2000);
|
||||
});
|
||||
proc.on('error', (err) => {
|
||||
logger.warn({ err }, 'Codex warm-up process error');
|
||||
finish({ ok: false, reason: 'process_error' });
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
finish({ ok: true, reason: 'ok' });
|
||||
return;
|
||||
}
|
||||
logger.warn(
|
||||
{ exitCode: code, stderr: stderr.trim() || undefined },
|
||||
'Codex warm-up command failed',
|
||||
);
|
||||
finish({ ok: false, reason: `exit_${code ?? 'unknown'}` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function runCodexWarmupCycle(
|
||||
config: CodexWarmupConfig,
|
||||
runtime: CodexWarmupRuntimeOptions = {},
|
||||
): Promise<CodexWarmupCycleResult> {
|
||||
if (!config.enabled) return { status: 'disabled' };
|
||||
if (runtime.shouldSkip?.())
|
||||
return { status: 'skipped', reason: 'runtime_busy' };
|
||||
|
||||
const nowMs = runtime.nowMs ?? Date.now();
|
||||
const nowIso = new Date(nowMs).toISOString();
|
||||
const statePath = runtime.statePath ?? DEFAULT_STATE_FILE;
|
||||
const state = readWarmupState(statePath);
|
||||
const selected = selectWarmupCandidate(config, state, nowMs);
|
||||
if ('reason' in selected) {
|
||||
return { status: 'skipped', reason: selected.reason };
|
||||
}
|
||||
|
||||
const authPath = getCodexAuthPath(selected.accountIndex);
|
||||
if (!authPath || !fs.existsSync(authPath)) {
|
||||
return { status: 'skipped', reason: 'missing_auth' };
|
||||
}
|
||||
const accountDir = path.dirname(authPath);
|
||||
const accountState = ensureAccountState(state, selected.accountIndex);
|
||||
state.lastAttemptAt = nowIso;
|
||||
accountState.lastAttemptAt = nowIso;
|
||||
|
||||
logger.info(
|
||||
{ account: selected.accountIndex + 1, model: config.model },
|
||||
'Starting Codex warm-up prompt',
|
||||
);
|
||||
const result = await runCodexWarmupCommand(accountDir, config);
|
||||
|
||||
if (result.ok) {
|
||||
state.lastWarmupAt = nowIso;
|
||||
state.consecutiveFailures = 0;
|
||||
delete state.disabledUntil;
|
||||
accountState.lastWarmupAt = nowIso;
|
||||
accountState.failures = 0;
|
||||
writeWarmupState(statePath, state);
|
||||
logger.info(
|
||||
{ account: selected.accountIndex + 1 },
|
||||
'Codex warm-up prompt completed',
|
||||
);
|
||||
return { status: 'warmed', accountIndex: selected.accountIndex };
|
||||
}
|
||||
|
||||
state.consecutiveFailures = (state.consecutiveFailures ?? 0) + 1;
|
||||
accountState.lastErrorAt = nowIso;
|
||||
accountState.failures = (accountState.failures ?? 0) + 1;
|
||||
if (state.consecutiveFailures >= config.maxConsecutiveFailures) {
|
||||
state.disabledUntil = new Date(
|
||||
nowMs + config.failureCooldownMs,
|
||||
).toISOString();
|
||||
}
|
||||
writeWarmupState(statePath, state);
|
||||
logger.warn(
|
||||
{
|
||||
account: selected.accountIndex + 1,
|
||||
reason: result.reason,
|
||||
consecutiveFailures: state.consecutiveFailures,
|
||||
disabledUntil: state.disabledUntil,
|
||||
},
|
||||
'Codex warm-up prompt failed',
|
||||
);
|
||||
return {
|
||||
status: 'failed',
|
||||
accountIndex: selected.accountIndex,
|
||||
reason: result.reason,
|
||||
};
|
||||
}
|
||||
@@ -31,6 +31,17 @@ describe('config/env loading', () => {
|
||||
delete process.env.DISCORD_CODEX_REVIEW_BOT_TOKEN;
|
||||
delete process.env.PAIRED_CARRY_FORWARD_LATEST_OWNER_FINAL;
|
||||
delete process.env.PAIRED_FORCE_FRESH_CLAUDE_REVIEWER_SESSION;
|
||||
delete process.env.CODEX_WARMUP_ENABLED;
|
||||
delete process.env.CODEX_WARMUP_PROMPT;
|
||||
delete process.env.CODEX_WARMUP_MODEL;
|
||||
delete process.env.CODEX_WARMUP_INTERVAL_MS;
|
||||
delete process.env.CODEX_WARMUP_MIN_INTERVAL_MS;
|
||||
delete process.env.CODEX_WARMUP_STAGGER_MS;
|
||||
delete process.env.CODEX_WARMUP_MAX_USAGE_PCT;
|
||||
delete process.env.CODEX_WARMUP_MAX_D7_USAGE_PCT;
|
||||
delete process.env.CODEX_WARMUP_COMMAND_TIMEOUT_MS;
|
||||
delete process.env.CODEX_WARMUP_FAILURE_COOLDOWN_MS;
|
||||
delete process.env.CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES;
|
||||
delete process.env.SESSION_COMMAND_USER_IDS;
|
||||
vi.resetModules();
|
||||
});
|
||||
@@ -124,6 +135,33 @@ describe('config/env loading', () => {
|
||||
expect(config.PAIRED_FORCE_FRESH_CLAUDE_REVIEWER_SESSION).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps Codex warm-up disabled by default and exposes conservative opt-in env config', async () => {
|
||||
let config = await import('./config.js');
|
||||
expect(config.CODEX_WARMUP_CONFIG.enabled).toBe(false);
|
||||
expect(config.CODEX_WARMUP_CONFIG.maxUsagePct).toBe(0);
|
||||
expect(
|
||||
config.CODEX_WARMUP_CONFIG.maxConsecutiveFailures,
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
|
||||
vi.resetModules();
|
||||
process.env.CODEX_MODEL = 'gpt-5.5';
|
||||
process.env.CODEX_WARMUP_ENABLED = 'true';
|
||||
process.env.CODEX_WARMUP_PROMPT = '.';
|
||||
process.env.CODEX_WARMUP_STAGGER_MS = '600000';
|
||||
process.env.CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES = '1';
|
||||
config = await import('./config.js');
|
||||
|
||||
expect(config.CODEX_WARMUP_CONFIG).toEqual(
|
||||
expect.objectContaining({
|
||||
enabled: true,
|
||||
prompt: '.',
|
||||
model: 'gpt-5.5',
|
||||
staggerMs: 600000,
|
||||
maxConsecutiveFailures: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('fails fast when a legacy Discord token alias is configured', async () => {
|
||||
process.env.DISCORD_BOT_TOKEN = 'legacy-owner-token';
|
||||
|
||||
|
||||
@@ -140,6 +140,7 @@ export const USAGE_UPDATE_INTERVAL = CONFIG.status.usageUpdateInterval;
|
||||
export const STATUS_SHOW_ROOMS = CONFIG.status.showRooms;
|
||||
export const STATUS_SHOW_ROOM_DETAILS = CONFIG.status.showRoomDetails;
|
||||
export const USAGE_DASHBOARD_ENABLED = CONFIG.status.usageDashboardEnabled;
|
||||
export const CODEX_WARMUP_CONFIG = CONFIG.codexWarmup;
|
||||
|
||||
// Timezone for scheduled tasks (cron expressions, etc.)
|
||||
// Uses system timezone by default
|
||||
|
||||
@@ -53,6 +53,11 @@ function readIntegerAtLeast(
|
||||
return Math.max(minimum, readInteger(key, fallback) || fallback);
|
||||
}
|
||||
|
||||
function readPercent(key: string, fallback: number): number {
|
||||
const value = readInteger(key, fallback);
|
||||
return Math.min(100, Math.max(0, value));
|
||||
}
|
||||
|
||||
function readAgentType(
|
||||
key: string,
|
||||
fallback?: AgentType,
|
||||
@@ -255,6 +260,40 @@ export function loadConfig(): AppConfig {
|
||||
timezone:
|
||||
readText('TZ') ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||
},
|
||||
codexWarmup: {
|
||||
enabled: readBoolean('CODEX_WARMUP_ENABLED', false),
|
||||
prompt:
|
||||
readNonEmptyText('CODEX_WARMUP_PROMPT') ??
|
||||
'Reply exactly OK. Do not run tools.',
|
||||
model:
|
||||
readNonEmptyText('CODEX_WARMUP_MODEL') ??
|
||||
readNonEmptyText('CODEX_MODEL') ??
|
||||
'codex',
|
||||
intervalMs: readIntegerAtLeast('CODEX_WARMUP_INTERVAL_MS', 300000, 60000),
|
||||
minIntervalMs: readIntegerAtLeast(
|
||||
'CODEX_WARMUP_MIN_INTERVAL_MS',
|
||||
18300000,
|
||||
60000,
|
||||
),
|
||||
staggerMs: Math.max(0, readInteger('CODEX_WARMUP_STAGGER_MS', 1800000)),
|
||||
maxUsagePct: readPercent('CODEX_WARMUP_MAX_USAGE_PCT', 0),
|
||||
maxD7UsagePct: readPercent('CODEX_WARMUP_MAX_D7_USAGE_PCT', 99),
|
||||
commandTimeoutMs: readIntegerAtLeast(
|
||||
'CODEX_WARMUP_COMMAND_TIMEOUT_MS',
|
||||
120000,
|
||||
10000,
|
||||
),
|
||||
failureCooldownMs: readIntegerAtLeast(
|
||||
'CODEX_WARMUP_FAILURE_COOLDOWN_MS',
|
||||
21600000,
|
||||
60000,
|
||||
),
|
||||
maxConsecutiveFailures: readIntegerAtLeast(
|
||||
'CODEX_WARMUP_MAX_CONSECUTIVE_FAILURES',
|
||||
2,
|
||||
1,
|
||||
),
|
||||
},
|
||||
sessionCommands: {
|
||||
allowedSenders: new Set(
|
||||
(readText('SESSION_COMMAND_ALLOWED_SENDERS') ?? '')
|
||||
|
||||
@@ -78,6 +78,19 @@ export interface AppConfig {
|
||||
usageDashboardEnabled: boolean;
|
||||
timezone: string;
|
||||
};
|
||||
codexWarmup: {
|
||||
enabled: boolean;
|
||||
prompt: string;
|
||||
model: string;
|
||||
intervalMs: number;
|
||||
minIntervalMs: number;
|
||||
staggerMs: number;
|
||||
maxUsagePct: number;
|
||||
maxD7UsagePct: number;
|
||||
commandTimeoutMs: number;
|
||||
failureCooldownMs: number;
|
||||
maxConsecutiveFailures: number;
|
||||
};
|
||||
sessionCommands: {
|
||||
allowedSenders: Set<string>;
|
||||
};
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
STATUS_SHOW_ROOM_DETAILS,
|
||||
STATUS_SHOW_ROOMS,
|
||||
USAGE_DASHBOARD_ENABLED,
|
||||
CODEX_WARMUP_CONFIG,
|
||||
getMoaConfig,
|
||||
} from './config.js';
|
||||
import { fetchKimiUsage, buildKimiUsageRows } from './kimi-usage.js';
|
||||
@@ -28,6 +29,7 @@ import {
|
||||
refreshActiveCodexUsage,
|
||||
refreshAllCodexAccountUsage,
|
||||
} from './codex-usage-collector.js';
|
||||
import { runCodexWarmupCycle } from './codex-warmup.js';
|
||||
import {
|
||||
composeDashboardContent,
|
||||
formatElapsed,
|
||||
@@ -735,18 +737,49 @@ export async function startUnifiedDashboard(
|
||||
cachedCodexUsageRows = result.rows;
|
||||
if (result.fetchedAt) codexUsageFetchedAt = result.fetchedAt;
|
||||
};
|
||||
void refreshAllCodexAccountUsage().then((r) => {
|
||||
applyCodexRefresh(r);
|
||||
return refreshActiveCodexUsage().then(applyCodexRefresh);
|
||||
});
|
||||
const isWarmupRuntimeBusy = () => {
|
||||
const groups = opts.roomBindings();
|
||||
return opts.queue
|
||||
.getStatuses(Object.keys(groups))
|
||||
.some((status) => status.status === 'processing');
|
||||
};
|
||||
let codexWarmupInFlight = false;
|
||||
const runCodexWarmup = async () => {
|
||||
if (!CODEX_WARMUP_CONFIG.enabled || codexWarmupInFlight) return;
|
||||
codexWarmupInFlight = true;
|
||||
try {
|
||||
const result = await runCodexWarmupCycle(CODEX_WARMUP_CONFIG, {
|
||||
shouldSkip: isWarmupRuntimeBusy,
|
||||
});
|
||||
if (result.status === 'warmed') {
|
||||
applyCodexRefresh(await refreshAllCodexAccountUsage());
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ err }, 'Codex warm-up cycle failed unexpectedly');
|
||||
} finally {
|
||||
codexWarmupInFlight = false;
|
||||
}
|
||||
};
|
||||
void refreshAllCodexAccountUsage()
|
||||
.then((r) => {
|
||||
applyCodexRefresh(r);
|
||||
return refreshActiveCodexUsage().then(applyCodexRefresh);
|
||||
})
|
||||
.then(() => runCodexWarmup());
|
||||
setInterval(
|
||||
() => void refreshActiveCodexUsage().then(applyCodexRefresh),
|
||||
opts.usageUpdateInterval,
|
||||
);
|
||||
setInterval(
|
||||
() => void refreshAllCodexAccountUsage().then(applyCodexRefresh),
|
||||
() =>
|
||||
void refreshAllCodexAccountUsage()
|
||||
.then(applyCodexRefresh)
|
||||
.then(() => runCodexWarmup()),
|
||||
CODEX_FULL_SCAN_INTERVAL,
|
||||
);
|
||||
if (CODEX_WARMUP_CONFIG.enabled) {
|
||||
setInterval(() => void runCodexWarmup(), CODEX_WARMUP_CONFIG.intervalMs);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user