perf: cut background polling and log noise

- Codex usage poll: run the repo-pinned CLI (0.159.3) instead of a global
  0.128 install, hold the account lease while polling (skip if a run holds
  it, so the poller cannot rotate a token under a live reviewer session), and
  poll every 5 min instead of every minute (~1,440 app-server spawns a day).
- Claude usage poll: 5 min instead of 1 min. The 1-minute cadence only served
  the auto-paired-mode gate, which is not wired into the runtime; it drew ~32
  HTTP 429s a day.
- Dashboard duplicate cleanup: was a 2 s poll fetching 100 messages (~43k
  Discord REST calls a day) plus a fetch after every edit. Clean up only when a
  fresh status message is posted or on first render, with a 5 min sweep.
- Logging: one line per event and no ANSI colors outside a TTY (events were
  split into ~11 journal lines). Progress-edit audits, Discord progress edits,
  streamed-activity and per-bucket Codex usage logs move to debug; sends,
  agent output and the usage summary stay at info.
- Two tests wrote under the real data/ipc and left fake room folders that the
  live 1 s IPC watcher kept polling; they now clean up after themselves.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-10-01 18:32:14 +09:00
parent de2de33c3e
commit f1abe037e9
15 changed files with 233 additions and 36 deletions

View File

@@ -417,7 +417,8 @@ function readPairedRolePrompt(
projectRoot: string,
role: PairedRoomRole | undefined,
): string | undefined {
if (role === 'reviewer') return readPairedRoomPrompt('claude-code', projectRoot);
if (role === 'reviewer')
return readPairedRoomPrompt('claude-code', projectRoot);
if (role === 'arbiter') return readArbiterPrompt(projectRoot);
return readOptionalPromptFile(projectRoot, 'owner-common-paired-room.md');
}

View File

@@ -801,7 +801,7 @@ export class DiscordChannel implements Channel {
});
}
}
logger.info(
logger.debug(
{
jid,
channelName: this.name,

View File

@@ -128,16 +128,17 @@ describe('Claude usage 429 Retry-After backoff', () => {
expect(held[0].usageRateLimited).toBe(true);
});
it('keeps backing off after the 60s throttle elapses but before the cooldown closes', async () => {
// This is the core regression: a 93s Retry-After must outlast the 60s
// MIN_FETCH_INTERVAL throttle. Without the cooldown the poller would refetch
// at ~60s — mid rate-limit window — and re-trip the 429 forever.
it('keeps backing off after the 5-minute throttle elapses but before the cooldown closes', async () => {
// This is the core regression: a Retry-After longer than the 5-minute
// MIN_FETCH_INTERVAL throttle must still be honoured. Without the cooldown
// the poller would refetch once the throttle elapses — mid rate-limit
// window — and re-trip the 429 forever.
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-06-20T00:00:00Z'));
const fetchMock = vi
.fn()
.mockResolvedValueOnce(makeResponse(429, { 'retry-after': '93' }))
.mockResolvedValueOnce(makeResponse(429, { 'retry-after': '393' }))
.mockResolvedValueOnce(makeResponse(200));
vi.stubGlobal('fetch', fetchMock);
@@ -146,13 +147,13 @@ describe('Claude usage 429 Retry-After backoff', () => {
await fetchAllClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
// +70s: past the 60s throttle, still inside the ~98s cooldown → no refetch.
vi.setSystemTime(Date.now() + 70_000);
// +370s: past the 300s throttle, still inside the ~398s cooldown → no refetch.
vi.setSystemTime(Date.now() + 370_000);
const held = await fetchAllClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(held[0].usageRateLimited).toBe(true);
// +100s total: cooldown window closed → poller is allowed to fetch again.
// +400s total: cooldown window closed → poller is allowed to fetch again.
vi.setSystemTime(Date.now() + 30_000);
const resumed = await fetchAllClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(2);

View File

@@ -149,10 +149,11 @@ export function getUsageCacheReadKeys(
return keys;
}
// Rate limit: at most one API call per token per 1 minute.
// We poll usage every minute (was 5) so the 95% exhaustion gate has tight
// resolution — at worst the gate fires ~60s after a window crosses 95%.
const MIN_FETCH_INTERVAL_MS = 60_000;
// Rate limit: at most one API call per token per 5 minutes.
// The 1-minute cadence existed for the 95% auto-paired-mode gate, which is no
// longer wired into the runtime. The dashboard only needs coarse numbers, and
// polling every minute drew ~32 HTTP 429s a day from the usage endpoint.
const MIN_FETCH_INTERVAL_MS = 300_000;
// Small margin added on top of a server-provided Retry-After so we wait until
// just *after* the cooldown window closes rather than racing its edge.

View File

@@ -625,6 +625,41 @@ export async function waitForCodexAuthLease(opts?: {
}
}
/**
* Lease one specific account for a short side task (usage polling).
*
* Unlike claimCodexAuthLease() this ignores rate limits (polling a limited
* account is how its reset time is learned) and never rotates to another
* account. Returns null when that account is already leased by a live run, so
* the caller can simply skip — the lease exists so two Codex processes never
* refresh the same token concurrently.
*/
export function claimCodexAuthLeaseForAccount(
accountIndex: number,
): CodexAuthLease | null {
const acct = accounts.find((a) => a.index === accountIndex);
if (!acct) return null;
const now = Date.now();
if (isCodexAccountLeasedByLiveRun(acct, now)) return null;
const leaseId = `${process.pid}-${now}-${acct.index}-poll-${Math.random()
.toString(36)
.slice(2)}`;
if (!tryAcquireDiskLease(acct, leaseId, now)) return null;
acct.leasedUntil = now + CODEX_AUTH_LEASE_MS;
acct.leaseId = leaseId;
return {
accountIndex: acct.index,
authPath: acct.authPath,
release: () => {
if (acct.leaseId === leaseId) {
acct.leasedUntil = null;
acct.leaseId = null;
}
releaseDiskLease(acct.authPath, leaseId);
},
};
}
export function claimCodexAuthLease(): CodexAuthLease | null {
if (accounts.length === 0) return null;
const now = Date.now();

View File

@@ -157,6 +157,66 @@ describe('codex-usage-collector fallback account usage', () => {
]);
});
it('runs the repo-pinned Codex CLI rather than a global codex binary', async () => {
createDefaultCodexAuth(tempHome);
const childProcess = await import('child_process');
let captured: { cmd: string; args: readonly string[] | undefined } | null =
null;
vi.mocked(childProcess.spawn).mockImplementation(((
cmd: string,
args: readonly string[] | undefined,
) => {
captured = { cmd, args };
return createFakeChildProcess({
codex: {
limitName: 'Codex',
primary: { usedPercent: 1, resetsAt: new Date().toISOString() },
secondary: { usedPercent: 1, resetsAt: new Date().toISOString() },
},
}) as never;
}) as unknown as typeof childProcess.spawn);
const rotation = await import('./codex-token-rotation.js');
const usage = await import('./codex-usage-collector.js');
rotation.initCodexTokenRotation();
await usage.refreshActiveCodexUsage();
expect(captured).not.toBeNull();
expect(captured!.cmd).toBe(process.execPath);
expect(captured!.args).toEqual([
path.join(
process.cwd(),
'node_modules',
'@openai',
'codex',
'bin',
'codex.js',
),
'app-server',
]);
});
it('skips the poll instead of spawning Codex while a run holds the account lease', async () => {
createDefaultCodexAuth(tempHome);
const childProcess = await import('child_process');
vi.mocked(childProcess.spawn).mockImplementation((() => {
throw new Error('must not spawn while the account is leased');
}) as unknown as typeof childProcess.spawn);
const rotation = await import('./codex-token-rotation.js');
const usage = await import('./codex-usage-collector.js');
rotation.initCodexTokenRotation();
const held = rotation.claimCodexAuthLease();
expect(held).not.toBeNull();
try {
const result = await usage.refreshActiveCodexUsage();
expect(result.fetchedAt).toBeNull();
expect(childProcess.spawn).not.toHaveBeenCalled();
} finally {
held?.release();
}
});
it('handles a lone weekly window (secondary: null) without throwing and routes it to 7d', async () => {
createDefaultCodexAuth(tempHome);
const childProcess = await import('child_process');
@@ -201,6 +261,9 @@ describe('codex-usage-collector fallback account usage', () => {
it('finds codex via ~/.hermes/node/bin when running under bun', async () => {
createDefaultCodexAuth(tempHome);
// PATH discovery only applies when the repo-pinned Codex package is absent.
const originalCwd = process.cwd();
process.chdir(tempHome);
const childProcess = await import('child_process');
const bunExecPath = path.join(tempHome, '.bun', 'bin', 'bun');
const hermesBin = path.join(tempHome, '.hermes', 'node', 'bin');
@@ -269,6 +332,7 @@ describe('codex-usage-collector fallback account usage', () => {
}),
]);
} finally {
process.chdir(originalCwd);
Object.defineProperty(process, 'execPath', {
configurable: true,
writable: true,

View File

@@ -4,6 +4,7 @@ import os from 'os';
import path from 'path';
import {
claimCodexAuthLeaseForAccount,
getAllCodexAccounts,
getCodexAuthPath,
updateCodexAccountUsage,
@@ -172,6 +173,27 @@ function findCodexBinary(): string {
return 'codex';
}
/**
* Run the Codex CLI that this repo pins (same package the reviewer runner
* loads), not whatever global `codex` happens to be first on the host. The
* global fnm install was 0.128 while the reviewer ran 0.159.3, so usage
* polling exercised a different CLI against the same account.
*/
function resolveCodexAppServerCommand(): { command: string; args: string[] } {
const pinnedEntry = path.join(
process.cwd(),
'node_modules',
'@openai',
'codex',
'bin',
'codex.js',
);
if (fs.existsSync(pinnedEntry)) {
return { command: process.execPath, args: [pinnedEntry, 'app-server'] };
}
return { command: findCodexBinary(), args: ['app-server'] };
}
function getCodexHomeForAccount(accountIndex?: number): string | null {
const authPath = getCodexAuthPath(accountIndex);
if (!authPath || !fs.existsSync(authPath)) return null;
@@ -181,7 +203,8 @@ function getCodexHomeForAccount(accountIndex?: number): string | null {
export async function fetchCodexUsage(
codexHomeOverride?: string,
): Promise<CodexRateLimit[] | null> {
const codexBin = findCodexBinary();
const { command: codexCommand, args: codexArgs } =
resolveCodexAppServerCommand();
return new Promise((resolve) => {
let done = false;
@@ -213,7 +236,7 @@ export async function fetchCodexUsage(
}
try {
proc = spawn(codexBin, ['app-server'], {
proc = spawn(codexCommand, codexArgs, {
stdio: ['pipe', 'pipe', 'pipe'],
env: spawnEnv,
});
@@ -293,8 +316,8 @@ export function applyCodexUsageToAccount(
): void {
if (usage.length === 0) return;
// Log all buckets for observability
logger.info(
// Per-bucket detail; the one-line usage summary below stays at info.
logger.debug(
{
account: accountIndex + 1,
buckets: usage.map((l) => ({
@@ -386,6 +409,34 @@ export function buildCodexUsageRowsFromState(): UsageRow[] {
* Scan ALL Codex accounts by spawning app-server with each auth.
* Returns refresh result — caller owns cache state.
*/
/**
* Poll one account's usage while holding that account's auth lease.
*
* The usage app-server runs with CODEX_HOME pointed at the account's canonical
* auth dir, so it may refresh the token there. Holding the lease keeps it from
* doing that while a reviewer run holds a session copy of the same token
* (which would leave the reviewer with an already-used refresh token). If a
* run holds the lease, skip this poll; the next one will catch up.
*/
async function fetchCodexUsageWithLease(
accountIndex: number,
accountDir: string,
): Promise<CodexRateLimit[] | null> {
const lease = claimCodexAuthLeaseForAccount(accountIndex);
if (!lease) {
logger.debug(
{ account: accountIndex + 1 },
'Skipped Codex usage poll: account is leased by a running Codex turn',
);
return null;
}
try {
return await fetchCodexUsage(accountDir);
} finally {
lease.release();
}
}
export async function refreshAllCodexAccountUsage(): Promise<CodexUsageRefreshResult> {
const codexAccounts = getAllCodexAccounts();
if (codexAccounts.length <= 1) {
@@ -403,7 +454,7 @@ export async function refreshAllCodexAccountUsage(): Promise<CodexUsageRefreshRe
if (!accountDir) continue;
try {
const usage = await fetchCodexUsage(accountDir);
const usage = await fetchCodexUsageWithLease(acct.index, accountDir);
if (usage && Array.isArray(usage) && usage.length > 0) {
applyCodexUsageToAccount(usage, acct.index);
anySuccess = true;
@@ -444,7 +495,7 @@ export async function refreshActiveCodexUsage(): Promise<CodexUsageRefreshResult
let fetchedAt: string | null = null;
try {
const usage = await fetchCodexUsage(accountDir);
const usage = await fetchCodexUsageWithLease(active.index, accountDir);
if (usage && Array.isArray(usage) && usage.length > 0) {
applyCodexUsageToAccount(usage, active.index);
fetchedAt = new Date().toISOString();

View File

@@ -102,6 +102,8 @@ import {
normalizeServiceId,
} from './config.js';
import {
resolveGroupIpcPath,
resolveGroupSessionsPath,
resolveTaskRuntimeIpcPath,
resolveTaskSessionsPath,
} from './group-folder.js';
@@ -1055,6 +1057,17 @@ Check the run.
expect(fs.existsSync(runtimeIpcDir)).toBe(false);
expect(fs.existsSync(taskSessionsDir)).toBe(false);
// Remove the group-level dirs too: this test writes under the real
// DATA_DIR, and a leftover data/ipc/<folder> is polled by the live IPC
// watcher as if it were a room.
fs.rmSync(resolveGroupIpcPath(groupFolder), {
recursive: true,
force: true,
});
fs.rmSync(resolveGroupSessionsPath(groupFolder), {
recursive: true,
force: true,
});
});
it('returns due tasks only for the requested agent type', () => {

View File

@@ -1,7 +1,7 @@
import fs from 'fs';
import path from 'path';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const { runHostEvidenceRequestMock } = vi.hoisted(() => ({
runHostEvidenceRequestMock: vi.fn(),
@@ -53,6 +53,15 @@ describe('host evidence IPC', () => {
});
});
afterEach(() => {
// Writes under the real DATA_DIR; don't leave a fake room folder behind
// for the live IPC watcher to poll.
fs.rmSync(resolveGroupIpcPath('other-group'), {
recursive: true,
force: true,
});
});
it('writes host evidence responses into the source group namespace', async () => {
runHostEvidenceRequestMock.mockResolvedValue({
ok: true,

View File

@@ -22,7 +22,12 @@ function createRootLogger(): Logger {
return pino({
...baseOptions,
transport: { target: 'pino-pretty', options: { colorize: true } },
// One line per event, colors only on an interactive terminal: multi-line
// colored output split each event into ~11 journal lines (~120k/day).
transport: {
target: 'pino-pretty',
options: { colorize: Boolean(process.stdout.isTTY), singleLine: true },
},
});
}

View File

@@ -210,7 +210,7 @@ class MessageAgentAttemptRunner {
if (outputPhase === 'final') {
return;
}
this.args.log.info(
this.args.log.debug(
{
provider: this.args.provider,
outputPhase,

View File

@@ -1697,7 +1697,7 @@ describe('runAgentForGroup room memory', () => {
onOutput: async () => {},
});
expect(logger.info).toHaveBeenCalledWith(
expect(logger.debug).toHaveBeenCalledWith(
expect.objectContaining({
provider: 'claude',
outputPhase: 'progress',

View File

@@ -133,7 +133,13 @@ export class MessageTurnController {
auditEvent: string,
extra: Record<string, unknown> = {},
): void {
this.log.info(
// Progress edits fire every few seconds per running turn; keep them out of
// the default log. Creates, fallbacks and final deliveries stay at info.
const log =
auditEvent === 'progress-edit'
? this.log.debug.bind(this.log)
: this.log.info.bind(this.log);
log(
this.buildOutboundAuditContext({
auditEvent,
...extra,

View File

@@ -114,9 +114,9 @@ describe('shouldPurgeDashboardChannelOnStart', () => {
});
describe('getDashboardDuplicateCleanupIntervalMs', () => {
it('polls duplicate cleanup faster than the status update interval', () => {
expect(getDashboardDuplicateCleanupIntervalMs(10_000)).toBe(2_000);
expect(getDashboardDuplicateCleanupIntervalMs(1_000)).toBe(1_000);
it('sweeps for stray duplicates only every 5 minutes', () => {
// Was 2s (~43k Discord fetches a day). Fresh posts clean up immediately.
expect(getDashboardDuplicateCleanupIntervalMs()).toBe(300_000);
});
});

View File

@@ -99,7 +99,12 @@ const USAGE_SNAPSHOT_MAX_AGE_MS = 600_000;
* so this only affects how quickly Codex snapshot data is picked up.
*/
const RENDERER_USAGE_REFRESH_MS = 30_000;
const DASHBOARD_DUPLICATE_CLEANUP_POLL_MS = 2_000;
const CODEX_ACTIVE_USAGE_POLL_MS = 300_000;
// Safety sweep for stray duplicate dashboard messages. Duplicates only appear
// when a new status message is posted (restart, failed edit), and that path
// cleans up immediately, so a slow sweep is enough. It used to poll every 2s:
// ~43k Discord REST fetches of 100 messages per day.
const DASHBOARD_DUPLICATE_CLEANUP_POLL_MS = 300_000;
let statusMessageId: string | null = null;
let cachedUsageContent = '';
@@ -196,10 +201,8 @@ export function shouldPurgeDashboardChannelOnStart(args: {
return args.purgeOnStart === true;
}
export function getDashboardDuplicateCleanupIntervalMs(
statusUpdateInterval: number,
): number {
return Math.min(statusUpdateInterval, DASHBOARD_DUPLICATE_CLEANUP_POLL_MS);
export function getDashboardDuplicateCleanupIntervalMs(): number {
return DASHBOARD_DUPLICATE_CLEANUP_POLL_MS;
}
async function refreshChannelMeta(
@@ -1013,17 +1016,23 @@ export async function startUnifiedDashboard(
}
}
let postedNewMessage = false;
if (!statusMessageId && channel.sendAndTrack) {
const id = await channel.sendAndTrack(statusJid, content);
if (id) {
statusMessageId = id;
postedNewMessage = true;
writeDashboardStatusMessageId(opts.statusChannelId, id);
}
}
// Duplicates only come from posting a fresh status message (restart or a
// failed edit), so clean up then and on the first render — not on every
// routine edit, which used to cost a 100-message fetch each minute.
const shouldCleanupDuplicates = postedNewMessage || firstStatusRender;
// A render (edit or repost) happened this tick; subsequent ticks use the
// steady-state retry policy.
firstStatusRender = false;
if (statusMessageId) {
if (statusMessageId && shouldCleanupDuplicates) {
await cleanupDashboardDuplicateMessages(opts, statusMessageId);
}
if (!dashboardUpdateLogged) {
@@ -1066,7 +1075,7 @@ export async function startUnifiedDashboard(
setInterval(() => {
if (!isRenderer || !statusMessageId) return;
void cleanupDashboardDuplicateMessages(opts, statusMessageId);
}, getDashboardDuplicateCleanupIntervalMs(opts.statusUpdateInterval));
}, getDashboardDuplicateCleanupIntervalMs());
await updateStatus();
if (isRenderer) {
@@ -1110,9 +1119,11 @@ export async function startUnifiedDashboard(
return refreshActiveCodexUsage().then(applyCodexRefresh);
})
.then(() => runCodexWarmup());
// Each poll spawns a Codex app-server; every 5 minutes is plenty for the
// dashboard (it used to run every minute, ~1,440 spawns a day).
setInterval(
() => void refreshActiveCodexUsage().then(applyCodexRefresh),
opts.usageUpdateInterval,
Math.max(opts.usageUpdateInterval, CODEX_ACTIVE_USAGE_POLL_MS),
);
setInterval(
() =>