Files
EJClaw/src/claude-usage-backoff.test.ts
Codex 97cc670e34 fix: show a rolled-over usage window as unknown, not 0%
91203d8 rendered an expired window as 0% unconditionally. If the post-reset
refetch fails (network error, 429) the 0% stayed for as long as fetches
kept failing, the same stale-value bug in the other direction. An expired
window now renders as -1 (the empty "─────" unknown cell) until a fetch
succeeds and brings the new window's real value.

Tests: expired window -> -1 for Claude and Codex rows; refetch failing at
+1 min and +60 min stays unknown, then success shows 3%; 429 shows the
429 indicator; renderer draws -1 as the placeholder cell.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 23:10:51 +09:00

276 lines
10 KiB
TypeScript

/**
* Regression tests for the Claude usage-API 429 `Retry-After` backoff.
*
* Root cause this guards against: the usage poller used to retry every
* MIN_FETCH_INTERVAL_MS (60s), but the endpoint hands back a longer rate-limit
* window (e.g. `retry-after: 93`). Retrying mid-cooldown re-trips the limit, so
* the 429s never clear. The fix records a `cooldownUntil` from the header and
* refuses to call the API until that window closes.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('./logger.js', () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
vi.mock('./config.js', () => ({ DATA_DIR: '/tmp/ejclaw-usage-backoff-data' }));
vi.mock('./claude-oauth-token.js', () => ({
getCurrentClaudeToken: () => 'sk-ant-oat01-testTokenAAAAAAAA',
}));
const { readJsonFileMock } = vi.hoisted(() => ({ readJsonFileMock: vi.fn() }));
vi.mock('./utils.js', async () => {
const actual =
await vi.importActual<typeof import('./utils.js')>('./utils.js');
return { ...actual, readJsonFile: readJsonFileMock, writeJsonFile: vi.fn() };
});
afterEach(() => {
vi.resetModules();
vi.clearAllMocks();
vi.unstubAllGlobals();
vi.useRealTimers();
});
function makeResponse(
status: number,
headers: Record<string, string> = {},
): Response {
return {
status,
ok: status >= 200 && status < 300,
headers: { get: (k: string) => headers[k.toLowerCase()] ?? null },
json: async () => ({}),
} as unknown as Response;
}
describe('parseRetryAfterMs', () => {
it('parses delta-seconds', async () => {
const { parseRetryAfterMs } = await import('./claude-usage.js');
expect(parseRetryAfterMs('93')).toBe(93_000);
expect(parseRetryAfterMs('0')).toBe(0);
});
it('parses an HTTP-date relative to now', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-06-20T00:00:00Z'));
const { parseRetryAfterMs } = await import('./claude-usage.js');
expect(parseRetryAfterMs('Sat, 20 Jun 2026 00:00:30 GMT')).toBe(30_000);
});
it('returns null for missing or junk values', async () => {
const { parseRetryAfterMs } = await import('./claude-usage.js');
expect(parseRetryAfterMs(null)).toBeNull();
expect(parseRetryAfterMs('soon')).toBeNull();
});
});
describe('Claude usage 429 Retry-After backoff', () => {
beforeEach(() => {
readJsonFileMock.mockReturnValue({}); // start from an empty disk cache
});
it('records a cooldown on 429 and skips the next fetch within the window', async () => {
const fetchMock = vi.fn(async () =>
makeResponse(429, { 'retry-after': '93' }),
);
vi.stubGlobal('fetch', fetchMock);
const { fetchClaudeUsage } = await import('./claude-usage.js');
const first = await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(first!.usageRateLimited).toBe(true);
expect(first!.usage).toBeNull();
// Immediate re-call: cooldown gate must hold, no second network call.
const second = await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(second!.usageRateLimited).toBe(true);
});
it('falls back to the default cooldown when Retry-After is 0 (does not disable backoff)', async () => {
// Production regression: the usage endpoint was returning 429 with
// `retry-after: 0`, which made `retryAfterMs ?? DEFAULT` evaluate to 0 — a
// ~5s cooldown. The poller then re-hit the endpoint every 60s and re-tripped
// the 429 indefinitely. A non-positive Retry-After must use the default.
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-06-20T00:00:00Z'));
const fetchMock = vi.fn(async () =>
makeResponse(429, { 'retry-after': '0' }),
);
vi.stubGlobal('fetch', fetchMock);
const { fetchClaudeUsage } = await import('./claude-usage.js');
await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
// +70s: past the 60s throttle. With the bug (0 cooldown) this would refetch;
// with the default 5-min cooldown it must still be held.
vi.setSystemTime(Date.now() + 70_000);
const held = await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(held!.usageRateLimited).toBe(true);
});
it('keeps backing off after the 5-minute throttle elapses but before the cooldown closes', async () => {
// This is the core regression: a Retry-After longer than the 5-minute
// MIN_FETCH_INTERVAL throttle must still be honoured. Without the cooldown
// the poller would refetch once the throttle elapses — mid rate-limit
// window — and re-trip the 429 forever.
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-06-20T00:00:00Z'));
const fetchMock = vi
.fn()
.mockResolvedValueOnce(makeResponse(429, { 'retry-after': '393' }))
.mockResolvedValueOnce(makeResponse(200));
vi.stubGlobal('fetch', fetchMock);
const { fetchClaudeUsage } = await import('./claude-usage.js');
await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
// +370s: past the 300s throttle, still inside the ~398s cooldown → no refetch.
vi.setSystemTime(Date.now() + 370_000);
const held = await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(held!.usageRateLimited).toBe(true);
// +400s total: cooldown window closed → poller is allowed to fetch again.
vi.setSystemTime(Date.now() + 30_000);
const resumed = await fetchClaudeUsage();
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(resumed!.usageRateLimited).toBe(false);
});
});
describe('Claude usage refetch after a window rolls over', () => {
// Production case (2026-10-19): fetched 99% at 22:59:04 KST for a 5h window
// resetting at 23:00:00; the 5-min throttle kept serving 99% until 23:04.
const staleCache = {
'account-0': {
fetchedAt: Date.parse('2026-10-19T13:59:04Z'),
lastAttemptAt: Date.parse('2026-10-19T13:59:04Z'),
usage: {
five_hour: { utilization: 99, resets_at: '2026-10-19T14:00:00Z' },
seven_day: { utilization: 27, resets_at: '2026-10-19T20:00:00Z' },
},
},
};
const freshBody = {
five_hour: { utilization: 3, resets_at: '2026-10-19T19:00:00Z' },
seven_day: { utilization: 27, resets_at: '2026-10-19T20:00:00Z' },
};
function okResponse(body: unknown): Response {
return { ...makeResponse(200), json: async () => body } as Response;
}
async function dashboardRow() {
const { fetchClaudeUsage } = await import('./claude-usage.js');
const { buildClaudeUsageRows } = await import('./dashboard-usage-rows.js');
return buildClaudeUsageRows(await fetchClaudeUsage())[0];
}
beforeEach(() => {
vi.useFakeTimers();
readJsonFileMock.mockReturnValue(structuredClone(staleCache));
});
it('skips the 5-minute throttle once when the cached window has reset', async () => {
vi.setSystemTime(new Date('2026-10-19T13:59:30Z'));
const fetchMock = vi.fn(async () => okResponse(freshBody));
vi.stubGlobal('fetch', fetchMock);
// Before the reset: throttled, no API call, cached value still valid.
expect(await dashboardRow()).toMatchObject({ h5pct: 99, d7pct: 27 });
expect(fetchMock).not.toHaveBeenCalled();
// Just after the reset: fetch the new window despite the throttle.
vi.setSystemTime(new Date('2026-10-19T14:00:10Z'));
expect(await dashboardRow()).toMatchObject({ h5pct: 3, d7pct: 27 });
expect(fetchMock).toHaveBeenCalledTimes(1);
// The throttle applies again after that one refetch.
vi.setSystemTime(new Date('2026-10-19T14:00:40Z'));
await dashboardRow();
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('shows the expired window as unknown while refetches fail, then the real value', async () => {
const fetchMock = vi
.fn()
.mockRejectedValueOnce(new Error('ECONNRESET'))
.mockRejectedValueOnce(new Error('ECONNRESET'))
.mockResolvedValueOnce(okResponse(freshBody));
vi.stubGlobal('fetch', fetchMock);
// +1 min after reset: refetch fails -> unknown, never 99% or 0%.
vi.setSystemTime(new Date('2026-10-19T14:01:00Z'));
expect(await dashboardRow()).toMatchObject({ h5pct: -1, h5reset: '' });
expect(fetchMock).toHaveBeenCalledTimes(1);
// +60 min: still failing -> still unknown.
vi.setSystemTime(new Date('2026-10-19T15:00:00Z'));
expect(await dashboardRow()).toMatchObject({ h5pct: -1, h5reset: '' });
expect(fetchMock).toHaveBeenCalledTimes(2);
// Next allowed attempt succeeds -> real new-window value.
vi.setSystemTime(new Date('2026-10-19T15:05:30Z'));
expect(await dashboardRow()).toMatchObject({ h5pct: 3, d7pct: 27 });
expect(fetchMock).toHaveBeenCalledTimes(3);
});
it('shows the 429 indicator (not a stale %) when the post-reset refetch is rate limited', async () => {
const fetchMock = vi.fn(async () =>
makeResponse(429, { 'retry-after': '120' }),
);
vi.stubGlobal('fetch', fetchMock);
vi.setSystemTime(new Date('2026-10-19T14:01:00Z'));
expect(await dashboardRow()).toMatchObject({
h5pct: -1,
d7pct: -1,
error: '429',
});
});
});
describe('getCachedClaudeUsageLimitResetAt', () => {
it('returns the later reset of the exhausted windows', async () => {
readJsonFileMock.mockReturnValue({
'account-0': {
fetchedAt: 1,
usage: {
five_hour: { utilization: 100, resets_at: '2026-10-01T12:00:00Z' },
seven_day: { utilization: 0.4, resets_at: '2026-10-05T00:00:00Z' },
},
},
});
const { getCachedClaudeUsageLimitResetAt } =
await import('./claude-usage.js');
expect(
getCachedClaudeUsageLimitResetAt(Date.parse('2026-10-01T10:00:00Z')),
).toEqual(new Date('2026-10-01T12:00:00Z'));
});
it('returns null when no window is exhausted', async () => {
readJsonFileMock.mockReturnValue({
'account-0': {
fetchedAt: 1,
usage: {
five_hour: { utilization: 42, resets_at: '2026-10-01T12:00:00Z' },
},
},
});
const { getCachedClaudeUsageLimitResetAt } =
await import('./claude-usage.js');
expect(
getCachedClaudeUsageLimitResetAt(Date.parse('2026-10-01T10:00:00Z')),
).toBeNull();
});
});