feat(dashboard): retry status edit before reposting
A transient Discord error (e.g. HTTP 503) on the periodic status-message edit previously caused an immediate repost of a fresh status message. Now the edit is retried up to 2 more times at 15s spacing, and a fresh message is only sent if every attempt fails. Retry logic is extracted into the testable editStatusMessageWithRetry helper (injectable sleep). Added a re-entrancy guard so overlapping interval ticks are skipped while a slow retry runs, preventing double-posts. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
|
|||||||
import type { UsageRow } from './dashboard-usage-rows.js';
|
import type { UsageRow } from './dashboard-usage-rows.js';
|
||||||
import {
|
import {
|
||||||
buildWebUsageRowsForSnapshot,
|
buildWebUsageRowsForSnapshot,
|
||||||
|
editStatusMessageWithRetry,
|
||||||
formatStatusHeader,
|
formatStatusHeader,
|
||||||
getDashboardDuplicateCleanupIntervalMs,
|
getDashboardDuplicateCleanupIntervalMs,
|
||||||
renderUsageTable,
|
renderUsageTable,
|
||||||
@@ -228,3 +229,86 @@ describe('buildWebUsageRowsForSnapshot', () => {
|
|||||||
expect(rows.map((row) => row.name)).toEqual(['Codex1']);
|
expect(rows.map((row) => row.name)).toEqual(['Codex1']);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('editStatusMessageWithRetry', () => {
|
||||||
|
it('succeeds on the first attempt without sleeping', async () => {
|
||||||
|
let calls = 0;
|
||||||
|
let slept = 0;
|
||||||
|
const ok = await editStatusMessageWithRetry({
|
||||||
|
editOnce: async () => {
|
||||||
|
calls++;
|
||||||
|
},
|
||||||
|
maxRetries: 2,
|
||||||
|
retryDelayMs: 15_000,
|
||||||
|
sleep: async () => {
|
||||||
|
slept++;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(ok).toBe(true);
|
||||||
|
expect(calls).toBe(1);
|
||||||
|
expect(slept).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('retries after a failure and succeeds on the second attempt', async () => {
|
||||||
|
let calls = 0;
|
||||||
|
const delays: number[] = [];
|
||||||
|
const ok = await editStatusMessageWithRetry({
|
||||||
|
editOnce: async () => {
|
||||||
|
calls++;
|
||||||
|
if (calls === 1) throw new Error('503');
|
||||||
|
},
|
||||||
|
maxRetries: 2,
|
||||||
|
retryDelayMs: 15_000,
|
||||||
|
sleep: async (ms) => {
|
||||||
|
delays.push(ms);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(ok).toBe(true);
|
||||||
|
expect(calls).toBe(2);
|
||||||
|
expect(delays).toEqual([15_000]); // one 15s wait between the two attempts
|
||||||
|
});
|
||||||
|
|
||||||
|
it('gives up after maxRetries (3 total attempts) and reports each failure', async () => {
|
||||||
|
let calls = 0;
|
||||||
|
const attempts: Array<{ attempt: number; willRetry: boolean }> = [];
|
||||||
|
let slept = 0;
|
||||||
|
const ok = await editStatusMessageWithRetry({
|
||||||
|
editOnce: async () => {
|
||||||
|
calls++;
|
||||||
|
throw new Error('503');
|
||||||
|
},
|
||||||
|
maxRetries: 2,
|
||||||
|
retryDelayMs: 15_000,
|
||||||
|
sleep: async () => {
|
||||||
|
slept++;
|
||||||
|
},
|
||||||
|
onAttemptFailed: (attempt, willRetry) =>
|
||||||
|
attempts.push({ attempt, willRetry }),
|
||||||
|
});
|
||||||
|
expect(ok).toBe(false);
|
||||||
|
expect(calls).toBe(3); // initial + 2 retries
|
||||||
|
expect(slept).toBe(2); // only between attempts, not after the last
|
||||||
|
expect(attempts).toEqual([
|
||||||
|
{ attempt: 1, willRetry: true },
|
||||||
|
{ attempt: 2, willRetry: true },
|
||||||
|
{ attempt: 3, willRetry: false },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses 15s spacing via the default sleep only when a retry is needed', async () => {
|
||||||
|
// Sanity: with maxRetries 0, a single failure returns false and never sleeps.
|
||||||
|
let slept = 0;
|
||||||
|
const ok = await editStatusMessageWithRetry({
|
||||||
|
editOnce: async () => {
|
||||||
|
throw new Error('nope');
|
||||||
|
},
|
||||||
|
maxRetries: 0,
|
||||||
|
retryDelayMs: 15_000,
|
||||||
|
sleep: async () => {
|
||||||
|
slept++;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(ok).toBe(false);
|
||||||
|
expect(slept).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -109,6 +109,14 @@ let usageUpdateInProgress = false;
|
|||||||
let channelMetaCache = new Map<string, ChannelMeta>();
|
let channelMetaCache = new Map<string, ChannelMeta>();
|
||||||
let channelMetaLastRefresh = 0;
|
let channelMetaLastRefresh = 0;
|
||||||
let dashboardUpdateLogged = false;
|
let dashboardUpdateLogged = false;
|
||||||
|
/** Guards against overlapping status updates while a slow edit-retry runs. */
|
||||||
|
let statusUpdateRunning = false;
|
||||||
|
/**
|
||||||
|
* On a failed status-message edit, retry the edit this many extra times at
|
||||||
|
* STATUS_EDIT_RETRY_DELAY_MS spacing before falling back to a fresh message.
|
||||||
|
*/
|
||||||
|
const STATUS_EDIT_MAX_RETRIES = 2;
|
||||||
|
const STATUS_EDIT_RETRY_DELAY_MS = 15_000;
|
||||||
/** Codex service only: cached usage rows written into the status snapshot. */
|
/** Codex service only: cached usage rows written into the status snapshot. */
|
||||||
let cachedCodexUsageRows: UsageRow[] = [];
|
let cachedCodexUsageRows: UsageRow[] = [];
|
||||||
/** Codex service only: ISO timestamp of last successful usage fetch. */
|
/** Codex service only: ISO timestamp of last successful usage fetch. */
|
||||||
@@ -226,7 +234,7 @@ function getAgentDisplayName(
|
|||||||
return serviceId === 'codex-review' ? '코리뷰' : '코덱스';
|
return serviceId === 'codex-review' ? '코리뷰' : '코덱스';
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatRoomName(
|
export function formatRoomName(
|
||||||
jid: string,
|
jid: string,
|
||||||
meta: ChannelMeta | undefined,
|
meta: ChannelMeta | undefined,
|
||||||
fallbackName: string | undefined,
|
fallbackName: string | undefined,
|
||||||
@@ -238,7 +246,12 @@ function formatRoomName(
|
|||||||
(fallbackName && fallbackName !== jid ? fallbackName : undefined) ||
|
(fallbackName && fallbackName !== jid ? fallbackName : undefined) ||
|
||||||
jid;
|
jid;
|
||||||
|
|
||||||
if (jid.startsWith('dc:') && base !== jid && !base.startsWith('#')) {
|
if (
|
||||||
|
jid.startsWith('dc:') &&
|
||||||
|
base !== jid &&
|
||||||
|
!base.startsWith('#') &&
|
||||||
|
!base.includes(' #')
|
||||||
|
) {
|
||||||
return `#${base}`;
|
return `#${base}`;
|
||||||
}
|
}
|
||||||
return base;
|
return base;
|
||||||
@@ -285,6 +298,9 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
|
|||||||
const groups = opts.roomBindings();
|
const groups = opts.roomBindings();
|
||||||
const statuses = opts.queue.getStatuses(Object.keys(groups));
|
const statuses = opts.queue.getStatuses(Object.keys(groups));
|
||||||
const usageSnapshot = buildUsageSnapshotRows(opts);
|
const usageSnapshot = buildUsageSnapshotRows(opts);
|
||||||
|
const chatNameByJid = new Map(
|
||||||
|
getAllChats().map((chat) => [chat.jid, chat.name]),
|
||||||
|
);
|
||||||
|
|
||||||
writeStatusSnapshot({
|
writeStatusSnapshot({
|
||||||
serviceId: opts.serviceId,
|
serviceId: opts.serviceId,
|
||||||
@@ -298,6 +314,9 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
|
|||||||
return {
|
return {
|
||||||
jid: status.jid,
|
jid: status.jid,
|
||||||
name: group.name,
|
name: group.name,
|
||||||
|
...(chatNameByJid.get(status.jid) && {
|
||||||
|
chatName: chatNameByJid.get(status.jid),
|
||||||
|
}),
|
||||||
folder: group.folder,
|
folder: group.folder,
|
||||||
agentType: (group.agentType || opts.serviceAgentType) as
|
agentType: (group.agentType || opts.serviceAgentType) as
|
||||||
| 'claude-code'
|
| 'claude-code'
|
||||||
@@ -311,6 +330,7 @@ function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void {
|
|||||||
.filter(Boolean) as Array<{
|
.filter(Boolean) as Array<{
|
||||||
jid: string;
|
jid: string;
|
||||||
name: string;
|
name: string;
|
||||||
|
chatName?: string;
|
||||||
folder: string;
|
folder: string;
|
||||||
agentType: 'claude-code' | 'codex';
|
agentType: 'claude-code' | 'codex';
|
||||||
status: 'processing' | 'waiting' | 'inactive';
|
status: 'processing' | 'waiting' | 'inactive';
|
||||||
@@ -342,6 +362,7 @@ function buildStatusContent(): string {
|
|||||||
pendingMessages: boolean;
|
pendingMessages: boolean;
|
||||||
pendingTasks: number;
|
pendingTasks: number;
|
||||||
name: string;
|
name: string;
|
||||||
|
chatName?: string;
|
||||||
meta: ChannelMeta | undefined;
|
meta: ChannelMeta | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -359,6 +380,7 @@ function buildStatusContent(): string {
|
|||||||
pendingMessages: entry.pendingMessages,
|
pendingMessages: entry.pendingMessages,
|
||||||
pendingTasks: entry.pendingTasks,
|
pendingTasks: entry.pendingTasks,
|
||||||
name: entry.name,
|
name: entry.name,
|
||||||
|
chatName: entry.chatName,
|
||||||
meta: channelMetaCache.get(entry.jid),
|
meta: channelMetaCache.get(entry.jid),
|
||||||
});
|
});
|
||||||
byJid.set(entry.jid, existing);
|
byJid.set(entry.jid, existing);
|
||||||
@@ -386,7 +408,8 @@ function buildStatusContent(): string {
|
|||||||
jid,
|
jid,
|
||||||
meta,
|
meta,
|
||||||
agents.find((agent) => agent.name && agent.name !== jid)?.name,
|
agents.find((agent) => agent.name && agent.name !== jid)?.name,
|
||||||
chatNameByJid.get(jid),
|
agents.find((agent) => agent.chatName && agent.chatName !== jid)
|
||||||
|
?.chatName ?? chatNameByJid.get(jid),
|
||||||
),
|
),
|
||||||
meta,
|
meta,
|
||||||
agents,
|
agents,
|
||||||
@@ -750,6 +773,34 @@ async function refreshUsageCache(): Promise<void> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt a status-message edit, retrying up to `maxRetries` extra times with
|
||||||
|
* `retryDelayMs` spacing before giving up. Returns true if an edit succeeded,
|
||||||
|
* false if every attempt failed (caller then reposts a fresh message).
|
||||||
|
* `sleep` is injectable so tests can run without real delays.
|
||||||
|
*/
|
||||||
|
export async function editStatusMessageWithRetry(args: {
|
||||||
|
editOnce: () => Promise<unknown>;
|
||||||
|
maxRetries: number;
|
||||||
|
retryDelayMs: number;
|
||||||
|
onAttemptFailed?: (attempt: number, willRetry: boolean, err: unknown) => void;
|
||||||
|
sleep?: (ms: number) => Promise<void>;
|
||||||
|
}): Promise<boolean> {
|
||||||
|
const sleep =
|
||||||
|
args.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
|
||||||
|
for (let attempt = 0; attempt <= args.maxRetries; attempt++) {
|
||||||
|
try {
|
||||||
|
await args.editOnce();
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
const willRetry = attempt < args.maxRetries;
|
||||||
|
args.onAttemptFailed?.(attempt + 1, willRetry, err);
|
||||||
|
if (willRetry) await sleep(args.retryDelayMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
export async function startUnifiedDashboard(
|
export async function startUnifiedDashboard(
|
||||||
opts: UnifiedDashboardOptions,
|
opts: UnifiedDashboardOptions,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
@@ -780,21 +831,25 @@ export async function startUnifiedDashboard(
|
|||||||
const updateStatus = async () => {
|
const updateStatus = async () => {
|
||||||
writeLocalStatusSnapshot(opts);
|
writeLocalStatusSnapshot(opts);
|
||||||
if (!isRenderer) return;
|
if (!isRenderer) return;
|
||||||
|
// A failed edit now retries with delays, so a single updateStatus can run
|
||||||
const channel = findDiscordChannel(opts.channels);
|
// for tens of seconds. Skip overlapping interval ticks to avoid double posts.
|
||||||
if (!channel) {
|
if (statusUpdateRunning) return;
|
||||||
logger.warn(
|
statusUpdateRunning = true;
|
||||||
{
|
|
||||||
channelCount: opts.channels.length,
|
|
||||||
names: opts.channels.map((c) => c.name),
|
|
||||||
connected: opts.channels.map((c) => c.isConnected()),
|
|
||||||
},
|
|
||||||
'Dashboard: no connected Discord channel found',
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const channel = findDiscordChannel(opts.channels);
|
||||||
|
if (!channel) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
channelCount: opts.channels.length,
|
||||||
|
names: opts.channels.map((c) => c.name),
|
||||||
|
connected: opts.channels.map((c) => c.isConnected()),
|
||||||
|
},
|
||||||
|
'Dashboard: no connected Discord channel found',
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
await refreshChannelMeta(opts);
|
await refreshChannelMeta(opts);
|
||||||
const content = buildUnifiedDashboardContent();
|
const content = buildUnifiedDashboardContent();
|
||||||
if (!content) {
|
if (!content) {
|
||||||
@@ -809,14 +864,32 @@ export async function startUnifiedDashboard(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (statusMessageId && channel.editMessage) {
|
if (statusMessageId && channel.editMessage) {
|
||||||
try {
|
// A transient Discord error (e.g. 503) on the periodic edit should not
|
||||||
await channel.editMessage(statusJid, statusMessageId, content);
|
// immediately spawn a fresh status message. Retry the edit a couple of
|
||||||
|
// times with a delay first; only give up (and repost) if all fail.
|
||||||
|
const editId = statusMessageId;
|
||||||
|
const editMessage = channel.editMessage;
|
||||||
|
const edited = await editStatusMessageWithRetry({
|
||||||
|
editOnce: () => editMessage(statusJid, editId, content),
|
||||||
|
maxRetries: STATUS_EDIT_MAX_RETRIES,
|
||||||
|
retryDelayMs: STATUS_EDIT_RETRY_DELAY_MS,
|
||||||
|
onAttemptFailed: (attempt, willRetry, err) =>
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
err,
|
||||||
|
messageId: editId,
|
||||||
|
attempt,
|
||||||
|
maxAttempts: STATUS_EDIT_MAX_RETRIES + 1,
|
||||||
|
willRetry,
|
||||||
|
},
|
||||||
|
willRetry
|
||||||
|
? 'Dashboard status message edit failed; retrying in 15s'
|
||||||
|
: 'Dashboard status message edit failed after retries; sending a fresh tracked message',
|
||||||
|
),
|
||||||
|
});
|
||||||
|
if (edited) {
|
||||||
writeDashboardStatusMessageId(opts.statusChannelId, statusMessageId);
|
writeDashboardStatusMessageId(opts.statusChannelId, statusMessageId);
|
||||||
} catch (err) {
|
} else {
|
||||||
logger.warn(
|
|
||||||
{ err, messageId: statusMessageId },
|
|
||||||
'Dashboard status message edit failed; sending a fresh tracked message',
|
|
||||||
);
|
|
||||||
statusMessageId = null;
|
statusMessageId = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -841,6 +914,8 @@ export async function startUnifiedDashboard(
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.warn({ err }, 'Dashboard update failed');
|
logger.warn({ err }, 'Dashboard update failed');
|
||||||
statusMessageId = null;
|
statusMessageId = null;
|
||||||
|
} finally {
|
||||||
|
statusUpdateRunning = false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user