import { execSync } from 'child_process'; import fs from 'fs'; import os from 'os'; import { ARBITER_AGENT_TYPE, ARBITER_MODEL_CONFIG, DEFAULT_CLAUDE_MODEL, DEFAULT_CODEX_MODEL, OWNER_AGENT_TYPE, OWNER_MODEL_CONFIG, REVIEWER_AGENT_TYPE, REVIEWER_MODEL_CONFIG, STATUS_SHOW_ROOM_DETAILS, STATUS_SHOW_ROOMS, USAGE_DASHBOARD_ENABLED, CODEX_WARMUP_CONFIG, getMoaConfig, } from './config.js'; import { fetchKimiUsage, buildKimiUsageRows, type KimiUsageData, } from './kimi-usage.js'; import { getGlobalFailoverInfo } from './service-routing.js'; import { fetchAllClaudeUsage, fetchAllClaudeProfiles, type ClaudeAccountUsage, } from './claude-usage.js'; import { CODEX_FULL_SCAN_INTERVAL, refreshActiveCodexUsage, refreshAllCodexAccountUsage, } from './codex-usage-collector.js'; import { runCodexWarmupCycle } from './codex-warmup.js'; import { composeDashboardContent, formatElapsed, getStatusLabel as formatDashboardStatusLabel, type DashboardRoomLine, renderCategorizedRoomSections, } from './dashboard-render.js'; import { cleanupDashboardDuplicateMessages, purgeDashboardMessages, } from './dashboard-message-cleanup.js'; import { buildClaudeUsageRows, extractCodexUsageRows, mergeClaudeDashboardAccounts, type UsageRow, } from './dashboard-usage-rows.js'; import { getAllChats, getAllTasks } from './db.js'; import type { GroupQueue } from './group-queue.js'; import { logger } from './logger.js'; import { isWatchCiTask } from './task-watch-status.js'; import { readDashboardStatusMessageId, readStatusSnapshots, writeDashboardStatusMessageId, writeStatusSnapshot, } from './status-dashboard.js'; import type { AgentType, Channel, ChannelMeta, RegisteredGroup, ScheduledTask, } from './types.js'; export interface UnifiedDashboardOptions { assistantName: string; serviceId: string; serviceAgentType: AgentType; statusChannelId: string; statusUpdateInterval: number; usageUpdateInterval: number; channels: Channel[]; queue: GroupQueue; roomBindings: () => Record; onGroupNameSynced?: (jid: string, name: string) => void; purgeOnStart?: boolean; } const STATUS_ICONS: Record = { processing: '🟑', waiting: 'πŸ”΅', inactive: 'βšͺ', }; const CHANNEL_META_REFRESH_MS = 300000; const STATUS_SNAPSHOT_MAX_AGE_MS = 60000; /** Usage data can be up to 10 min old before considered stale. */ const USAGE_SNAPSHOT_MAX_AGE_MS = 600_000; /** * Renderer refreshes usage cache every 30s (not 5min). * Claude API calls are internally rate-limited to 5min per token, * so this only affects how quickly Codex snapshot data is picked up. */ const RENDERER_USAGE_REFRESH_MS = 30_000; const DASHBOARD_DUPLICATE_CLEANUP_POLL_MS = 2_000; let statusMessageId: string | null = null; let cachedUsageContent = ''; let cachedClaudeAccounts: ClaudeAccountUsage[] = []; let cachedKimiUsage: KimiUsageData | null = null; let usageUpdateInProgress = false; let channelMetaCache = new Map(); let channelMetaLastRefresh = 0; let dashboardUpdateLogged = false; /** Guards against overlapping status updates while a slow edit-retry runs. */ let statusUpdateRunning = false; /** A refresh was requested while one was already running; run once more after. */ let statusUpdatePending = false; /** * On a failed status-message edit, retry the edit this many extra times at * STATUS_EDIT_RETRY_DELAY_MS spacing before falling back to a fresh message. */ const STATUS_EDIT_MAX_RETRIES = 2; const STATUS_EDIT_RETRY_DELAY_MS = 15_000; /** Coalesce bursts of event-driven refresh requests into one update. */ const IMMEDIATE_UPDATE_DEBOUNCE_MS = 1_500; /** Set by the renderer's startUnifiedDashboard so external events can nudge it. */ let immediateUpdateTrigger: (() => void) | null = null; /** * Request an out-of-band status refresh (e.g. a new chat message arrived or an * agent's activity changed) so the dashboard updates immediately instead of * waiting for the next periodic tick. No-op until the renderer dashboard starts; * bursts are debounced. */ export function requestImmediateStatusUpdate(): void { immediateUpdateTrigger?.(); } /** Codex service only: cached usage rows written into the status snapshot. */ let cachedCodexUsageRows: UsageRow[] = []; /** Codex service only: ISO timestamp of last successful usage fetch. */ let codexUsageFetchedAt: string | null = null; /** Renderer service only: ISO timestamp of last successful Claude/Kimi usage render. */ let rendererUsageFetchedAt: string | null = null; export interface WatcherTaskSummary { active: number; paused: number; } export function summarizeWatcherTasks( tasks: Array>, ): WatcherTaskSummary { let active = 0; let paused = 0; for (const task of tasks) { if (!isWatchCiTask(task)) continue; if (task.status === 'active') active += 1; if (task.status === 'paused') paused += 1; } return { active, paused }; } export function formatStatusHeader(args: { totalActive: number; totalRooms: number; watchers: WatcherTaskSummary; }): string { const parts = [ `**πŸ“Š μ—μ΄μ „νŠΈ μƒνƒœ** β€” ν™œμ„± ${args.totalActive} / ${args.totalRooms}`, `κ°μ‹œ ${args.watchers.active}`, ]; if (args.watchers.paused > 0) { parts.push(`μΌμ‹œμ •μ§€ ${args.watchers.paused}`); } return parts.join(' | '); } function findDiscordChannel(channels: Channel[]): Channel | undefined { return channels.find( (channel) => channel.name.startsWith('discord') && channel.isConnected(), ); } export async function purgeDashboardChannel( opts: Pick, ): Promise { await purgeDashboardMessages(opts); } export function shouldPurgeDashboardChannelOnStart(args: { purgeOnStart?: boolean; storedMessageId: string | null; }): boolean { return args.purgeOnStart === true; } export function getDashboardDuplicateCleanupIntervalMs( statusUpdateInterval: number, ): number { return Math.min(statusUpdateInterval, DASHBOARD_DUPLICATE_CLEANUP_POLL_MS); } async function refreshChannelMeta( opts: UnifiedDashboardOptions, ): Promise { const now = Date.now(); if (now - channelMetaLastRefresh < CHANNEL_META_REFRESH_MS) return; const channel = opts.channels.find( (item) => item.name.startsWith('discord') && item.isConnected() && item.getChannelMeta, ); if (!channel?.getChannelMeta) return; const localJids = Object.keys(opts.roomBindings()).filter((jid) => jid.startsWith('dc:'), ); const snapshotJids = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS) .flatMap((snapshot) => snapshot.entries.map((entry) => entry.jid)) .filter((jid) => jid.startsWith('dc:')); const jids = [...new Set([...localJids, ...snapshotJids])]; try { channelMetaCache = await channel.getChannelMeta(jids); channelMetaLastRefresh = now; for (const [jid, meta] of channelMetaCache) { if (!meta.name) continue; const group = opts.roomBindings()[jid]; if (!group || group.name === meta.name) continue; logger.debug( { jid, oldName: group.name, newName: meta.name }, 'Keeping registered group name distinct from Discord channel name', ); } } catch (err) { logger.debug({ err }, 'Failed to refresh channel metadata'); } } function getAgentDisplayName( agentType: 'claude-code' | 'codex', serviceId: string, ): string { if (agentType === 'claude-code') return '클코'; return serviceId === 'codex-review' ? '코리뷰' : 'μ½”λ±μŠ€'; } export function formatRoomName( jid: string, meta: ChannelMeta | undefined, fallbackName: string | undefined, chatName: string | undefined, ): string { const base = meta?.name || (chatName && chatName !== jid ? chatName : undefined) || (fallbackName && fallbackName !== jid ? fallbackName : undefined) || jid; if ( jid.startsWith('dc:') && base !== jid && !base.startsWith('#') && !base.includes(' #') ) { return `#${base}`; } return base; } export function buildWebUsageRowsForSnapshot(args: { serviceAgentType: AgentType; claudeAccounts: ClaudeAccountUsage[]; kimiUsage: KimiUsageData | null; codexRows: UsageRow[]; }): UsageRow[] { const rows: UsageRow[] = []; if (args.serviceAgentType === 'claude-code') { rows.push(...buildClaudeUsageRows(args.claudeAccounts)); rows.push(...buildKimiUsageRows(args.kimiUsage)); } rows.push(...args.codexRows); return rows; } function buildUsageSnapshotRows(opts: UnifiedDashboardOptions): { rows: UsageRow[]; fetchedAt: string | null; } { const rows = buildWebUsageRowsForSnapshot({ serviceAgentType: opts.serviceAgentType, claudeAccounts: cachedClaudeAccounts, kimiUsage: cachedKimiUsage, codexRows: cachedCodexUsageRows, }); const fetchedAt = [rendererUsageFetchedAt, codexUsageFetchedAt] .filter((value): value is string => !!value) .sort() .at(-1) ?? null; return { rows, fetchedAt }; } function writeLocalStatusSnapshot(opts: UnifiedDashboardOptions): void { const groups = opts.roomBindings(); const statuses = opts.queue.getStatuses(Object.keys(groups)); const usageSnapshot = buildUsageSnapshotRows(opts); const chatNameByJid = new Map( getAllChats().map((chat) => [chat.jid, chat.name]), ); writeStatusSnapshot({ serviceId: opts.serviceId, agentType: opts.serviceAgentType, assistantName: opts.assistantName, updatedAt: new Date().toISOString(), entries: statuses .map((status) => { const group = groups[status.jid]; if (!group) return null; return { jid: status.jid, name: group.name, ...(chatNameByJid.get(status.jid) && { chatName: chatNameByJid.get(status.jid), }), folder: group.folder, agentType: (group.agentType || opts.serviceAgentType) as | 'claude-code' | 'codex', status: status.status, elapsedMs: status.elapsedMs, pendingMessages: status.pendingMessages, pendingTasks: status.pendingTasks, }; }) .filter(Boolean) as Array<{ jid: string; name: string; chatName?: string; folder: string; agentType: 'claude-code' | 'codex'; status: 'processing' | 'waiting' | 'inactive'; elapsedMs: number | null; pendingMessages: boolean; pendingTasks: number; }>, ...(usageSnapshot.rows.length > 0 && { usageRows: usageSnapshot.rows }), ...(usageSnapshot.fetchedAt && { usageRowsFetchedAt: usageSnapshot.fetchedAt, }), }); } function buildStatusContent(): string { if (!STATUS_SHOW_ROOMS) return ''; const snapshots = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS); const watcherSummary = summarizeWatcherTasks(getAllTasks()); const chatNameByJid = new Map( getAllChats().map((chat) => [chat.jid, chat.name]), ); interface RoomEntry { serviceId: string; agentType: 'claude-code' | 'codex'; status: 'processing' | 'waiting' | 'inactive'; elapsedMs: number | null; pendingMessages: boolean; pendingTasks: number; name: string; chatName?: string; meta: ChannelMeta | undefined; } const byJid = new Map(); for (const snapshot of snapshots) { const agentType = snapshot.agentType as 'claude-code' | 'codex'; for (const entry of snapshot.entries) { const existing = byJid.get(entry.jid) || []; existing.push({ serviceId: snapshot.serviceId, agentType, status: entry.status, elapsedMs: entry.elapsedMs, pendingMessages: entry.pendingMessages, pendingTasks: entry.pendingTasks, name: entry.name, chatName: entry.chatName, meta: channelMetaCache.get(entry.jid), }); byJid.set(entry.jid, existing); } } interface RoomInfo { name: string; meta: ChannelMeta | undefined; agents: RoomEntry[]; } const categoryMap = new Map(); let totalActive = 0; let totalRooms = 0; for (const [jid, agents] of byJid) { const meta = agents[0]?.meta; const category = meta?.category || '기타'; if (!categoryMap.has(category)) { categoryMap.set(category, []); } categoryMap.get(category)!.push({ name: formatRoomName( jid, meta, agents.find((agent) => agent.name && agent.name !== jid)?.name, agents.find((agent) => agent.chatName && agent.chatName !== jid) ?.chatName ?? chatNameByJid.get(jid), ), meta, agents, }); totalRooms++; if (agents.some((agent) => agent.status === 'processing')) { totalActive++; } } const sortedCategories = [...categoryMap.entries()].sort((a, b) => { const posA = a[1][0]?.meta?.categoryPosition ?? 999; const posB = b[1][0]?.meta?.categoryPosition ?? 999; return posA - posB; }); const roomLines: DashboardRoomLine[] = []; for (const [categoryName, rooms] of sortedCategories) { rooms.sort((a, b) => (a.meta?.position ?? 999) - (b.meta?.position ?? 999)); for (const room of rooms) { room.agents.sort((a, b) => a.agentType === b.agentType ? 0 : a.agentType === 'claude-code' ? -1 : 1, ); const agentParts = room.agents.map((agent) => { const icon = STATUS_ICONS[agent.status] || 'βšͺ'; const label = formatDashboardStatusLabel({ status: agent.status, elapsedMs: agent.elapsedMs, pendingTasks: agent.pendingTasks, }); const tag = getAgentDisplayName(agent.agentType, agent.serviceId); return `${tag} ${icon} ${label}`; }); roomLines.push({ category: categoryName, categoryPosition: room.meta?.categoryPosition ?? 999, position: room.meta?.position ?? 999, line: ` **${room.name}** β€” ${agentParts.join(' | ')}`, }); } } const header = formatStatusHeader({ totalActive, totalRooms, watchers: watcherSummary, }); if (!STATUS_SHOW_ROOM_DETAILS) { return header; } const sections = renderCategorizedRoomSections({ lines: roomLines, showCategoryHeaders: channelMetaCache.size > 0, }); return `${header}\n\n${sections}`; } /** Previous /proc/stat sample for CPU utilization deltas. */ let lastCpuSample: { idle: number; total: number } | null = null; /** * Real CPU utilization percent from /proc/stat deltas between calls. * First call (no previous sample) and non-Linux hosts return null. * Exported for testing alongside resetCpuUtilizationSample. */ export function readCpuUtilizationPct( readStat: () => string = () => fs.readFileSync('/proc/stat', 'utf-8'), ): number | null { try { const cpuLine = readStat() .split('\n') .find((line) => line.startsWith('cpu ')); if (!cpuLine) return null; const fields = cpuLine.trim().split(/\s+/).slice(1).map(Number); if (fields.length < 5 || fields.some((n) => !Number.isFinite(n))) { return null; } // user nice system idle iowait irq softirq steal ... const idle = fields[3] + (fields[4] ?? 0); // idle + iowait const total = fields.reduce((a, b) => a + b, 0); const prev = lastCpuSample; lastCpuSample = { idle, total }; if (!prev || total <= prev.total) return null; const totalDelta = total - prev.total; const idleDelta = idle - prev.idle; return Math.round( Math.min(100, Math.max(0, ((totalDelta - idleDelta) / totalDelta) * 100)), ); } catch { return null; } } /** Test-only: reset the CPU sample so the next read starts fresh. */ export function resetCpuUtilizationSample(): void { lastCpuSample = null; } /** * Render usage table lines from two row groups (Claude and Codex). * Returns rendered lines including code block markers. * Ordering: Claude rows β†’ separator β†’ Codex rows. * Exported for testing. */ export function renderUsageTable( claudeBotRows: UsageRow[], codexBotRows: UsageRow[], ): string[] { const allRows = [...claudeBotRows, ...codexBotRows]; if (allRows.length === 0) return ['_쑰회 λΆˆκ°€_']; const bar = (pct: number) => { const filled = Math.max(0, Math.min(5, Math.round(pct / 20))); return 'β–ˆ'.repeat(filled) + 'β–‘'.repeat(5 - filled); }; const visualWidth = (s: string) => [...s].reduce((w, c) => w + (c.codePointAt(0)! > 0x7f ? 2 : 1), 0); const maxNameWidth = Math.max(8, ...allRows.map((r) => visualWidth(r.name))) + 1; const padName = (s: string) => s + ' '.repeat(Math.max(0, maxNameWidth - visualWidth(s))); const compactReset = (s: string) => s ? s.replace(/\s+/g, '').replace(/m$/, '') : ''; const lines: string[] = []; // Missing-window placeholder. Must mirror a real cell's char composition // (5 ambiguous-width bar chars + 4 ASCII): mobile fonts render block/box // chars double-width, so a plain 'β€”' cell breaks column alignment. const emptyCell = '─'.repeat(5) + ' '; const renderRows = (rows: UsageRow[]) => { for (const row of rows) { if (row.error) { // Live error (e.g. 429) β€” show the indicator in both columns instead // of a bar or a stale value. const cell = row.error.padEnd(6); lines.push(`${padName(row.name)}${cell} ${cell}`); continue; } const h5 = row.h5pct >= 0 ? `${bar(row.h5pct)}${String(row.h5pct).padStart(3)}%` : emptyCell; const d7 = row.d7pct >= 0 ? `${bar(row.d7pct)}${String(row.d7pct).padStart(3)}%` : emptyCell; lines.push(`${padName(row.name)}${h5} ${d7}`); const r5 = compactReset(row.h5reset); const r7 = compactReset(row.d7reset); if (r5 || r7) { const d7ColStart = maxNameWidth + 10; let resetLine = ' '.repeat(maxNameWidth); if (r5) resetLine += r5; resetLine = resetLine.padEnd(d7ColStart); if (r7) resetLine += r7; lines.push(resetLine); } } }; lines.push('```'); lines.push(`${' '.repeat(maxNameWidth)}5h 7d`); renderRows(claudeBotRows); if (claudeBotRows.length > 0 && codexBotRows.length > 0) { const separatorWidth = maxNameWidth + 20; lines.push('─'.repeat(separatorWidth)); } renderRows(codexBotRows); lines.push('```'); return lines; } async function buildUsageContent(): Promise { const shouldFetchClaudeUsage = USAGE_DASHBOARD_ENABLED; let liveClaudeAccounts: ClaudeAccountUsage[] | null = null; if (shouldFetchClaudeUsage) { try { liveClaudeAccounts = await fetchAllClaudeUsage(); } catch (err) { logger.warn({ err }, 'Failed to fetch Claude usage for dashboard'); } } // Kimi usage try { cachedKimiUsage = await fetchKimiUsage(); } catch (err) { logger.warn({ err }, 'Failed to fetch Kimi usage for dashboard'); } const lines: string[] = ['πŸ“Š *μ‚¬μš©λŸ‰*']; const bar = (pct: number) => { const filled = Math.max(0, Math.min(5, Math.round(pct / 20))); return 'β–ˆ'.repeat(filled) + 'β–‘'.repeat(5 - filled); }; // Group 1: Claude bot const claudeBotRows: UsageRow[] = []; if (shouldFetchClaudeUsage) { cachedClaudeAccounts = mergeClaudeDashboardAccounts( liveClaudeAccounts, cachedClaudeAccounts, ); claudeBotRows.push(...buildClaudeUsageRows(cachedClaudeAccounts)); } // Group 2: Codex bot β€” use in-process cache (unified service) // or fall back to snapshot from separate Codex service. const codexBotRows: UsageRow[] = []; if (cachedCodexUsageRows.length > 0) { codexBotRows.push(...cachedCodexUsageRows); } else { const codexSnapshot = readStatusSnapshots(STATUS_SNAPSHOT_MAX_AGE_MS).find( (s) => s.serviceId === 'codex-main' || s.serviceId === 'codex', ); codexBotRows.push( ...extractCodexUsageRows(codexSnapshot, USAGE_SNAPSHOT_MAX_AGE_MS), ); } // Group 3: Kimi coding plan const kimiRows = buildKimiUsageRows(cachedKimiUsage); claudeBotRows.push(...kimiRows); lines.push(...renderUsageTable(claudeBotRows, codexBotRows)); lines.push(''); lines.push('πŸ–₯️ *μ„œλ²„*'); const loadAvg = os.loadavg(); const cpuCount = os.cpus().length; // Real CPU utilization from /proc/stat deltas between renders. // Load-average-based percent counts D-state (I/O-wait) processes, so a // writeback storm once rendered as "CPU 3104%" despite idle CPUs. const cpuPct = readCpuUtilizationPct() ?? 0; const loadPerCore = loadAvg[0] / cpuCount; const totalMem = os.totalmem(); // os.freemem() includes buffers/cache as "used" β€” misleading. // Read MemAvailable from /proc/meminfo for actual available memory. let availableMem = os.freemem(); try { const meminfo = fs.readFileSync('/proc/meminfo', 'utf-8'); const match = meminfo.match(/MemAvailable:\s+(\d+)\s+kB/); if (match) availableMem = parseInt(match[1], 10) * 1024; } catch { /* non-Linux or unreadable β€” fall back to os.freemem() */ } const usedMem = totalMem - availableMem; const memPct = Math.round((usedMem / totalMem) * 100); const memUsedGB = (usedMem / 1073741824).toFixed(1); const memTotalGB = (totalMem / 1073741824).toFixed(1); let diskPct = 0; let diskUsedGB = '?'; let diskTotalGB = '?'; try { const df = execSync('df -B1 / | tail -1', { encoding: 'utf-8', timeout: 5000, }).trim(); const parts = df.split(/\s+/); const diskUsed = parseInt(parts[2], 10); const diskTotal = parseInt(parts[1], 10); diskPct = Math.round((diskUsed / diskTotal) * 100); diskUsedGB = (diskUsed / 1073741824).toFixed(0); diskTotalGB = (diskTotal / 1073741824).toFixed(0); } catch { /* ignore */ } // GPU (optional β€” only shown when an NVIDIA GPU + nvidia-smi is present). let gpuLine: string | null = null; let vramLine: string | null = null; try { const out = execSync( 'nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits', { encoding: 'utf-8', timeout: 5000 }, ).trim(); const first = out.split('\n')[0]?.trim(); if (first) { const [utilStr, usedStr, totalStr] = first .split(',') .map((s) => s.trim()); const gpuPct = parseInt(utilStr, 10); const vramUsed = parseInt(usedStr, 10); const vramTotal = parseInt(totalStr, 10); if ( Number.isFinite(gpuPct) && Number.isFinite(vramUsed) && Number.isFinite(vramTotal) && vramTotal > 0 ) { const vramPct = Math.round((vramUsed / vramTotal) * 100); const vramUsedGB = (vramUsed / 1024).toFixed(1); const vramTotalGB = (vramTotal / 1024).toFixed(1); gpuLine = `${'GPU'.padEnd(8)}${bar(gpuPct)} ${String(gpuPct).padStart(3)}%`; vramLine = `${'VRAM'.padEnd(8)}${bar(vramPct)} ${String(vramPct).padStart(3)}% ${vramUsedGB}/${vramTotalGB}GB`; } } } catch { /* no GPU or nvidia-smi unavailable */ } lines.push('```'); lines.push(`${'CPU'.padEnd(8)}${bar(cpuPct)} ${String(cpuPct).padStart(3)}%`); // Raw load average with core count: I/O storms (D-state pileups) show up // here without masquerading as CPU usage. Flag when load exceeds cores. const loadFlag = loadPerCore > 1 ? ' β–²' : ''; lines.push( `${'Load'.padEnd(8)}${loadAvg[0] >= 100 ? loadAvg[0].toFixed(0) : loadAvg[0].toFixed(1)}/${cpuCount}cpu${loadFlag}`, ); lines.push( `${'Memory'.padEnd(8)}${bar(memPct)} ${String(memPct).padStart(3)}% ${memUsedGB}/${memTotalGB}GB`, ); lines.push( `${'Disk'.padEnd(8)}${bar(diskPct)} ${String(diskPct).padStart(3)}% ${diskUsedGB}/${diskTotalGB}GB`, ); if (gpuLine) lines.push(gpuLine); if (vramLine) lines.push(vramLine); lines.push(`${'Uptime'.padEnd(8)}${formatElapsed(os.uptime() * 1000)}`); lines.push('```'); return lines.join('\n'); } function buildModelConfigSection(): string { const roleConfigs = [ { label: 'Owner', agentType: OWNER_AGENT_TYPE, model: OWNER_MODEL_CONFIG.model, }, { label: 'Reviewer', agentType: REVIEWER_AGENT_TYPE, model: REVIEWER_MODEL_CONFIG.model, }, { label: 'Arbiter', agentType: ARBITER_AGENT_TYPE, model: ARBITER_MODEL_CONFIG.model, }, ]; const failover = getGlobalFailoverInfo(); const lines = ['πŸ€– *λͺ¨λΈ ꡬ성*']; for (const role of roleConfigs) { if (!role.agentType && role.label === 'Arbiter') continue; const type = role.agentType || 'β€”'; const defaultModel = type === 'codex' ? DEFAULT_CODEX_MODEL : DEFAULT_CLAUDE_MODEL; const model = role.model || defaultModel; // Show fallback status for claude-code roles when global failover is active const isFallback = failover.active && type === 'claude-code'; if (isFallback) { const fallbackModel = DEFAULT_CODEX_MODEL; lines.push(` **${role.label}** β€” codex \`${fallbackModel}\` (fallback)`); } else { lines.push(` **${role.label}** β€” ${type} \`${model}\``); } } // MoA status const moaConfig = getMoaConfig(); if (moaConfig.enabled) { const refs = moaConfig.referenceModels .map((m) => `${m.name} \`${m.model}\``) .join(', '); lines.push(` **MoA** β€” ${refs}`); } if (failover.active) { lines.push(` ⚠️ **Failover ν™œμ„±** β€” ${failover.reason || 'μ•Œ 수 μ—†μŒ'}`); } return lines.join('\n'); } function buildUnifiedDashboardContent(): string { const sections: string[] = []; sections.push(buildModelConfigSection()); if (STATUS_SHOW_ROOMS) { sections.push(buildStatusContent()); } if (cachedUsageContent) { sections.push(cachedUsageContent); } return composeDashboardContent(sections); } async function refreshUsageCache(): Promise { if (usageUpdateInProgress) return; usageUpdateInProgress = true; try { cachedUsageContent = await buildUsageContent(); rendererUsageFetchedAt = new Date().toISOString(); } catch (err) { logger.warn({ err }, 'Failed to build usage content'); } finally { usageUpdateInProgress = false; } } /** * Attempt a status-message edit, retrying up to `maxRetries` extra times with * `retryDelayMs` spacing before giving up. Returns true if an edit succeeded, * false if every attempt failed (caller then reposts a fresh message). * `sleep` is injectable so tests can run without real delays. */ export async function editStatusMessageWithRetry(args: { editOnce: () => Promise; maxRetries: number; retryDelayMs: number; onAttemptFailed?: (attempt: number, willRetry: boolean, err: unknown) => void; sleep?: (ms: number) => Promise; }): Promise { const sleep = args.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); for (let attempt = 0; attempt <= args.maxRetries; attempt++) { try { await args.editOnce(); return true; } catch (err) { const willRetry = attempt < args.maxRetries; args.onAttemptFailed?.(attempt + 1, willRetry, err); if (willRetry) await sleep(args.retryDelayMs); } } return false; } /** * Returns a trigger that runs `run` at most once per `delayMs` window: the * first call schedules a run after `delayMs`, and further calls within that * window are folded into the same pending run. Timer fns are injectable for * tests. */ export function createCoalescingTrigger( run: () => void, delayMs: number, timer: { set: (cb: () => void, ms: number) => ReturnType; } = { set: (cb, ms) => setTimeout(cb, ms) }, ): () => void { let scheduled: ReturnType | null = null; return () => { if (scheduled) return; scheduled = timer.set(() => { scheduled = null; run(); }, delayMs); }; } /** * Milliseconds from `nowMs` until the next wall-clock minute boundary (:00). * Minute boundaries align across timezones (offsets are whole minutes), so this * is timezone-agnostic. Exactly on a boundary returns a full minute. */ export function msUntilNextMinuteBoundary(nowMs: number): number { const rem = nowMs % 60_000; return rem === 0 ? 60_000 : 60_000 - rem; } export async function startUnifiedDashboard( opts: UnifiedDashboardOptions, ): Promise { if (!opts.statusChannelId) return; const isRenderer = opts.serviceAgentType === 'claude-code'; const statusJid = `dc:${opts.statusChannelId}`; // The first render after a (re)start should show the status message // immediately: edit the stored message if possible, but if that edit fails, // repost right away instead of waiting through the steady-state retry cycle. let firstStatusRender = true; if (isRenderer) { statusMessageId = readDashboardStatusMessageId(opts.statusChannelId); } if ( isRenderer && shouldPurgeDashboardChannelOnStart({ purgeOnStart: opts.purgeOnStart, storedMessageId: statusMessageId, }) ) { await purgeDashboardChannel(opts); statusMessageId = null; } if (isRenderer) { await fetchAllClaudeProfiles(); await refreshUsageCache(); } const updateStatus = async () => { writeLocalStatusSnapshot(opts); if (!isRenderer) return; // A failed edit now retries with delays, so a single updateStatus can run // for tens of seconds. Skip overlapping ticks (this is also why the base // periodic refresh does not fire mid-retry); remember the request so it runs // once the in-flight update finishes. if (statusUpdateRunning) { statusUpdatePending = true; return; } statusUpdateRunning = true; try { const channel = findDiscordChannel(opts.channels); if (!channel) { logger.warn( { channelCount: opts.channels.length, names: opts.channels.map((c) => c.name), connected: opts.channels.map((c) => c.isConnected()), }, 'Dashboard: no connected Discord channel found', ); return; } await refreshChannelMeta(opts); const content = buildUnifiedDashboardContent(); if (!content) { logger.warn( { cachedUsageLength: cachedUsageContent.length, statusShowRooms: STATUS_SHOW_ROOMS, }, 'Dashboard content empty, skipping render', ); return; } if (statusMessageId && channel.editMessage) { // A transient Discord error (e.g. 503) on the periodic edit should not // immediately spawn a fresh status message. Retry the edit a couple of // times with a delay first; only give up (and repost) if all fail. const editId = statusMessageId; // Bind to the channel: a detached method reference loses `this` and the // edit throws ("this.client is undefined"), which would make every // update fail and repost a fresh (notifying) message. const editMessage = channel.editMessage.bind(channel); // First render after start: 0 retries β†’ repost immediately if the edit // fails. Steady state: retry twice at 15s before reposting. const maxRetries = firstStatusRender ? 0 : STATUS_EDIT_MAX_RETRIES; const edited = await editStatusMessageWithRetry({ editOnce: () => editMessage(statusJid, editId, content), maxRetries, retryDelayMs: STATUS_EDIT_RETRY_DELAY_MS, onAttemptFailed: (attempt, willRetry, err) => logger.warn( { err, messageId: editId, attempt, maxAttempts: maxRetries + 1, willRetry, }, willRetry ? 'Dashboard status message edit failed; retrying in 15s' : 'Dashboard status message edit failed after retries; sending a fresh tracked message', ), }); if (edited) { writeDashboardStatusMessageId(opts.statusChannelId, statusMessageId); } else { statusMessageId = null; } } if (!statusMessageId && channel.sendAndTrack) { const id = await channel.sendAndTrack(statusJid, content); if (id) { statusMessageId = id; writeDashboardStatusMessageId(opts.statusChannelId, id); } } // A render (edit or repost) happened this tick; subsequent ticks use the // steady-state retry policy. firstStatusRender = false; if (statusMessageId) { await cleanupDashboardDuplicateMessages(opts, statusMessageId); } if (!dashboardUpdateLogged) { logger.info( { messageId: statusMessageId, contentLength: content.length }, 'Dashboard updated successfully (first)', ); dashboardUpdateLogged = true; } } catch (err) { logger.warn({ err }, 'Dashboard update failed'); statusMessageId = null; } finally { statusUpdateRunning = false; if (statusUpdatePending) { statusUpdatePending = false; immediateUpdateTrigger?.(); } } }; if (isRenderer) { // Event-driven refresh: external callers (new chat message, agent activity // change) call requestImmediateStatusUpdate(); bursts are coalesced. immediateUpdateTrigger = createCoalescingTrigger( () => void updateStatus(), IMMEDIATE_UPDATE_DEBOUNCE_MS, ); } // Base periodic refresh fires on each wall-clock minute boundary (:00) so the // minute-precision timestamp shown in the status message stays accurate. const scheduleMinuteBoundaryUpdate = () => { setTimeout(() => { void updateStatus(); scheduleMinuteBoundaryUpdate(); }, msUntilNextMinuteBoundary(Date.now())); }; scheduleMinuteBoundaryUpdate(); setInterval(() => { if (!isRenderer || !statusMessageId) return; void cleanupDashboardDuplicateMessages(opts, statusMessageId); }, getDashboardDuplicateCleanupIntervalMs(opts.statusUpdateInterval)); await updateStatus(); if (isRenderer) { setInterval(refreshUsageCache, RENDERER_USAGE_REFRESH_MS); } // Codex usage collection β€” runs in unified service regardless of renderer role. const applyCodexRefresh = (result: { rows: UsageRow[]; fetchedAt: string | null; }) => { cachedCodexUsageRows = result.rows; if (result.fetchedAt) codexUsageFetchedAt = result.fetchedAt; }; const isWarmupRuntimeBusy = () => { const groups = opts.roomBindings(); return opts.queue .getStatuses(Object.keys(groups)) .some((status) => status.status === 'processing'); }; let codexWarmupInFlight = false; const runCodexWarmup = async () => { if (!CODEX_WARMUP_CONFIG.enabled || codexWarmupInFlight) return; codexWarmupInFlight = true; try { const result = await runCodexWarmupCycle(CODEX_WARMUP_CONFIG, { shouldSkip: isWarmupRuntimeBusy, }); if (result.status === 'warmed') { applyCodexRefresh(await refreshAllCodexAccountUsage()); } } catch (err) { logger.warn({ err }, 'Codex warm-up cycle failed unexpectedly'); } finally { codexWarmupInFlight = false; } }; void refreshAllCodexAccountUsage() .then((r) => { applyCodexRefresh(r); return refreshActiveCodexUsage().then(applyCodexRefresh); }) .then(() => runCodexWarmup()); setInterval( () => void refreshActiveCodexUsage().then(applyCodexRefresh), opts.usageUpdateInterval, ); setInterval( () => void refreshAllCodexAccountUsage() .then(applyCodexRefresh) .then(() => runCodexWarmup()), CODEX_FULL_SCAN_INTERVAL, ); if (CODEX_WARMUP_CONFIG.enabled) { setInterval(() => void runCodexWarmup(), CODEX_WARMUP_CONFIG.intervalMs); } logger.info( { channelId: opts.statusChannelId, isRenderer, agentType: opts.serviceAgentType, }, isRenderer ? 'Unified dashboard started' : 'Status snapshot updater started', ); }