From 68901e5da9cf6435115502a24c08346caa404561 Mon Sep 17 00:00:00 2001 From: Eyejoker Date: Tue, 31 Mar 2026 00:38:29 +0900 Subject: [PATCH] feat: global failover + dashboard model config section - Failover is now global (account-level, not per-channel) - Dashboard shows role model config (Owner/Reviewer/Arbiter + MoA refs) - Dashboard shows failover status when active - Auto-clear failover on successful Claude rotation - Remove per-channel lease writes from failover path --- src/config.ts | 3 +- src/index.ts | 55 ++++++------------- src/message-agent-executor.test.ts | 1 + src/provider-retry.ts | 2 + src/service-routing.test.ts | 28 +++++----- src/service-routing.ts | 85 +++++++++++++++++++++--------- src/unified-dashboard.ts | 55 +++++++++++++++++++ 7 files changed, 152 insertions(+), 77 deletions(-) diff --git a/src/config.ts b/src/config.ts index 2794c37..05123e2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -191,8 +191,7 @@ function parseMoaReferenceModels(): MoaModelConfig[] { export function getMoaConfig(): MoaConfig { const referenceModels = parseMoaReferenceModels(); return { - enabled: - getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0, + enabled: getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0, referenceModels, }; } diff --git a/src/index.ts b/src/index.ts index 059a017..c1c78ba 100644 --- a/src/index.ts +++ b/src/index.ts @@ -83,8 +83,8 @@ import { stopTokenRefreshLoop, } from './token-refresh.js'; import { - getActiveCodexFailoverLeases, - restoreDefaultChannelLease, + clearGlobalFailover, + getGlobalFailoverInfo, } from './service-routing.js'; import { FAILOVER_MIN_DURATION_MS } from './config.js'; import { startCredentialProxy } from './credential-proxy.js'; @@ -536,43 +536,20 @@ async function main(): Promise { }); leaseRecoveryTimer = setInterval(() => { - if (!hasAvailableClaudeToken()) { - return; - } - const now = Date.now(); - for (const lease of getActiveCodexFailoverLeases()) { - const activatedMs = lease.activatedAt - ? new Date(lease.activatedAt).getTime() - : NaN; - if (Number.isNaN(activatedMs)) { - logger.warn( - { chatJid: lease.chatJid, activatedAt: lease.activatedAt }, - 'Failover lease has unparseable activated_at, skipping auto-restore', - ); - continue; - } - const elapsed = now - activatedMs; - if (elapsed < FAILOVER_MIN_DURATION_MS) { - logger.debug( - { - chatJid: lease.chatJid, - elapsedMin: Math.round(elapsed / 60_000), - minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000), - }, - 'Failover lease still within minimum hold period, skipping restore', - ); - continue; - } - restoreDefaultChannelLease(lease.chatJid); - logger.info( - { - chatJid: lease.chatJid, - serviceId: SERVICE_ID, - elapsedMin: Math.round(elapsed / 60_000), - }, - 'Claude token available and failover hold period elapsed, restored default channel lease', - ); - } + const failover = getGlobalFailoverInfo(); + if (!failover.active) return; + if (!hasAvailableClaudeToken()) return; + const activatedMs = failover.activatedAt + ? new Date(failover.activatedAt).getTime() + : NaN; + if (Number.isNaN(activatedMs)) return; + const elapsed = Date.now() - activatedMs; + if (elapsed < FAILOVER_MIN_DURATION_MS) return; + clearGlobalFailover(); + logger.info( + { elapsedMin: Math.round(elapsed / 60_000) }, + 'Claude token available and hold period elapsed, global failover cleared', + ); }, 5_000); runtime.startMessageLoop().catch((err) => { logger.fatal({ err }, 'Message loop crashed unexpectedly'); diff --git a/src/message-agent-executor.test.ts b/src/message-agent-executor.test.ts index 547d239..14acf25 100644 --- a/src/message-agent-executor.test.ts +++ b/src/message-agent-executor.test.ts @@ -53,6 +53,7 @@ vi.mock('./db.js', () => ({ vi.mock('./service-routing.js', () => ({ activateCodexFailover: vi.fn(), + clearGlobalFailover: vi.fn(), getEffectiveChannelLease: vi.fn(() => ({ chat_jid: 'group@test', owner_service_id: 'claude', diff --git a/src/provider-retry.ts b/src/provider-retry.ts index e374ed4..1b21fcb 100644 --- a/src/provider-retry.ts +++ b/src/provider-retry.ts @@ -19,6 +19,7 @@ import { getTokenCount, markTokenHealthy, } from './token-rotation.js'; +import { clearGlobalFailover } from './service-routing.js'; // ── Types ──────────────────────────────────────────────────────── @@ -159,6 +160,7 @@ export async function runClaudeRotationLoop( // ── Success ── markTokenHealthy(); + clearGlobalFailover(); return { type: 'success' }; } diff --git a/src/service-routing.test.ts b/src/service-routing.test.ts index 97da7fd..06e5b4c 100644 --- a/src/service-routing.test.ts +++ b/src/service-routing.test.ts @@ -3,19 +3,20 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { _initTestDatabase, setRegisteredGroup } from './db.js'; import { activateCodexFailover, - getActiveCodexFailoverLeases, + clearGlobalFailover, getEffectiveChannelLease, + getGlobalFailoverInfo, refreshChannelOwnerCache, - restoreDefaultChannelLease, } from './service-routing.js'; beforeEach(() => { _initTestDatabase(); refreshChannelOwnerCache(true); + clearGlobalFailover(); }); -describe('service-routing failover leases', () => { - it('uses codex-review as owner and codex-main as reviewer during failover', () => { +describe('service-routing global failover', () => { + it('uses codex-review as owner during global failover', () => { setRegisteredGroup('dc:paired', { name: 'Paired Room Claude', folder: 'paired-claude', @@ -33,6 +34,8 @@ describe('service-routing failover leases', () => { activateCodexFailover('dc:paired', 'claude-429'); + // Global failover applies to ALL channels + expect(getGlobalFailoverInfo().active).toBe(true); expect(getEffectiveChannelLease('dc:paired')).toMatchObject({ chat_jid: 'dc:paired', owner_service_id: 'codex-review', @@ -40,15 +43,15 @@ describe('service-routing failover leases', () => { reason: 'claude-429', explicit: true, }); - expect(getActiveCodexFailoverLeases()).toEqual([ - { - chatJid: 'dc:paired', - activatedAt: expect.any(String), - }, - ]); + // Any other channel is also affected + expect(getEffectiveChannelLease('dc:other')).toMatchObject({ + owner_service_id: 'codex-review', + reviewer_service_id: 'codex-main', + explicit: true, + }); }); - it('restores the default lease after failover is cleared', () => { + it('restores default lease after global failover is cleared', () => { setRegisteredGroup('dc:paired', { name: 'Paired Room Claude', folder: 'paired-claude', @@ -65,8 +68,9 @@ describe('service-routing failover leases', () => { }); activateCodexFailover('dc:paired', 'claude-429'); - restoreDefaultChannelLease('dc:paired'); + clearGlobalFailover(); + expect(getGlobalFailoverInfo().active).toBe(false); expect(getEffectiveChannelLease('dc:paired')).toMatchObject({ chat_jid: 'dc:paired', owner_service_id: 'codex-main', diff --git a/src/service-routing.ts b/src/service-routing.ts index f0fc7dd..d18e7c9 100644 --- a/src/service-routing.ts +++ b/src/service-routing.ts @@ -16,6 +16,7 @@ import { setChannelOwnerLease, type ChannelOwnerLeaseRow, } from './db.js'; +import { logger } from './logger.js'; export interface EffectiveChannelLease { chat_jid: string; @@ -122,6 +123,18 @@ export function refreshChannelOwnerCache(force = false): void { export function getEffectiveChannelLease( chatJid: string, ): EffectiveChannelLease { + // Global failover overrides all per-channel leases + if (globalFailoverActive) { + return { + chat_jid: chatJid, + owner_service_id: CODEX_REVIEW_SERVICE_ID, + reviewer_service_id: CODEX_MAIN_SERVICE_ID, + arbiter_service_id: null, + activated_at: globalFailoverActivatedAt, + reason: globalFailoverReason, + explicit: true, + }; + } refreshChannelOwnerCache(); const row = leaseCache.get(chatJid); if (row) { @@ -167,19 +180,48 @@ export function shouldServiceProcessChat( return true; } -export function activateCodexFailover(chatJid: string, reason: string): void { - const now = new Date().toISOString(); - const row: ChannelOwnerLeaseRow = { - chat_jid: chatJid, - owner_service_id: CODEX_REVIEW_SERVICE_ID, - reviewer_service_id: CODEX_MAIN_SERVICE_ID, - arbiter_service_id: null, - activated_at: now, - reason, +// ── Global failover ────────────────────────────────────────────── +// Claude API limits are account-level, so failover applies to all channels. + +let globalFailoverActive = false; +let globalFailoverReason: string | null = null; +let globalFailoverActivatedAt: string | null = null; + +export function activateCodexFailover( + _chatJid: string, + reason: string, +): void { + globalFailoverActive = true; + globalFailoverReason = reason; + globalFailoverActivatedAt = new Date().toISOString(); + logger.warn( + { reason, activatedAt: globalFailoverActivatedAt }, + 'Global failover activated — all channels switching to codex', + ); +} + +export function isGlobalFailoverActive(): boolean { + return globalFailoverActive; +} + +export function getGlobalFailoverInfo(): { + active: boolean; + reason: string | null; + activatedAt: string | null; +} { + return { + active: globalFailoverActive, + reason: globalFailoverReason, + activatedAt: globalFailoverActivatedAt, }; - setChannelOwnerLease(row); - leaseCache.set(chatJid, row); - lastLeaseRefreshAt = Date.now(); +} + +export function clearGlobalFailover(): void { + if (!globalFailoverActive) return; + globalFailoverActive = false; + globalFailoverReason = null; + globalFailoverActivatedAt = null; + logger.info('Global failover cleared — resuming normal routing'); } export function restoreDefaultChannelLease(chatJid: string): void { @@ -194,18 +236,13 @@ export interface ActiveFailoverLease { } export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] { - refreshChannelOwnerCache(true); - return [...leaseCache.values()] - .filter( - (row) => - normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID && - normalizeServiceId(row.reviewer_service_id || '') === - CODEX_MAIN_SERVICE_ID, - ) - .map((row) => ({ - chatJid: row.chat_jid, - activatedAt: row.activated_at ?? null, - })); + // Global failover: report as a single pseudo-lease + if (globalFailoverActive) { + return [ + { chatJid: '*', activatedAt: globalFailoverActivatedAt }, + ]; + } + return []; } /** @deprecated Use getActiveCodexFailoverLeases() instead */ diff --git a/src/unified-dashboard.ts b/src/unified-dashboard.ts index 706a78f..ad167df 100644 --- a/src/unified-dashboard.ts +++ b/src/unified-dashboard.ts @@ -2,10 +2,18 @@ import { execSync } from 'child_process'; import os from 'os'; import { + ARBITER_AGENT_TYPE, + ARBITER_MODEL_CONFIG, + OWNER_AGENT_TYPE, + OWNER_MODEL_CONFIG, + REVIEWER_AGENT_TYPE, + REVIEWER_MODEL_CONFIG, STATUS_SHOW_ROOM_DETAILS, STATUS_SHOW_ROOMS, USAGE_DASHBOARD_ENABLED, + getMoaConfig, } from './config.js'; +import { getGlobalFailoverInfo } from './service-routing.js'; import { fetchAllClaudeUsage, fetchAllClaudeProfiles, @@ -540,8 +548,55 @@ async function buildUsageContent(): Promise { return lines.join('\n'); } +function buildModelConfigSection(): string { + const roleConfigs = [ + { + label: 'Owner', + agentType: OWNER_AGENT_TYPE, + model: OWNER_MODEL_CONFIG.model, + }, + { + label: 'Reviewer', + agentType: REVIEWER_AGENT_TYPE, + model: REVIEWER_MODEL_CONFIG.model, + }, + { + label: 'Arbiter', + agentType: ARBITER_AGENT_TYPE, + model: ARBITER_MODEL_CONFIG.model, + }, + ]; + + const lines = ['🤖 *모델 구성*']; + for (const role of roleConfigs) { + if (!role.agentType && role.label === 'Arbiter') continue; + const type = role.agentType || '—'; + const defaultModel = type === 'codex' + ? (process.env.CODEX_MODEL || 'codex') + : (process.env.CLAUDE_MODEL || 'claude'); + const model = role.model || defaultModel; + lines.push(` **${role.label}** — ${type} \`${model}\``); + } + + // MoA status + const moaConfig = getMoaConfig(); + if (moaConfig.enabled) { + const refs = moaConfig.referenceModels.map((m) => m.name).join(', '); + lines.push(` **MoA** — ${refs}`); + } + + // Global failover + const failover = getGlobalFailoverInfo(); + if (failover.active) { + lines.push(` ⚠️ **Failover 활성** — ${failover.reason || '알 수 없음'}`); + } + + return lines.join('\n'); +} + function buildUnifiedDashboardContent(): string { const sections: string[] = []; + sections.push(buildModelConfigSection()); if (STATUS_SHOW_ROOMS) { sections.push(buildStatusContent()); }