feat: global failover + dashboard model config section

- Failover is now global (account-level, not per-channel)
- Dashboard shows role model config (Owner/Reviewer/Arbiter + MoA refs)
- Dashboard shows failover status when active
- Auto-clear failover on successful Claude rotation
- Remove per-channel lease writes from failover path
This commit is contained in:
Eyejoker
2026-03-31 00:38:29 +09:00
parent f98dd27712
commit 68901e5da9
7 changed files with 152 additions and 77 deletions

View File

@@ -191,8 +191,7 @@ function parseMoaReferenceModels(): MoaModelConfig[] {
export function getMoaConfig(): MoaConfig { export function getMoaConfig(): MoaConfig {
const referenceModels = parseMoaReferenceModels(); const referenceModels = parseMoaReferenceModels();
return { return {
enabled: enabled: getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
referenceModels, referenceModels,
}; };
} }

View File

@@ -83,8 +83,8 @@ import {
stopTokenRefreshLoop, stopTokenRefreshLoop,
} from './token-refresh.js'; } from './token-refresh.js';
import { import {
getActiveCodexFailoverLeases, clearGlobalFailover,
restoreDefaultChannelLease, getGlobalFailoverInfo,
} from './service-routing.js'; } from './service-routing.js';
import { FAILOVER_MIN_DURATION_MS } from './config.js'; import { FAILOVER_MIN_DURATION_MS } from './config.js';
import { startCredentialProxy } from './credential-proxy.js'; import { startCredentialProxy } from './credential-proxy.js';
@@ -536,43 +536,20 @@ async function main(): Promise<void> {
}); });
leaseRecoveryTimer = setInterval(() => { leaseRecoveryTimer = setInterval(() => {
if (!hasAvailableClaudeToken()) { const failover = getGlobalFailoverInfo();
return; if (!failover.active) return;
} if (!hasAvailableClaudeToken()) return;
const now = Date.now(); const activatedMs = failover.activatedAt
for (const lease of getActiveCodexFailoverLeases()) { ? new Date(failover.activatedAt).getTime()
const activatedMs = lease.activatedAt : NaN;
? new Date(lease.activatedAt).getTime() if (Number.isNaN(activatedMs)) return;
: NaN; const elapsed = Date.now() - activatedMs;
if (Number.isNaN(activatedMs)) { if (elapsed < FAILOVER_MIN_DURATION_MS) return;
logger.warn( clearGlobalFailover();
{ chatJid: lease.chatJid, activatedAt: lease.activatedAt }, logger.info(
'Failover lease has unparseable activated_at, skipping auto-restore', { elapsedMin: Math.round(elapsed / 60_000) },
); 'Claude token available and hold period elapsed, global failover cleared',
continue; );
}
const elapsed = now - activatedMs;
if (elapsed < FAILOVER_MIN_DURATION_MS) {
logger.debug(
{
chatJid: lease.chatJid,
elapsedMin: Math.round(elapsed / 60_000),
minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000),
},
'Failover lease still within minimum hold period, skipping restore',
);
continue;
}
restoreDefaultChannelLease(lease.chatJid);
logger.info(
{
chatJid: lease.chatJid,
serviceId: SERVICE_ID,
elapsedMin: Math.round(elapsed / 60_000),
},
'Claude token available and failover hold period elapsed, restored default channel lease',
);
}
}, 5_000); }, 5_000);
runtime.startMessageLoop().catch((err) => { runtime.startMessageLoop().catch((err) => {
logger.fatal({ err }, 'Message loop crashed unexpectedly'); logger.fatal({ err }, 'Message loop crashed unexpectedly');

View File

@@ -53,6 +53,7 @@ vi.mock('./db.js', () => ({
vi.mock('./service-routing.js', () => ({ vi.mock('./service-routing.js', () => ({
activateCodexFailover: vi.fn(), activateCodexFailover: vi.fn(),
clearGlobalFailover: vi.fn(),
getEffectiveChannelLease: vi.fn(() => ({ getEffectiveChannelLease: vi.fn(() => ({
chat_jid: 'group@test', chat_jid: 'group@test',
owner_service_id: 'claude', owner_service_id: 'claude',

View File

@@ -19,6 +19,7 @@ import {
getTokenCount, getTokenCount,
markTokenHealthy, markTokenHealthy,
} from './token-rotation.js'; } from './token-rotation.js';
import { clearGlobalFailover } from './service-routing.js';
// ── Types ──────────────────────────────────────────────────────── // ── Types ────────────────────────────────────────────────────────
@@ -159,6 +160,7 @@ export async function runClaudeRotationLoop(
// ── Success ── // ── Success ──
markTokenHealthy(); markTokenHealthy();
clearGlobalFailover();
return { type: 'success' }; return { type: 'success' };
} }

View File

@@ -3,19 +3,20 @@ import { beforeEach, describe, expect, it } from 'vitest';
import { _initTestDatabase, setRegisteredGroup } from './db.js'; import { _initTestDatabase, setRegisteredGroup } from './db.js';
import { import {
activateCodexFailover, activateCodexFailover,
getActiveCodexFailoverLeases, clearGlobalFailover,
getEffectiveChannelLease, getEffectiveChannelLease,
getGlobalFailoverInfo,
refreshChannelOwnerCache, refreshChannelOwnerCache,
restoreDefaultChannelLease,
} from './service-routing.js'; } from './service-routing.js';
beforeEach(() => { beforeEach(() => {
_initTestDatabase(); _initTestDatabase();
refreshChannelOwnerCache(true); refreshChannelOwnerCache(true);
clearGlobalFailover();
}); });
describe('service-routing failover leases', () => { describe('service-routing global failover', () => {
it('uses codex-review as owner and codex-main as reviewer during failover', () => { it('uses codex-review as owner during global failover', () => {
setRegisteredGroup('dc:paired', { setRegisteredGroup('dc:paired', {
name: 'Paired Room Claude', name: 'Paired Room Claude',
folder: 'paired-claude', folder: 'paired-claude',
@@ -33,6 +34,8 @@ describe('service-routing failover leases', () => {
activateCodexFailover('dc:paired', 'claude-429'); activateCodexFailover('dc:paired', 'claude-429');
// Global failover applies to ALL channels
expect(getGlobalFailoverInfo().active).toBe(true);
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({ expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
chat_jid: 'dc:paired', chat_jid: 'dc:paired',
owner_service_id: 'codex-review', owner_service_id: 'codex-review',
@@ -40,15 +43,15 @@ describe('service-routing failover leases', () => {
reason: 'claude-429', reason: 'claude-429',
explicit: true, explicit: true,
}); });
expect(getActiveCodexFailoverLeases()).toEqual([ // Any other channel is also affected
{ expect(getEffectiveChannelLease('dc:other')).toMatchObject({
chatJid: 'dc:paired', owner_service_id: 'codex-review',
activatedAt: expect.any(String), reviewer_service_id: 'codex-main',
}, explicit: true,
]); });
}); });
it('restores the default lease after failover is cleared', () => { it('restores default lease after global failover is cleared', () => {
setRegisteredGroup('dc:paired', { setRegisteredGroup('dc:paired', {
name: 'Paired Room Claude', name: 'Paired Room Claude',
folder: 'paired-claude', folder: 'paired-claude',
@@ -65,8 +68,9 @@ describe('service-routing failover leases', () => {
}); });
activateCodexFailover('dc:paired', 'claude-429'); activateCodexFailover('dc:paired', 'claude-429');
restoreDefaultChannelLease('dc:paired'); clearGlobalFailover();
expect(getGlobalFailoverInfo().active).toBe(false);
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({ expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
chat_jid: 'dc:paired', chat_jid: 'dc:paired',
owner_service_id: 'codex-main', owner_service_id: 'codex-main',

View File

@@ -16,6 +16,7 @@ import {
setChannelOwnerLease, setChannelOwnerLease,
type ChannelOwnerLeaseRow, type ChannelOwnerLeaseRow,
} from './db.js'; } from './db.js';
import { logger } from './logger.js';
export interface EffectiveChannelLease { export interface EffectiveChannelLease {
chat_jid: string; chat_jid: string;
@@ -122,6 +123,18 @@ export function refreshChannelOwnerCache(force = false): void {
export function getEffectiveChannelLease( export function getEffectiveChannelLease(
chatJid: string, chatJid: string,
): EffectiveChannelLease { ): EffectiveChannelLease {
// Global failover overrides all per-channel leases
if (globalFailoverActive) {
return {
chat_jid: chatJid,
owner_service_id: CODEX_REVIEW_SERVICE_ID,
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
arbiter_service_id: null,
activated_at: globalFailoverActivatedAt,
reason: globalFailoverReason,
explicit: true,
};
}
refreshChannelOwnerCache(); refreshChannelOwnerCache();
const row = leaseCache.get(chatJid); const row = leaseCache.get(chatJid);
if (row) { if (row) {
@@ -167,19 +180,48 @@ export function shouldServiceProcessChat(
return true; return true;
} }
export function activateCodexFailover(chatJid: string, reason: string): void { // ── Global failover ──────────────────────────────────────────────
const now = new Date().toISOString(); // Claude API limits are account-level, so failover applies to all channels.
const row: ChannelOwnerLeaseRow = {
chat_jid: chatJid, let globalFailoverActive = false;
owner_service_id: CODEX_REVIEW_SERVICE_ID, let globalFailoverReason: string | null = null;
reviewer_service_id: CODEX_MAIN_SERVICE_ID, let globalFailoverActivatedAt: string | null = null;
arbiter_service_id: null,
activated_at: now, export function activateCodexFailover(
reason, _chatJid: string,
reason: string,
): void {
globalFailoverActive = true;
globalFailoverReason = reason;
globalFailoverActivatedAt = new Date().toISOString();
logger.warn(
{ reason, activatedAt: globalFailoverActivatedAt },
'Global failover activated — all channels switching to codex',
);
}
export function isGlobalFailoverActive(): boolean {
return globalFailoverActive;
}
export function getGlobalFailoverInfo(): {
active: boolean;
reason: string | null;
activatedAt: string | null;
} {
return {
active: globalFailoverActive,
reason: globalFailoverReason,
activatedAt: globalFailoverActivatedAt,
}; };
setChannelOwnerLease(row); }
leaseCache.set(chatJid, row);
lastLeaseRefreshAt = Date.now(); export function clearGlobalFailover(): void {
if (!globalFailoverActive) return;
globalFailoverActive = false;
globalFailoverReason = null;
globalFailoverActivatedAt = null;
logger.info('Global failover cleared — resuming normal routing');
} }
export function restoreDefaultChannelLease(chatJid: string): void { export function restoreDefaultChannelLease(chatJid: string): void {
@@ -194,18 +236,13 @@ export interface ActiveFailoverLease {
} }
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] { export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
refreshChannelOwnerCache(true); // Global failover: report as a single pseudo-lease
return [...leaseCache.values()] if (globalFailoverActive) {
.filter( return [
(row) => { chatJid: '*', activatedAt: globalFailoverActivatedAt },
normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID && ];
normalizeServiceId(row.reviewer_service_id || '') === }
CODEX_MAIN_SERVICE_ID, return [];
)
.map((row) => ({
chatJid: row.chat_jid,
activatedAt: row.activated_at ?? null,
}));
} }
/** @deprecated Use getActiveCodexFailoverLeases() instead */ /** @deprecated Use getActiveCodexFailoverLeases() instead */

View File

@@ -2,10 +2,18 @@ import { execSync } from 'child_process';
import os from 'os'; import os from 'os';
import { import {
ARBITER_AGENT_TYPE,
ARBITER_MODEL_CONFIG,
OWNER_AGENT_TYPE,
OWNER_MODEL_CONFIG,
REVIEWER_AGENT_TYPE,
REVIEWER_MODEL_CONFIG,
STATUS_SHOW_ROOM_DETAILS, STATUS_SHOW_ROOM_DETAILS,
STATUS_SHOW_ROOMS, STATUS_SHOW_ROOMS,
USAGE_DASHBOARD_ENABLED, USAGE_DASHBOARD_ENABLED,
getMoaConfig,
} from './config.js'; } from './config.js';
import { getGlobalFailoverInfo } from './service-routing.js';
import { import {
fetchAllClaudeUsage, fetchAllClaudeUsage,
fetchAllClaudeProfiles, fetchAllClaudeProfiles,
@@ -540,8 +548,55 @@ async function buildUsageContent(): Promise<string> {
return lines.join('\n'); return lines.join('\n');
} }
function buildModelConfigSection(): string {
const roleConfigs = [
{
label: 'Owner',
agentType: OWNER_AGENT_TYPE,
model: OWNER_MODEL_CONFIG.model,
},
{
label: 'Reviewer',
agentType: REVIEWER_AGENT_TYPE,
model: REVIEWER_MODEL_CONFIG.model,
},
{
label: 'Arbiter',
agentType: ARBITER_AGENT_TYPE,
model: ARBITER_MODEL_CONFIG.model,
},
];
const lines = ['🤖 *모델 구성*'];
for (const role of roleConfigs) {
if (!role.agentType && role.label === 'Arbiter') continue;
const type = role.agentType || '—';
const defaultModel = type === 'codex'
? (process.env.CODEX_MODEL || 'codex')
: (process.env.CLAUDE_MODEL || 'claude');
const model = role.model || defaultModel;
lines.push(` **${role.label}** — ${type} \`${model}\``);
}
// MoA status
const moaConfig = getMoaConfig();
if (moaConfig.enabled) {
const refs = moaConfig.referenceModels.map((m) => m.name).join(', ');
lines.push(` **MoA** — ${refs}`);
}
// Global failover
const failover = getGlobalFailoverInfo();
if (failover.active) {
lines.push(` ⚠️ **Failover 활성** — ${failover.reason || '알 수 없음'}`);
}
return lines.join('\n');
}
function buildUnifiedDashboardContent(): string { function buildUnifiedDashboardContent(): string {
const sections: string[] = []; const sections: string[] = [];
sections.push(buildModelConfigSection());
if (STATUS_SHOW_ROOMS) { if (STATUS_SHOW_ROOMS) {
sections.push(buildStatusContent()); sections.push(buildStatusContent());
} }