feat: global failover + dashboard model config section

- Failover is now global (account-level, not per-channel)
- Dashboard shows role model config (Owner/Reviewer/Arbiter + MoA refs)
- Dashboard shows failover status when active
- Auto-clear failover on successful Claude rotation
- Remove per-channel lease writes from failover path
This commit is contained in:
Eyejoker
2026-03-31 00:38:29 +09:00
parent f98dd27712
commit 68901e5da9
7 changed files with 152 additions and 77 deletions

View File

@@ -191,8 +191,7 @@ function parseMoaReferenceModels(): MoaModelConfig[] {
export function getMoaConfig(): MoaConfig {
const referenceModels = parseMoaReferenceModels();
return {
enabled:
getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
enabled: getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
referenceModels,
};
}

View File

@@ -83,8 +83,8 @@ import {
stopTokenRefreshLoop,
} from './token-refresh.js';
import {
getActiveCodexFailoverLeases,
restoreDefaultChannelLease,
clearGlobalFailover,
getGlobalFailoverInfo,
} from './service-routing.js';
import { FAILOVER_MIN_DURATION_MS } from './config.js';
import { startCredentialProxy } from './credential-proxy.js';
@@ -536,43 +536,20 @@ async function main(): Promise<void> {
});
leaseRecoveryTimer = setInterval(() => {
if (!hasAvailableClaudeToken()) {
return;
}
const now = Date.now();
for (const lease of getActiveCodexFailoverLeases()) {
const activatedMs = lease.activatedAt
? new Date(lease.activatedAt).getTime()
: NaN;
if (Number.isNaN(activatedMs)) {
logger.warn(
{ chatJid: lease.chatJid, activatedAt: lease.activatedAt },
'Failover lease has unparseable activated_at, skipping auto-restore',
);
continue;
}
const elapsed = now - activatedMs;
if (elapsed < FAILOVER_MIN_DURATION_MS) {
logger.debug(
{
chatJid: lease.chatJid,
elapsedMin: Math.round(elapsed / 60_000),
minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000),
},
'Failover lease still within minimum hold period, skipping restore',
);
continue;
}
restoreDefaultChannelLease(lease.chatJid);
logger.info(
{
chatJid: lease.chatJid,
serviceId: SERVICE_ID,
elapsedMin: Math.round(elapsed / 60_000),
},
'Claude token available and failover hold period elapsed, restored default channel lease',
);
}
const failover = getGlobalFailoverInfo();
if (!failover.active) return;
if (!hasAvailableClaudeToken()) return;
const activatedMs = failover.activatedAt
? new Date(failover.activatedAt).getTime()
: NaN;
if (Number.isNaN(activatedMs)) return;
const elapsed = Date.now() - activatedMs;
if (elapsed < FAILOVER_MIN_DURATION_MS) return;
clearGlobalFailover();
logger.info(
{ elapsedMin: Math.round(elapsed / 60_000) },
'Claude token available and hold period elapsed, global failover cleared',
);
}, 5_000);
runtime.startMessageLoop().catch((err) => {
logger.fatal({ err }, 'Message loop crashed unexpectedly');

View File

@@ -53,6 +53,7 @@ vi.mock('./db.js', () => ({
vi.mock('./service-routing.js', () => ({
activateCodexFailover: vi.fn(),
clearGlobalFailover: vi.fn(),
getEffectiveChannelLease: vi.fn(() => ({
chat_jid: 'group@test',
owner_service_id: 'claude',

View File

@@ -19,6 +19,7 @@ import {
getTokenCount,
markTokenHealthy,
} from './token-rotation.js';
import { clearGlobalFailover } from './service-routing.js';
// ── Types ────────────────────────────────────────────────────────
@@ -159,6 +160,7 @@ export async function runClaudeRotationLoop(
// ── Success ──
markTokenHealthy();
clearGlobalFailover();
return { type: 'success' };
}

View File

@@ -3,19 +3,20 @@ import { beforeEach, describe, expect, it } from 'vitest';
import { _initTestDatabase, setRegisteredGroup } from './db.js';
import {
activateCodexFailover,
getActiveCodexFailoverLeases,
clearGlobalFailover,
getEffectiveChannelLease,
getGlobalFailoverInfo,
refreshChannelOwnerCache,
restoreDefaultChannelLease,
} from './service-routing.js';
beforeEach(() => {
_initTestDatabase();
refreshChannelOwnerCache(true);
clearGlobalFailover();
});
describe('service-routing failover leases', () => {
it('uses codex-review as owner and codex-main as reviewer during failover', () => {
describe('service-routing global failover', () => {
it('uses codex-review as owner during global failover', () => {
setRegisteredGroup('dc:paired', {
name: 'Paired Room Claude',
folder: 'paired-claude',
@@ -33,6 +34,8 @@ describe('service-routing failover leases', () => {
activateCodexFailover('dc:paired', 'claude-429');
// Global failover applies to ALL channels
expect(getGlobalFailoverInfo().active).toBe(true);
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
chat_jid: 'dc:paired',
owner_service_id: 'codex-review',
@@ -40,15 +43,15 @@ describe('service-routing failover leases', () => {
reason: 'claude-429',
explicit: true,
});
expect(getActiveCodexFailoverLeases()).toEqual([
{
chatJid: 'dc:paired',
activatedAt: expect.any(String),
},
]);
// Any other channel is also affected
expect(getEffectiveChannelLease('dc:other')).toMatchObject({
owner_service_id: 'codex-review',
reviewer_service_id: 'codex-main',
explicit: true,
});
});
it('restores the default lease after failover is cleared', () => {
it('restores default lease after global failover is cleared', () => {
setRegisteredGroup('dc:paired', {
name: 'Paired Room Claude',
folder: 'paired-claude',
@@ -65,8 +68,9 @@ describe('service-routing failover leases', () => {
});
activateCodexFailover('dc:paired', 'claude-429');
restoreDefaultChannelLease('dc:paired');
clearGlobalFailover();
expect(getGlobalFailoverInfo().active).toBe(false);
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
chat_jid: 'dc:paired',
owner_service_id: 'codex-main',

View File

@@ -16,6 +16,7 @@ import {
setChannelOwnerLease,
type ChannelOwnerLeaseRow,
} from './db.js';
import { logger } from './logger.js';
export interface EffectiveChannelLease {
chat_jid: string;
@@ -122,6 +123,18 @@ export function refreshChannelOwnerCache(force = false): void {
export function getEffectiveChannelLease(
chatJid: string,
): EffectiveChannelLease {
// Global failover overrides all per-channel leases
if (globalFailoverActive) {
return {
chat_jid: chatJid,
owner_service_id: CODEX_REVIEW_SERVICE_ID,
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
arbiter_service_id: null,
activated_at: globalFailoverActivatedAt,
reason: globalFailoverReason,
explicit: true,
};
}
refreshChannelOwnerCache();
const row = leaseCache.get(chatJid);
if (row) {
@@ -167,19 +180,48 @@ export function shouldServiceProcessChat(
return true;
}
export function activateCodexFailover(chatJid: string, reason: string): void {
const now = new Date().toISOString();
const row: ChannelOwnerLeaseRow = {
chat_jid: chatJid,
owner_service_id: CODEX_REVIEW_SERVICE_ID,
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
arbiter_service_id: null,
activated_at: now,
reason,
// ── Global failover ──────────────────────────────────────────────
// Claude API limits are account-level, so failover applies to all channels.
let globalFailoverActive = false;
let globalFailoverReason: string | null = null;
let globalFailoverActivatedAt: string | null = null;
export function activateCodexFailover(
_chatJid: string,
reason: string,
): void {
globalFailoverActive = true;
globalFailoverReason = reason;
globalFailoverActivatedAt = new Date().toISOString();
logger.warn(
{ reason, activatedAt: globalFailoverActivatedAt },
'Global failover activated — all channels switching to codex',
);
}
export function isGlobalFailoverActive(): boolean {
return globalFailoverActive;
}
export function getGlobalFailoverInfo(): {
active: boolean;
reason: string | null;
activatedAt: string | null;
} {
return {
active: globalFailoverActive,
reason: globalFailoverReason,
activatedAt: globalFailoverActivatedAt,
};
setChannelOwnerLease(row);
leaseCache.set(chatJid, row);
lastLeaseRefreshAt = Date.now();
}
export function clearGlobalFailover(): void {
if (!globalFailoverActive) return;
globalFailoverActive = false;
globalFailoverReason = null;
globalFailoverActivatedAt = null;
logger.info('Global failover cleared — resuming normal routing');
}
export function restoreDefaultChannelLease(chatJid: string): void {
@@ -194,18 +236,13 @@ export interface ActiveFailoverLease {
}
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
refreshChannelOwnerCache(true);
return [...leaseCache.values()]
.filter(
(row) =>
normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID &&
normalizeServiceId(row.reviewer_service_id || '') ===
CODEX_MAIN_SERVICE_ID,
)
.map((row) => ({
chatJid: row.chat_jid,
activatedAt: row.activated_at ?? null,
}));
// Global failover: report as a single pseudo-lease
if (globalFailoverActive) {
return [
{ chatJid: '*', activatedAt: globalFailoverActivatedAt },
];
}
return [];
}
/** @deprecated Use getActiveCodexFailoverLeases() instead */

View File

@@ -2,10 +2,18 @@ import { execSync } from 'child_process';
import os from 'os';
import {
ARBITER_AGENT_TYPE,
ARBITER_MODEL_CONFIG,
OWNER_AGENT_TYPE,
OWNER_MODEL_CONFIG,
REVIEWER_AGENT_TYPE,
REVIEWER_MODEL_CONFIG,
STATUS_SHOW_ROOM_DETAILS,
STATUS_SHOW_ROOMS,
USAGE_DASHBOARD_ENABLED,
getMoaConfig,
} from './config.js';
import { getGlobalFailoverInfo } from './service-routing.js';
import {
fetchAllClaudeUsage,
fetchAllClaudeProfiles,
@@ -540,8 +548,55 @@ async function buildUsageContent(): Promise<string> {
return lines.join('\n');
}
function buildModelConfigSection(): string {
const roleConfigs = [
{
label: 'Owner',
agentType: OWNER_AGENT_TYPE,
model: OWNER_MODEL_CONFIG.model,
},
{
label: 'Reviewer',
agentType: REVIEWER_AGENT_TYPE,
model: REVIEWER_MODEL_CONFIG.model,
},
{
label: 'Arbiter',
agentType: ARBITER_AGENT_TYPE,
model: ARBITER_MODEL_CONFIG.model,
},
];
const lines = ['🤖 *모델 구성*'];
for (const role of roleConfigs) {
if (!role.agentType && role.label === 'Arbiter') continue;
const type = role.agentType || '—';
const defaultModel = type === 'codex'
? (process.env.CODEX_MODEL || 'codex')
: (process.env.CLAUDE_MODEL || 'claude');
const model = role.model || defaultModel;
lines.push(` **${role.label}** — ${type} \`${model}\``);
}
// MoA status
const moaConfig = getMoaConfig();
if (moaConfig.enabled) {
const refs = moaConfig.referenceModels.map((m) => m.name).join(', ');
lines.push(` **MoA** — ${refs}`);
}
// Global failover
const failover = getGlobalFailoverInfo();
if (failover.active) {
lines.push(` ⚠️ **Failover 활성** — ${failover.reason || '알 수 없음'}`);
}
return lines.join('\n');
}
function buildUnifiedDashboardContent(): string {
const sections: string[] = [];
sections.push(buildModelConfigSection());
if (STATUS_SHOW_ROOMS) {
sections.push(buildStatusContent());
}