feat: global failover + dashboard model config section
- Failover is now global (account-level, not per-channel) - Dashboard shows role model config (Owner/Reviewer/Arbiter + MoA refs) - Dashboard shows failover status when active - Auto-clear failover on successful Claude rotation - Remove per-channel lease writes from failover path
This commit is contained in:
@@ -191,8 +191,7 @@ function parseMoaReferenceModels(): MoaModelConfig[] {
|
||||
export function getMoaConfig(): MoaConfig {
|
||||
const referenceModels = parseMoaReferenceModels();
|
||||
return {
|
||||
enabled:
|
||||
getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
|
||||
enabled: getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
|
||||
referenceModels,
|
||||
};
|
||||
}
|
||||
|
||||
55
src/index.ts
55
src/index.ts
@@ -83,8 +83,8 @@ import {
|
||||
stopTokenRefreshLoop,
|
||||
} from './token-refresh.js';
|
||||
import {
|
||||
getActiveCodexFailoverLeases,
|
||||
restoreDefaultChannelLease,
|
||||
clearGlobalFailover,
|
||||
getGlobalFailoverInfo,
|
||||
} from './service-routing.js';
|
||||
import { FAILOVER_MIN_DURATION_MS } from './config.js';
|
||||
import { startCredentialProxy } from './credential-proxy.js';
|
||||
@@ -536,43 +536,20 @@ async function main(): Promise<void> {
|
||||
});
|
||||
|
||||
leaseRecoveryTimer = setInterval(() => {
|
||||
if (!hasAvailableClaudeToken()) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
for (const lease of getActiveCodexFailoverLeases()) {
|
||||
const activatedMs = lease.activatedAt
|
||||
? new Date(lease.activatedAt).getTime()
|
||||
: NaN;
|
||||
if (Number.isNaN(activatedMs)) {
|
||||
logger.warn(
|
||||
{ chatJid: lease.chatJid, activatedAt: lease.activatedAt },
|
||||
'Failover lease has unparseable activated_at, skipping auto-restore',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const elapsed = now - activatedMs;
|
||||
if (elapsed < FAILOVER_MIN_DURATION_MS) {
|
||||
logger.debug(
|
||||
{
|
||||
chatJid: lease.chatJid,
|
||||
elapsedMin: Math.round(elapsed / 60_000),
|
||||
minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000),
|
||||
},
|
||||
'Failover lease still within minimum hold period, skipping restore',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
restoreDefaultChannelLease(lease.chatJid);
|
||||
logger.info(
|
||||
{
|
||||
chatJid: lease.chatJid,
|
||||
serviceId: SERVICE_ID,
|
||||
elapsedMin: Math.round(elapsed / 60_000),
|
||||
},
|
||||
'Claude token available and failover hold period elapsed, restored default channel lease',
|
||||
);
|
||||
}
|
||||
const failover = getGlobalFailoverInfo();
|
||||
if (!failover.active) return;
|
||||
if (!hasAvailableClaudeToken()) return;
|
||||
const activatedMs = failover.activatedAt
|
||||
? new Date(failover.activatedAt).getTime()
|
||||
: NaN;
|
||||
if (Number.isNaN(activatedMs)) return;
|
||||
const elapsed = Date.now() - activatedMs;
|
||||
if (elapsed < FAILOVER_MIN_DURATION_MS) return;
|
||||
clearGlobalFailover();
|
||||
logger.info(
|
||||
{ elapsedMin: Math.round(elapsed / 60_000) },
|
||||
'Claude token available and hold period elapsed, global failover cleared',
|
||||
);
|
||||
}, 5_000);
|
||||
runtime.startMessageLoop().catch((err) => {
|
||||
logger.fatal({ err }, 'Message loop crashed unexpectedly');
|
||||
|
||||
@@ -53,6 +53,7 @@ vi.mock('./db.js', () => ({
|
||||
|
||||
vi.mock('./service-routing.js', () => ({
|
||||
activateCodexFailover: vi.fn(),
|
||||
clearGlobalFailover: vi.fn(),
|
||||
getEffectiveChannelLease: vi.fn(() => ({
|
||||
chat_jid: 'group@test',
|
||||
owner_service_id: 'claude',
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
getTokenCount,
|
||||
markTokenHealthy,
|
||||
} from './token-rotation.js';
|
||||
import { clearGlobalFailover } from './service-routing.js';
|
||||
|
||||
// ── Types ────────────────────────────────────────────────────────
|
||||
|
||||
@@ -159,6 +160,7 @@ export async function runClaudeRotationLoop(
|
||||
|
||||
// ── Success ──
|
||||
markTokenHealthy();
|
||||
clearGlobalFailover();
|
||||
return { type: 'success' };
|
||||
}
|
||||
|
||||
|
||||
@@ -3,19 +3,20 @@ import { beforeEach, describe, expect, it } from 'vitest';
|
||||
import { _initTestDatabase, setRegisteredGroup } from './db.js';
|
||||
import {
|
||||
activateCodexFailover,
|
||||
getActiveCodexFailoverLeases,
|
||||
clearGlobalFailover,
|
||||
getEffectiveChannelLease,
|
||||
getGlobalFailoverInfo,
|
||||
refreshChannelOwnerCache,
|
||||
restoreDefaultChannelLease,
|
||||
} from './service-routing.js';
|
||||
|
||||
beforeEach(() => {
|
||||
_initTestDatabase();
|
||||
refreshChannelOwnerCache(true);
|
||||
clearGlobalFailover();
|
||||
});
|
||||
|
||||
describe('service-routing failover leases', () => {
|
||||
it('uses codex-review as owner and codex-main as reviewer during failover', () => {
|
||||
describe('service-routing global failover', () => {
|
||||
it('uses codex-review as owner during global failover', () => {
|
||||
setRegisteredGroup('dc:paired', {
|
||||
name: 'Paired Room Claude',
|
||||
folder: 'paired-claude',
|
||||
@@ -33,6 +34,8 @@ describe('service-routing failover leases', () => {
|
||||
|
||||
activateCodexFailover('dc:paired', 'claude-429');
|
||||
|
||||
// Global failover applies to ALL channels
|
||||
expect(getGlobalFailoverInfo().active).toBe(true);
|
||||
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
||||
chat_jid: 'dc:paired',
|
||||
owner_service_id: 'codex-review',
|
||||
@@ -40,15 +43,15 @@ describe('service-routing failover leases', () => {
|
||||
reason: 'claude-429',
|
||||
explicit: true,
|
||||
});
|
||||
expect(getActiveCodexFailoverLeases()).toEqual([
|
||||
{
|
||||
chatJid: 'dc:paired',
|
||||
activatedAt: expect.any(String),
|
||||
},
|
||||
]);
|
||||
// Any other channel is also affected
|
||||
expect(getEffectiveChannelLease('dc:other')).toMatchObject({
|
||||
owner_service_id: 'codex-review',
|
||||
reviewer_service_id: 'codex-main',
|
||||
explicit: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('restores the default lease after failover is cleared', () => {
|
||||
it('restores default lease after global failover is cleared', () => {
|
||||
setRegisteredGroup('dc:paired', {
|
||||
name: 'Paired Room Claude',
|
||||
folder: 'paired-claude',
|
||||
@@ -65,8 +68,9 @@ describe('service-routing failover leases', () => {
|
||||
});
|
||||
|
||||
activateCodexFailover('dc:paired', 'claude-429');
|
||||
restoreDefaultChannelLease('dc:paired');
|
||||
clearGlobalFailover();
|
||||
|
||||
expect(getGlobalFailoverInfo().active).toBe(false);
|
||||
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
||||
chat_jid: 'dc:paired',
|
||||
owner_service_id: 'codex-main',
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
setChannelOwnerLease,
|
||||
type ChannelOwnerLeaseRow,
|
||||
} from './db.js';
|
||||
import { logger } from './logger.js';
|
||||
|
||||
export interface EffectiveChannelLease {
|
||||
chat_jid: string;
|
||||
@@ -122,6 +123,18 @@ export function refreshChannelOwnerCache(force = false): void {
|
||||
export function getEffectiveChannelLease(
|
||||
chatJid: string,
|
||||
): EffectiveChannelLease {
|
||||
// Global failover overrides all per-channel leases
|
||||
if (globalFailoverActive) {
|
||||
return {
|
||||
chat_jid: chatJid,
|
||||
owner_service_id: CODEX_REVIEW_SERVICE_ID,
|
||||
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
|
||||
arbiter_service_id: null,
|
||||
activated_at: globalFailoverActivatedAt,
|
||||
reason: globalFailoverReason,
|
||||
explicit: true,
|
||||
};
|
||||
}
|
||||
refreshChannelOwnerCache();
|
||||
const row = leaseCache.get(chatJid);
|
||||
if (row) {
|
||||
@@ -167,19 +180,48 @@ export function shouldServiceProcessChat(
|
||||
return true;
|
||||
}
|
||||
|
||||
export function activateCodexFailover(chatJid: string, reason: string): void {
|
||||
const now = new Date().toISOString();
|
||||
const row: ChannelOwnerLeaseRow = {
|
||||
chat_jid: chatJid,
|
||||
owner_service_id: CODEX_REVIEW_SERVICE_ID,
|
||||
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
|
||||
arbiter_service_id: null,
|
||||
activated_at: now,
|
||||
reason,
|
||||
// ── Global failover ──────────────────────────────────────────────
|
||||
// Claude API limits are account-level, so failover applies to all channels.
|
||||
|
||||
let globalFailoverActive = false;
|
||||
let globalFailoverReason: string | null = null;
|
||||
let globalFailoverActivatedAt: string | null = null;
|
||||
|
||||
export function activateCodexFailover(
|
||||
_chatJid: string,
|
||||
reason: string,
|
||||
): void {
|
||||
globalFailoverActive = true;
|
||||
globalFailoverReason = reason;
|
||||
globalFailoverActivatedAt = new Date().toISOString();
|
||||
logger.warn(
|
||||
{ reason, activatedAt: globalFailoverActivatedAt },
|
||||
'Global failover activated — all channels switching to codex',
|
||||
);
|
||||
}
|
||||
|
||||
export function isGlobalFailoverActive(): boolean {
|
||||
return globalFailoverActive;
|
||||
}
|
||||
|
||||
export function getGlobalFailoverInfo(): {
|
||||
active: boolean;
|
||||
reason: string | null;
|
||||
activatedAt: string | null;
|
||||
} {
|
||||
return {
|
||||
active: globalFailoverActive,
|
||||
reason: globalFailoverReason,
|
||||
activatedAt: globalFailoverActivatedAt,
|
||||
};
|
||||
setChannelOwnerLease(row);
|
||||
leaseCache.set(chatJid, row);
|
||||
lastLeaseRefreshAt = Date.now();
|
||||
}
|
||||
|
||||
export function clearGlobalFailover(): void {
|
||||
if (!globalFailoverActive) return;
|
||||
globalFailoverActive = false;
|
||||
globalFailoverReason = null;
|
||||
globalFailoverActivatedAt = null;
|
||||
logger.info('Global failover cleared — resuming normal routing');
|
||||
}
|
||||
|
||||
export function restoreDefaultChannelLease(chatJid: string): void {
|
||||
@@ -194,18 +236,13 @@ export interface ActiveFailoverLease {
|
||||
}
|
||||
|
||||
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
|
||||
refreshChannelOwnerCache(true);
|
||||
return [...leaseCache.values()]
|
||||
.filter(
|
||||
(row) =>
|
||||
normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID &&
|
||||
normalizeServiceId(row.reviewer_service_id || '') ===
|
||||
CODEX_MAIN_SERVICE_ID,
|
||||
)
|
||||
.map((row) => ({
|
||||
chatJid: row.chat_jid,
|
||||
activatedAt: row.activated_at ?? null,
|
||||
}));
|
||||
// Global failover: report as a single pseudo-lease
|
||||
if (globalFailoverActive) {
|
||||
return [
|
||||
{ chatJid: '*', activatedAt: globalFailoverActivatedAt },
|
||||
];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** @deprecated Use getActiveCodexFailoverLeases() instead */
|
||||
|
||||
@@ -2,10 +2,18 @@ import { execSync } from 'child_process';
|
||||
import os from 'os';
|
||||
|
||||
import {
|
||||
ARBITER_AGENT_TYPE,
|
||||
ARBITER_MODEL_CONFIG,
|
||||
OWNER_AGENT_TYPE,
|
||||
OWNER_MODEL_CONFIG,
|
||||
REVIEWER_AGENT_TYPE,
|
||||
REVIEWER_MODEL_CONFIG,
|
||||
STATUS_SHOW_ROOM_DETAILS,
|
||||
STATUS_SHOW_ROOMS,
|
||||
USAGE_DASHBOARD_ENABLED,
|
||||
getMoaConfig,
|
||||
} from './config.js';
|
||||
import { getGlobalFailoverInfo } from './service-routing.js';
|
||||
import {
|
||||
fetchAllClaudeUsage,
|
||||
fetchAllClaudeProfiles,
|
||||
@@ -540,8 +548,55 @@ async function buildUsageContent(): Promise<string> {
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function buildModelConfigSection(): string {
|
||||
const roleConfigs = [
|
||||
{
|
||||
label: 'Owner',
|
||||
agentType: OWNER_AGENT_TYPE,
|
||||
model: OWNER_MODEL_CONFIG.model,
|
||||
},
|
||||
{
|
||||
label: 'Reviewer',
|
||||
agentType: REVIEWER_AGENT_TYPE,
|
||||
model: REVIEWER_MODEL_CONFIG.model,
|
||||
},
|
||||
{
|
||||
label: 'Arbiter',
|
||||
agentType: ARBITER_AGENT_TYPE,
|
||||
model: ARBITER_MODEL_CONFIG.model,
|
||||
},
|
||||
];
|
||||
|
||||
const lines = ['🤖 *모델 구성*'];
|
||||
for (const role of roleConfigs) {
|
||||
if (!role.agentType && role.label === 'Arbiter') continue;
|
||||
const type = role.agentType || '—';
|
||||
const defaultModel = type === 'codex'
|
||||
? (process.env.CODEX_MODEL || 'codex')
|
||||
: (process.env.CLAUDE_MODEL || 'claude');
|
||||
const model = role.model || defaultModel;
|
||||
lines.push(` **${role.label}** — ${type} \`${model}\``);
|
||||
}
|
||||
|
||||
// MoA status
|
||||
const moaConfig = getMoaConfig();
|
||||
if (moaConfig.enabled) {
|
||||
const refs = moaConfig.referenceModels.map((m) => m.name).join(', ');
|
||||
lines.push(` **MoA** — ${refs}`);
|
||||
}
|
||||
|
||||
// Global failover
|
||||
const failover = getGlobalFailoverInfo();
|
||||
if (failover.active) {
|
||||
lines.push(` ⚠️ **Failover 활성** — ${failover.reason || '알 수 없음'}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function buildUnifiedDashboardContent(): string {
|
||||
const sections: string[] = [];
|
||||
sections.push(buildModelConfigSection());
|
||||
if (STATUS_SHOW_ROOMS) {
|
||||
sections.push(buildStatusContent());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user