feat: global failover + dashboard model config section
- Failover is now global (account-level, not per-channel) - Dashboard shows role model config (Owner/Reviewer/Arbiter + MoA refs) - Dashboard shows failover status when active - Auto-clear failover on successful Claude rotation - Remove per-channel lease writes from failover path
This commit is contained in:
@@ -191,8 +191,7 @@ function parseMoaReferenceModels(): MoaModelConfig[] {
|
|||||||
export function getMoaConfig(): MoaConfig {
|
export function getMoaConfig(): MoaConfig {
|
||||||
const referenceModels = parseMoaReferenceModels();
|
const referenceModels = parseMoaReferenceModels();
|
||||||
return {
|
return {
|
||||||
enabled:
|
enabled: getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
|
||||||
getEnv('MOA_ENABLED') === 'true' && referenceModels.length > 0,
|
|
||||||
referenceModels,
|
referenceModels,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
55
src/index.ts
55
src/index.ts
@@ -83,8 +83,8 @@ import {
|
|||||||
stopTokenRefreshLoop,
|
stopTokenRefreshLoop,
|
||||||
} from './token-refresh.js';
|
} from './token-refresh.js';
|
||||||
import {
|
import {
|
||||||
getActiveCodexFailoverLeases,
|
clearGlobalFailover,
|
||||||
restoreDefaultChannelLease,
|
getGlobalFailoverInfo,
|
||||||
} from './service-routing.js';
|
} from './service-routing.js';
|
||||||
import { FAILOVER_MIN_DURATION_MS } from './config.js';
|
import { FAILOVER_MIN_DURATION_MS } from './config.js';
|
||||||
import { startCredentialProxy } from './credential-proxy.js';
|
import { startCredentialProxy } from './credential-proxy.js';
|
||||||
@@ -536,43 +536,20 @@ async function main(): Promise<void> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
leaseRecoveryTimer = setInterval(() => {
|
leaseRecoveryTimer = setInterval(() => {
|
||||||
if (!hasAvailableClaudeToken()) {
|
const failover = getGlobalFailoverInfo();
|
||||||
return;
|
if (!failover.active) return;
|
||||||
}
|
if (!hasAvailableClaudeToken()) return;
|
||||||
const now = Date.now();
|
const activatedMs = failover.activatedAt
|
||||||
for (const lease of getActiveCodexFailoverLeases()) {
|
? new Date(failover.activatedAt).getTime()
|
||||||
const activatedMs = lease.activatedAt
|
: NaN;
|
||||||
? new Date(lease.activatedAt).getTime()
|
if (Number.isNaN(activatedMs)) return;
|
||||||
: NaN;
|
const elapsed = Date.now() - activatedMs;
|
||||||
if (Number.isNaN(activatedMs)) {
|
if (elapsed < FAILOVER_MIN_DURATION_MS) return;
|
||||||
logger.warn(
|
clearGlobalFailover();
|
||||||
{ chatJid: lease.chatJid, activatedAt: lease.activatedAt },
|
logger.info(
|
||||||
'Failover lease has unparseable activated_at, skipping auto-restore',
|
{ elapsedMin: Math.round(elapsed / 60_000) },
|
||||||
);
|
'Claude token available and hold period elapsed, global failover cleared',
|
||||||
continue;
|
);
|
||||||
}
|
|
||||||
const elapsed = now - activatedMs;
|
|
||||||
if (elapsed < FAILOVER_MIN_DURATION_MS) {
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
chatJid: lease.chatJid,
|
|
||||||
elapsedMin: Math.round(elapsed / 60_000),
|
|
||||||
minDurationMin: Math.round(FAILOVER_MIN_DURATION_MS / 60_000),
|
|
||||||
},
|
|
||||||
'Failover lease still within minimum hold period, skipping restore',
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
restoreDefaultChannelLease(lease.chatJid);
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
chatJid: lease.chatJid,
|
|
||||||
serviceId: SERVICE_ID,
|
|
||||||
elapsedMin: Math.round(elapsed / 60_000),
|
|
||||||
},
|
|
||||||
'Claude token available and failover hold period elapsed, restored default channel lease',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}, 5_000);
|
}, 5_000);
|
||||||
runtime.startMessageLoop().catch((err) => {
|
runtime.startMessageLoop().catch((err) => {
|
||||||
logger.fatal({ err }, 'Message loop crashed unexpectedly');
|
logger.fatal({ err }, 'Message loop crashed unexpectedly');
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ vi.mock('./db.js', () => ({
|
|||||||
|
|
||||||
vi.mock('./service-routing.js', () => ({
|
vi.mock('./service-routing.js', () => ({
|
||||||
activateCodexFailover: vi.fn(),
|
activateCodexFailover: vi.fn(),
|
||||||
|
clearGlobalFailover: vi.fn(),
|
||||||
getEffectiveChannelLease: vi.fn(() => ({
|
getEffectiveChannelLease: vi.fn(() => ({
|
||||||
chat_jid: 'group@test',
|
chat_jid: 'group@test',
|
||||||
owner_service_id: 'claude',
|
owner_service_id: 'claude',
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import {
|
|||||||
getTokenCount,
|
getTokenCount,
|
||||||
markTokenHealthy,
|
markTokenHealthy,
|
||||||
} from './token-rotation.js';
|
} from './token-rotation.js';
|
||||||
|
import { clearGlobalFailover } from './service-routing.js';
|
||||||
|
|
||||||
// ── Types ────────────────────────────────────────────────────────
|
// ── Types ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -159,6 +160,7 @@ export async function runClaudeRotationLoop(
|
|||||||
|
|
||||||
// ── Success ──
|
// ── Success ──
|
||||||
markTokenHealthy();
|
markTokenHealthy();
|
||||||
|
clearGlobalFailover();
|
||||||
return { type: 'success' };
|
return { type: 'success' };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,19 +3,20 @@ import { beforeEach, describe, expect, it } from 'vitest';
|
|||||||
import { _initTestDatabase, setRegisteredGroup } from './db.js';
|
import { _initTestDatabase, setRegisteredGroup } from './db.js';
|
||||||
import {
|
import {
|
||||||
activateCodexFailover,
|
activateCodexFailover,
|
||||||
getActiveCodexFailoverLeases,
|
clearGlobalFailover,
|
||||||
getEffectiveChannelLease,
|
getEffectiveChannelLease,
|
||||||
|
getGlobalFailoverInfo,
|
||||||
refreshChannelOwnerCache,
|
refreshChannelOwnerCache,
|
||||||
restoreDefaultChannelLease,
|
|
||||||
} from './service-routing.js';
|
} from './service-routing.js';
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
_initTestDatabase();
|
_initTestDatabase();
|
||||||
refreshChannelOwnerCache(true);
|
refreshChannelOwnerCache(true);
|
||||||
|
clearGlobalFailover();
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('service-routing failover leases', () => {
|
describe('service-routing global failover', () => {
|
||||||
it('uses codex-review as owner and codex-main as reviewer during failover', () => {
|
it('uses codex-review as owner during global failover', () => {
|
||||||
setRegisteredGroup('dc:paired', {
|
setRegisteredGroup('dc:paired', {
|
||||||
name: 'Paired Room Claude',
|
name: 'Paired Room Claude',
|
||||||
folder: 'paired-claude',
|
folder: 'paired-claude',
|
||||||
@@ -33,6 +34,8 @@ describe('service-routing failover leases', () => {
|
|||||||
|
|
||||||
activateCodexFailover('dc:paired', 'claude-429');
|
activateCodexFailover('dc:paired', 'claude-429');
|
||||||
|
|
||||||
|
// Global failover applies to ALL channels
|
||||||
|
expect(getGlobalFailoverInfo().active).toBe(true);
|
||||||
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
||||||
chat_jid: 'dc:paired',
|
chat_jid: 'dc:paired',
|
||||||
owner_service_id: 'codex-review',
|
owner_service_id: 'codex-review',
|
||||||
@@ -40,15 +43,15 @@ describe('service-routing failover leases', () => {
|
|||||||
reason: 'claude-429',
|
reason: 'claude-429',
|
||||||
explicit: true,
|
explicit: true,
|
||||||
});
|
});
|
||||||
expect(getActiveCodexFailoverLeases()).toEqual([
|
// Any other channel is also affected
|
||||||
{
|
expect(getEffectiveChannelLease('dc:other')).toMatchObject({
|
||||||
chatJid: 'dc:paired',
|
owner_service_id: 'codex-review',
|
||||||
activatedAt: expect.any(String),
|
reviewer_service_id: 'codex-main',
|
||||||
},
|
explicit: true,
|
||||||
]);
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('restores the default lease after failover is cleared', () => {
|
it('restores default lease after global failover is cleared', () => {
|
||||||
setRegisteredGroup('dc:paired', {
|
setRegisteredGroup('dc:paired', {
|
||||||
name: 'Paired Room Claude',
|
name: 'Paired Room Claude',
|
||||||
folder: 'paired-claude',
|
folder: 'paired-claude',
|
||||||
@@ -65,8 +68,9 @@ describe('service-routing failover leases', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
activateCodexFailover('dc:paired', 'claude-429');
|
activateCodexFailover('dc:paired', 'claude-429');
|
||||||
restoreDefaultChannelLease('dc:paired');
|
clearGlobalFailover();
|
||||||
|
|
||||||
|
expect(getGlobalFailoverInfo().active).toBe(false);
|
||||||
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
expect(getEffectiveChannelLease('dc:paired')).toMatchObject({
|
||||||
chat_jid: 'dc:paired',
|
chat_jid: 'dc:paired',
|
||||||
owner_service_id: 'codex-main',
|
owner_service_id: 'codex-main',
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import {
|
|||||||
setChannelOwnerLease,
|
setChannelOwnerLease,
|
||||||
type ChannelOwnerLeaseRow,
|
type ChannelOwnerLeaseRow,
|
||||||
} from './db.js';
|
} from './db.js';
|
||||||
|
import { logger } from './logger.js';
|
||||||
|
|
||||||
export interface EffectiveChannelLease {
|
export interface EffectiveChannelLease {
|
||||||
chat_jid: string;
|
chat_jid: string;
|
||||||
@@ -122,6 +123,18 @@ export function refreshChannelOwnerCache(force = false): void {
|
|||||||
export function getEffectiveChannelLease(
|
export function getEffectiveChannelLease(
|
||||||
chatJid: string,
|
chatJid: string,
|
||||||
): EffectiveChannelLease {
|
): EffectiveChannelLease {
|
||||||
|
// Global failover overrides all per-channel leases
|
||||||
|
if (globalFailoverActive) {
|
||||||
|
return {
|
||||||
|
chat_jid: chatJid,
|
||||||
|
owner_service_id: CODEX_REVIEW_SERVICE_ID,
|
||||||
|
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
|
||||||
|
arbiter_service_id: null,
|
||||||
|
activated_at: globalFailoverActivatedAt,
|
||||||
|
reason: globalFailoverReason,
|
||||||
|
explicit: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
refreshChannelOwnerCache();
|
refreshChannelOwnerCache();
|
||||||
const row = leaseCache.get(chatJid);
|
const row = leaseCache.get(chatJid);
|
||||||
if (row) {
|
if (row) {
|
||||||
@@ -167,19 +180,48 @@ export function shouldServiceProcessChat(
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function activateCodexFailover(chatJid: string, reason: string): void {
|
// ── Global failover ──────────────────────────────────────────────
|
||||||
const now = new Date().toISOString();
|
// Claude API limits are account-level, so failover applies to all channels.
|
||||||
const row: ChannelOwnerLeaseRow = {
|
|
||||||
chat_jid: chatJid,
|
let globalFailoverActive = false;
|
||||||
owner_service_id: CODEX_REVIEW_SERVICE_ID,
|
let globalFailoverReason: string | null = null;
|
||||||
reviewer_service_id: CODEX_MAIN_SERVICE_ID,
|
let globalFailoverActivatedAt: string | null = null;
|
||||||
arbiter_service_id: null,
|
|
||||||
activated_at: now,
|
export function activateCodexFailover(
|
||||||
reason,
|
_chatJid: string,
|
||||||
|
reason: string,
|
||||||
|
): void {
|
||||||
|
globalFailoverActive = true;
|
||||||
|
globalFailoverReason = reason;
|
||||||
|
globalFailoverActivatedAt = new Date().toISOString();
|
||||||
|
logger.warn(
|
||||||
|
{ reason, activatedAt: globalFailoverActivatedAt },
|
||||||
|
'Global failover activated — all channels switching to codex',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isGlobalFailoverActive(): boolean {
|
||||||
|
return globalFailoverActive;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getGlobalFailoverInfo(): {
|
||||||
|
active: boolean;
|
||||||
|
reason: string | null;
|
||||||
|
activatedAt: string | null;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
active: globalFailoverActive,
|
||||||
|
reason: globalFailoverReason,
|
||||||
|
activatedAt: globalFailoverActivatedAt,
|
||||||
};
|
};
|
||||||
setChannelOwnerLease(row);
|
}
|
||||||
leaseCache.set(chatJid, row);
|
|
||||||
lastLeaseRefreshAt = Date.now();
|
export function clearGlobalFailover(): void {
|
||||||
|
if (!globalFailoverActive) return;
|
||||||
|
globalFailoverActive = false;
|
||||||
|
globalFailoverReason = null;
|
||||||
|
globalFailoverActivatedAt = null;
|
||||||
|
logger.info('Global failover cleared — resuming normal routing');
|
||||||
}
|
}
|
||||||
|
|
||||||
export function restoreDefaultChannelLease(chatJid: string): void {
|
export function restoreDefaultChannelLease(chatJid: string): void {
|
||||||
@@ -194,18 +236,13 @@ export interface ActiveFailoverLease {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
|
export function getActiveCodexFailoverLeases(): ActiveFailoverLease[] {
|
||||||
refreshChannelOwnerCache(true);
|
// Global failover: report as a single pseudo-lease
|
||||||
return [...leaseCache.values()]
|
if (globalFailoverActive) {
|
||||||
.filter(
|
return [
|
||||||
(row) =>
|
{ chatJid: '*', activatedAt: globalFailoverActivatedAt },
|
||||||
normalizeServiceId(row.owner_service_id) === CODEX_REVIEW_SERVICE_ID &&
|
];
|
||||||
normalizeServiceId(row.reviewer_service_id || '') ===
|
}
|
||||||
CODEX_MAIN_SERVICE_ID,
|
return [];
|
||||||
)
|
|
||||||
.map((row) => ({
|
|
||||||
chatJid: row.chat_jid,
|
|
||||||
activatedAt: row.activated_at ?? null,
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @deprecated Use getActiveCodexFailoverLeases() instead */
|
/** @deprecated Use getActiveCodexFailoverLeases() instead */
|
||||||
|
|||||||
@@ -2,10 +2,18 @@ import { execSync } from 'child_process';
|
|||||||
import os from 'os';
|
import os from 'os';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
ARBITER_AGENT_TYPE,
|
||||||
|
ARBITER_MODEL_CONFIG,
|
||||||
|
OWNER_AGENT_TYPE,
|
||||||
|
OWNER_MODEL_CONFIG,
|
||||||
|
REVIEWER_AGENT_TYPE,
|
||||||
|
REVIEWER_MODEL_CONFIG,
|
||||||
STATUS_SHOW_ROOM_DETAILS,
|
STATUS_SHOW_ROOM_DETAILS,
|
||||||
STATUS_SHOW_ROOMS,
|
STATUS_SHOW_ROOMS,
|
||||||
USAGE_DASHBOARD_ENABLED,
|
USAGE_DASHBOARD_ENABLED,
|
||||||
|
getMoaConfig,
|
||||||
} from './config.js';
|
} from './config.js';
|
||||||
|
import { getGlobalFailoverInfo } from './service-routing.js';
|
||||||
import {
|
import {
|
||||||
fetchAllClaudeUsage,
|
fetchAllClaudeUsage,
|
||||||
fetchAllClaudeProfiles,
|
fetchAllClaudeProfiles,
|
||||||
@@ -540,8 +548,55 @@ async function buildUsageContent(): Promise<string> {
|
|||||||
return lines.join('\n');
|
return lines.join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildModelConfigSection(): string {
|
||||||
|
const roleConfigs = [
|
||||||
|
{
|
||||||
|
label: 'Owner',
|
||||||
|
agentType: OWNER_AGENT_TYPE,
|
||||||
|
model: OWNER_MODEL_CONFIG.model,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Reviewer',
|
||||||
|
agentType: REVIEWER_AGENT_TYPE,
|
||||||
|
model: REVIEWER_MODEL_CONFIG.model,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Arbiter',
|
||||||
|
agentType: ARBITER_AGENT_TYPE,
|
||||||
|
model: ARBITER_MODEL_CONFIG.model,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const lines = ['🤖 *모델 구성*'];
|
||||||
|
for (const role of roleConfigs) {
|
||||||
|
if (!role.agentType && role.label === 'Arbiter') continue;
|
||||||
|
const type = role.agentType || '—';
|
||||||
|
const defaultModel = type === 'codex'
|
||||||
|
? (process.env.CODEX_MODEL || 'codex')
|
||||||
|
: (process.env.CLAUDE_MODEL || 'claude');
|
||||||
|
const model = role.model || defaultModel;
|
||||||
|
lines.push(` **${role.label}** — ${type} \`${model}\``);
|
||||||
|
}
|
||||||
|
|
||||||
|
// MoA status
|
||||||
|
const moaConfig = getMoaConfig();
|
||||||
|
if (moaConfig.enabled) {
|
||||||
|
const refs = moaConfig.referenceModels.map((m) => m.name).join(', ');
|
||||||
|
lines.push(` **MoA** — ${refs}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Global failover
|
||||||
|
const failover = getGlobalFailoverInfo();
|
||||||
|
if (failover.active) {
|
||||||
|
lines.push(` ⚠️ **Failover 활성** — ${failover.reason || '알 수 없음'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
function buildUnifiedDashboardContent(): string {
|
function buildUnifiedDashboardContent(): string {
|
||||||
const sections: string[] = [];
|
const sections: string[] = [];
|
||||||
|
sections.push(buildModelConfigSection());
|
||||||
if (STATUS_SHOW_ROOMS) {
|
if (STATUS_SHOW_ROOMS) {
|
||||||
sections.push(buildStatusContent());
|
sections.push(buildStatusContent());
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user