feat: add Mixture of Agents (MoA) for arbiter verdicts
When MOA_ENABLED=true, arbiter queries multiple LLM models in parallel via OpenAI-compatible API, then an aggregator model synthesizes the final verdict from all opinions. Falls back to single-agent arbiter when MoA is disabled. Config: MOA_BASE_URL, MOA_API_KEY, MOA_REFERENCE_MODELS, MOA_AGGREGATOR_MODEL
This commit is contained in:
@@ -57,6 +57,15 @@ STATUS_CHANNEL_ID= # Discord channel ID for live status updat
|
||||
# ARBITER_EFFORT=high # Arbiter effort override
|
||||
# ARBITER_FALLBACK_ENABLED=true # Fall back to codex on Claude failure (default: true)
|
||||
|
||||
# --- Mixture of Agents (MoA) ---
|
||||
# Queries multiple models in parallel for arbiter verdicts, then aggregates.
|
||||
# Requires OpenAI-compatible API (OpenRouter recommended for multi-model access).
|
||||
# MOA_ENABLED=true
|
||||
# MOA_BASE_URL=https://openrouter.ai/api/v1
|
||||
# MOA_API_KEY=sk-or-xxx
|
||||
# MOA_REFERENCE_MODELS=anthropic/claude-sonnet-4-6,openai/gpt-5.4,deepseek/deepseek-chat
|
||||
# MOA_AGGREGATOR_MODEL=anthropic/claude-opus-4-6
|
||||
|
||||
# --- Advanced ---
|
||||
# MAX_CONCURRENT_AGENTS=5 # Max parallel agent processes
|
||||
# SESSION_COMMAND_ALLOWED_SENDERS= # Comma-separated Discord user IDs for session commands
|
||||
|
||||
@@ -161,6 +161,46 @@ export function getRoleModelConfig(
|
||||
}
|
||||
}
|
||||
|
||||
// ── Mixture of Agents (MoA) ──────────────────────────────────────
|
||||
|
||||
import type { MoaConfig, MoaModelConfig } from './moa.js';
|
||||
|
||||
const MOA_BASE_URL = getEnv('MOA_BASE_URL') || 'https://openrouter.ai/api/v1';
|
||||
const MOA_API_KEY = getEnv('MOA_API_KEY') || '';
|
||||
|
||||
function parseMoaModels(envKey: string): MoaModelConfig[] {
|
||||
const raw = getEnv(envKey) || '';
|
||||
return raw
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
.map((model) => ({
|
||||
name: model.split('/').pop() || model,
|
||||
model,
|
||||
baseUrl: MOA_BASE_URL,
|
||||
apiKey: MOA_API_KEY,
|
||||
}));
|
||||
}
|
||||
|
||||
export function getMoaConfig(): MoaConfig {
|
||||
const referenceModels = parseMoaModels('MOA_REFERENCE_MODELS');
|
||||
const aggregatorModel = getEnv('MOA_AGGREGATOR_MODEL') || '';
|
||||
return {
|
||||
enabled:
|
||||
getEnv('MOA_ENABLED') === 'true' &&
|
||||
referenceModels.length > 0 &&
|
||||
!!aggregatorModel &&
|
||||
!!MOA_API_KEY,
|
||||
referenceModels,
|
||||
aggregator: {
|
||||
name: aggregatorModel.split('/').pop() || aggregatorModel,
|
||||
model: aggregatorModel,
|
||||
baseUrl: MOA_BASE_URL,
|
||||
apiKey: MOA_API_KEY,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Max owner↔reviewer round trips per task. 0 = unlimited.
|
||||
const rawMaxRoundTrips = getEnv('PAIRED_MAX_ROUND_TRIPS') || '10';
|
||||
export const PAIRED_MAX_ROUND_TRIPS =
|
||||
|
||||
@@ -25,6 +25,20 @@ vi.mock('./config.js', () => ({
|
||||
effort: undefined,
|
||||
fallbackEnabled: true,
|
||||
})),
|
||||
getMoaConfig: vi.fn(() => ({ enabled: false, referenceModels: [], aggregator: {} })),
|
||||
TIMEZONE: 'Asia/Seoul',
|
||||
}));
|
||||
|
||||
vi.mock('./arbiter-context.js', () => ({
|
||||
buildArbiterContextPrompt: vi.fn(() => ''),
|
||||
}));
|
||||
|
||||
vi.mock('./moa.js', () => ({
|
||||
runMoaArbiter: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./platform-prompts.js', () => ({
|
||||
readArbiterPrompt: vi.fn(() => ''),
|
||||
}));
|
||||
|
||||
vi.mock('./db.js', () => ({
|
||||
|
||||
@@ -41,9 +41,14 @@ import {
|
||||
import {
|
||||
CODEX_REVIEW_SERVICE_ID,
|
||||
SERVICE_SESSION_SCOPE,
|
||||
TIMEZONE,
|
||||
isClaudeService,
|
||||
getRoleModelConfig,
|
||||
getMoaConfig,
|
||||
} from './config.js';
|
||||
import { buildArbiterContextPrompt } from './arbiter-context.js';
|
||||
import { runMoaArbiter } from './moa.js';
|
||||
import { readArbiterPrompt } from './platform-prompts.js';
|
||||
import {
|
||||
activateCodexFailover,
|
||||
getEffectiveChannelLease,
|
||||
@@ -323,6 +328,75 @@ export async function runAgentForGroup(
|
||||
return 'success';
|
||||
}
|
||||
|
||||
// ── MoA arbiter path ────────────────────────────────────────────
|
||||
// When MoA is enabled and we're in arbiter mode, query multiple
|
||||
// models in parallel instead of spawning a single agent process.
|
||||
const moaConfig = getMoaConfig();
|
||||
if (arbiterMode && moaConfig.enabled && pairedExecutionContext) {
|
||||
logger.info(
|
||||
{
|
||||
chatJid,
|
||||
group: group.name,
|
||||
runId,
|
||||
referenceModels: moaConfig.referenceModels.map((m) => m.model),
|
||||
aggregator: moaConfig.aggregator.model,
|
||||
},
|
||||
'Running MoA arbiter instead of single agent',
|
||||
);
|
||||
|
||||
const systemPrompt =
|
||||
readArbiterPrompt(process.cwd()) || 'You are an arbiter.';
|
||||
const contextPrompt = buildArbiterContextPrompt({
|
||||
chatJid,
|
||||
taskId: pairedExecutionContext.task.id,
|
||||
roundTripCount: pairedExecutionContext.task.round_trip_count,
|
||||
timezone: TIMEZONE,
|
||||
});
|
||||
|
||||
try {
|
||||
const moaResult = await runMoaArbiter({
|
||||
config: moaConfig,
|
||||
systemPrompt,
|
||||
contextPrompt,
|
||||
});
|
||||
|
||||
pairedExecutionSummary = moaResult.verdict.slice(0, 500);
|
||||
pairedExecutionStatus = 'succeeded';
|
||||
|
||||
// Build display text with reference model opinions
|
||||
const referenceSection = moaResult.referenceResponses
|
||||
.filter((r) => !r.error)
|
||||
.map((r) => `**${r.model}**: ${r.response.split('\n')[0]}`)
|
||||
.join('\n');
|
||||
const displayText = referenceSection
|
||||
? `${moaResult.verdict}\n\n---\n*MoA references: ${moaResult.referenceResponses.filter((r) => !r.error).length} models queried*\n${referenceSection}`
|
||||
: moaResult.verdict;
|
||||
|
||||
await onOutput?.({
|
||||
status: 'success',
|
||||
result: null,
|
||||
output: { visibility: 'public', text: displayText },
|
||||
phase: 'final',
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ chatJid, group: group.name, runId, error },
|
||||
'MoA arbiter failed',
|
||||
);
|
||||
pairedExecutionSummary = 'ESCALATE\nMoA arbiter failed';
|
||||
pairedExecutionStatus = 'failed';
|
||||
}
|
||||
|
||||
completePairedExecutionContext({
|
||||
taskId: pairedExecutionContext.task.id,
|
||||
role: 'arbiter',
|
||||
status: pairedExecutionStatus,
|
||||
summary: pairedExecutionSummary,
|
||||
});
|
||||
pairedExecutionCompleted = true;
|
||||
return pairedExecutionStatus === 'succeeded' ? 'success' : 'error';
|
||||
}
|
||||
|
||||
const runAttempt = async (
|
||||
provider: string,
|
||||
): Promise<{
|
||||
|
||||
@@ -23,6 +23,8 @@ vi.mock('./config.js', () => ({
|
||||
isClaudeService: vi.fn(() => true),
|
||||
isReviewService: vi.fn(() => false),
|
||||
isSessionCommandSenderAllowed: vi.fn(() => false),
|
||||
getMoaConfig: vi.fn(() => ({ enabled: false, referenceModels: [], aggregator: {} })),
|
||||
TIMEZONE: 'Asia/Seoul',
|
||||
}));
|
||||
|
||||
vi.mock('./paired-execution-context.js', () => ({
|
||||
|
||||
171
src/moa.ts
Normal file
171
src/moa.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Mixture of Agents (MoA) for arbiter verdicts.
|
||||
*
|
||||
* Queries multiple LLM models in parallel, then aggregates their
|
||||
* opinions into a single binding verdict. Uses OpenAI-compatible
|
||||
* chat completions API (works with OpenRouter, direct providers, etc.)
|
||||
*/
|
||||
import { logger } from './logger.js';
|
||||
|
||||
export interface MoaModelConfig {
|
||||
name: string;
|
||||
model: string;
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
}
|
||||
|
||||
export interface MoaConfig {
|
||||
enabled: boolean;
|
||||
referenceModels: MoaModelConfig[];
|
||||
aggregator: MoaModelConfig;
|
||||
}
|
||||
|
||||
async function queryModel(
|
||||
model: MoaModelConfig,
|
||||
systemPrompt: string,
|
||||
userPrompt: string,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<string> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${model.baseUrl.replace(/\/+$/, '')}/chat/completions`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${model.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model.model,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
max_tokens: 2048,
|
||||
}),
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
throw new Error(`${response.status} ${response.statusText}: ${body.slice(0, 200)}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
};
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (!content) throw new Error('Empty response from model');
|
||||
return content;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMoaArbiter(args: {
|
||||
config: MoaConfig;
|
||||
systemPrompt: string;
|
||||
contextPrompt: string;
|
||||
}): Promise<{
|
||||
verdict: string;
|
||||
referenceResponses: { model: string; response: string; error?: string }[];
|
||||
}> {
|
||||
const { config, systemPrompt, contextPrompt } = args;
|
||||
|
||||
// Phase 1: Query reference models in parallel
|
||||
logger.info(
|
||||
{ modelCount: config.referenceModels.length },
|
||||
'MoA: querying reference models',
|
||||
);
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
config.referenceModels.map((model) =>
|
||||
queryModel(model, systemPrompt, contextPrompt),
|
||||
),
|
||||
);
|
||||
|
||||
const referenceResponses = results.map((result, i) => {
|
||||
const model = config.referenceModels[i].name;
|
||||
if (result.status === 'fulfilled') {
|
||||
logger.info(
|
||||
{ model, responseLen: result.value.length },
|
||||
'MoA: reference model responded',
|
||||
);
|
||||
return { model, response: result.value };
|
||||
}
|
||||
const error =
|
||||
result.reason instanceof Error
|
||||
? result.reason.message
|
||||
: String(result.reason);
|
||||
logger.warn({ model, error }, 'MoA: reference model failed');
|
||||
return { model, response: '', error };
|
||||
});
|
||||
|
||||
const successfulResponses = referenceResponses.filter((r) => !r.error);
|
||||
|
||||
if (successfulResponses.length === 0) {
|
||||
logger.error('MoA: all reference models failed, using ESCALATE');
|
||||
return {
|
||||
verdict:
|
||||
'ESCALATE\n\nAll reference models failed to respond. Human judgment required.',
|
||||
referenceResponses,
|
||||
};
|
||||
}
|
||||
|
||||
// Phase 2: Aggregate via aggregator model
|
||||
const opinions = successfulResponses
|
||||
.map((r, i) => `### Opinion ${i + 1} (${r.model}):\n${r.response}`)
|
||||
.join('\n\n---\n\n');
|
||||
|
||||
const aggregatorPrompt = [
|
||||
contextPrompt,
|
||||
'',
|
||||
'---',
|
||||
'',
|
||||
`The following ${successfulResponses.length} independent AI models have each reviewed the deadlock and provided their analysis:`,
|
||||
'',
|
||||
opinions,
|
||||
'',
|
||||
'---',
|
||||
'',
|
||||
'Consider all perspectives above. Where they agree, that strengthens the case.',
|
||||
'Where they disagree, weigh the evidence each side presents.',
|
||||
'Render your final verdict. Start your first line with: PROCEED, REVISE, RESET, or ESCALATE.',
|
||||
].join('\n');
|
||||
|
||||
logger.info(
|
||||
{ aggregator: config.aggregator.name },
|
||||
'MoA: running aggregator',
|
||||
);
|
||||
|
||||
try {
|
||||
const verdict = await queryModel(
|
||||
config.aggregator,
|
||||
systemPrompt,
|
||||
aggregatorPrompt,
|
||||
90_000,
|
||||
);
|
||||
logger.info(
|
||||
{
|
||||
aggregator: config.aggregator.name,
|
||||
verdictPreview: verdict.slice(0, 100),
|
||||
},
|
||||
'MoA: aggregator verdict rendered',
|
||||
);
|
||||
return { verdict, referenceResponses };
|
||||
} catch (error) {
|
||||
// Aggregator failed — fall back to majority vote from reference models
|
||||
logger.warn(
|
||||
{ error },
|
||||
'MoA: aggregator failed, falling back to first successful reference',
|
||||
);
|
||||
return {
|
||||
verdict: successfulResponses[0].response,
|
||||
referenceResponses,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user