feat(bot+dashboard): bot info, server/voice-channel picker, participants, speaker
Adds a dashboard<->bot control plane (bot pushes state + polls commands, keeping the bot's single outbound-HTTP direction): - New bot_control.BotControl + endpoints: GET /api/bot/state, /api/bot/commands; POST /api/bot/report, /api/bot/select. - Dashboard header bar: bot identity/connection, server dropdown (top "없음"), voice-channel dropdown (top "없음"), and live participant list. - Turns record who spoke (Turn.speaker, via X-User-Name on the voice-turn POST). - dave/bot.mjs: reports identity/guilds/voice-channels/members, polls join/leave commands and joins dynamically, and sends the speaker's display name. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
226
dave/bot.mjs
226
dave/bot.mjs
@@ -55,6 +55,10 @@ const RUN_MS = process.env.RUN_MS != null ? Number(process.env.RUN_MS) : 0; // 0
|
||||
// Python STT+TTS voice-turn endpoint (run `python -m wsai --voice-server`).
|
||||
const VOICE_ENDPOINT = process.env.WSAI_VOICE_ENDPOINT || env.WSAI_VOICE_ENDPOINT
|
||||
|| 'http://127.0.0.1:8787/api/voice-turn';
|
||||
// Dashboard control plane (state push + command poll), same host as voice-turn.
|
||||
const API_BASE = VOICE_ENDPOINT.replace(/\/api\/voice-turn\/?$/, '');
|
||||
const REPORT_ENDPOINT = API_BASE + '/api/bot/report';
|
||||
const REPORT_INTERVAL_MS = Number(process.env.WSAI_REPORT_INTERVAL_MS || 2500);
|
||||
// Ignore utterances shorter than this many PCM bytes (48kHz*2ch*2B = 192000 B/s),
|
||||
// so key clicks / brief noise don't trigger a turn. ~0.35s.
|
||||
const MIN_UTTERANCE_BYTES = Number(process.env.WSAI_MIN_UTTERANCE_BYTES || 67000);
|
||||
@@ -109,10 +113,19 @@ async function handleUtterance(userId, pcm) {
|
||||
return;
|
||||
}
|
||||
const wav = Buffer.concat([wavHeader(pcm.length), pcm]);
|
||||
// Resolve who spoke (Discord display name) so the dashboard log can show it.
|
||||
let speaker = userId;
|
||||
try {
|
||||
const g = currentGuildId && client.guilds.cache.get(currentGuildId);
|
||||
const m = g && (g.members.cache.get(userId) || await g.members.fetch(userId).catch(() => null));
|
||||
if (m) speaker = m.displayName || m.user.username;
|
||||
} catch {}
|
||||
let resp;
|
||||
try {
|
||||
resp = await fetch(VOICE_ENDPOINT, {
|
||||
method: 'POST', headers: { 'Content-Type': 'audio/wav' }, body: wav,
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'audio/wav', 'X-User-Name': encodeURIComponent(speaker) },
|
||||
body: wav,
|
||||
});
|
||||
} catch (e) {
|
||||
log(`voice-turn POST failed (is \`python -m wsai --voice-server\` running?): ${e.message}`);
|
||||
@@ -147,11 +160,131 @@ const client = new Client({
|
||||
|
||||
const perUser = new Map(); // userId -> { opusPackets, pcmFrames }
|
||||
|
||||
// --- control-plane state (dashboard drives which channel we're in) --------- #
|
||||
let currentGuildId = null, currentChannelId = null, currentChannelName = null;
|
||||
const speakingSet = new Set(); // userIds currently speaking (for participant list)
|
||||
const activeSubs = new Set(); // userIds with an in-flight receive subscription
|
||||
|
||||
// Attach the bot's audio player (so it can speak) to a fresh connection.
|
||||
function setupPlayer(connection) {
|
||||
voicePlayer = createAudioPlayer({ behaviors: { noSubscriber: NoSubscriberBehavior.Play } });
|
||||
voicePlayer.on('error', (e) => log(`player error: ${e.message}`));
|
||||
connection.subscribe(voicePlayer);
|
||||
}
|
||||
|
||||
// Attach the receive path: capture each utterance and run the voice turn.
|
||||
function setupReceiver(connection) {
|
||||
const receiver = connection.receiver;
|
||||
receiver.speaking.on('start', (userId) => {
|
||||
speakingSet.add(userId);
|
||||
if (userId === client.user.id || activeSubs.has(userId)) return;
|
||||
activeSubs.add(userId);
|
||||
if (!perUser.has(userId)) perUser.set(userId, { opusPackets: 0, pcmFrames: 0 });
|
||||
const opusStream = receiver.subscribe(userId, {
|
||||
end: { behavior: EndBehaviorType.AfterSilence, duration: 800 },
|
||||
});
|
||||
const decoder = new prism.opus.Decoder({ rate: 48000, channels: 2, frameSize: 960 });
|
||||
const chunks = [];
|
||||
opusStream.on('data', () => { perUser.get(userId).opusPackets++; });
|
||||
opusStream.on('error', (e) => { logThrottled(`recv:${e.message}`, `recv stream error user=${userId}: ${e.message}`); activeSubs.delete(userId); });
|
||||
opusStream.pipe(decoder);
|
||||
decoder.on('data', (d) => { chunks.push(d); perUser.get(userId).pcmFrames++; });
|
||||
decoder.on('error', (e) => log(`decode error user=${userId}: ${e.message}`));
|
||||
decoder.on('end', () => {
|
||||
activeSubs.delete(userId);
|
||||
const pcm = Buffer.concat(chunks);
|
||||
log(`utterance end user=${userId} pcm=${pcm.length}B — running voice turn`);
|
||||
handleUtterance(userId, pcm).catch((e) => log(`voice turn error: ${e.message}`));
|
||||
});
|
||||
});
|
||||
receiver.speaking.on('end', (userId) => speakingSet.delete(userId));
|
||||
}
|
||||
|
||||
// Join (or switch to) a voice channel on command from the dashboard.
|
||||
async function joinChannel(guildId, channelId) {
|
||||
const guild = await client.guilds.fetch(guildId).catch(() => null);
|
||||
const channel = guild && await guild.channels.fetch(channelId).catch(() => null);
|
||||
if (!channel || !channel.isVoiceBased()) { log(`join failed: ${guildId}/${channelId} not a voice channel`); return; }
|
||||
if (currentGuildId && currentGuildId !== guildId) leaveChannel();
|
||||
const connection = joinVoiceChannel({
|
||||
channelId, guildId, adapterCreator: guild.voiceAdapterCreator,
|
||||
selfDeaf: false, selfMute: false,
|
||||
});
|
||||
connection.on('error', (e) => log(`voice connection error: ${e.message}`));
|
||||
try {
|
||||
await entersState(connection, VoiceConnectionStatus.Ready, 40_000);
|
||||
} catch (e) { log(`join not Ready in 40s: ${e.message}`); return; }
|
||||
currentGuildId = guildId; currentChannelId = channelId; currentChannelName = channel.name;
|
||||
speakingSet.clear(); activeSubs.clear();
|
||||
setupPlayer(connection);
|
||||
setupReceiver(connection);
|
||||
connection.on(VoiceConnectionStatus.Disconnected, () => {
|
||||
log('voice: disconnected — attempting to resume…');
|
||||
Promise.race([
|
||||
entersState(connection, VoiceConnectionStatus.Signalling, 5_000),
|
||||
entersState(connection, VoiceConnectionStatus.Connecting, 5_000),
|
||||
]).catch(() => { log('voice: could not resume'); leaveChannel(); });
|
||||
});
|
||||
log(`✅ joined voice: "${guild.name}" / "${channel.name}"`);
|
||||
}
|
||||
|
||||
function leaveChannel() {
|
||||
try { getVoiceConnection(currentGuildId)?.destroy(); } catch {}
|
||||
currentGuildId = currentChannelId = currentChannelName = null;
|
||||
speakingSet.clear(); activeSubs.clear();
|
||||
}
|
||||
|
||||
// Build the state snapshot the dashboard shows (identity, joinable servers +
|
||||
// voice channels, current channel, and who is in it).
|
||||
function buildState() {
|
||||
const guilds = [...client.guilds.cache.values()].map((g) => ({
|
||||
id: g.id, name: g.name,
|
||||
voiceChannels: [...g.channels.cache.values()]
|
||||
.filter((c) => c.isVoiceBased())
|
||||
.map((c) => ({ id: c.id, name: c.name })),
|
||||
}));
|
||||
let members = [];
|
||||
if (currentGuildId && currentChannelId) {
|
||||
const ch = client.guilds.cache.get(currentGuildId)?.channels.cache.get(currentChannelId);
|
||||
if (ch && ch.members) {
|
||||
members = [...ch.members.values()].map((m) => ({
|
||||
id: m.id, name: m.displayName, speaking: speakingSet.has(m.id),
|
||||
}));
|
||||
}
|
||||
}
|
||||
return {
|
||||
identity: { id: client.user.id, username: client.user.username, tag: client.user.tag },
|
||||
guilds,
|
||||
current: { guildId: currentGuildId, channelId: currentChannelId, channelName: currentChannelName },
|
||||
members,
|
||||
};
|
||||
}
|
||||
|
||||
async function handleCommand(cmd) {
|
||||
if (cmd.type === 'join') await joinChannel(cmd.guildId, cmd.channelId);
|
||||
else if (cmd.type === 'leave') { leaveChannel(); log('left voice on command'); }
|
||||
}
|
||||
|
||||
// Push state to the dashboard and apply any commands it hands back.
|
||||
async function reportLoop() {
|
||||
let j;
|
||||
try {
|
||||
const r = await fetch(REPORT_ENDPOINT, {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(buildState()),
|
||||
});
|
||||
j = await r.json();
|
||||
} catch { return; } // dashboard down: keep running, retry next tick
|
||||
for (const cmd of (j?.commands || [])) {
|
||||
try { await handleCommand(cmd); } catch (e) { log(`command ${cmd?.type} failed: ${e.message}`); }
|
||||
}
|
||||
}
|
||||
|
||||
let leaving = false;
|
||||
function leaveAndExit(code = 0) {
|
||||
if (leaving) return;
|
||||
leaving = true;
|
||||
try { getVoiceConnection(GUILD_ID)?.destroy(); } catch {}
|
||||
try { getVoiceConnection(currentGuildId || GUILD_ID)?.destroy(); } catch {}
|
||||
const summary = [...perUser.entries()].map(([u, s]) => `${u}:opus=${s.opusPackets},pcm=${s.pcmFrames}`);
|
||||
log(`leaving. speakers heard: ${summary.length ? summary.join(' ') : '(none)'}`);
|
||||
try { client.destroy(); } catch {}
|
||||
@@ -165,84 +298,21 @@ if (RUN_MS > 0) setTimeout(() => { log(`RUN_MS=${RUN_MS} hard ceiling elapsed
|
||||
|
||||
client.once('clientReady', async () => {
|
||||
log(`logged in as ${client.user.tag} (${client.user.id})`);
|
||||
let guild, channel;
|
||||
try {
|
||||
guild = await client.guilds.fetch(GUILD_ID);
|
||||
channel = await guild.channels.fetch(CHANNEL_ID);
|
||||
} catch (e) {
|
||||
log(`FATAL: cannot access guild/channel — is the bot invited to guild ${GUILD_ID}? (${e.message})`);
|
||||
log('run `node bot.mjs --invite` and have a server admin authorise the bot, then retry.');
|
||||
return leaveAndExit(1);
|
||||
log(`voice endpoint: ${VOICE_ENDPOINT} · control: ${REPORT_ENDPOINT}`);
|
||||
|
||||
// Backward-compat: if a default guild/channel is configured, auto-join it.
|
||||
// Otherwise idle and wait for the dashboard to pick a channel.
|
||||
if (GUILD_ID && CHANNEL_ID) {
|
||||
await joinChannel(GUILD_ID, CHANNEL_ID).catch((e) => log(`initial join failed: ${e.message}`));
|
||||
} else {
|
||||
log('no default channel — waiting for the dashboard to select a server/voice channel…');
|
||||
}
|
||||
if (!channel || !channel.isVoiceBased()) { log(`FATAL: channel ${CHANNEL_ID} is not a voice channel`); return leaveAndExit(1); }
|
||||
|
||||
log(`joining voice: guild="${guild.name}" channel="${channel.name}"`);
|
||||
const connection = joinVoiceChannel({
|
||||
channelId: CHANNEL_ID,
|
||||
guildId: GUILD_ID,
|
||||
adapterCreator: guild.voiceAdapterCreator,
|
||||
selfDeaf: false, // MUST be false to receive audio (the STT input path)
|
||||
selfMute: false, // false so we can also speak later (M5 TTS)
|
||||
});
|
||||
connection.on('error', (e) => log(`voice connection error: ${e.message}`));
|
||||
|
||||
try {
|
||||
// The DAVE/MLS handshake here cycles signalling<->connecting several times
|
||||
// and can take ~25s, so give it a generous ceiling before declaring failure.
|
||||
await entersState(connection, VoiceConnectionStatus.Ready, 40_000);
|
||||
} catch (e) {
|
||||
log(`FATAL: voice connection did not become Ready in 40s (${e.message})`);
|
||||
return leaveAndExit(1);
|
||||
}
|
||||
log(`✅ JOINED & READY. channel=${CHANNEL_ID} — staying connected, listening for speakers…`);
|
||||
|
||||
// Playback path (bot speaks): one player, subscribed to the connection.
|
||||
voicePlayer = createAudioPlayer({ behaviors: { noSubscriber: NoSubscriberBehavior.Play } });
|
||||
voicePlayer.on('error', (e) => log(`player error: ${e.message}`));
|
||||
connection.subscribe(voicePlayer);
|
||||
log(`voice endpoint: ${VOICE_ENDPOINT}`);
|
||||
|
||||
// ---------- receive path: capture each utterance and run the voice turn ----
|
||||
const receiver = connection.receiver;
|
||||
const active = new Set(); // userIds with an in-flight subscription (avoid dupes)
|
||||
receiver.speaking.on('start', (userId) => {
|
||||
if (userId === client.user.id || active.has(userId)) return; // skip self / dupes
|
||||
active.add(userId);
|
||||
if (!perUser.has(userId)) perUser.set(userId, { opusPackets: 0, pcmFrames: 0 });
|
||||
log(`SPEAKING start user=${userId}`);
|
||||
const opusStream = receiver.subscribe(userId, {
|
||||
// End the utterance after a short silence so natural pauses don't cut words.
|
||||
end: { behavior: EndBehaviorType.AfterSilence, duration: 800 },
|
||||
});
|
||||
// Decode Opus -> 48kHz stereo s16le PCM, buffered until the utterance ends.
|
||||
const decoder = new prism.opus.Decoder({ rate: 48000, channels: 2, frameSize: 960 });
|
||||
const chunks = [];
|
||||
opusStream.on('data', () => { perUser.get(userId).opusPackets++; });
|
||||
// A receive-stream error (e.g. a DAVE decrypt/UDP GenericFailure on one
|
||||
// packet) must NOT crash the process — log it and free the slot so the
|
||||
// next utterance still works.
|
||||
opusStream.on('error', (e) => { logThrottled(`recv:${e.message}`, `recv stream error user=${userId}: ${e.message}`); active.delete(userId); });
|
||||
opusStream.pipe(decoder);
|
||||
decoder.on('data', (d) => {
|
||||
chunks.push(d);
|
||||
const s = perUser.get(userId); s.pcmFrames++;
|
||||
});
|
||||
decoder.on('error', (e) => log(`decode error user=${userId}: ${e.message}`));
|
||||
decoder.on('end', () => {
|
||||
active.delete(userId);
|
||||
const pcm = Buffer.concat(chunks);
|
||||
log(`utterance end user=${userId} pcm=${pcm.length}B — running voice turn`);
|
||||
handleUtterance(userId, pcm).catch((e) => log(`voice turn error: ${e.message}`));
|
||||
});
|
||||
});
|
||||
|
||||
connection.on(VoiceConnectionStatus.Disconnected, () => {
|
||||
log('voice: disconnected — attempting to resume…');
|
||||
Promise.race([
|
||||
entersState(connection, VoiceConnectionStatus.Signalling, 5_000),
|
||||
entersState(connection, VoiceConnectionStatus.Connecting, 5_000),
|
||||
]).catch(() => { log('voice: could not resume, leaving'); leaveAndExit(0); });
|
||||
});
|
||||
// Report state + poll commands forever. This is what powers the dashboard's
|
||||
// bot info, server/voice-channel pickers, participant list, and join/leave.
|
||||
reportLoop();
|
||||
const reportTimer = setInterval(reportLoop, REPORT_INTERVAL_MS);
|
||||
if (typeof reportTimer.unref === 'function') reportTimer.unref();
|
||||
});
|
||||
|
||||
client.on('error', (e) => log('client error', e.message));
|
||||
|
||||
Reference in New Issue
Block a user