tkrmagid/watch_sceen_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): real Discord listen+speak loop (STT->echo->TTS)

Browse Source 
The bot (dave/bot.mjs) previously only joined the channel and counted audio
frames — it never fed STT or spoke back. Wire the real loop:

- Node bot: buffer each speaker's Opus->PCM utterance until AfterSilence,
  wrap as WAV, POST to the Python voice-turn endpoint, then play the returned
  reply wav into the channel via an AudioPlayer (ffmpeg->Opus). Skips its own
  audio, dedupes overlapping subscriptions, and ignores sub-0.35s noise.
- Python: new `python -m wsai --voice-server` serves /api/voice-turn — decode
  the uploaded utterance, GPU faster-whisper STT, produce a reply (echo of what
  was heard for now), GPU MeloTTS synth, return the reply wav (recognised/reply
  text ride along as X-Heard/X-Reply headers). Both engines pre-warmed; turns
  show in the dashboard feed. MeloTTS.synth() extracted for direct wav reuse.

Echo mode verifies listening+speaking+GPU recognition entirely in Discord; the
Claude brain is the next slice. Verified the endpoint round-trip: utterance wav
-> correct Korean X-Heard/X-Reply + a WAVE reply on device=cuda. 12 tests pass,
node --check clean.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-18 21:59:01 +09:00
parent b9c929a73f
commit 0f94245d5f
4 changed files with 244 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
dave/bot.mjs
View File

@@ -19,6 +19,7 @@
// "Speak" voice permissions. See dave/README or run `node bot.mjs --invite`.
import fs from 'node:fs';
import { Readable } from 'node:stream';
import {
Client,
GatewayIntentBits,
@@ -29,6 +30,10 @@ import {
entersState,
VoiceConnectionStatus,
EndBehaviorType,
createAudioPlayer,
createAudioResource,
StreamType,
NoSubscriberBehavior,
} from '@discordjs/voice';
import prism from 'prism-media';
@@ -47,10 +52,61 @@ const TOKEN = process.env.DISCORD_BOT_TOKEN || env.DISCORD_BOT_TOKEN;
const GUILD_ID = process.env.GUILD_ID || env.GUILD_ID || '1352269198297923648';
const CHANNEL_ID = process.env.CHANNEL_ID || env.CHANNEL_ID || '1352269198914621465';
const RUN_MS = process.env.RUN_MS != null ? Number(process.env.RUN_MS) : 0; // 0 = stay forever
// Python STT+TTS voice-turn endpoint (run `python -m wsai --voice-server`).
const VOICE_ENDPOINT = process.env.WSAI_VOICE_ENDPOINT || env.WSAI_VOICE_ENDPOINT
|| 'http://127.0.0.1:8787/api/voice-turn';
// Ignore utterances shorter than this many PCM bytes (48kHz*2ch*2B = 192000 B/s),
// so key clicks / brief noise don't trigger a turn. ~0.35s.
const MIN_UTTERANCE_BYTES = Number(process.env.WSAI_MIN_UTTERANCE_BYTES || 67000);
const t0 = Date.now();
const log = (...a) => console.log(`[+${String(Date.now() - t0).padStart(6)}ms]`, ...a);
// PCM s16le -> WAV container (so the Python side can ffmpeg-decode it).
function wavHeader(dataLen, sampleRate = 48000, channels = 2, bits = 16) {
const blockAlign = channels * bits / 8;
const b = Buffer.alloc(44);
b.write('RIFF', 0); b.writeUInt32LE(36 + dataLen, 4); b.write('WAVE', 8);
b.write('fmt ', 12); b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20);
b.writeUInt16LE(channels, 22); b.writeUInt32LE(sampleRate, 24);
b.writeUInt32LE(sampleRate * blockAlign, 28); b.writeUInt16LE(blockAlign, 32);
b.writeUInt16LE(bits, 34); b.write('data', 36); b.writeUInt32LE(dataLen, 40);
return b;
}
// Speak audio into the voice channel. Feed the reply wav bytes through ffmpeg
// (StreamType.Arbitrary) so @discordjs/voice re-encodes to Opus.
let voicePlayer = null;
function playReply(wavBytes) {
if (!voicePlayer || !wavBytes || wavBytes.length === 0) return;
const resource = createAudioResource(Readable.from(wavBytes), { inputType: StreamType.Arbitrary });
voicePlayer.play(resource);
}
// One utterance: PCM -> WAV -> POST to Python -> play the reply back.
async function handleUtterance(userId, pcm) {
if (pcm.length < MIN_UTTERANCE_BYTES) {
log(`utterance too short user=${userId} bytes=${pcm.length} — skip`);
return;
}
const wav = Buffer.concat([wavHeader(pcm.length), pcm]);
let resp;
try {
resp = await fetch(VOICE_ENDPOINT, {
method: 'POST', headers: { 'Content-Type': 'audio/wav' }, body: wav,
});
} catch (e) {
log(`voice-turn POST failed (is \`python -m wsai --voice-server\` running?): ${e.message}`);
return;
}
if (!resp.ok) { log(`voice-turn HTTP ${resp.status}`); return; }
const heard = decodeURIComponent(resp.headers.get('X-Heard') || '');
const reply = decodeURIComponent(resp.headers.get('X-Reply') || '');
const buf = Buffer.from(await resp.arrayBuffer());
log(`heard="${heard}" reply="${reply}" replyWav=${buf.length}B`);
playReply(buf);
}
// ---------- invite URL helper ----------
// A bot cannot add itself to a server; a server admin must click an OAuth2 invite
// URL once. Print it so the user can authorise the bot with voice permissions.
@@ -118,24 +174,40 @@ client.once('clientReady', async () => {
}
log(`✅ JOINED & READY. channel=${CHANNEL_ID} — staying connected, listening for speakers…`);
// ---------- receive path (partial M2) ----------
// Playback path (bot speaks): one player, subscribed to the connection.
voicePlayer = createAudioPlayer({ behaviors: { noSubscriber: NoSubscriberBehavior.Play } });
voicePlayer.on('error', (e) => log(`player error: ${e.message}`));
connection.subscribe(voicePlayer);
log(`voice endpoint: ${VOICE_ENDPOINT}`);
// ---------- receive path: capture each utterance and run the voice turn ----
const receiver = connection.receiver;
const active = new Set(); // userIds with an in-flight subscription (avoid dupes)
receiver.speaking.on('start', (userId) => {
if (userId === client.user.id || active.has(userId)) return; // skip self / dupes
active.add(userId);
if (!perUser.has(userId)) perUser.set(userId, { opusPackets: 0, pcmFrames: 0 });
log(`SPEAKING start user=${userId}`);
const opusStream = receiver.subscribe(userId, {
end: { behavior: EndBehaviorType.AfterSilence, duration: 500 },
// End the utterance after a short silence so natural pauses don't cut words.
end: { behavior: EndBehaviorType.AfterSilence, duration: 800 },
});
// Decode Opus -> 48kHz stereo s16le PCM (what faster-whisper will consume, downsampled to 16k in M3).
// Decode Opus -> 48kHz stereo s16le PCM, buffered until the utterance ends.
const decoder = new prism.opus.Decoder({ rate: 48000, channels: 2, frameSize: 960 });
const chunks = [];
opusStream.on('data', () => { perUser.get(userId).opusPackets++; });
opusStream.pipe(decoder);
decoder.on('data', () => {
const s = perUser.get(userId);
s.pcmFrames++;
if (s.pcmFrames === 1 || s.pcmFrames % 200 === 0) log(`rtp: user=${userId} opus=${s.opusPackets} pcm=${s.pcmFrames}`);
decoder.on('data', (d) => {
chunks.push(d);
const s = perUser.get(userId); s.pcmFrames++;
});
decoder.on('error', (e) => log(`decode error user=${userId}: ${e.message}`));
decoder.on('end', () => {
active.delete(userId);
const pcm = Buffer.concat(chunks);
log(`utterance end user=${userId} pcm=${pcm.length}B — running voice turn`);
handleUtterance(userId, pcm).catch((e) => log(`voice turn error: ${e.message}`));
});
});
connection.on(VoiceConnectionStatus.Disconnected, () => {