feat(voice): real Discord listen+speak loop (STT->echo->TTS)
The bot (dave/bot.mjs) previously only joined the channel and counted audio frames — it never fed STT or spoke back. Wire the real loop: - Node bot: buffer each speaker's Opus->PCM utterance until AfterSilence, wrap as WAV, POST to the Python voice-turn endpoint, then play the returned reply wav into the channel via an AudioPlayer (ffmpeg->Opus). Skips its own audio, dedupes overlapping subscriptions, and ignores sub-0.35s noise. - Python: new `python -m wsai --voice-server` serves /api/voice-turn — decode the uploaded utterance, GPU faster-whisper STT, produce a reply (echo of what was heard for now), GPU MeloTTS synth, return the reply wav (recognised/reply text ride along as X-Heard/X-Reply headers). Both engines pre-warmed; turns show in the dashboard feed. MeloTTS.synth() extracted for direct wav reuse. Echo mode verifies listening+speaking+GPU recognition entirely in Discord; the Claude brain is the next slice. Verified the endpoint round-trip: utterance wav -> correct Korean X-Heard/X-Reply + a WAVE reply on device=cuda. 12 tests pass, node --check clean. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -58,9 +58,51 @@ def _make_handler(dash: "Dashboard"):
|
||||
path = self.path.split("?", 1)[0]
|
||||
if path == "/api/stt":
|
||||
self._handle_stt()
|
||||
elif path == "/api/voice-turn":
|
||||
self._handle_voice_turn()
|
||||
else:
|
||||
self._send(404, b"not found", "text/plain; charset=utf-8")
|
||||
|
||||
def _read_body(self) -> bytes:
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
length = 0
|
||||
return self.rfile.read(length) if length > 0 else b""
|
||||
|
||||
def _handle_voice_turn(self) -> None:
|
||||
"""Discord voice bridge: utterance wav in -> reply wav out. The
|
||||
recognised/reply text ride along as URL-encoded response headers so
|
||||
the bot can log them; the body is the reply audio to play back."""
|
||||
import urllib.parse
|
||||
|
||||
if dash.stt is None or dash.tts is None:
|
||||
self._send(503, json.dumps({"ok": False, "error": "voice loop not enabled"}).encode(),
|
||||
"application/json; charset=utf-8")
|
||||
return
|
||||
raw = self._read_body()
|
||||
if not raw:
|
||||
self._send(400, json.dumps({"ok": False, "error": "empty upload"}).encode(),
|
||||
"application/json; charset=utf-8")
|
||||
return
|
||||
try:
|
||||
res = dash.voice_turn(raw)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.exception("voice-turn failed")
|
||||
self._send(500, json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"},
|
||||
ensure_ascii=False).encode(), "application/json; charset=utf-8")
|
||||
return
|
||||
body = res["wav"]
|
||||
self.send_response(200 if body else 204)
|
||||
self.send_header("Content-Type", "audio/wav")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Heard", urllib.parse.quote(res.get("heard", "")))
|
||||
self.send_header("X-Reply", urllib.parse.quote(res.get("reply", "")))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def _handle_stt(self) -> None:
|
||||
"""Accept an uploaded audio blob (mic recording or file), run it
|
||||
through the real GPU STT, and return the recognised text."""
|
||||
@@ -130,18 +172,19 @@ class Dashboard:
|
||||
"""
|
||||
|
||||
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
|
||||
stt=None) -> None:
|
||||
stt=None, tts=None) -> None:
|
||||
self.monitor = monitor
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.stt = stt
|
||||
self.tts = tts
|
||||
self._server: ThreadingHTTPServer | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._loop = None
|
||||
self._loop_thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self.stt is not None:
|
||||
if self.stt is not None or self.tts is not None:
|
||||
self._start_loop()
|
||||
handler = _make_handler(self)
|
||||
self._server = ThreadingHTTPServer((self.host, self.port), handler)
|
||||
@@ -178,10 +221,72 @@ class Dashboard:
|
||||
return fut.result(timeout=timeout)
|
||||
|
||||
def warm(self) -> None:
|
||||
"""Pre-start the STT worker (loads + warms the GPU) so the first web
|
||||
recognition is instant instead of paying model-load + CUDA autotune."""
|
||||
"""Pre-start the STT/TTS workers (loads + warms the GPU) so the first
|
||||
recognition/synth is instant instead of paying model-load + CUDA autotune."""
|
||||
if self.stt is not None:
|
||||
self._submit(self.stt._ensure())
|
||||
if self.tts is not None:
|
||||
self._submit(self.tts._ensure())
|
||||
|
||||
def voice_turn(self, audio_bytes: bytes) -> dict:
|
||||
"""One Discord voice turn: decode the uploaded utterance, recognise it
|
||||
on the GPU, produce a reply (echo of what was heard for now), synthesise
|
||||
it on the GPU, and return {heard, reply, wav} where wav is the reply
|
||||
audio bytes for the bot to play back into the channel."""
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
if self.stt is None or self.tts is None:
|
||||
raise RuntimeError("voice_turn needs both STT and TTS")
|
||||
updir = os.path.expanduser("~/.cache/wsai/uploads")
|
||||
os.makedirs(updir, exist_ok=True)
|
||||
stem = os.path.join(updir, uuid.uuid4().hex)
|
||||
src, wav = stem + ".bin", stem + ".wav"
|
||||
with open(src, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
turn = self.monitor.turn(source="discord")
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
heard = self._submit(self.stt.transcribe(wav)) or ""
|
||||
turn.heard(heard or "(빈 결과)")
|
||||
reply_text = heard.strip()
|
||||
if not reply_text:
|
||||
# Nothing recognised (silence/noise): skip TTS, tell the bot.
|
||||
turn.finish()
|
||||
return {"heard": heard, "reply": "", "wav": b""}
|
||||
out_path = self._submit(self.tts.synth(reply_text))
|
||||
with open(out_path, "rb") as f:
|
||||
reply_wav = f.read()
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
turn.replied(reply_text)
|
||||
step = turn.step("STT+TTS(GPU)")
|
||||
step.ok, step.ms = True, float(ms)
|
||||
turn._steps.append(step)
|
||||
turn.finish()
|
||||
try:
|
||||
os.remove(out_path)
|
||||
except OSError:
|
||||
pass
|
||||
return {"heard": heard, "reply": reply_text, "wav": reply_wav}
|
||||
except subprocess.CalledProcessError as exc:
|
||||
turn.finish(error="ffmpeg decode failed")
|
||||
err = exc.stderr.decode("utf-8", "replace")[-300:] if exc.stderr else str(exc)
|
||||
raise RuntimeError(f"ffmpeg: {err}") from exc
|
||||
except Exception as exc:
|
||||
turn.finish(error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
for p in (src, wav):
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def transcribe_upload(self, audio_bytes: bytes) -> dict:
|
||||
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it
|
||||
|
||||
Reference in New Issue
Block a user