feat(voice): real Discord listen+speak loop (STT->echo->TTS)

The bot (dave/bot.mjs) previously only joined the channel and counted audio
frames — it never fed STT or spoke back. Wire the real loop:

- Node bot: buffer each speaker's Opus->PCM utterance until AfterSilence,
  wrap as WAV, POST to the Python voice-turn endpoint, then play the returned
  reply wav into the channel via an AudioPlayer (ffmpeg->Opus). Skips its own
  audio, dedupes overlapping subscriptions, and ignores sub-0.35s noise.
- Python: new `python -m wsai --voice-server` serves /api/voice-turn — decode
  the uploaded utterance, GPU faster-whisper STT, produce a reply (echo of what
  was heard for now), GPU MeloTTS synth, return the reply wav (recognised/reply
  text ride along as X-Heard/X-Reply headers). Both engines pre-warmed; turns
  show in the dashboard feed. MeloTTS.synth() extracted for direct wav reuse.

Echo mode verifies listening+speaking+GPU recognition entirely in Discord; the
Claude brain is the next slice. Verified the endpoint round-trip: utterance wav
-> correct Korean X-Heard/X-Reply + a WAVE reply on device=cuda. 12 tests pass,
node --check clean.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
EJClaw
2026-08-18 21:59:01 +09:00
parent b9c929a73f
commit 0f94245d5f
4 changed files with 244 additions and 14 deletions

View File

@@ -58,9 +58,51 @@ def _make_handler(dash: "Dashboard"):
path = self.path.split("?", 1)[0]
if path == "/api/stt":
self._handle_stt()
elif path == "/api/voice-turn":
self._handle_voice_turn()
else:
self._send(404, b"not found", "text/plain; charset=utf-8")
def _read_body(self) -> bytes:
try:
length = int(self.headers.get("Content-Length", "0"))
except ValueError:
length = 0
return self.rfile.read(length) if length > 0 else b""
def _handle_voice_turn(self) -> None:
"""Discord voice bridge: utterance wav in -> reply wav out. The
recognised/reply text ride along as URL-encoded response headers so
the bot can log them; the body is the reply audio to play back."""
import urllib.parse
if dash.stt is None or dash.tts is None:
self._send(503, json.dumps({"ok": False, "error": "voice loop not enabled"}).encode(),
"application/json; charset=utf-8")
return
raw = self._read_body()
if not raw:
self._send(400, json.dumps({"ok": False, "error": "empty upload"}).encode(),
"application/json; charset=utf-8")
return
try:
res = dash.voice_turn(raw)
except Exception as exc: # noqa: BLE001
log.exception("voice-turn failed")
self._send(500, json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"},
ensure_ascii=False).encode(), "application/json; charset=utf-8")
return
body = res["wav"]
self.send_response(200 if body else 204)
self.send_header("Content-Type", "audio/wav")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Heard", urllib.parse.quote(res.get("heard", "")))
self.send_header("X-Reply", urllib.parse.quote(res.get("reply", "")))
self.send_header("Cache-Control", "no-store")
self.end_headers()
if body:
self.wfile.write(body)
def _handle_stt(self) -> None:
"""Accept an uploaded audio blob (mic recording or file), run it
through the real GPU STT, and return the recognised text."""
@@ -130,18 +172,19 @@ class Dashboard:
"""
def __init__(self, monitor: Monitor, host: str = "0.0.0.0", port: int = 8787,
stt=None) -> None:
stt=None, tts=None) -> None:
self.monitor = monitor
self.host = host
self.port = port
self.stt = stt
self.tts = tts
self._server: ThreadingHTTPServer | None = None
self._thread: threading.Thread | None = None
self._loop = None
self._loop_thread: threading.Thread | None = None
def start(self) -> None:
if self.stt is not None:
if self.stt is not None or self.tts is not None:
self._start_loop()
handler = _make_handler(self)
self._server = ThreadingHTTPServer((self.host, self.port), handler)
@@ -178,10 +221,72 @@ class Dashboard:
return fut.result(timeout=timeout)
def warm(self) -> None:
"""Pre-start the STT worker (loads + warms the GPU) so the first web
recognition is instant instead of paying model-load + CUDA autotune."""
"""Pre-start the STT/TTS workers (loads + warms the GPU) so the first
recognition/synth is instant instead of paying model-load + CUDA autotune."""
if self.stt is not None:
self._submit(self.stt._ensure())
if self.tts is not None:
self._submit(self.tts._ensure())
def voice_turn(self, audio_bytes: bytes) -> dict:
"""One Discord voice turn: decode the uploaded utterance, recognise it
on the GPU, produce a reply (echo of what was heard for now), synthesise
it on the GPU, and return {heard, reply, wav} where wav is the reply
audio bytes for the bot to play back into the channel."""
import os
import subprocess
import time
import uuid
if self.stt is None or self.tts is None:
raise RuntimeError("voice_turn needs both STT and TTS")
updir = os.path.expanduser("~/.cache/wsai/uploads")
os.makedirs(updir, exist_ok=True)
stem = os.path.join(updir, uuid.uuid4().hex)
src, wav = stem + ".bin", stem + ".wav"
with open(src, "wb") as f:
f.write(audio_bytes)
turn = self.monitor.turn(source="discord")
t0 = time.monotonic()
try:
subprocess.run(
["ffmpeg", "-y", "-i", src, "-ar", "16000", "-ac", "1", wav],
check=True, capture_output=True,
)
heard = self._submit(self.stt.transcribe(wav)) or ""
turn.heard(heard or "(빈 결과)")
reply_text = heard.strip()
if not reply_text:
# Nothing recognised (silence/noise): skip TTS, tell the bot.
turn.finish()
return {"heard": heard, "reply": "", "wav": b""}
out_path = self._submit(self.tts.synth(reply_text))
with open(out_path, "rb") as f:
reply_wav = f.read()
ms = int((time.monotonic() - t0) * 1000)
turn.replied(reply_text)
step = turn.step("STT+TTS(GPU)")
step.ok, step.ms = True, float(ms)
turn._steps.append(step)
turn.finish()
try:
os.remove(out_path)
except OSError:
pass
return {"heard": heard, "reply": reply_text, "wav": reply_wav}
except subprocess.CalledProcessError as exc:
turn.finish(error="ffmpeg decode failed")
err = exc.stderr.decode("utf-8", "replace")[-300:] if exc.stderr else str(exc)
raise RuntimeError(f"ffmpeg: {err}") from exc
except Exception as exc:
turn.finish(error=str(exc))
raise
finally:
for p in (src, wav):
try:
os.remove(p)
except OSError:
pass
def transcribe_upload(self, audio_bytes: bytes) -> dict:
"""ffmpeg-normalise an uploaded blob to 16 kHz mono wav, transcribe it