feat(voice): real Discord listen+speak loop (STT->echo->TTS)
The bot (dave/bot.mjs) previously only joined the channel and counted audio frames — it never fed STT or spoke back. Wire the real loop: - Node bot: buffer each speaker's Opus->PCM utterance until AfterSilence, wrap as WAV, POST to the Python voice-turn endpoint, then play the returned reply wav into the channel via an AudioPlayer (ffmpeg->Opus). Skips its own audio, dedupes overlapping subscriptions, and ignores sub-0.35s noise. - Python: new `python -m wsai --voice-server` serves /api/voice-turn — decode the uploaded utterance, GPU faster-whisper STT, produce a reply (echo of what was heard for now), GPU MeloTTS synth, return the reply wav (recognised/reply text ride along as X-Heard/X-Reply headers). Both engines pre-warmed; turns show in the dashboard feed. MeloTTS.synth() extracted for direct wav reuse. Echo mode verifies listening+speaking+GPU recognition entirely in Discord; the Claude brain is the next slice. Verified the endpoint round-trip: utterance wav -> correct Korean X-Heard/X-Reply + a WAVE reply on device=cuda. 12 tests pass, node --check clean. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -100,6 +100,47 @@ def _run_stt_test(host: str, port: int) -> None:
|
||||
dash.stop()
|
||||
|
||||
|
||||
def _run_voice_server(host: str, port: int) -> None:
|
||||
"""Serve the STT+TTS voice-turn endpoint that the Discord bot (dave/bot.mjs)
|
||||
calls: it POSTs a captured utterance wav and gets back the reply wav to play
|
||||
into the voice channel. Both STT and TTS run on the GPU and are pre-warmed.
|
||||
The same page also shows the live turn feed. Echo mode for now (the reply is
|
||||
what was heard); the Claude brain can be added as the next slice."""
|
||||
import time
|
||||
|
||||
from .backends.melo import MeloTTS
|
||||
from .backends.whisper import WhisperSTT
|
||||
from .dashboard import Dashboard
|
||||
from .monitor import Monitor
|
||||
|
||||
monitor = Monitor()
|
||||
stt = WhisperSTT()
|
||||
tts = MeloTTS()
|
||||
dash = Dashboard(monitor, host=host, port=port, stt=stt, tts=tts)
|
||||
dash.start()
|
||||
monitor.set_components({"source": "none", "vision": "none", "stt": "whisper",
|
||||
"brain": "echo", "tts": "melo"})
|
||||
monitor.set_status(running=True, listening=False)
|
||||
monitor.log("info", "디스코드 음성 서버 시작 — STT+TTS GPU 워밍업 중…")
|
||||
print("\n STT+TTS 워밍업 중… (모델 로드 + CUDA 예열)")
|
||||
dash.warm()
|
||||
sdev = getattr(stt, "resolved_device", None) or "?"
|
||||
monitor.set_status(listening=True)
|
||||
monitor.log("info", f"음성 서버 준비 완료 (STT device={sdev}). 디스코드 봇 연결 대기.")
|
||||
|
||||
shown = host if host not in ("0.0.0.0", "") else _lan_ip()
|
||||
print(f"\n 음성 서버 준비 완료 (STT device: {sdev})")
|
||||
print(f" 대시보드/상태: http://{shown}:{port}")
|
||||
print(f" 봇 연결 엔드포인트: http://127.0.0.1:{port}/api/voice-turn\n")
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
dash.stop()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(prog="wsai")
|
||||
ap.add_argument("--voice", action="store_true", help="eyes-free voice loop (STT -> Brain -> TTS)")
|
||||
@@ -108,6 +149,8 @@ def main() -> None:
|
||||
help="keep generating mock demo utterances forever (off by default)")
|
||||
ap.add_argument("--stt-test", action="store_true",
|
||||
help="serve the dashboard with a live GPU STT recognition test")
|
||||
ap.add_argument("--voice-server", action="store_true",
|
||||
help="serve STT+TTS voice-turn endpoint for the Discord bot (dave/bot.mjs)")
|
||||
ap.add_argument("--live", action="store_true", help="capture screen + Claude backends")
|
||||
ap.add_argument("--env", action="store_true", help="build from WSAI_* env vars")
|
||||
ap.add_argument("--port", type=int, default=int(os.environ.get("WSAI_DASHBOARD_PORT", "8787")),
|
||||
@@ -126,6 +169,10 @@ def main() -> None:
|
||||
_run_stt_test(args.host, args.port)
|
||||
return
|
||||
|
||||
if args.voice_server:
|
||||
_run_voice_server(args.host, args.port)
|
||||
return
|
||||
|
||||
if args.dashboard:
|
||||
# Default to the eyes-free voice preset for the demo; env can override.
|
||||
settings = Settings.from_env() if args.env else Settings.voice()
|
||||
|
||||
Reference in New Issue
Block a user