Files
javis_bot/src/voice_worker/server.py
javis-bot a7685e9078 feat(voice_worker): M1 multi-speaker streaming STT and sentence TTS over a local WebSocket
- One shared faster-whisper model for every speaker, batched; finals go before partials and
  partial batches are capped at two so a final never waits behind a ten-speaker pass.
- Per-speaker 48k stereo to 16k mono conversion, streaming Silero VAD, utterance ends on VAD
  silence, Discord speaking_end, missing audio, or the 28 s Whisper window.
- MeloTTS in its own venv through a child process; sentences stream out one chunk each, with
  incremental text, cancel and failure reporting.
- Loopback-only server with a token from the environment; logs carry no transcript text.
- Spec in src/voice_worker/voice_worker.spec.md; unit tests with fake engines and an opt-in
  real-GPU test that streams ten speakers at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 10:00:06 +09:00

311 lines
13 KiB
Python

"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md)."""
from __future__ import annotations
import asyncio
import hmac
import ipaddress
import itertools
import json
import time
from dataclasses import dataclass
from typing import Callable
import numpy as np
from jarvis.debug import debug_log
from .audio import InputConverter
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
from .settings import SttSettings
from .tts import TtsEngine, TtsRequest, TtsService
CLOSE_BAD_HELLO = 4400
CLOSE_BAD_TOKEN = 4401
MAX_SPEAKERS = 32
def is_loopback(host: str) -> bool:
if host == "localhost":
return True
try:
return ipaddress.ip_address(host).is_loopback
except ValueError:
return False
class VoiceWorker:
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05):
if not token:
raise ValueError("a connection token is required")
self.token = token
self.transcriber = transcriber
self.vad_factory = vad_factory
self.tts_engine = tts_engine
self.hello_timeout_s = hello_timeout_s
self.tick_s = tick_s
self.scheduler: SttScheduler | None = None
self.tts: TtsService | None = None
self._ids = itertools.count(1)
async def start(self) -> None:
if self.transcriber is not None:
self.scheduler = SttScheduler(self.transcriber)
self.scheduler.start()
if self.tts_engine is not None:
self.tts = TtsService(self.tts_engine)
self.tts.start()
async def stop(self) -> None:
if self.scheduler:
await self.scheduler.stop()
if self.tts:
await self.tts.stop()
async def serve(self, host: str, port: int):
"""Start listening; returns the websockets server. Loopback addresses only."""
if not is_loopback(host):
raise ValueError("the voice worker only listens on a loopback address")
from websockets.asyncio.server import serve
await self.start()
return await serve(self.handle, host, port, max_size=2 ** 20)
async def handle(self, ws) -> None:
try:
first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s)
hello = json.loads(first) if isinstance(first, str) else None
except (asyncio.TimeoutError, ValueError):
hello = None
except Exception:
return
if not isinstance(hello, dict) or hello.get("type") != "hello":
debug_log("connection refused: no hello", "voice_worker")
await ws.close(CLOSE_BAD_HELLO, "hello expected")
return
token = hello.get("token")
if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()):
debug_log("connection refused: bad token", "voice_worker")
await ws.close(CLOSE_BAD_TOKEN, "bad token")
return
try:
session = Session(self, ws, next(self._ids), hello)
except ValueError as e:
await ws.close(CLOSE_BAD_HELLO, str(e)[:100])
return
await session.run()
@dataclass
class _Speaker:
converter: InputConverter
segmenter: Segmenter
def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict:
if value is None:
return dict(default)
if not isinstance(value, dict):
raise ValueError("format must be an object")
rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"])
ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000)
if not ok_rate or channels not in (1, 2) or isinstance(channels, bool):
raise ValueError("unsupported audio format")
return {"rate": rate, "channels": channels}
class Session:
def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict):
self.worker = worker
self.ws = ws
self.sid = sid
self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000))
self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None)
self.settings = SttSettings()
stt = hello.get("stt")
if stt is not None:
if not isinstance(stt, dict):
raise ValueError("stt must be an object")
self.settings.update(stt)
self.speakers: dict[str, _Speaker] = {}
self.tts_requests: dict[str, TtsRequest] = {}
self.out: asyncio.Queue = asyncio.Queue()
self.closed = False
# --- outbound -------------------------------------------------------------------------------
def send(self, msg) -> None:
if not self.closed:
self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False))
def error(self, code: str, message: str, **extra) -> None:
self.send({"type": "error", "code": code, "message": message, **extra})
async def _writer(self) -> None:
while True:
await self.ws.send(await self.out.get())
# --- lifecycle ------------------------------------------------------------------------------
async def run(self) -> None:
w = self.worker
debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker")
self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None,
"audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()})
writer = asyncio.create_task(self._writer())
ticker = asyncio.create_task(self._ticker())
try:
async for frame in self.ws:
if isinstance(frame, bytes):
self._on_binary(frame)
else:
self._on_text(frame)
except Exception as e: # connection dropped
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
finally:
self.closed = True
ticker.cancel()
writer.cancel()
if w.scheduler:
w.scheduler.drop_partials(lambda key: key[0] == self.sid)
if w.tts:
for req in list(self.tts_requests.values()):
w.tts.cancel(req)
debug_log(f"session {self.sid} closed", "voice_worker")
async def _ticker(self) -> None:
while True:
await asyncio.sleep(self.worker.tick_s)
now = time.monotonic()
for speaker, st in list(self.speakers.items()):
self._handle_events(speaker, st, st.segmenter.tick(now))
# --- STT ------------------------------------------------------------------------------------
def _on_binary(self, frame: bytes) -> None:
if not frame or frame[0] != KIND_AUDIO:
self.error("bad_frame", "unknown binary frame kind")
return
try:
speaker, pcm = decode_audio_frame(frame)
except ProtocolError as e:
self.error("bad_frame", str(e))
return
st = self.speakers.get(speaker)
if st is None:
if len(self.speakers) >= MAX_SPEAKERS:
self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers")
return
st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]),
Segmenter(self.worker.vad_factory(), self.settings))
self.speakers[speaker] = st
samples = st.converter.push(pcm)
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
def _opts(self, beam: int) -> DecodeOptions:
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None:
sched = self.worker.scheduler
for ev in events:
if isinstance(ev, SpeechStart):
self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt})
elif isinstance(ev, Partial):
if sched and self.settings.partials:
sched.submit_partial((self.sid, speaker), ev.audio, self._opts(1),
self._partial_cb(speaker, st, ev.utt))
elif isinstance(ev, Final):
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
f"ms={len(ev.audio) * 1000 // SR}", "voice_worker")
if sched is None:
self._send_final(speaker, ev, SttResult("", 0, "stt unavailable"))
else:
sched.submit_final((self.sid, speaker), ev.audio, self._opts(self.settings.final_beam),
lambda res, ev=ev: self._send_final(speaker, ev, res))
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
def cb(res: SttResult) -> None:
# Drop it when the utterance already ended or captions were turned off meanwhile.
if st.segmenter.open_utt == utt and self.settings.partials and not res.error:
self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text})
return cb
def _send_final(self, speaker: str, ev: Final, res: SttResult) -> None:
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
# --- text messages --------------------------------------------------------------------------
def _on_text(self, raw: str) -> None:
try:
msg = json.loads(raw)
except ValueError:
self.error("bad_message", "not JSON")
return
if not isinstance(msg, dict):
self.error("bad_message", "not an object")
return
kind = msg.get("type")
handler = {"speaking_end": self._on_speaking_end, "config": self._on_config, "tts": self._on_tts,
"tts_cancel": self._on_tts_cancel}.get(kind)
if handler is None:
self.error("bad_message", f"unknown type: {kind!r}")
return
handler(msg)
def _on_speaking_end(self, msg: dict) -> None:
st = self.speakers.get(msg.get("speaker"))
if st is not None:
self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic()))
def _on_config(self, msg: dict) -> None:
changes = {k: v for k, v in msg.items() if k != "type"}
try:
self.settings.update(changes)
except ValueError as e:
self.error("bad_message", str(e))
return
if not self.settings.partials and self.worker.scheduler:
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
def _on_tts(self, msg: dict) -> None:
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
or not isinstance(more, bool):
self.error("bad_message", "tts needs a string id and text, and a boolean more")
return
tts = self.worker.tts
if tts is None:
self.error("tts_unavailable", "no TTS engine is loaded", id=rid)
return
req = self.tts_requests.get(rid)
if req is None:
req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done,
self._tts_error)
self.tts_requests[rid] = req
elif req.closed:
self.error("bad_message", "this tts id is already closed", id=rid)
return
tts.feed(req, text, more)
def _on_tts_cancel(self, msg: dict) -> None:
req = self.tts_requests.get(msg.get("id"))
if req is not None and self.worker.tts:
self.worker.tts.cancel(req)
def _tts_chunk_cb(self, rid: str):
def cb(seq: int, pcm: np.ndarray) -> None:
pcm = np.asarray(pcm, dtype="<i2")
if self.tts_format["channels"] == 2:
pcm = np.repeat(pcm, 2)
self.send(encode_tts_frame(rid, seq, pcm.tobytes()))
return cb
def _tts_done(self, req: TtsRequest) -> None:
self.tts_requests.pop(req.id, None)
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
"failed": req.failed})
def _tts_error(self, req: TtsRequest, message: str) -> None:
self.error("tts_failed", message, id=req.id)