- One shared faster-whisper model for every speaker, batched; finals go before partials and partial batches are capped at two so a final never waits behind a ten-speaker pass. - Per-speaker 48k stereo to 16k mono conversion, streaming Silero VAD, utterance ends on VAD silence, Discord speaking_end, missing audio, or the 28 s Whisper window. - MeloTTS in its own venv through a child process; sentences stream out one chunk each, with incremental text, cancel and failure reporting. - Loopback-only server with a token from the environment; logs carry no transcript text. - Spec in src/voice_worker/voice_worker.spec.md; unit tests with fake engines and an opt-in real-GPU test that streams ten speakers at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
311 lines
13 KiB
Python
311 lines
13 KiB
Python
"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hmac
|
|
import ipaddress
|
|
import itertools
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Callable
|
|
|
|
import numpy as np
|
|
|
|
from jarvis.debug import debug_log
|
|
|
|
from .audio import InputConverter
|
|
from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame
|
|
from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber
|
|
from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream
|
|
from .settings import SttSettings
|
|
from .tts import TtsEngine, TtsRequest, TtsService
|
|
|
|
CLOSE_BAD_HELLO = 4400
|
|
CLOSE_BAD_TOKEN = 4401
|
|
MAX_SPEAKERS = 32
|
|
|
|
|
|
def is_loopback(host: str) -> bool:
|
|
if host == "localhost":
|
|
return True
|
|
try:
|
|
return ipaddress.ip_address(host).is_loopback
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
class VoiceWorker:
|
|
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
|
|
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05):
|
|
if not token:
|
|
raise ValueError("a connection token is required")
|
|
self.token = token
|
|
self.transcriber = transcriber
|
|
self.vad_factory = vad_factory
|
|
self.tts_engine = tts_engine
|
|
self.hello_timeout_s = hello_timeout_s
|
|
self.tick_s = tick_s
|
|
self.scheduler: SttScheduler | None = None
|
|
self.tts: TtsService | None = None
|
|
self._ids = itertools.count(1)
|
|
|
|
async def start(self) -> None:
|
|
if self.transcriber is not None:
|
|
self.scheduler = SttScheduler(self.transcriber)
|
|
self.scheduler.start()
|
|
if self.tts_engine is not None:
|
|
self.tts = TtsService(self.tts_engine)
|
|
self.tts.start()
|
|
|
|
async def stop(self) -> None:
|
|
if self.scheduler:
|
|
await self.scheduler.stop()
|
|
if self.tts:
|
|
await self.tts.stop()
|
|
|
|
async def serve(self, host: str, port: int):
|
|
"""Start listening; returns the websockets server. Loopback addresses only."""
|
|
if not is_loopback(host):
|
|
raise ValueError("the voice worker only listens on a loopback address")
|
|
from websockets.asyncio.server import serve
|
|
await self.start()
|
|
return await serve(self.handle, host, port, max_size=2 ** 20)
|
|
|
|
async def handle(self, ws) -> None:
|
|
try:
|
|
first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s)
|
|
hello = json.loads(first) if isinstance(first, str) else None
|
|
except (asyncio.TimeoutError, ValueError):
|
|
hello = None
|
|
except Exception:
|
|
return
|
|
if not isinstance(hello, dict) or hello.get("type") != "hello":
|
|
debug_log("connection refused: no hello", "voice_worker")
|
|
await ws.close(CLOSE_BAD_HELLO, "hello expected")
|
|
return
|
|
token = hello.get("token")
|
|
if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()):
|
|
debug_log("connection refused: bad token", "voice_worker")
|
|
await ws.close(CLOSE_BAD_TOKEN, "bad token")
|
|
return
|
|
try:
|
|
session = Session(self, ws, next(self._ids), hello)
|
|
except ValueError as e:
|
|
await ws.close(CLOSE_BAD_HELLO, str(e)[:100])
|
|
return
|
|
await session.run()
|
|
|
|
|
|
@dataclass
|
|
class _Speaker:
|
|
converter: InputConverter
|
|
segmenter: Segmenter
|
|
|
|
|
|
def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict:
|
|
if value is None:
|
|
return dict(default)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("format must be an object")
|
|
rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"])
|
|
ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000)
|
|
if not ok_rate or channels not in (1, 2) or isinstance(channels, bool):
|
|
raise ValueError("unsupported audio format")
|
|
return {"rate": rate, "channels": channels}
|
|
|
|
|
|
class Session:
|
|
def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict):
|
|
self.worker = worker
|
|
self.ws = ws
|
|
self.sid = sid
|
|
self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000))
|
|
self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None)
|
|
self.settings = SttSettings()
|
|
stt = hello.get("stt")
|
|
if stt is not None:
|
|
if not isinstance(stt, dict):
|
|
raise ValueError("stt must be an object")
|
|
self.settings.update(stt)
|
|
self.speakers: dict[str, _Speaker] = {}
|
|
self.tts_requests: dict[str, TtsRequest] = {}
|
|
self.out: asyncio.Queue = asyncio.Queue()
|
|
self.closed = False
|
|
|
|
# --- outbound -------------------------------------------------------------------------------
|
|
def send(self, msg) -> None:
|
|
if not self.closed:
|
|
self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False))
|
|
|
|
def error(self, code: str, message: str, **extra) -> None:
|
|
self.send({"type": "error", "code": code, "message": message, **extra})
|
|
|
|
async def _writer(self) -> None:
|
|
while True:
|
|
await self.ws.send(await self.out.get())
|
|
|
|
# --- lifecycle ------------------------------------------------------------------------------
|
|
async def run(self) -> None:
|
|
w = self.worker
|
|
debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker")
|
|
self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None,
|
|
"audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()})
|
|
writer = asyncio.create_task(self._writer())
|
|
ticker = asyncio.create_task(self._ticker())
|
|
try:
|
|
async for frame in self.ws:
|
|
if isinstance(frame, bytes):
|
|
self._on_binary(frame)
|
|
else:
|
|
self._on_text(frame)
|
|
except Exception as e: # connection dropped
|
|
debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker")
|
|
finally:
|
|
self.closed = True
|
|
ticker.cancel()
|
|
writer.cancel()
|
|
if w.scheduler:
|
|
w.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
if w.tts:
|
|
for req in list(self.tts_requests.values()):
|
|
w.tts.cancel(req)
|
|
debug_log(f"session {self.sid} closed", "voice_worker")
|
|
|
|
async def _ticker(self) -> None:
|
|
while True:
|
|
await asyncio.sleep(self.worker.tick_s)
|
|
now = time.monotonic()
|
|
for speaker, st in list(self.speakers.items()):
|
|
self._handle_events(speaker, st, st.segmenter.tick(now))
|
|
|
|
# --- STT ------------------------------------------------------------------------------------
|
|
def _on_binary(self, frame: bytes) -> None:
|
|
if not frame or frame[0] != KIND_AUDIO:
|
|
self.error("bad_frame", "unknown binary frame kind")
|
|
return
|
|
try:
|
|
speaker, pcm = decode_audio_frame(frame)
|
|
except ProtocolError as e:
|
|
self.error("bad_frame", str(e))
|
|
return
|
|
st = self.speakers.get(speaker)
|
|
if st is None:
|
|
if len(self.speakers) >= MAX_SPEAKERS:
|
|
self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers")
|
|
return
|
|
st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]),
|
|
Segmenter(self.worker.vad_factory(), self.settings))
|
|
self.speakers[speaker] = st
|
|
samples = st.converter.push(pcm)
|
|
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
|
|
|
|
def _opts(self, beam: int) -> DecodeOptions:
|
|
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
|
|
|
|
def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None:
|
|
sched = self.worker.scheduler
|
|
for ev in events:
|
|
if isinstance(ev, SpeechStart):
|
|
self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt})
|
|
elif isinstance(ev, Partial):
|
|
if sched and self.settings.partials:
|
|
sched.submit_partial((self.sid, speaker), ev.audio, self._opts(1),
|
|
self._partial_cb(speaker, st, ev.utt))
|
|
elif isinstance(ev, Final):
|
|
debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} "
|
|
f"ms={len(ev.audio) * 1000 // SR}", "voice_worker")
|
|
if sched is None:
|
|
self._send_final(speaker, ev, SttResult("", 0, "stt unavailable"))
|
|
else:
|
|
sched.submit_final((self.sid, speaker), ev.audio, self._opts(self.settings.final_beam),
|
|
lambda res, ev=ev: self._send_final(speaker, ev, res))
|
|
|
|
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
|
|
def cb(res: SttResult) -> None:
|
|
# Drop it when the utterance already ended or captions were turned off meanwhile.
|
|
if st.segmenter.open_utt == utt and self.settings.partials and not res.error:
|
|
self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text})
|
|
return cb
|
|
|
|
def _send_final(self, speaker: str, ev: Final, res: SttResult) -> None:
|
|
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
|
|
"end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR,
|
|
"wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms})
|
|
|
|
# --- text messages --------------------------------------------------------------------------
|
|
def _on_text(self, raw: str) -> None:
|
|
try:
|
|
msg = json.loads(raw)
|
|
except ValueError:
|
|
self.error("bad_message", "not JSON")
|
|
return
|
|
if not isinstance(msg, dict):
|
|
self.error("bad_message", "not an object")
|
|
return
|
|
kind = msg.get("type")
|
|
handler = {"speaking_end": self._on_speaking_end, "config": self._on_config, "tts": self._on_tts,
|
|
"tts_cancel": self._on_tts_cancel}.get(kind)
|
|
if handler is None:
|
|
self.error("bad_message", f"unknown type: {kind!r}")
|
|
return
|
|
handler(msg)
|
|
|
|
def _on_speaking_end(self, msg: dict) -> None:
|
|
st = self.speakers.get(msg.get("speaker"))
|
|
if st is not None:
|
|
self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic()))
|
|
|
|
def _on_config(self, msg: dict) -> None:
|
|
changes = {k: v for k, v in msg.items() if k != "type"}
|
|
try:
|
|
self.settings.update(changes)
|
|
except ValueError as e:
|
|
self.error("bad_message", str(e))
|
|
return
|
|
if not self.settings.partials and self.worker.scheduler:
|
|
self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid)
|
|
debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker")
|
|
|
|
def _on_tts(self, msg: dict) -> None:
|
|
rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False)
|
|
if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \
|
|
or not isinstance(more, bool):
|
|
self.error("bad_message", "tts needs a string id and text, and a boolean more")
|
|
return
|
|
tts = self.worker.tts
|
|
if tts is None:
|
|
self.error("tts_unavailable", "no TTS engine is loaded", id=rid)
|
|
return
|
|
req = self.tts_requests.get(rid)
|
|
if req is None:
|
|
req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done,
|
|
self._tts_error)
|
|
self.tts_requests[rid] = req
|
|
elif req.closed:
|
|
self.error("bad_message", "this tts id is already closed", id=rid)
|
|
return
|
|
tts.feed(req, text, more)
|
|
|
|
def _on_tts_cancel(self, msg: dict) -> None:
|
|
req = self.tts_requests.get(msg.get("id"))
|
|
if req is not None and self.worker.tts:
|
|
self.worker.tts.cancel(req)
|
|
|
|
def _tts_chunk_cb(self, rid: str):
|
|
def cb(seq: int, pcm: np.ndarray) -> None:
|
|
pcm = np.asarray(pcm, dtype="<i2")
|
|
if self.tts_format["channels"] == 2:
|
|
pcm = np.repeat(pcm, 2)
|
|
self.send(encode_tts_frame(rid, seq, pcm.tobytes()))
|
|
return cb
|
|
|
|
def _tts_done(self, req: TtsRequest) -> None:
|
|
self.tts_requests.pop(req.id, None)
|
|
self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled,
|
|
"failed": req.failed})
|
|
|
|
def _tts_error(self, req: TtsRequest, message: str) -> None:
|
|
self.error("tts_failed", message, id=req.id)
|