"""Local WebSocket server: one session per connection (protocol in voice_worker.spec.md).""" from __future__ import annotations import asyncio import hmac import ipaddress import itertools import json import time from dataclasses import dataclass from typing import Callable import numpy as np from jarvis.debug import debug_log from .audio import InputConverter from .protocol import KIND_AUDIO, ProtocolError, decode_audio_frame, encode_tts_frame from .scheduler import DecodeOptions, SttResult, SttScheduler, Transcriber from .segmenter import SR, Final, Partial, Segmenter, SpeechStart, VadStream from .settings import SttSettings from .tts import TtsEngine, TtsRequest, TtsService CLOSE_BAD_HELLO = 4400 CLOSE_BAD_TOKEN = 4401 MAX_SPEAKERS = 32 def is_loopback(host: str) -> bool: if host == "localhost": return True try: return ipaddress.ip_address(host).is_loopback except ValueError: return False class VoiceWorker: def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream], tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05): if not token: raise ValueError("a connection token is required") self.token = token self.transcriber = transcriber self.vad_factory = vad_factory self.tts_engine = tts_engine self.hello_timeout_s = hello_timeout_s self.tick_s = tick_s self.scheduler: SttScheduler | None = None self.tts: TtsService | None = None self._ids = itertools.count(1) async def start(self) -> None: if self.transcriber is not None: self.scheduler = SttScheduler(self.transcriber) self.scheduler.start() if self.tts_engine is not None: self.tts = TtsService(self.tts_engine) self.tts.start() async def stop(self) -> None: if self.scheduler: await self.scheduler.stop() if self.tts: await self.tts.stop() async def serve(self, host: str, port: int): """Start listening; returns the websockets server. Loopback addresses only.""" if not is_loopback(host): raise ValueError("the voice worker only listens on a loopback address") from websockets.asyncio.server import serve await self.start() return await serve(self.handle, host, port, max_size=2 ** 20) async def handle(self, ws) -> None: try: first = await asyncio.wait_for(ws.recv(), self.hello_timeout_s) hello = json.loads(first) if isinstance(first, str) else None except (asyncio.TimeoutError, ValueError): hello = None except Exception: return if not isinstance(hello, dict) or hello.get("type") != "hello": debug_log("connection refused: no hello", "voice_worker") await ws.close(CLOSE_BAD_HELLO, "hello expected") return token = hello.get("token") if not isinstance(token, str) or not hmac.compare_digest(token.encode(), self.token.encode()): debug_log("connection refused: bad token", "voice_worker") await ws.close(CLOSE_BAD_TOKEN, "bad token") return try: session = Session(self, ws, next(self._ids), hello) except ValueError as e: await ws.close(CLOSE_BAD_HELLO, str(e)[:100]) return await session.run() @dataclass class _Speaker: converter: InputConverter segmenter: Segmenter def _format(value, default: dict, rates: tuple[int, ...] | None) -> dict: if value is None: return dict(default) if not isinstance(value, dict): raise ValueError("format must be an object") rate, channels = value.get("rate", default["rate"]), value.get("channels", default["channels"]) ok_rate = isinstance(rate, int) and (rate in rates if rates else 8000 <= rate <= 96000) if not ok_rate or channels not in (1, 2) or isinstance(channels, bool): raise ValueError("unsupported audio format") return {"rate": rate, "channels": channels} class Session: def __init__(self, worker: VoiceWorker, ws, sid: int, hello: dict): self.worker = worker self.ws = ws self.sid = sid self.audio = _format(hello.get("audio"), {"rate": 48000, "channels": 2}, (16000, 48000)) self.tts_format = _format(hello.get("tts"), {"rate": 48000, "channels": 2}, None) self.settings = SttSettings() stt = hello.get("stt") if stt is not None: if not isinstance(stt, dict): raise ValueError("stt must be an object") self.settings.update(stt) self.speakers: dict[str, _Speaker] = {} self.tts_requests: dict[str, TtsRequest] = {} self.out: asyncio.Queue = asyncio.Queue() self.closed = False # --- outbound ------------------------------------------------------------------------------- def send(self, msg) -> None: if not self.closed: self.out.put_nowait(msg if isinstance(msg, bytes) else json.dumps(msg, ensure_ascii=False)) def error(self, code: str, message: str, **extra) -> None: self.send({"type": "error", "code": code, "message": message, **extra}) async def _writer(self) -> None: while True: await self.ws.send(await self.out.get()) # --- lifecycle ------------------------------------------------------------------------------ async def run(self) -> None: w = self.worker debug_log(f"session {self.sid} connected audio={self.audio} tts={self.tts_format}", "voice_worker") self.send({"type": "hello_ok", "stt_ready": w.scheduler is not None, "tts_ready": w.tts is not None, "audio": self.audio, "tts": self.tts_format, "stt": self.settings.as_dict()}) writer = asyncio.create_task(self._writer()) ticker = asyncio.create_task(self._ticker()) try: async for frame in self.ws: if isinstance(frame, bytes): self._on_binary(frame) else: self._on_text(frame) except Exception as e: # connection dropped debug_log(f"session {self.sid} receive ended: {type(e).__name__}", "voice_worker") finally: self.closed = True ticker.cancel() writer.cancel() if w.scheduler: w.scheduler.drop_partials(lambda key: key[0] == self.sid) if w.tts: for req in list(self.tts_requests.values()): w.tts.cancel(req) debug_log(f"session {self.sid} closed", "voice_worker") async def _ticker(self) -> None: while True: await asyncio.sleep(self.worker.tick_s) now = time.monotonic() for speaker, st in list(self.speakers.items()): self._handle_events(speaker, st, st.segmenter.tick(now)) # --- STT ------------------------------------------------------------------------------------ def _on_binary(self, frame: bytes) -> None: if not frame or frame[0] != KIND_AUDIO: self.error("bad_frame", "unknown binary frame kind") return try: speaker, pcm = decode_audio_frame(frame) except ProtocolError as e: self.error("bad_frame", str(e)) return st = self.speakers.get(speaker) if st is None: if len(self.speakers) >= MAX_SPEAKERS: self.error("bad_frame", f"more than {MAX_SPEAKERS} speakers") return st = _Speaker(InputConverter(self.audio["rate"], self.audio["channels"]), Segmenter(self.worker.vad_factory(), self.settings)) self.speakers[speaker] = st samples = st.converter.push(pcm) self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic())) def _opts(self, beam: int) -> DecodeOptions: return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords) def _handle_events(self, speaker: str, st: _Speaker, events: list) -> None: sched = self.worker.scheduler for ev in events: if isinstance(ev, SpeechStart): self.send({"type": "speech_start", "speaker": speaker, "utt": ev.utt}) elif isinstance(ev, Partial): if sched and self.settings.partials: sched.submit_partial((self.sid, speaker), ev.audio, self._opts(1), self._partial_cb(speaker, st, ev.utt)) elif isinstance(ev, Final): debug_log(f"utterance end speaker={speaker} utt={ev.utt} reason={ev.reason} " f"ms={len(ev.audio) * 1000 // SR}", "voice_worker") if sched is None: self._send_final(speaker, ev, SttResult("", 0, "stt unavailable")) else: sched.submit_final((self.sid, speaker), ev.audio, self._opts(self.settings.final_beam), lambda res, ev=ev: self._send_final(speaker, ev, res)) def _partial_cb(self, speaker: str, st: _Speaker, utt: int): def cb(res: SttResult) -> None: # Drop it when the utterance already ended or captions were turned off meanwhile. if st.segmenter.open_utt == utt and self.settings.partials and not res.error: self.send({"type": "partial", "speaker": speaker, "utt": utt, "text": res.text}) return cb def _send_final(self, speaker: str, ev: Final, res: SttResult) -> None: self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text, "end_reason": ev.reason, "audio_ms": len(ev.audio) * 1000 // SR, "wait_ms": int((time.monotonic() - ev.decided_at) * 1000), "stt_ms": res.stt_ms}) # --- text messages -------------------------------------------------------------------------- def _on_text(self, raw: str) -> None: try: msg = json.loads(raw) except ValueError: self.error("bad_message", "not JSON") return if not isinstance(msg, dict): self.error("bad_message", "not an object") return kind = msg.get("type") handler = {"speaking_end": self._on_speaking_end, "config": self._on_config, "tts": self._on_tts, "tts_cancel": self._on_tts_cancel}.get(kind) if handler is None: self.error("bad_message", f"unknown type: {kind!r}") return handler(msg) def _on_speaking_end(self, msg: dict) -> None: st = self.speakers.get(msg.get("speaker")) if st is not None: self._handle_events(msg["speaker"], st, st.segmenter.speaking_end(time.monotonic())) def _on_config(self, msg: dict) -> None: changes = {k: v for k, v in msg.items() if k != "type"} try: self.settings.update(changes) except ValueError as e: self.error("bad_message", str(e)) return if not self.settings.partials and self.worker.scheduler: self.worker.scheduler.drop_partials(lambda key: key[0] == self.sid) debug_log(f"session {self.sid} settings changed: {sorted(changes)}", "voice_worker") def _on_tts(self, msg: dict) -> None: rid, text, more = msg.get("id"), msg.get("text", ""), msg.get("more", False) if not isinstance(rid, str) or not 0 < len(rid.encode()) < 256 or not isinstance(text, str) \ or not isinstance(more, bool): self.error("bad_message", "tts needs a string id and text, and a boolean more") return tts = self.worker.tts if tts is None: self.error("tts_unavailable", "no TTS engine is loaded", id=rid) return req = self.tts_requests.get(rid) if req is None: req = TtsRequest(rid, self.tts_format["rate"], self._tts_chunk_cb(rid), self._tts_done, self._tts_error) self.tts_requests[rid] = req elif req.closed: self.error("bad_message", "this tts id is already closed", id=rid) return tts.feed(req, text, more) def _on_tts_cancel(self, msg: dict) -> None: req = self.tts_requests.get(msg.get("id")) if req is not None and self.worker.tts: self.worker.tts.cancel(req) def _tts_chunk_cb(self, rid: str): def cb(seq: int, pcm: np.ndarray) -> None: pcm = np.asarray(pcm, dtype=" None: self.tts_requests.pop(req.id, None) self.send({"type": "tts_done", "id": req.id, "chunks": req.chunks, "cancelled": req.cancelled, "failed": req.failed}) def _tts_error(self, req: TtsRequest, message: str) -> None: self.error("tts_failed", message, id=req.id)