Files
javis_bot/src/voice_worker/segmenter.py
javis-bot a7685e9078 feat(voice_worker): M1 multi-speaker streaming STT and sentence TTS over a local WebSocket
- One shared faster-whisper model for every speaker, batched; finals go before partials and
  partial batches are capped at two so a final never waits behind a ten-speaker pass.
- Per-speaker 48k stereo to 16k mono conversion, streaming Silero VAD, utterance ends on VAD
  silence, Discord speaking_end, missing audio, or the 28 s Whisper window.
- MeloTTS in its own venv through a child process; sentences stream out one chunk each, with
  incremental text, cancel and failure reporting.
- Loopback-only server with a token from the environment; logs carry no transcript text.
- Spec in src/voice_worker/voice_worker.spec.md; unit tests with fake engines and an opt-in
  real-GPU test that streams ten speakers at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 10:00:06 +09:00

139 lines
4.3 KiB
Python

"""Per-speaker utterance segmentation on 16 kHz mono audio (rules in voice_worker.spec.md)."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
import numpy as np
from .settings import SttSettings
SR = 16000
WINDOW = 512 # Silero window at 16 kHz
WINDOW_MS = WINDOW * 1000 / SR # 32 ms
START_PROB = 0.5
KEEP_PROB = 0.35
PRE_ROLL_MS = 200
MIN_SPEECH_MS = 250
TAIL_KEEP_MS = 200
MAX_UTTERANCE_MS = 28000
class VadStream(Protocol):
def prob(self, window: np.ndarray) -> float: ...
@dataclass
class SpeechStart:
utt: int
@dataclass
class Partial:
utt: int
audio: np.ndarray
@dataclass
class Final:
utt: int
audio: np.ndarray
reason: str # silence | speaking_end | idle | max_length
decided_at: float
def _windows(ms: float) -> int:
return int(ms // WINDOW_MS)
class Segmenter:
def __init__(self, vad: VadStream, settings: SttSettings):
self.vad = vad
self.settings = settings
self._pending = np.zeros(0, dtype=np.float32)
self._pre: list[np.ndarray] = []
self._utt_count = 0
self._last_audio_at: float | None = None
self._reset()
def _reset(self) -> None:
self._active = False
self._win: list[np.ndarray] = []
self._speech = 0
self._trailing = 0
self._started = False
self._last_partial_at = 0.0
self._last_partial_len = 0
@property
def open_utt(self) -> int | None:
"""The utterance that sent speech_start and has not ended yet."""
return self._utt_count if self._active and self._started else None
def push(self, samples: np.ndarray, now: float) -> list:
self._last_audio_at = now
data = np.concatenate([self._pending, samples.astype(np.float32, copy=False)])
n = len(data) // WINDOW * WINDOW
self._pending = data[n:]
events: list = []
for i in range(0, n, WINDOW):
events += self._window(data[i:i + WINDOW], now)
if self.open_utt is not None and len(self._win) > self._last_partial_len \
and now - self._last_partial_at >= self.settings.partial_interval_ms / 1000:
self._last_partial_at = now
self._last_partial_len = len(self._win)
events.append(Partial(self._utt_count, np.concatenate(self._win)))
return events
def _window(self, w: np.ndarray, now: float) -> list:
p = self.vad.prob(w)
if not self._active:
if p >= START_PROB:
self._active = True
self._win = self._pre + [w]
self._pre = []
self._speech = 1
self._trailing = 0
else:
self._pre.append(w)
del self._pre[:-_windows(PRE_ROLL_MS)]
return []
self._win.append(w)
if p >= KEEP_PROB:
self._speech += 1
self._trailing = 0
else:
self._trailing += 1
events: list = []
if not self._started and self._speech * WINDOW_MS >= MIN_SPEECH_MS:
self._started = True
self._utt_count += 1
self._last_partial_at = now
events.append(SpeechStart(self._utt_count))
if self._trailing * WINDOW_MS >= self.settings.end_silence_ms:
events += self._end("silence", now)
elif len(self._win) * WINDOW_MS >= MAX_UTTERANCE_MS:
events += self._end("max_length", now)
return events
def _end(self, reason: str, now: float) -> list:
events: list = []
if self._active and self._started:
drop = max(0, self._trailing - _windows(TAIL_KEEP_MS))
kept = self._win[:len(self._win) - drop]
events.append(Final(self._utt_count, np.concatenate(kept), reason, now))
self._reset()
return events
def speaking_end(self, now: float) -> list:
self._pending = np.zeros(0, dtype=np.float32)
return self._end("speaking_end", now) if self._active else []
def tick(self, now: float) -> list:
if self._active and self._last_audio_at is not None \
and now - self._last_audio_at >= self.settings.end_silence_ms / 1000:
self._pending = np.zeros(0, dtype=np.float32)
return self._end("idle", now)
return []