- One shared faster-whisper model for every speaker, batched; finals go before partials and partial batches are capped at two so a final never waits behind a ten-speaker pass. - Per-speaker 48k stereo to 16k mono conversion, streaming Silero VAD, utterance ends on VAD silence, Discord speaking_end, missing audio, or the 28 s Whisper window. - MeloTTS in its own venv through a child process; sentences stream out one chunk each, with incremental text, cancel and failure reporting. - Loopback-only server with a token from the environment; logs carry no transcript text. - Spec in src/voice_worker/voice_worker.spec.md; unit tests with fake engines and an opt-in real-GPU test that streams ten speakers at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
139 lines
4.3 KiB
Python
139 lines
4.3 KiB
Python
"""Per-speaker utterance segmentation on 16 kHz mono audio (rules in voice_worker.spec.md)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Protocol
|
|
|
|
import numpy as np
|
|
|
|
from .settings import SttSettings
|
|
|
|
SR = 16000
|
|
WINDOW = 512 # Silero window at 16 kHz
|
|
WINDOW_MS = WINDOW * 1000 / SR # 32 ms
|
|
START_PROB = 0.5
|
|
KEEP_PROB = 0.35
|
|
PRE_ROLL_MS = 200
|
|
MIN_SPEECH_MS = 250
|
|
TAIL_KEEP_MS = 200
|
|
MAX_UTTERANCE_MS = 28000
|
|
|
|
|
|
class VadStream(Protocol):
|
|
def prob(self, window: np.ndarray) -> float: ...
|
|
|
|
|
|
@dataclass
|
|
class SpeechStart:
|
|
utt: int
|
|
|
|
|
|
@dataclass
|
|
class Partial:
|
|
utt: int
|
|
audio: np.ndarray
|
|
|
|
|
|
@dataclass
|
|
class Final:
|
|
utt: int
|
|
audio: np.ndarray
|
|
reason: str # silence | speaking_end | idle | max_length
|
|
decided_at: float
|
|
|
|
|
|
def _windows(ms: float) -> int:
|
|
return int(ms // WINDOW_MS)
|
|
|
|
|
|
class Segmenter:
|
|
def __init__(self, vad: VadStream, settings: SttSettings):
|
|
self.vad = vad
|
|
self.settings = settings
|
|
self._pending = np.zeros(0, dtype=np.float32)
|
|
self._pre: list[np.ndarray] = []
|
|
self._utt_count = 0
|
|
self._last_audio_at: float | None = None
|
|
self._reset()
|
|
|
|
def _reset(self) -> None:
|
|
self._active = False
|
|
self._win: list[np.ndarray] = []
|
|
self._speech = 0
|
|
self._trailing = 0
|
|
self._started = False
|
|
self._last_partial_at = 0.0
|
|
self._last_partial_len = 0
|
|
|
|
@property
|
|
def open_utt(self) -> int | None:
|
|
"""The utterance that sent speech_start and has not ended yet."""
|
|
return self._utt_count if self._active and self._started else None
|
|
|
|
def push(self, samples: np.ndarray, now: float) -> list:
|
|
self._last_audio_at = now
|
|
data = np.concatenate([self._pending, samples.astype(np.float32, copy=False)])
|
|
n = len(data) // WINDOW * WINDOW
|
|
self._pending = data[n:]
|
|
events: list = []
|
|
for i in range(0, n, WINDOW):
|
|
events += self._window(data[i:i + WINDOW], now)
|
|
if self.open_utt is not None and len(self._win) > self._last_partial_len \
|
|
and now - self._last_partial_at >= self.settings.partial_interval_ms / 1000:
|
|
self._last_partial_at = now
|
|
self._last_partial_len = len(self._win)
|
|
events.append(Partial(self._utt_count, np.concatenate(self._win)))
|
|
return events
|
|
|
|
def _window(self, w: np.ndarray, now: float) -> list:
|
|
p = self.vad.prob(w)
|
|
if not self._active:
|
|
if p >= START_PROB:
|
|
self._active = True
|
|
self._win = self._pre + [w]
|
|
self._pre = []
|
|
self._speech = 1
|
|
self._trailing = 0
|
|
else:
|
|
self._pre.append(w)
|
|
del self._pre[:-_windows(PRE_ROLL_MS)]
|
|
return []
|
|
self._win.append(w)
|
|
if p >= KEEP_PROB:
|
|
self._speech += 1
|
|
self._trailing = 0
|
|
else:
|
|
self._trailing += 1
|
|
events: list = []
|
|
if not self._started and self._speech * WINDOW_MS >= MIN_SPEECH_MS:
|
|
self._started = True
|
|
self._utt_count += 1
|
|
self._last_partial_at = now
|
|
events.append(SpeechStart(self._utt_count))
|
|
if self._trailing * WINDOW_MS >= self.settings.end_silence_ms:
|
|
events += self._end("silence", now)
|
|
elif len(self._win) * WINDOW_MS >= MAX_UTTERANCE_MS:
|
|
events += self._end("max_length", now)
|
|
return events
|
|
|
|
def _end(self, reason: str, now: float) -> list:
|
|
events: list = []
|
|
if self._active and self._started:
|
|
drop = max(0, self._trailing - _windows(TAIL_KEEP_MS))
|
|
kept = self._win[:len(self._win) - drop]
|
|
events.append(Final(self._utt_count, np.concatenate(kept), reason, now))
|
|
self._reset()
|
|
return events
|
|
|
|
def speaking_end(self, now: float) -> list:
|
|
self._pending = np.zeros(0, dtype=np.float32)
|
|
return self._end("speaking_end", now) if self._active else []
|
|
|
|
def tick(self, now: float) -> list:
|
|
if self._active and self._last_audio_at is not None \
|
|
and now - self._last_audio_at >= self.settings.end_silence_ms / 1000:
|
|
self._pending = np.zeros(0, dtype=np.float32)
|
|
return self._end("idle", now)
|
|
return []
|