- One shared faster-whisper model for every speaker, batched; finals go before partials and partial batches are capped at two so a final never waits behind a ten-speaker pass. - Per-speaker 48k stereo to 16k mono conversion, streaming Silero VAD, utterance ends on VAD silence, Discord speaking_end, missing audio, or the 28 s Whisper window. - MeloTTS in its own venv through a child process; sentences stream out one chunk each, with incremental text, cancel and failure reporting. - Loopback-only server with a token from the environment; logs carry no transcript text. - Spec in src/voice_worker/voice_worker.spec.md; unit tests with fake engines and an opt-in real-GPU test that streams ten speakers at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
"""Voice worker: the streaming Silero VAD (real ONNX model bundled with faster-whisper, CPU only)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import wave
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
pytest.importorskip("onnxruntime")
|
|
pytest.importorskip("faster_whisper")
|
|
|
|
from voice_worker.segmenter import WINDOW # noqa: E402
|
|
from voice_worker.vad import SileroVad # noqa: E402
|
|
|
|
SAMPLE = Path(__file__).resolve().parent.parent / "scripts" / "m0" / "assets" / "ko_samples" / "ko_01.wav"
|
|
|
|
|
|
def _speech() -> np.ndarray:
|
|
with wave.open(str(SAMPLE), "rb") as w:
|
|
return np.frombuffer(w.readframes(w.getnframes()), dtype="<i2").astype(np.float32) / 32768.0
|
|
|
|
|
|
def _probs(stream, audio):
|
|
n = len(audio) // WINDOW * WINDOW
|
|
return np.array([stream.prob(audio[i:i + WINDOW]) for i in range(0, n, WINDOW)])
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def vad():
|
|
return SileroVad()
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_speech_is_detected_and_silence_is_not(vad):
|
|
speech = _probs(vad.stream(), _speech())
|
|
silence = _probs(vad.stream(), np.zeros(16000, dtype=np.float32))
|
|
assert (speech >= 0.5).mean() > 0.4
|
|
assert silence.max() < 0.2
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_each_speaker_stream_keeps_its_own_state(vad):
|
|
audio = _speech()
|
|
alone = _probs(vad.stream(), audio)
|
|
a, b = vad.stream(), vad.stream()
|
|
n = len(audio) // WINDOW * WINDOW
|
|
interleaved = []
|
|
for i in range(0, n, WINDOW):
|
|
interleaved.append(a.prob(audio[i:i + WINDOW]))
|
|
b.prob(np.zeros(WINDOW, dtype=np.float32)) # another speaker in silence between every window
|
|
assert np.allclose(alone, interleaved, atol=1e-5)
|