Files
javis_bot/src/voice_worker/segmenter.py
javis-bot 590cd07268 fix(voice_worker): even out speaker loudness and stop transcribing noise into words
- Loudness equalisation replaces peak normalisation: rumble below 60 Hz removed, level measured on
  the VAD-voiced windows only, brought to -20 dBFS (gain -20..+40 dB), soft limiter below 0.98.
- Only clearly voiced windows (VAD >= 0.5) count towards the 250 ms needed before an utterance
  reaches the model, so noise the VAD half-believes never calls Whisper.
- Hallucination gate: mean token log-probability >= -0.7 (the old len/(len+1) correction let short
  hallucinations through), compression ratio <= 2.4; no_speech_prob dropped (always 0.0 on turbo).
- With hotwords, the plain decode of the same encoder pass decides; the hint may fix a wake word's
  spelling but cannot add or drop one (it wrote the wake word on laughter and after real sentences).
- Opt-in real-model evals: voice-like babble with the hint, speech over babble, quiet and clipped
  speakers, and a JARVIS_NOISE_DIR folder of real noise recordings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 10:27:38 +09:00

148 lines
4.9 KiB
Python

"""Per-speaker utterance segmentation on 16 kHz mono audio (rules in voice_worker.spec.md)."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
import numpy as np
from .settings import SttSettings
SR = 16000
WINDOW = 512 # Silero window at 16 kHz
WINDOW_MS = WINDOW * 1000 / SR # 32 ms
START_PROB = 0.5
KEEP_PROB = 0.35
PRE_ROLL_MS = 200
MIN_SPEECH_MS = 250
TAIL_KEEP_MS = 200
MAX_UTTERANCE_MS = 28000
class VadStream(Protocol):
def prob(self, window: np.ndarray) -> float: ...
@dataclass
class SpeechStart:
utt: int
@dataclass
class Partial:
utt: int
audio: np.ndarray
voiced: np.ndarray # one bool per 512-sample window: the VAD heard speech there
@dataclass
class Final:
utt: int
audio: np.ndarray
reason: str # silence | speaking_end | idle | max_length
decided_at: float
voiced: np.ndarray
def _windows(ms: float) -> int:
return int(ms // WINDOW_MS)
class Segmenter:
def __init__(self, vad: VadStream, settings: SttSettings):
self.vad = vad
self.settings = settings
self._pending = np.zeros(0, dtype=np.float32)
self._pre: list[np.ndarray] = []
self._utt_count = 0
self._last_audio_at: float | None = None
self._reset()
def _reset(self) -> None:
self._active = False
self._win: list[np.ndarray] = []
self._voiced: list[bool] = []
self._speech = 0 # windows that clearly held speech (>= START_PROB)
self._trailing = 0
self._started = False
self._last_partial_at = 0.0
self._last_partial_len = 0
@property
def open_utt(self) -> int | None:
"""The utterance that sent speech_start and has not ended yet."""
return self._utt_count if self._active and self._started else None
def push(self, samples: np.ndarray, now: float) -> list:
self._last_audio_at = now
data = np.concatenate([self._pending, samples.astype(np.float32, copy=False)])
n = len(data) // WINDOW * WINDOW
self._pending = data[n:]
events: list = []
for i in range(0, n, WINDOW):
events += self._window(data[i:i + WINDOW], now)
if self.open_utt is not None and len(self._win) > self._last_partial_len \
and now - self._last_partial_at >= self.settings.partial_interval_ms / 1000:
self._last_partial_at = now
self._last_partial_len = len(self._win)
events.append(Partial(self._utt_count, np.concatenate(self._win), np.array(self._voiced)))
return events
def _window(self, w: np.ndarray, now: float) -> list:
p = self.vad.prob(w)
if not self._active:
if p >= START_PROB:
self._active = True
self._win = self._pre + [w]
self._voiced = [False] * len(self._pre) + [True]
self._pre = []
self._speech = 1
self._trailing = 0
else:
self._pre.append(w)
del self._pre[:-_windows(PRE_ROLL_MS)]
return []
self._win.append(w)
self._voiced.append(p >= START_PROB)
# "Maybe speech" (between the two thresholds) keeps an utterance going but is not evidence
# of a voice, so noise the VAD half-believes never adds up to a call to the model.
if p >= START_PROB:
self._speech += 1
if p >= KEEP_PROB:
self._trailing = 0
else:
self._trailing += 1
events: list = []
if not self._started and self._speech * WINDOW_MS >= MIN_SPEECH_MS:
self._started = True
self._utt_count += 1
self._last_partial_at = now
events.append(SpeechStart(self._utt_count))
if self._trailing * WINDOW_MS >= self.settings.end_silence_ms:
events += self._end("silence", now)
elif len(self._win) * WINDOW_MS >= MAX_UTTERANCE_MS:
events += self._end("max_length", now)
return events
def _end(self, reason: str, now: float) -> list:
events: list = []
if self._active and self._started:
drop = max(0, self._trailing - _windows(TAIL_KEEP_MS))
keep = len(self._win) - drop
events.append(Final(self._utt_count, np.concatenate(self._win[:keep]), reason, now,
np.array(self._voiced[:keep])))
self._reset()
return events
def speaking_end(self, now: float) -> list:
self._pending = np.zeros(0, dtype=np.float32)
return self._end("speaking_end", now) if self._active else []
def tick(self, now: float) -> list:
if self._active and self._last_audio_at is not None \
and now - self._last_audio_at >= self.settings.end_silence_ms / 1000:
self._pending = np.zeros(0, dtype=np.float32)
return self._end("idle", now)
return []