Files
javis_bot/src/voice_worker/audio.py
javis-bot 590cd07268 fix(voice_worker): even out speaker loudness and stop transcribing noise into words
- Loudness equalisation replaces peak normalisation: rumble below 60 Hz removed, level measured on
  the VAD-voiced windows only, brought to -20 dBFS (gain -20..+40 dB), soft limiter below 0.98.
- Only clearly voiced windows (VAD >= 0.5) count towards the 250 ms needed before an utterance
  reaches the model, so noise the VAD half-believes never calls Whisper.
- Hallucination gate: mean token log-probability >= -0.7 (the old len/(len+1) correction let short
  hallucinations through), compression ratio <= 2.4; no_speech_prob dropped (always 0.0 on turbo).
- With hotwords, the plain decode of the same encoder pass decides; the hint may fix a wake word's
  spelling but cannot add or drop one (it wrote the wake word on laughter and after real sentences).
- Opt-in real-model evals: voice-like babble with the hint, speech over babble, quiet and clipped
  speakers, and a JARVIS_NOISE_DIR folder of real noise recordings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 10:27:38 +09:00

89 lines
3.6 KiB
Python

"""Incoming PCM to 16 kHz mono float32, and clip normalisation before transcription."""
from __future__ import annotations
import numpy as np
TARGET_RATE = 16000
_TAPS = 97 # windowed-sinc low-pass for the 48 kHz -> 16 kHz decimation
_CUTOFF_HZ = 7200 # below the new 8 kHz Nyquist
def _lowpass(rate: int) -> np.ndarray:
n = np.arange(_TAPS) - (_TAPS - 1) / 2
fc = _CUTOFF_HZ / rate
h = 2 * fc * np.sinc(2 * fc * n) * np.blackman(_TAPS)
return (h / h.sum()).astype(np.float64)
class InputConverter:
"""Streaming converter for one speaker. Keeps filter history and odd bytes between frames."""
def __init__(self, rate: int, channels: int):
if rate not in (16000, 48000):
raise ValueError("audio rate must be 16000 or 48000")
if channels not in (1, 2):
raise ValueError("audio channels must be 1 or 2")
self.channels = channels
self.factor = rate // TARGET_RATE
self._carry = b""
if self.factor > 1:
self._h = _lowpass(rate)
self._hist = np.zeros(_TAPS - 1, dtype=np.float64)
self._phase = 0 # input samples to skip before the next kept output
def push(self, pcm: bytes) -> np.ndarray:
data = self._carry + pcm
frame_bytes = 2 * self.channels
usable = len(data) - len(data) % frame_bytes
self._carry = data[usable:]
x = np.frombuffer(data[:usable], dtype="<i2").astype(np.float64) / 32768.0
if self.channels == 2:
x = x.reshape(-1, 2).mean(axis=1)
if self.factor == 1:
return x.astype(np.float32)
buf = np.concatenate([self._hist, x])
self._hist = buf[-(_TAPS - 1):]
y = np.convolve(buf, self._h, mode="valid") # one output per new input sample
out = y[self._phase::self.factor]
self._phase = (self._phase - len(y)) % self.factor
return out.astype(np.float32)
TARGET_DB = -20.0 # loudness every speaker is brought to, dBFS RMS over the voiced windows
MAX_GAIN_DB = 40.0 # a whisper at -60 dBFS still reaches the target; pure hiss is not blown up
MIN_GAIN_DB = -20.0
HIGHPASS_HZ = 60.0 # mains hum and desk rumble carry no speech
_KNEE = 0.8 # soft limiter: peaks above this bend smoothly towards the ceiling instead of clipping
_CEILING = 0.98
_WINDOW = 512
def loudness_equalise(x: np.ndarray, voiced: np.ndarray | None = None) -> np.ndarray:
"""Bring a clip to the same loudness whoever spoke it, without clipping.
``voiced`` marks the 512-sample windows the VAD heard as speech; the level is measured on them so
silence, a click or a knock does not decide the gain. Without it the louder half of the windows is
used. Low rumble is removed first and peaks are soft-limited after the gain.
"""
x = np.asarray(x, dtype=np.float32)
if len(x) == 0:
return x
spec = np.fft.rfft(x.astype(np.float64))
spec[np.fft.rfftfreq(len(x), 1 / TARGET_RATE) < HIGHPASS_HZ] = 0
y = np.fft.irfft(spec, len(x))
n_win = -(-len(y) // _WINDOW)
frames = np.pad(y, (0, n_win * _WINDOW - len(y))).reshape(n_win, _WINDOW)
energy = (frames ** 2).mean(axis=1)
if voiced is not None and len(voiced) == n_win and voiced.any():
sel = energy[voiced]
else:
sel = energy[energy >= np.median(energy)]
level_db = 10 * np.log10(sel.mean() + 1e-20)
gain_db = float(np.clip(TARGET_DB - level_db, MIN_GAIN_DB, MAX_GAIN_DB))
y = y * 10 ** (gain_db / 20)
over = np.abs(y) > _KNEE
room = _CEILING - _KNEE
y[over] = np.sign(y[over]) * (_KNEE + room * np.tanh((np.abs(y[over]) - _KNEE) / room))
return y.astype(np.float32)