- Loudness equalisation replaces peak normalisation: rumble below 60 Hz removed, level measured on the VAD-voiced windows only, brought to -20 dBFS (gain -20..+40 dB), soft limiter below 0.98. - Only clearly voiced windows (VAD >= 0.5) count towards the 250 ms needed before an utterance reaches the model, so noise the VAD half-believes never calls Whisper. - Hallucination gate: mean token log-probability >= -0.7 (the old len/(len+1) correction let short hallucinations through), compression ratio <= 2.4; no_speech_prob dropped (always 0.0 on turbo). - With hotwords, the plain decode of the same encoder pass decides; the hint may fix a wake word's spelling but cannot add or drop one (it wrote the wake word on laughter and after real sentences). - Opt-in real-model evals: voice-like babble with the hint, speech over babble, quiet and clipped speakers, and a JARVIS_NOISE_DIR folder of real noise recordings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
89 lines
3.6 KiB
Python
89 lines
3.6 KiB
Python
"""Incoming PCM to 16 kHz mono float32, and clip normalisation before transcription."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
TARGET_RATE = 16000
|
|
_TAPS = 97 # windowed-sinc low-pass for the 48 kHz -> 16 kHz decimation
|
|
_CUTOFF_HZ = 7200 # below the new 8 kHz Nyquist
|
|
|
|
|
|
def _lowpass(rate: int) -> np.ndarray:
|
|
n = np.arange(_TAPS) - (_TAPS - 1) / 2
|
|
fc = _CUTOFF_HZ / rate
|
|
h = 2 * fc * np.sinc(2 * fc * n) * np.blackman(_TAPS)
|
|
return (h / h.sum()).astype(np.float64)
|
|
|
|
|
|
class InputConverter:
|
|
"""Streaming converter for one speaker. Keeps filter history and odd bytes between frames."""
|
|
|
|
def __init__(self, rate: int, channels: int):
|
|
if rate not in (16000, 48000):
|
|
raise ValueError("audio rate must be 16000 or 48000")
|
|
if channels not in (1, 2):
|
|
raise ValueError("audio channels must be 1 or 2")
|
|
self.channels = channels
|
|
self.factor = rate // TARGET_RATE
|
|
self._carry = b""
|
|
if self.factor > 1:
|
|
self._h = _lowpass(rate)
|
|
self._hist = np.zeros(_TAPS - 1, dtype=np.float64)
|
|
self._phase = 0 # input samples to skip before the next kept output
|
|
|
|
def push(self, pcm: bytes) -> np.ndarray:
|
|
data = self._carry + pcm
|
|
frame_bytes = 2 * self.channels
|
|
usable = len(data) - len(data) % frame_bytes
|
|
self._carry = data[usable:]
|
|
x = np.frombuffer(data[:usable], dtype="<i2").astype(np.float64) / 32768.0
|
|
if self.channels == 2:
|
|
x = x.reshape(-1, 2).mean(axis=1)
|
|
if self.factor == 1:
|
|
return x.astype(np.float32)
|
|
buf = np.concatenate([self._hist, x])
|
|
self._hist = buf[-(_TAPS - 1):]
|
|
y = np.convolve(buf, self._h, mode="valid") # one output per new input sample
|
|
out = y[self._phase::self.factor]
|
|
self._phase = (self._phase - len(y)) % self.factor
|
|
return out.astype(np.float32)
|
|
|
|
|
|
TARGET_DB = -20.0 # loudness every speaker is brought to, dBFS RMS over the voiced windows
|
|
MAX_GAIN_DB = 40.0 # a whisper at -60 dBFS still reaches the target; pure hiss is not blown up
|
|
MIN_GAIN_DB = -20.0
|
|
HIGHPASS_HZ = 60.0 # mains hum and desk rumble carry no speech
|
|
_KNEE = 0.8 # soft limiter: peaks above this bend smoothly towards the ceiling instead of clipping
|
|
_CEILING = 0.98
|
|
_WINDOW = 512
|
|
|
|
|
|
def loudness_equalise(x: np.ndarray, voiced: np.ndarray | None = None) -> np.ndarray:
|
|
"""Bring a clip to the same loudness whoever spoke it, without clipping.
|
|
|
|
``voiced`` marks the 512-sample windows the VAD heard as speech; the level is measured on them so
|
|
silence, a click or a knock does not decide the gain. Without it the louder half of the windows is
|
|
used. Low rumble is removed first and peaks are soft-limited after the gain.
|
|
"""
|
|
x = np.asarray(x, dtype=np.float32)
|
|
if len(x) == 0:
|
|
return x
|
|
spec = np.fft.rfft(x.astype(np.float64))
|
|
spec[np.fft.rfftfreq(len(x), 1 / TARGET_RATE) < HIGHPASS_HZ] = 0
|
|
y = np.fft.irfft(spec, len(x))
|
|
n_win = -(-len(y) // _WINDOW)
|
|
frames = np.pad(y, (0, n_win * _WINDOW - len(y))).reshape(n_win, _WINDOW)
|
|
energy = (frames ** 2).mean(axis=1)
|
|
if voiced is not None and len(voiced) == n_win and voiced.any():
|
|
sel = energy[voiced]
|
|
else:
|
|
sel = energy[energy >= np.median(energy)]
|
|
level_db = 10 * np.log10(sel.mean() + 1e-20)
|
|
gain_db = float(np.clip(TARGET_DB - level_db, MIN_GAIN_DB, MAX_GAIN_DB))
|
|
y = y * 10 ** (gain_db / 20)
|
|
over = np.abs(y) > _KNEE
|
|
room = _CEILING - _KNEE
|
|
y[over] = np.sign(y[over]) * (_KNEE + room * np.tanh((np.abs(y[over]) - _KNEE) / room))
|
|
return y.astype(np.float32)
|