- Loudness equalisation replaces peak normalisation: rumble below 60 Hz removed, level measured on the VAD-voiced windows only, brought to -20 dBFS (gain -20..+40 dB), soft limiter below 0.98. - Only clearly voiced windows (VAD >= 0.5) count towards the 250 ms needed before an utterance reaches the model, so noise the VAD half-believes never calls Whisper. - Hallucination gate: mean token log-probability >= -0.7 (the old len/(len+1) correction let short hallucinations through), compression ratio <= 2.4; no_speech_prob dropped (always 0.0 on turbo). - With hotwords, the plain decode of the same encoder pass decides; the hint may fix a wake word's spelling but cannot add or drop one (it wrote the wake word on laughter and after real sentences). - Opt-in real-model evals: voice-like babble with the hint, speech over babble, quiet and clipped speakers, and a JARVIS_NOISE_DIR folder of real noise recordings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
174 lines
5.8 KiB
Python
174 lines
5.8 KiB
Python
"""Voice worker: binary frames, input conversion to 16 kHz mono, clip normalisation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from voice_worker.audio import InputConverter, loudness_equalise
|
|
from voice_worker.protocol import (KIND_AUDIO, KIND_TTS, ProtocolError, decode_audio_frame, encode_audio_frame,
|
|
encode_tts_frame, decode_tts_frame)
|
|
|
|
|
|
def _pcm16(x: np.ndarray) -> bytes:
|
|
return (np.clip(x, -1, 1) * 32767).astype("<i2").tobytes()
|
|
|
|
|
|
def _dominant_hz(x: np.ndarray, rate: int) -> float:
|
|
spec = np.abs(np.fft.rfft(x * np.hanning(len(x))))
|
|
return float(np.fft.rfftfreq(len(x), 1 / rate)[int(np.argmax(spec))])
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_audio_frame_round_trip_keeps_speaker_and_pcm():
|
|
pcm = _pcm16(np.linspace(-0.5, 0.5, 960))
|
|
frame = encode_audio_frame("123456789012345678", pcm)
|
|
assert frame[0] == KIND_AUDIO
|
|
speaker, payload = decode_audio_frame(frame)
|
|
assert speaker == "123456789012345678"
|
|
assert payload == pcm
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_tts_frame_round_trip_keeps_id_and_sequence():
|
|
pcm = b"\x01\x00\x02\x00"
|
|
frame = encode_tts_frame("reply-7", 3, pcm)
|
|
assert frame[0] == KIND_TTS
|
|
assert decode_tts_frame(frame) == ("reply-7", 3, pcm)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("frame", [b"", b"\x01", b"\x09\x01a\x00\x00", b"\x01\x05ab", b"\x01\x01a\x00",
|
|
b"\x01\x02\xff\xfe\x00\x00"])
|
|
def test_malformed_audio_frames_are_rejected(frame):
|
|
with pytest.raises(ProtocolError):
|
|
decode_audio_frame(frame)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_48k_stereo_becomes_16k_mono_and_keeps_the_tone():
|
|
rate = 48000
|
|
t = np.arange(rate) / rate
|
|
tone = 0.5 * np.sin(2 * np.pi * 1000 * t)
|
|
stereo = np.stack([tone, tone], axis=1).reshape(-1)
|
|
out = InputConverter(rate, 2).push(_pcm16(stereo))
|
|
assert out.dtype == np.float32
|
|
assert abs(len(out) - rate // 3) <= 1
|
|
assert abs(_dominant_hz(out, 16000) - 1000) < 5
|
|
assert 0.4 < np.max(np.abs(out[500:])) < 0.6
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_tones_above_the_new_nyquist_are_filtered_out():
|
|
rate = 48000
|
|
t = np.arange(rate) / rate
|
|
high = 0.5 * np.sin(2 * np.pi * 11000 * t)
|
|
out = InputConverter(rate, 1).push(_pcm16(high))
|
|
# At least 30 dB down instead of folding back to 5 kHz.
|
|
assert np.sqrt(np.mean(out[500:] ** 2)) < 0.5 / np.sqrt(2) * 10 ** (-30 / 20)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_streaming_in_20ms_frames_matches_one_shot_conversion():
|
|
rng = np.random.default_rng(1)
|
|
sig = rng.uniform(-0.5, 0.5, 48000 * 2 // 2 * 2)
|
|
one = InputConverter(48000, 2).push(_pcm16(sig))
|
|
conv = InputConverter(48000, 2)
|
|
raw = _pcm16(sig)
|
|
step = 960 * 2 * 2 # 20 ms of 48 kHz stereo PCM16
|
|
parts = [conv.push(raw[i:i + step]) for i in range(0, len(raw), step)]
|
|
assert np.allclose(np.concatenate(parts), one, atol=1e-6)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_16k_mono_passes_through():
|
|
sig = np.linspace(-0.25, 0.25, 1600)
|
|
out = InputConverter(16000, 1).push(_pcm16(sig))
|
|
assert np.allclose(out, sig, atol=1e-4)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_unsupported_input_formats_are_refused():
|
|
with pytest.raises(ValueError):
|
|
InputConverter(44100, 2)
|
|
with pytest.raises(ValueError):
|
|
InputConverter(48000, 6)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_odd_byte_counts_are_carried_to_the_next_frame():
|
|
sig = _pcm16(np.full(10, 0.5))
|
|
conv = InputConverter(16000, 1)
|
|
a = conv.push(sig[:5])
|
|
b = conv.push(sig[5:])
|
|
assert len(a) + len(b) == 10
|
|
|
|
|
|
def _db(x: np.ndarray) -> float:
|
|
return 20 * np.log10(np.sqrt(np.mean(np.asarray(x, dtype=np.float64) ** 2)) + 1e-12)
|
|
|
|
|
|
def _voice(seconds: float, db: float) -> np.ndarray:
|
|
"""A stand-in for a voice: a 220 Hz tone with harmonics, at ``db`` dBFS RMS."""
|
|
t = np.arange(int(16000 * seconds)) / 16000
|
|
x = sum(np.sin(2 * np.pi * 220 * k * t) / k for k in (1, 2, 3, 4))
|
|
return (x * 10 ** (db / 20) / np.sqrt(np.mean(x ** 2))).astype(np.float32)
|
|
|
|
|
|
def _mask(n_samples: int, voiced: slice) -> np.ndarray:
|
|
m = np.zeros(-(-n_samples // 512), dtype=bool)
|
|
m[voiced] = True
|
|
return m
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("db", [-60, -45, -30, -10, -3])
|
|
def test_every_speaker_level_comes_out_at_the_same_loudness(db):
|
|
x = _voice(2.0, db)
|
|
y = loudness_equalise(x)
|
|
assert y.dtype == np.float32 and len(y) == len(x)
|
|
assert abs(_db(y) - (-20)) < 1.0
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_loudness_is_measured_on_the_voice_not_the_silence_around_it():
|
|
x = np.concatenate([np.zeros(16000, dtype=np.float32), _voice(1.0, -40)])
|
|
y = loudness_equalise(x, _mask(len(x), slice(32, None)))
|
|
assert abs(_db(y[16000:]) - (-20)) < 1.0
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_loud_click_neither_keeps_a_quiet_voice_quiet_nor_clips():
|
|
x = _voice(2.0, -50)
|
|
burst = np.random.default_rng(3).normal(0, 1, 200) * np.exp(-np.arange(200) / 30)
|
|
x[8000:8200] += (0.95 * burst / np.max(np.abs(burst))).astype(np.float32) # a mouse click, far louder
|
|
voiced = _mask(len(x), slice(None))
|
|
voiced[8000 // 512] = False
|
|
y = loudness_equalise(x, voiced)
|
|
assert abs(_db(y[:7000]) - (-20)) < 1.0
|
|
assert np.max(np.abs(y)) < 1.0
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_clipped_shouting_is_brought_down_without_new_clipping():
|
|
x = np.clip(_voice(2.0, -3) * 4, -1, 1)
|
|
y = loudness_equalise(x)
|
|
assert abs(_db(y) - (-20)) < 1.5
|
|
assert np.max(np.abs(y)) < 1.0
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_near_silence_is_not_boosted_past_the_gain_cap():
|
|
x = _voice(1.0, -90)
|
|
assert abs(_db(loudness_equalise(x)) - (-90 + 40)) < 1.0
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_low_rumble_is_removed_before_levelling():
|
|
t = np.arange(32000) / 16000
|
|
rumble = (0.3 * np.sin(2 * np.pi * 30 * t)).astype(np.float32)
|
|
y = loudness_equalise(_voice(2.0, -40) + rumble)
|
|
spec = np.abs(np.fft.rfft(y))
|
|
freqs = np.fft.rfftfreq(len(y), 1 / 16000)
|
|
assert spec[np.argmin(np.abs(freqs - 30))] < 0.01 * spec[np.argmin(np.abs(freqs - 220))]
|