Files
javis_bot/tests/test_voice_worker_audio.py
javis-bot 9f62bfb059 fix(voice): raise the consonant band by 12 dB before transcription
A dark microphone or a speaker who swallows first consonants and
sentence ends loses the band above 1 kHz first: in the live
recordings "자비스" came out as "아비스"/"다비스" and "서울" as "어우".
loudness_equalise now raises it with a one-octave shelf at 1 kHz.

Measured with large-v3-turbo: 25 sure live utterances 5.0% -> 3.4% CER,
FLEURS Korean test split (382 clips, held out) 5.16% -> 4.98%. The real
noise check stays at 108 clips with no text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 18:09:05 +09:00

187 lines
6.6 KiB
Python

"""Voice worker: binary frames, input conversion to 16 kHz mono, clip normalisation."""
from __future__ import annotations
import numpy as np
import pytest
from voice_worker.audio import HIGH_SHELF_DB, HIGH_SHELF_HZ, InputConverter, loudness_equalise
from voice_worker.protocol import (KIND_AUDIO, KIND_TTS, ProtocolError, decode_audio_frame, encode_audio_frame,
encode_tts_frame, decode_tts_frame)
def _pcm16(x: np.ndarray) -> bytes:
return (np.clip(x, -1, 1) * 32767).astype("<i2").tobytes()
def _dominant_hz(x: np.ndarray, rate: int) -> float:
spec = np.abs(np.fft.rfft(x * np.hanning(len(x))))
return float(np.fft.rfftfreq(len(x), 1 / rate)[int(np.argmax(spec))])
@pytest.mark.unit
def test_audio_frame_round_trip_keeps_speaker_and_pcm():
pcm = _pcm16(np.linspace(-0.5, 0.5, 960))
frame = encode_audio_frame("123456789012345678", pcm)
assert frame[0] == KIND_AUDIO
speaker, payload = decode_audio_frame(frame)
assert speaker == "123456789012345678"
assert payload == pcm
@pytest.mark.unit
def test_tts_frame_round_trip_keeps_id_and_sequence():
pcm = b"\x01\x00\x02\x00"
frame = encode_tts_frame("reply-7", 3, pcm)
assert frame[0] == KIND_TTS
assert decode_tts_frame(frame) == ("reply-7", 3, pcm)
@pytest.mark.unit
@pytest.mark.parametrize("frame", [b"", b"\x01", b"\x09\x01a\x00\x00", b"\x01\x05ab", b"\x01\x01a\x00",
b"\x01\x02\xff\xfe\x00\x00"])
def test_malformed_audio_frames_are_rejected(frame):
with pytest.raises(ProtocolError):
decode_audio_frame(frame)
@pytest.mark.unit
def test_48k_stereo_becomes_16k_mono_and_keeps_the_tone():
rate = 48000
t = np.arange(rate) / rate
tone = 0.5 * np.sin(2 * np.pi * 1000 * t)
stereo = np.stack([tone, tone], axis=1).reshape(-1)
out = InputConverter(rate, 2).push(_pcm16(stereo))
assert out.dtype == np.float32
assert abs(len(out) - rate // 3) <= 1
assert abs(_dominant_hz(out, 16000) - 1000) < 5
assert 0.4 < np.max(np.abs(out[500:])) < 0.6
@pytest.mark.unit
def test_tones_above_the_new_nyquist_are_filtered_out():
rate = 48000
t = np.arange(rate) / rate
high = 0.5 * np.sin(2 * np.pi * 11000 * t)
out = InputConverter(rate, 1).push(_pcm16(high))
# At least 30 dB down instead of folding back to 5 kHz.
assert np.sqrt(np.mean(out[500:] ** 2)) < 0.5 / np.sqrt(2) * 10 ** (-30 / 20)
@pytest.mark.unit
def test_streaming_in_20ms_frames_matches_one_shot_conversion():
rng = np.random.default_rng(1)
sig = rng.uniform(-0.5, 0.5, 48000 * 2 // 2 * 2)
one = InputConverter(48000, 2).push(_pcm16(sig))
conv = InputConverter(48000, 2)
raw = _pcm16(sig)
step = 960 * 2 * 2 # 20 ms of 48 kHz stereo PCM16
parts = [conv.push(raw[i:i + step]) for i in range(0, len(raw), step)]
assert np.allclose(np.concatenate(parts), one, atol=1e-6)
@pytest.mark.unit
def test_16k_mono_passes_through():
sig = np.linspace(-0.25, 0.25, 1600)
out = InputConverter(16000, 1).push(_pcm16(sig))
assert np.allclose(out, sig, atol=1e-4)
@pytest.mark.unit
def test_unsupported_input_formats_are_refused():
with pytest.raises(ValueError):
InputConverter(44100, 2)
with pytest.raises(ValueError):
InputConverter(48000, 6)
@pytest.mark.unit
def test_odd_byte_counts_are_carried_to_the_next_frame():
sig = _pcm16(np.full(10, 0.5))
conv = InputConverter(16000, 1)
a = conv.push(sig[:5])
b = conv.push(sig[5:])
assert len(a) + len(b) == 10
def _db(x: np.ndarray) -> float:
return 20 * np.log10(np.sqrt(np.mean(np.asarray(x, dtype=np.float64) ** 2)) + 1e-12)
def _voice(seconds: float, db: float) -> np.ndarray:
"""A stand-in for a voice: a 220 Hz tone with harmonics, at ``db`` dBFS RMS."""
t = np.arange(int(16000 * seconds)) / 16000
x = sum(np.sin(2 * np.pi * 220 * k * t) / k for k in (1, 2, 3, 4))
return (x * 10 ** (db / 20) / np.sqrt(np.mean(x ** 2))).astype(np.float32)
def _mask(n_samples: int, voiced: slice) -> np.ndarray:
m = np.zeros(-(-n_samples // 512), dtype=bool)
m[voiced] = True
return m
@pytest.mark.unit
@pytest.mark.parametrize("db", [-60, -45, -30, -10, -3])
def test_every_speaker_level_comes_out_at_the_same_loudness(db):
x = _voice(2.0, db)
y = loudness_equalise(x)
assert y.dtype == np.float32 and len(y) == len(x)
assert abs(_db(y) - (-20)) < 1.0
@pytest.mark.unit
def test_loudness_is_measured_on_the_voice_not_the_silence_around_it():
x = np.concatenate([np.zeros(16000, dtype=np.float32), _voice(1.0, -40)])
y = loudness_equalise(x, _mask(len(x), slice(32, None)))
assert abs(_db(y[16000:]) - (-20)) < 1.0
@pytest.mark.unit
def test_a_loud_click_neither_keeps_a_quiet_voice_quiet_nor_clips():
x = _voice(2.0, -50)
burst = np.random.default_rng(3).normal(0, 1, 200) * np.exp(-np.arange(200) / 30)
x[8000:8200] += (0.95 * burst / np.max(np.abs(burst))).astype(np.float32) # a mouse click, far louder
voiced = _mask(len(x), slice(None))
voiced[8000 // 512] = False
y = loudness_equalise(x, voiced)
assert abs(_db(y[:7000]) - (-20)) < 1.0
assert np.max(np.abs(y)) < 1.0
@pytest.mark.unit
def test_clipped_shouting_is_brought_down_without_new_clipping():
x = np.clip(_voice(2.0, -3) * 4, -1, 1)
y = loudness_equalise(x)
assert abs(_db(y) - (-20)) < 1.5
assert np.max(np.abs(y)) < 1.0
@pytest.mark.unit
def test_near_silence_is_not_boosted_past_the_gain_cap():
x = _voice(1.0, -90)
assert abs(_db(loudness_equalise(x)) - (-90 + 40)) < 1.0
@pytest.mark.unit
def test_low_rumble_is_removed_before_levelling():
t = np.arange(32000) / 16000
rumble = (0.3 * np.sin(2 * np.pi * 30 * t)).astype(np.float32)
y = loudness_equalise(_voice(2.0, -40) + rumble)
spec = np.abs(np.fft.rfft(y))
freqs = np.fft.rfftfreq(len(y), 1 / 16000)
assert spec[np.argmin(np.abs(freqs - 30))] < 0.01 * spec[np.argmin(np.abs(freqs - 220))]
@pytest.mark.unit
def test_consonant_band_is_raised_against_the_vowel_band():
"""A dark microphone (or a speaker who swallows consonants) loses the band above 1 kHz where
"ㅈ" and "ㅅ" live; it is raised by the shelf, the band below it is left as it was."""
t = np.arange(32000) / 16000
low, high = HIGH_SHELF_HZ / 4, HIGH_SHELF_HZ * 4
x = (0.05 * np.sin(2 * np.pi * low * t) + 0.05 * np.sin(2 * np.pi * high * t)).astype(np.float32)
spec = np.abs(np.fft.rfft(loudness_equalise(x)))
freqs = np.fft.rfftfreq(len(x), 1 / 16000)
ratio_db = 20 * np.log10(spec[np.argmin(np.abs(freqs - high))] / spec[np.argmin(np.abs(freqs - low))])
assert abs(ratio_db - HIGH_SHELF_DB) < 0.5