- Loudness equalisation replaces peak normalisation: rumble below 60 Hz removed, level measured on the VAD-voiced windows only, brought to -20 dBFS (gain -20..+40 dB), soft limiter below 0.98. - Only clearly voiced windows (VAD >= 0.5) count towards the 250 ms needed before an utterance reaches the model, so noise the VAD half-believes never calls Whisper. - Hallucination gate: mean token log-probability >= -0.7 (the old len/(len+1) correction let short hallucinations through), compression ratio <= 2.4; no_speech_prob dropped (always 0.0 on turbo). - With hotwords, the plain decode of the same encoder pass decides; the hint may fix a wake word's spelling but cannot add or drop one (it wrote the wake word on laughter and after real sentences). - Opt-in real-model evals: voice-like babble with the hint, speech over babble, quiet and clipped speakers, and a JARVIS_NOISE_DIR folder of real noise recordings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
81 lines
3.0 KiB
Python
81 lines
3.0 KiB
Python
"""Voice worker: which Whisper transcripts are kept (the hallucination gate), model-free."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from voice_worker.stt_whisper import MIN_MEAN_LOGPROB, choose_with_hint, compression_ratio, keep_transcript
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_confident_transcript_is_kept():
|
|
assert keep_transcript("자비스, 오늘 서울 날씨 어때?", -0.05)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_low_confidence_is_rejected_even_for_a_short_output():
|
|
# A three-token hallucination on laughter scored a mean of -0.73 per token; the length
|
|
# correction used before (len / (len + 1)) lifted it to -0.55 and let it through.
|
|
assert not keep_transcript("자비스", MIN_MEAN_LOGPROB - 0.03)
|
|
assert keep_transcript("자비스", MIN_MEAN_LOGPROB + 0.03)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_empty_and_punctuation_only_outputs_are_rejected():
|
|
assert not keep_transcript("", -0.01)
|
|
assert not keep_transcript(" . . ", -0.01)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_repetition_loop_is_rejected():
|
|
loop = "자막 제공 및 " * 12
|
|
assert compression_ratio(loop) > 2.4
|
|
assert not keep_transcript(loop, -0.1)
|
|
assert compression_ratio("오늘 회의는 오후 세 시에 시작해서 다섯 시쯤 끝날 것 같아요.") < 2.4
|
|
|
|
|
|
HINT = ("자비스",)
|
|
ASK = "지금 두 번째 모니터에 뭐 떠 있는지 알려줄래?"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_voice_like_noise_that_the_hint_turns_into_the_wake_word_gives_nothing():
|
|
# Laughter, decoded with the wake word as a hint and without it.
|
|
assert choose_with_hint("자비스", -0.4, "감사합니다.", -1.4, HINT) == ""
|
|
assert choose_with_hint("자비스", -0.4, "", -0.1, HINT) == ""
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_wake_word_the_hint_adds_to_real_speech_is_dropped():
|
|
assert choose_with_hint(ASK + " 자비스", -0.1, ASK, -0.1, HINT) == ASK
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_hint_may_fix_how_a_spoken_wake_word_is_spelt():
|
|
fixed = "자비스, 오늘 서울 날씨 어때?"
|
|
assert choose_with_hint(fixed, -0.1, "자비수, 오늘 서울 날씨 어때?", -0.2, HINT) == fixed
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_wake_word_the_hint_loses_is_kept_from_the_plain_decode():
|
|
plain = "자비스, 오늘 서울 날씨 어때?"
|
|
assert choose_with_hint("오늘 서울 날씨 어때?", -0.2, plain, -0.07, HINT) == plain
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_unsure_hint_decode_falls_back_to_the_plain_one():
|
|
assert choose_with_hint(ASK, MIN_MEAN_LOGPROB - 0.2, ASK, -0.1, HINT) == ASK
|
|
assert choose_with_hint("zzz", -0.1, ASK, -0.1, HINT) == ASK
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
|
assert choose_with_hint("Hey Jarvis, lights off.", -0.1, "hey jarvis lights off", -0.1, ("jarvis",)) \
|
|
== "Hey Jarvis, lights off."
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
|
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
|
== "자비스 오늘 서울 날씨 어때"
|