Hotword occurrences were counted with fuzzy matching, which never counted a one-letter wake word (so a hinted "봇 ..." was accepted over a plain decode without it) and counted the real word "서비스" as "자비스" (so the hint could turn it into the wake word). Occurrences are now counted exactly (case, spacing and punctuation ignored); near misses are left as heard for call detection. The real-noise eval now plays each recording as recorded and at -25 dBFS and reports how many clips reached the model. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
99 lines
3.9 KiB
Python
99 lines
3.9 KiB
Python
"""Voice worker: which Whisper transcripts are kept (the hallucination gate), model-free."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from voice_worker.stt_whisper import MIN_MEAN_LOGPROB, choose_with_hint, compression_ratio, keep_transcript
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_confident_transcript_is_kept():
|
|
assert keep_transcript("자비스, 오늘 서울 날씨 어때?", -0.05)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_low_confidence_is_rejected_even_for_a_short_output():
|
|
# A three-token hallucination on laughter scored a mean of -0.73 per token; the length
|
|
# correction used before (len / (len + 1)) lifted it to -0.55 and let it through.
|
|
assert not keep_transcript("자비스", MIN_MEAN_LOGPROB - 0.03)
|
|
assert keep_transcript("자비스", MIN_MEAN_LOGPROB + 0.03)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_empty_and_punctuation_only_outputs_are_rejected():
|
|
assert not keep_transcript("", -0.01)
|
|
assert not keep_transcript(" . . ", -0.01)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_repetition_loop_is_rejected():
|
|
loop = "자막 제공 및 " * 12
|
|
assert compression_ratio(loop) > 2.4
|
|
assert not keep_transcript(loop, -0.1)
|
|
assert compression_ratio("오늘 회의는 오후 세 시에 시작해서 다섯 시쯤 끝날 것 같아요.") < 2.4
|
|
|
|
|
|
HINT = ("자비스",)
|
|
ASK = "지금 두 번째 모니터에 뭐 떠 있는지 알려줄래?"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_voice_like_noise_that_the_hint_turns_into_the_wake_word_gives_nothing():
|
|
# Laughter, decoded with the wake word as a hint and without it.
|
|
assert choose_with_hint("자비스", -0.4, "감사합니다.", -1.4, HINT) == ""
|
|
assert choose_with_hint("자비스", -0.4, "", -0.1, HINT) == ""
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_wake_word_the_hint_adds_to_real_speech_is_dropped():
|
|
assert choose_with_hint(ASK + " 자비스", -0.1, ASK, -0.1, HINT) == ASK
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_one_letter_wake_word_the_hint_adds_is_dropped():
|
|
assert choose_with_hint("봇 오늘 날씨 알려 줘", -0.1, "오늘 날씨 알려 줘", -0.1, ("봇",)) == "오늘 날씨 알려 줘"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_hint_may_not_turn_a_similar_real_word_into_the_wake_word():
|
|
assert choose_with_hint("자비스 신청해 줘", -0.1, "서비스 신청해 줘", -0.1, HINT) == "서비스 신청해 줘"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_near_miss_spelling_is_left_as_heard_rather_than_guessed():
|
|
# "자비수" may be the wake word or another word; text alone cannot tell, so the plain decode stands
|
|
# and fuzzy wake word matching is left to the call detection that reads it.
|
|
plain = "자비수, 오늘 서울 날씨 어때?"
|
|
assert choose_with_hint("자비스, 오늘 서울 날씨 어때?", -0.1, plain, -0.2, HINT) == plain
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_hint_is_used_when_it_keeps_the_wake_words_as_they_were():
|
|
hinted = "자비스, 철수 화면 공유 보여 줘"
|
|
assert choose_with_hint(hinted, -0.1, "자비스 철수 화면공유 보여줘", -0.15, HINT) == hinted
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_wake_word_the_hint_loses_is_kept_from_the_plain_decode():
|
|
plain = "자비스, 오늘 서울 날씨 어때?"
|
|
assert choose_with_hint("오늘 서울 날씨 어때?", -0.2, plain, -0.07, HINT) == plain
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_unsure_hint_decode_falls_back_to_the_plain_one():
|
|
assert choose_with_hint(ASK, MIN_MEAN_LOGPROB - 0.2, ASK, -0.1, HINT) == ASK
|
|
assert choose_with_hint("zzz", -0.1, ASK, -0.1, HINT) == ASK
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
|
assert choose_with_hint("Hey Jarvis, lights off.", -0.1, "hey jarvis lights off", -0.1, ("jarvis",)) \
|
|
== "Hey Jarvis, lights off."
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
|
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
|
== "자비스 오늘 서울 날씨 어때"
|