Harsh copies of the user's speech (weak voice, noise at SNR 10/5 dB, headset band, Opus 12 kbps, reverb) with the worker's own VAD for noise: lowering the gate itself let laughter through (VAD-cut noise 0/5 -> 1/5 at -0.8), and three noise suppressors (DeepFilterNet3, RNNoise, noisereduce) made every condition worse, so neither is used. Rescuing a plain decode at -0.8..-0.7 with the prompted text when that passed the gate and stays close: headset 13.8% -> 7.5%, no more noise with text. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
164 lines
6.8 KiB
Python
164 lines
6.8 KiB
Python
"""Voice worker: which Whisper transcripts are kept (the hallucination gate), model-free."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from voice_worker.stt_whisper import MIN_MEAN_LOGPROB, choose_with_hint, compression_ratio, keep_transcript
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_confident_transcript_is_kept():
|
|
assert keep_transcript("자비스, 오늘 서울 날씨 어때?", -0.05)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_low_confidence_is_rejected_even_for_a_short_output():
|
|
# A three-token hallucination on laughter scored a mean of -0.73 per token; the length
|
|
# correction used before (len / (len + 1)) lifted it to -0.55 and let it through.
|
|
assert not keep_transcript("자비스", MIN_MEAN_LOGPROB - 0.03)
|
|
assert keep_transcript("자비스", MIN_MEAN_LOGPROB + 0.03)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_empty_and_punctuation_only_outputs_are_rejected():
|
|
assert not keep_transcript("", -0.01)
|
|
assert not keep_transcript(" . . ", -0.01)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_repetition_loop_is_rejected():
|
|
loop = "자막 제공 및 " * 12
|
|
assert compression_ratio(loop) > 2.4
|
|
assert not keep_transcript(loop, -0.1)
|
|
assert compression_ratio("오늘 회의는 오후 세 시에 시작해서 다섯 시쯤 끝날 것 같아요.") < 2.4
|
|
|
|
|
|
HINT = ("자비스",)
|
|
ASK = "지금 두 번째 모니터에 뭐 떠 있는지 알려줄래?"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_voice_like_noise_that_the_hint_turns_into_the_wake_word_gives_nothing():
|
|
# Laughter, decoded with the wake word as a hint and without it.
|
|
assert choose_with_hint("자비스", -0.4, "감사합니다.", -1.4, HINT) == ""
|
|
assert choose_with_hint("자비스", -0.4, "", -0.1, HINT) == ""
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_wake_word_the_hint_adds_to_real_speech_is_dropped():
|
|
assert choose_with_hint(ASK + " 자비스", -0.1, ASK, -0.1, HINT) == ASK
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_one_letter_wake_word_the_hint_adds_is_dropped():
|
|
assert choose_with_hint("봇 오늘 날씨 알려 줘", -0.1, "오늘 날씨 알려 줘", -0.1, ("봇",)) == "오늘 날씨 알려 줘"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_hint_may_not_turn_a_similar_real_word_into_the_wake_word():
|
|
assert choose_with_hint("자비스 신청해 줘", -0.1, "서비스 신청해 줘", -0.1, HINT) == "서비스 신청해 줘"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_near_miss_spelling_is_left_as_heard_rather_than_guessed():
|
|
# "자비수" may be the wake word or another word; text alone cannot tell, so the plain decode stands
|
|
# and fuzzy wake word matching is left to the call detection that reads it.
|
|
plain = "자비수, 오늘 서울 날씨 어때?"
|
|
assert choose_with_hint("자비스, 오늘 서울 날씨 어때?", -0.1, plain, -0.2, HINT) == plain
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_hint_is_used_when_it_keeps_the_wake_words_as_they_were():
|
|
hinted = "자비스, 철수 화면 공유 보여 줘"
|
|
assert choose_with_hint(hinted, -0.1, "자비스 철수 화면공유 보여줘", -0.15, HINT) == hinted
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_wake_word_the_hint_loses_is_kept_from_the_plain_decode():
|
|
plain = "자비스, 오늘 서울 날씨 어때?"
|
|
assert choose_with_hint("오늘 서울 날씨 어때?", -0.2, plain, -0.07, HINT) == plain
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_unsure_hint_decode_falls_back_to_the_plain_one():
|
|
assert choose_with_hint(ASK, MIN_MEAN_LOGPROB - 0.2, ASK, -0.1, HINT) == ASK
|
|
assert choose_with_hint("zzz", -0.1, ASK, -0.1, HINT) == ASK
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_wake_word_matching_ignores_case_spacing_and_punctuation():
|
|
assert choose_with_hint("Hey Jarvis, lights off.", -0.1, "hey jarvis lights off", -0.1, ("jarvis",)) \
|
|
== "Hey Jarvis, lights off."
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
|
|
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
|
|
== "자비스 오늘 서울 날씨 어때"
|
|
|
|
|
|
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
|
|
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
|
|
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
|
|
from voice_worker.stt_whisper import choose_with_context # noqa: E402
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_context_may_fix_words_of_what_was_said():
|
|
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_nothing_heard_without_the_context_stays_nothing():
|
|
# Noise: the prompted decode wrote words of the conversation.
|
|
assert choose_with_context("", "화면에 디스코드", HINT) == ""
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
|
|
heard = "가상화면을 못 봐"
|
|
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
|
|
assert choose_with_context(heard, copied, HINT) == heard
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_context_never_changes_how_many_wake_words_were_heard():
|
|
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
|
|
== "라비스 지금 영상 재생되고 있어?"
|
|
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_empty_prompted_decode_keeps_what_was_heard():
|
|
assert choose_with_context("네.", "", HINT) == "네."
|
|
|
|
|
|
# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for
|
|
# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes.
|
|
from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one():
|
|
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \
|
|
== "디코 어디 갔어?"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_unsure_decode_well_below_the_gate_is_not_rescued():
|
|
assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == ""
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_noise_with_a_prompt_far_from_it_is_not_rescued():
|
|
# The prompted decode wrote words of the conversation over laughter.
|
|
assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
|
assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == ""
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_rescue_never_adds_a_wake_word():
|
|
assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT,
|
|
("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""
|