From b30aa4a0db4a416bd8ce80a52a6bcfd15ba59c99 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 19 Nov 2026 21:02:05 +0900 Subject: [PATCH] feat(voice_worker): a plain decode just short of the gate is rescued by a confident close prompted one Harsh copies of the user's speech (weak voice, noise at SNR 10/5 dB, headset band, Opus 12 kbps, reverb) with the worker's own VAD for noise: lowering the gate itself let laughter through (VAD-cut noise 0/5 -> 1/5 at -0.8), and three noise suppressors (DeepFilterNet3, RNNoise, noisereduce) made every condition worse, so neither is used. Rescuing a plain decode at -0.8..-0.7 with the prompted text when that passed the gate and stays close: headset 13.8% -> 7.5%, no more noise with text. Co-Authored-By: Claude Opus 5.5 --- docs/v2/stt-accuracy-plan.md | 5 ++++ src/voice_worker/stt_whisper.py | 41 ++++++++++++++++++--------- src/voice_worker/voice_worker.spec.md | 6 ++++ tests/test_voice_worker_gate.py | 29 +++++++++++++++++++ 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docs/v2/stt-accuracy-plan.md b/docs/v2/stt-accuracy-plan.md index 4d2d39d..31a45a3 100644 --- a/docs/v2/stt-accuracy-plan.md +++ b/docs/v2/stt-accuracy-plan.md @@ -96,3 +96,8 @@ - 빈 결과의 원인은 확신도 기준(-0.7). -0.85/-1.0으로 낮추면 헤드셋 7.5%, 잡음 10dB 15.5%, 5dB 30.2%, 빈 결과 원본 17→15/10개. 대신 잡음 108개(VAD 없이 바로 넣음)에서 글자가 나온 것이 16→27/32개. 실제 워커는 VAD가 먼저 거르므로(M1: 108개 중 5개만 모델까지 감) 기준 변경은 VAD를 포함한 잡음 시험으로 다시 재고 사용자가 정할 것. 이번에는 바꾸지 않음. - 작은 목소리+마이크 잡음(약 SNR 5dB)은 어떤 설정으로도 50% 안팎이라, 소리 앞단의 잡음 제거(모델) 같은 다른 방향이 필요함. 아직 미검증. - 근거: groups/javis_bot/evidence/2026-11-19/stt-plan/harsh/ (build.py 조건 정의, harsh.py, gate.py, 결과 JSON). +- 2026-11-19 밤 (0.1.50): 악조건 두 갈래를 같은 자료(녹음 100개 × 7조건, 정답 16개)로 비교. 근거: evidence/2026-11-19/stt-plan/harsh2/. + - 앞단 잡음 제거 3종(DeepFilterNet3, RNNoise, noisereduce)은 전부 나빠서 넣지 않음. 원본 7.0% → 13.8/9.6/11.0%, 싼 헤드셋 13.8% → 21.8/11.4/19.5%, 잡음 10dB 18.9% → 44.9/48.3/31.6%. DeepFilterNet은 작은 목소리에서 없는 말을 길게 지어냈고(108.6%), VAD를 지난 잡음 5개를 모두 글자로 만듦. 디스코드가 이미 보내는 쪽에서 잡음 제거(Krisp)를 하고, Whisper는 잡음 섞인 소리로 학습되어 제거 흔적이 더 해로움. + - 확신도 기준: 실제 워커처럼 VAD와 분할기로 자른 잡음만 넣으면 모델까지 가는 것은 108개 중 5개(웃음). 기준을 -0.8로만 낮춰도 그중 1개가 "감사합니다."가 되고, VAD 없이 넣은 108개도 16→20개. 그래서 기준 자체는 -0.7 그대로. + - 대신 살리기: 문맥 없는 결과가 -0.7에 조금 못 미쳐도(-0.8 이상) 문맥을 준 결과가 기준을 넘고 가까우면 그것을 씀. 실제 코드로 다시 잼(verify.py): 원본 7.0%, 헤드셋 13.8→7.5%, 빈 결과 대부분 1~2개 줄어듦, 잡음 0/5와 16/108 그대로, 최종 전사 평균 178~180ms(p95 246~254ms), 10명 한 묶음 1.1초. + - 남은 문제(수치 그대로): 작은 목소리+마이크 잡음 51.4%(빈 결과 48/100), 잡음 5dB 33.9%(34/100), 잡음 10dB 18.9%(23/100). 이번에 본 방법(모델 7종, 문맥, 기준, 잡음 제거 3종)으로는 나아지지 않음. 다음 후보는 이런 조건의 실제 녹음으로 미세조정한 모델이나, 보내는 사람 쪽 디스코드 설정(입력 감도, 잡음 제거) 안내. diff --git a/src/voice_worker/stt_whisper.py b/src/voice_worker/stt_whisper.py index 7df879e..2e5052f 100644 --- a/src/voice_worker/stt_whisper.py +++ b/src/voice_worker/stt_whisper.py @@ -19,9 +19,13 @@ SR = 16000 # over it: every hallucination stayed below, real speech above except two short words in loud noise. # Whisper's no_speech_prob is not used: large-v3-turbo reports 0.0 for silence and noise alike. MIN_MEAN_LOGPROB = -0.7 -MAX_COMPRESSION_RATIO = 2.4 +MAX_COMPRESSION_RATIO = 2.4 # repetition loops ("A B A B A B ...") compress far better than speech # faster-whisper keeps at most this many previous-text tokens in a prompt (max_length // 2 - 1). -MAX_PREVIOUS_TOKENS = 223 # repetition loops ("A B A B A B ...") compress far better than speech +MAX_PREVIOUS_TOKENS = 223 +# A plain decode at or above this, though below MIN_MEAN_LOGPROB, may be rescued by the context +# (choose_with_context). Measured 2026-11-19 on harsh copies of the user's speech: down to -0.8 no +# more noise came out as text (VAD-cut noise 0/5, raw noise 16/108 as without); -0.9 let 3 more through. +UNSURE_LOGPROB = -0.8 def _letters(text: str) -> str: @@ -75,16 +79,24 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp: return plain_text -def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str: +def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...], + plain: tuple[str, float] | None = None) -> str: """Pick between the decode without the conversation context (``heard``, already through the hint - choice) and the one that had it as Whisper's previous-text prompt (``prompted``). + choice) and the one that had it as Whisper's previous-text prompt (``prompted``, through the same + choice, so it passed the gate itself). The context fixes words of the topic the model misheard, but on noise or unsure audio it can write - words of the prompt, so the decode without it decides, as for the hint: nothing heard stays - nothing, and the prompted text is used only when it stays close to what was heard and holds every - hotword exactly as often. + words of the prompt, so the decode without it decides, as for the hint: the prompted text is used + only when it stays close to what was heard and holds every hotword exactly as often. When nothing + was heard, ``plain`` (the plain decode's text and mean log-probability) may still be just short of + the gate (at least UNSURE_LOGPROB): then the prompted text is used if it is that close to it. """ - if not heard or not prompted: + if not heard: + if not prompted or plain is None or plain[1] < UNSURE_LOGPROB \ + or compression_ratio(plain[0]) > MAX_COMPRESSION_RATIO: + return "" + return prompted if choose_with_context(plain[0], prompted, hotwords) == prompted else "" + if not prompted: return heard h, p = _letters(heard), _letters(prompted) if _edit_distance(p, h) > len(h) // 2: @@ -130,13 +142,14 @@ class WhisperTranscriber: """Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent conversation to prompt with ("" for none); see ``choose_with_context``.""" enc, langs = self._encode(clips, opts) - heard = self._choose(enc, langs, opts, None) + plain = self._decode(enc, langs, opts.beam, None) + heard = self._choose(enc, langs, opts, None, plain) prev = self._previous(langs, contexts) if prev is None: return heard - prompted = self._choose(enc, langs, opts, prev) - return [choose_with_context(h, p, opts.hotwords) if c else h - for h, p, c in zip(heard, prompted, contexts)] + prompted = self._choose(enc, langs, opts, prev, self._decode(enc, langs, opts.beam, None, prev)) + return [choose_with_context(h, p, opts.hotwords, pl) if c else h + for h, p, pl, c in zip(heard, prompted, plain, contexts)] def _previous(self, langs: list[str], contexts: list[str] | None) -> list[list[int]] | None: """The contexts as previous-text tokens of one length for the whole batch: the decoder needs @@ -166,8 +179,8 @@ class WhisperTranscriber: langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)] return enc, langs - def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None) -> list[str]: - plain = self._decode(enc, langs, opts.beam, None, prev) + def _choose(self, enc, langs: list[str], opts: DecodeOptions, prev: list[list[int]] | None, + plain: list[tuple[str, float]]) -> list[str]: if not opts.hotwords: return [text if keep_transcript(text, lp) else "" for text, lp in plain] hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), prev) diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index 1c089f9..894d785 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -171,6 +171,12 @@ decoded with the recent conversation as its previous-text prompt: was kept, it stays within half that text's letters of it, and it holds every hotword exactly as often; otherwise the text without context is used. On noise or unsure audio the prompt makes Whisper write words of the conversation; this rule keeps that out, as for the hint. +- One rescue: when the decode without context was dropped by the gate but its mean log-probability + is at least -0.8 (just short of -0.7), the prompted text (which passed the gate itself) is used if + it is that close to it and keeps every hotword count. The unsure text itself is never used. + Measured on harsh copies of the user's speech: a cheap headset 13.8% to 7.5% error, empty results + fewer by 1-2 per 100 in most conditions, and no more noise with text (VAD-cut noise 0 of 5, noise + decoded straight 16 of 108, as before); -0.9 let 3 more noise clips through. - Finals stay batched. The decoder needs the start token at the same position in every prompt of a batch, so each clip's context is cut from its end to the shortest one in the batch (a channel's clips share most of it); a clip without a context borrows another's and keeps its plain result. diff --git a/tests/test_voice_worker_gate.py b/tests/test_voice_worker_gate.py index 7419be1..27833fb 100644 --- a/tests/test_voice_worker_gate.py +++ b/tests/test_voice_worker_gate.py @@ -132,3 +132,32 @@ def test_the_context_never_changes_how_many_wake_words_were_heard(): @pytest.mark.unit def test_an_empty_prompted_decode_keeps_what_was_heard(): assert choose_with_context("네.", "", HINT) == "네." + + +# A plain decode just short of the gate (live 2026-11-19 on a headset copy: "니코 어디갔어" at -0.72 for +# "디코 어디 갔어") is rescued by a confident prompted decode close to it; nothing else changes. +from voice_worker.stt_whisper import UNSURE_LOGPROB # noqa: E402 + + +@pytest.mark.unit +def test_a_nearly_sure_decode_is_rescued_by_a_confident_close_prompted_one(): + assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", MIN_MEAN_LOGPROB - 0.02)) \ + == "디코 어디 갔어?" + + +@pytest.mark.unit +def test_an_unsure_decode_well_below_the_gate_is_not_rescued(): + assert choose_with_context("", "디코 어디 갔어?", HINT, ("니코 어디갔어", UNSURE_LOGPROB - 0.05)) == "" + + +@pytest.mark.unit +def test_noise_with_a_prompt_far_from_it_is_not_rescued(): + # The prompted decode wrote words of the conversation over laughter. + assert choose_with_context("", "화면에 디스코드", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == "" + assert choose_with_context("", "", HINT, ("감사합니다.", MIN_MEAN_LOGPROB - 0.05)) == "" + + +@pytest.mark.unit +def test_a_rescue_never_adds_a_wake_word(): + assert choose_with_context("", "자비스 지금 영상 재생되고 있어?", HINT, + ("라비스 지금 영상 재생되고 있어", MIN_MEAN_LOGPROB - 0.05)) == ""