feat(voice_worker): finals are also decoded with the recent conversation as Whisper's prompt

The words of the topic were the ones misheard live (화면공이 for 화면 공유, 계산지 for 계산기), and
five models made the same mistakes. Each final is decoded as before and again with the last 180 s
of the channel's conversation (everyone's finals and Jarvis's speech, at most context_chars=200
characters) as the previous-text prompt; the prompted text is used only when the plain one was kept,
stays within half its letters and keeps every hotword count, as for the hint.

Measured on the game PC (RTX 5070 Ti, 100 recordings of the user, 16 with their own text): CER
12.0% -> 7.5%, empty results unchanged, noise clips with text unchanged (16/108), final decode
p95 170 -> 253 ms. Finals with a context are decoded one clip at a time (ctranslate2 needs the
start token at one position in a batch).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-19 19:38:10 +09:00
parent 478968949c
commit 628d89c4f1
10 changed files with 234 additions and 17 deletions

View File

@@ -82,3 +82,9 @@
## 진행 기록
- 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능.
- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개).
- 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤).
- H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함).
- H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR은 VRAM(Strata와 함께 남는 약 2.5GB)에 들어가지 않아 서버 CPU에서 따로 봄.
- H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context").
- 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음.

View File

@@ -32,7 +32,8 @@ class SttResult:
class Transcriber(Protocol):
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ...
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
contexts: list[str] | None = None) -> list[str]: ...
Callback = Callable[[SttResult], None]
@@ -44,6 +45,7 @@ class _Job:
audio: np.ndarray
opts: DecodeOptions
callback: Callback
context: str = "" # finals only: the recent conversation, as the model's prompt
class SttScheduler:
@@ -74,8 +76,9 @@ class SttScheduler:
pass
self._executor.shutdown(wait=False, cancel_futures=True)
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
self._finals.append(_Job(key, audio, opts, callback))
def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback,
context: str = "") -> None:
self._finals.append(_Job(key, audio, opts, callback, context))
self._wake.set()
def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None:
@@ -121,7 +124,7 @@ class SttScheduler:
t0 = time.perf_counter()
try:
texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips,
batch[0].opts)
batch[0].opts, [j.context for j in batch])
error = None
except Exception as e: # the model failing must not stop every speaker's STT
texts, error = [""] * len(batch), f"{type(e).__name__}: {e}"

View File

@@ -10,6 +10,7 @@ import itertools
import json
import secrets
import time
from collections import deque
from dataclasses import dataclass
from typing import Callable
@@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService
CLOSE_BAD_HELLO = 4400
CLOSE_BAD_TOKEN = 4401
MAX_SPEAKERS = 32
CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches
def is_loopback(host: str) -> bool:
@@ -274,6 +276,9 @@ class Session:
self.chat_waiting: dict[str, asyncio.Future] = {}
self._chat_ids = itertools.count(1)
self.names: dict[str, str] = {}
# What was said and answered lately in Jarvis's voice channel: (time, text), the STT's
# conversation context (voice_worker.spec.md, "Conversation context"). Never logged.
self.lines: deque[tuple[float, str]] = deque(maxlen=50)
self.speakers: dict[str, _Speaker] = {}
self.tts_requests: dict[str, TtsRequest] = {}
# With recordings on: each spoken answer's text and mono samples, kept until it is made and the
@@ -364,6 +369,15 @@ class Session:
samples = st.converter.push(pcm)
self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic()))
def _context(self) -> str:
"""The recent conversation for a final's prompt: the channel's lines of the last
CONTEXT_SECONDS, at most `context_chars` characters from the end."""
n = self.settings.context_chars
if not n or not self.voice_channel:
return ""
since = time.monotonic() - CONTEXT_SECONDS
return " ".join(t for at, t in self.lines if at >= since)[-n:].strip()
def _opts(self, beam: int) -> DecodeOptions:
return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords)
@@ -388,7 +402,8 @@ class Session:
else:
sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced),
self._opts(self.settings.final_beam),
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said))
lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said),
context=self._context())
def _partial_cb(self, speaker: str, st: _Speaker, utt: int):
def cb(res: SttResult) -> None:
@@ -402,6 +417,8 @@ class Session:
store = self.worker.history
if store is not None and channel and res.text.strip():
store.add_voice(channel, speaker, name, res.text, ts)
if channel and channel == self.voice_channel and res.text.strip():
self.lines.append((time.monotonic(), res.text.strip()))
if self.worker.recorder is not None:
self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR)
self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text,
@@ -499,6 +516,8 @@ class Session:
rec["text"] += text
while len(self.tts_recorded) > 50: # never reported as played: not kept forever
self.tts_recorded.pop(next(iter(self.tts_recorded)))
if self.voice_channel and text.strip():
self.lines.append((time.monotonic(), text.strip()))
tts.feed(req, text, more)
def _on_tts_cancel(self, msg: dict) -> None:
@@ -523,6 +542,7 @@ class Session:
if self.voice_channel:
debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker")
self.voice_channel = None
self.lines.clear()
def _on_voice_join(self, msg: dict) -> None:
got = self._strings(msg, "channel")

View File

@@ -13,6 +13,7 @@ class SttSettings:
final_beam: int = 2
language: str | None = "ko"
hotwords: tuple[str, ...] = ()
context_chars: int = 200
def update(self, changes: dict) -> None:
"""Apply ``changes``; raises ValueError and changes nothing if any value is bad."""
@@ -52,6 +53,8 @@ def _check(key: str, value):
if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8):
raise ValueError("language must be a language code or null")
return value
if key == "context_chars":
return _int_in(key, value, 0, 500)
if key == "hotwords":
if (not isinstance(value, (list, tuple)) or len(value) > 50
or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)):

View File

@@ -73,6 +73,25 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp:
return plain_text
def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str:
"""Pick between the decode without the conversation context (``heard``, already through the hint
choice) and the one that had it as Whisper's previous-text prompt (``prompted``).
The context fixes words of the topic the model misheard, but on noise or unsure audio it can write
words of the prompt, so the decode without it decides, as for the hint: nothing heard stays
nothing, and the prompted text is used only when it stays close to what was heard and holds every
hotword exactly as often.
"""
if not heard or not prompted:
return heard
h, p = _letters(heard), _letters(prompted)
if _edit_distance(p, h) > len(h) // 2:
return heard
if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords):
return heard
return prompted
def add_cuda_dll_dirs() -> None:
"""On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin."""
if not hasattr(os, "add_dll_directory"):
@@ -104,8 +123,25 @@ class WhisperTranscriber:
def warm_up(self) -> None:
self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=()))
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]:
"""Clips arrive loudness-equalised by the session."""
def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions,
contexts: list[str] | None = None) -> list[str]:
"""Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent
conversation to prompt with ("" for none); see ``choose_with_context``."""
if contexts and any(contexts):
# Prompts of different lengths cannot share one generate call (the start token must sit
# at the same position): one clip at a time. The encoder costs the same per clip anyway.
return [self._with_context(c, opts, ctx) for c, ctx in zip(clips, contexts)]
enc, langs = self._encode(clips, opts)
return self._choose(enc, langs, opts, None)
def _with_context(self, clip: np.ndarray, opts: DecodeOptions, context: str) -> str:
enc, langs = self._encode([clip], opts)
heard = self._choose(enc, langs, opts, None)[0]
if not context:
return heard
return choose_with_context(heard, self._choose(enc, langs, opts, [context])[0], opts.hotwords)
def _encode(self, clips: list[np.ndarray], opts: DecodeOptions):
m = self.model
n_frames = m.feature_extractor.nb_max_frames
feats = []
@@ -118,16 +154,22 @@ class WhisperTranscriber:
langs = [opts.language] * len(clips)
else:
langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)]
plain = self._decode(enc, langs, opts.beam, None)
return enc, langs
def _choose(self, enc, langs: list[str], opts: DecodeOptions, contexts: list[str] | None) -> list[str]:
plain = self._decode(enc, langs, opts.beam, None, contexts)
if not opts.hotwords:
return [text if keep_transcript(text, lp) else "" for text, lp in plain]
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords))
hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), contexts)
return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)]
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]:
def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None,
contexts: list[str] | None = None) -> list[tuple[str, float]]:
m = self.model
prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords)
for lang in langs]
prompts = [m.get_prompt(self._tokenizer(lang),
self._tokenizer(lang).encode(" " + ctx.strip()) if ctx else [],
without_timestamps=True, hotwords=hotwords)
for lang, ctx in zip(langs, contexts or [""] * len(langs))]
results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True,
suppress_tokens=[-1], return_scores=True)
# ``scores`` is the sequence log-probability divided by its length: the mean per token.

View File

@@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`):
| `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. |
| `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. |
| `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. |
| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. |
| `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). |
### Utterance rules
@@ -152,6 +153,35 @@ text is empty:
`no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike.
### Conversation context
People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기")
were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이"
and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made
the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also
decoded with the recent conversation as its previous-text prompt:
- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said
(the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined
by spaces, at most `context_chars` characters from the end. Joining another channel or leaving
clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt.
- Partials have none (they stay one cheap decode).
- The clip is decoded as before (with the hotword choice above) and again with the context (the
same choice, each decode prompted). The prompted text is used only when the one without context
was kept, it stays within half that text's letters of it, and it holds every hotword exactly as
often; otherwise the text without context is used. On noise or unsure audio the prompt makes
Whisper write words of the conversation; this rule keeps that out, as for the hint.
- Prompts of different lengths cannot share one decoder call, so finals with a context are decoded
one clip at a time (the encoder costs the same per clip).
Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16,
beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own
text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more
empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms).
On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as
without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote
whole sentences of the prompt. Evidence and the model comparison: `docs/v2/stt-accuracy-plan.md`.
Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing,
coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only
clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD

View File

@@ -303,7 +303,7 @@ class SlowTranscriber(LevelTranscriber):
super().__init__()
self.delay = delay
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
import time as _t
_t.sleep(self.delay)
return super().transcribe(clips, opts)

View File

@@ -96,3 +96,39 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation():
def test_spacing_and_punctuation_differences_do_not_change_the_choice():
assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \
== "자비스 오늘 서울 날씨 어때"
# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation
# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" ->
# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint.
from voice_worker.stt_whisper import choose_with_context # noqa: E402
@pytest.mark.unit
def test_the_context_may_fix_words_of_what_was_said():
assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?"
@pytest.mark.unit
def test_nothing_heard_without_the_context_stays_nothing():
# Noise: the prompted decode wrote words of the conversation.
assert choose_with_context("", "화면에 디스코드", HINT) == ""
@pytest.mark.unit
def test_a_prompted_decode_far_from_what_was_heard_is_not_used():
heard = "가상화면을 못 봐"
copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다."
assert choose_with_context(heard, copied, HINT) == heard
@pytest.mark.unit
def test_the_context_never_changes_how_many_wake_words_were_heard():
assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \
== "라비스 지금 영상 재생되고 있어?"
assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘"
@pytest.mark.unit
def test_an_empty_prompted_decode_keeps_what_was_heard():
assert choose_with_context("네.", "", HINT) == "네."

View File

@@ -25,7 +25,7 @@ class FakeTranscriber:
self.gate.set()
self.fail_on_beam = fail_on_beam
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
self.entered.set()
self.gate.wait(5)
self.calls.append((opts, [len(c) for c in clips]))
@@ -218,12 +218,45 @@ class StepTranscriber:
def step(self):
self._steps.release()
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
self.calls.append([len(c) for c in clips])
self._steps.acquire(timeout=5)
return ["" for _ in clips]
@pytest.mark.unit
def test_each_final_reaches_the_model_with_its_own_context():
class Ctx:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts, contexts=None):
self.calls.append(list(contexts or []))
return [f"n{len(c)}" for c in clips]
eng = Ctx()
async def body():
s = SttScheduler(eng)
s.start()
got = []
done = asyncio.Event()
def cb(r):
got.append(r.text)
if len(got) == 3:
done.set()
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
s.submit_final("b", clip(200), FINAL, cb)
s.submit_partial("c", clip(300), PARTIAL, cb)
await asyncio.wait_for(done.wait(), 5)
await s.stop()
asyncio.run(body())
assert eng.calls[0] == ["화면 공유", ""]
assert eng.calls[1] == [""], "partials go without context"
@pytest.mark.unit
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
t = StepTranscriber()

View File

@@ -32,7 +32,7 @@ class LevelTranscriber:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts):
def transcribe(self, clips, opts, contexts=None):
self.calls.append((opts, len(clips)))
return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips]
@@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k():
lengths = []
orig = t.transcribe
def spy(clips, opts):
def spy(clips, opts, contexts=None):
lengths.extend(len(c) for c in clips)
return orig(clips, opts)
@@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path
r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600))
r.close()
assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()]
class ContextTranscriber(LevelTranscriber):
def __init__(self):
super().__init__()
self.contexts = []
def transcribe(self, clips, opts, contexts=None):
if opts.beam > 1: # finals
self.contexts += list(contexts or [""] * len(clips))
return super().transcribe(clips, opts)
async def _said(ws, speaker, level):
await stream(ws, speaker, level, 600)
await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker)
@pytest.mark.unit
def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel():
"""voice_worker.spec.md, "Conversation context": what was said and answered just before, in the
channel Jarvis is in, is the model's prompt for the next final."""
t = ContextTranscriber()
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"}))
await _said(ws, "100", 0.9)
await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.",
"more": False}))
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done")
await _said(ws, "200", 0.6)
await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel
await _said(ws, "100", 0.9)
await ws.send(json.dumps({"type": "config", "context_chars": 0}))
await _said(ws, "100", 0.6)
asyncio.run(with_worker(body, transcriber=t, tts=FakeTts()))
assert t.contexts[0] == ""
assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다."
assert t.contexts[2] == "", "a new channel starts without the old channel's conversation"
assert t.contexts[3] == "", "context_chars 0 turns it off"