From 628d89c4f1edae34567512083b9cf5ffad084525 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 19 Nov 2026 19:38:10 +0900 Subject: [PATCH] feat(voice_worker): finals are also decoded with the recent conversation as Whisper's prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The words of the topic were the ones misheard live (화면공이 for 화면 공유, 계산지 for 계산기), and five models made the same mistakes. Each final is decoded as before and again with the last 180 s of the channel's conversation (everyone's finals and Jarvis's speech, at most context_chars=200 characters) as the previous-text prompt; the prompted text is used only when the plain one was kept, stays within half its letters and keeps every hotword count, as for the hint. Measured on the game PC (RTX 5070 Ti, 100 recordings of the user, 16 with their own text): CER 12.0% -> 7.5%, empty results unchanged, noise clips with text unchanged (16/108), final decode p95 170 -> 253 ms. Finals with a context are decoded one clip at a time (ctranslate2 needs the start token at one position in a batch). Co-Authored-By: Claude Opus 5.5 --- docs/v2/stt-accuracy-plan.md | 6 +++ src/voice_worker/scheduler.py | 11 ++++-- src/voice_worker/server.py | 22 ++++++++++- src/voice_worker/settings.py | 3 ++ src/voice_worker/stt_whisper.py | 56 +++++++++++++++++++++++---- src/voice_worker/voice_worker.spec.md | 30 ++++++++++++++ tests/test_voice_worker_brain.py | 2 +- tests/test_voice_worker_gate.py | 36 +++++++++++++++++ tests/test_voice_worker_scheduler.py | 37 +++++++++++++++++- tests/test_voice_worker_server.py | 48 ++++++++++++++++++++++- 10 files changed, 234 insertions(+), 17 deletions(-) diff --git a/docs/v2/stt-accuracy-plan.md b/docs/v2/stt-accuracy-plan.md index cc74a41..f6ec940 100644 --- a/docs/v2/stt-accuracy-plan.md +++ b/docs/v2/stt-accuracy-plan.md @@ -82,3 +82,9 @@ ## 진행 기록 - 2026-11-19: 서버 GPU 사용 불가. 15:19 재부팅으로 커널 7.0.0-43이 올라왔는데 이 커널용 nvidia 모듈 패키지(`linux-modules-nvidia-595-open-7.0.0-43-generic`)가 설치되지 않음. 서버 측정은 CPU로만 가능. +- 2026-11-19 저녁 (게임PC, 사용자가 "지금 컴퓨터 사용해도 됨"): 사용자가 내보낸 녹음과 대화 기록을 받음. 사용자 녹음 100개(11-16 11개, 11-18 53개, 11-19 36개), 정답 16개(11-16 시험 문장 11개 + 사용자가 준 5개). + - 불만 2(끊어 말하기)의 원인은 STT가 아니었음: 한 생각 안의 쉼이 0.1~4.7초라 조각마다 따로 요청이 되었고, 앱이 새 조각으로 앞 답을 멈춰도 워커는 그 답을 끝까지 만들어 다음 답이 13~35초 기다림. 0.1.47에서 고침(conversation.spec.md "Going on", voice_worker.spec.md: 취소된 답은 시작하지 않거나 다음 모델·도구 호출 전에 멈춤). + - H2: 녹음을 다시 봐도 소리 손상은 없었음(채널 64kbps, 4~7kHz 성분 정상). 빈 결과 17개는 모델이 확신하지 못해 버린 것(평균 확신도 -0.7 미만, "감사합니다" 같은 지어낸 말 포함). + - H3 모델 비교(게임PC GPU, 워커와 같은 경로, int8_float16, 정답 16개 기준 문자 오류율): large-v3-turbo 12.0%, large-v3 12.7%, BuzzASR/korean 13.3%, batisay-ko-turbo 11.2%, ghost613 turbo-korean 33.1%. 모두 같은 곳을 틀림("기사야", "화면군이", "기구", "화면" 빠짐). 상대 20% 넘게 나은 모델이 없어서 모델은 바꾸지 않음. Qwen3-ASR은 VRAM(Strata와 함께 남는 약 2.5GB)에 들어가지 않아 서버 CPU에서 따로 봄. + - H4 앞 대화 문맥: 12.0% → 7.5%(상대 38%), 빈 결과 그대로, 잡음 108개의 지어낸 말 그대로(16개), 최종 전사 평균 182ms(p95 253ms). 문맥만 주면 잡음에서 앞 대화 낱말을 씀(turbo 22개, large-v3 63개)이라 호출어 힌트와 같은 규칙으로 막음. 0.1.48에 넣음(voice_worker.spec.md "Conversation context"). + - 남은 것: 첫 낱말(호출어)과 짧은 말("기사야"→비서야, "기구"→디코)은 앞 대화가 없거나 너무 짧아 그대로. 호출 판정의 비슷한 말 허용이 호출어 쪽을 덮음. diff --git a/src/voice_worker/scheduler.py b/src/voice_worker/scheduler.py index 6174d69..3d10835 100644 --- a/src/voice_worker/scheduler.py +++ b/src/voice_worker/scheduler.py @@ -32,7 +32,8 @@ class SttResult: class Transcriber(Protocol): - def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: ... + def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions, + contexts: list[str] | None = None) -> list[str]: ... Callback = Callable[[SttResult], None] @@ -44,6 +45,7 @@ class _Job: audio: np.ndarray opts: DecodeOptions callback: Callback + context: str = "" # finals only: the recent conversation, as the model's prompt class SttScheduler: @@ -74,8 +76,9 @@ class SttScheduler: pass self._executor.shutdown(wait=False, cancel_futures=True) - def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None: - self._finals.append(_Job(key, audio, opts, callback)) + def submit_final(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback, + context: str = "") -> None: + self._finals.append(_Job(key, audio, opts, callback, context)) self._wake.set() def submit_partial(self, key: Hashable, audio: np.ndarray, opts: DecodeOptions, callback: Callback) -> None: @@ -121,7 +124,7 @@ class SttScheduler: t0 = time.perf_counter() try: texts = await loop.run_in_executor(self._executor, self.transcriber.transcribe, clips, - batch[0].opts) + batch[0].opts, [j.context for j in batch]) error = None except Exception as e: # the model failing must not stop every speaker's STT texts, error = [""] * len(batch), f"{type(e).__name__}: {e}" diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index c36809d..15ca385 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -10,6 +10,7 @@ import itertools import json import secrets import time +from collections import deque from dataclasses import dataclass from typing import Callable @@ -28,6 +29,7 @@ from .tts import TtsEngine, TtsRequest, TtsService CLOSE_BAD_HELLO = 4400 CLOSE_BAD_TOKEN = 4401 MAX_SPEAKERS = 32 +CONTEXT_SECONDS = 180 # how far back the STT's conversation context reaches def is_loopback(host: str) -> bool: @@ -274,6 +276,9 @@ class Session: self.chat_waiting: dict[str, asyncio.Future] = {} self._chat_ids = itertools.count(1) self.names: dict[str, str] = {} + # What was said and answered lately in Jarvis's voice channel: (time, text), the STT's + # conversation context (voice_worker.spec.md, "Conversation context"). Never logged. + self.lines: deque[tuple[float, str]] = deque(maxlen=50) self.speakers: dict[str, _Speaker] = {} self.tts_requests: dict[str, TtsRequest] = {} # With recordings on: each spoken answer's text and mono samples, kept until it is made and the @@ -364,6 +369,15 @@ class Session: samples = st.converter.push(pcm) self._handle_events(speaker, st, st.segmenter.push(samples, time.monotonic())) + def _context(self) -> str: + """The recent conversation for a final's prompt: the channel's lines of the last + CONTEXT_SECONDS, at most `context_chars` characters from the end.""" + n = self.settings.context_chars + if not n or not self.voice_channel: + return "" + since = time.monotonic() - CONTEXT_SECONDS + return " ".join(t for at, t in self.lines if at >= since)[-n:].strip() + def _opts(self, beam: int) -> DecodeOptions: return DecodeOptions(beam=beam, language=self.settings.language, hotwords=self.settings.hotwords) @@ -388,7 +402,8 @@ class Session: else: sched.submit_final((self.sid, speaker), loudness_equalise(ev.audio, ev.voiced), self._opts(self.settings.final_beam), - lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said)) + lambda res, ev=ev, said=said: self._send_final(speaker, ev, res, said), + context=self._context()) def _partial_cb(self, speaker: str, st: _Speaker, utt: int): def cb(res: SttResult) -> None: @@ -402,6 +417,8 @@ class Session: store = self.worker.history if store is not None and channel and res.text.strip(): store.add_voice(channel, speaker, name, res.text, ts) + if channel and channel == self.voice_channel and res.text.strip(): + self.lines.append((time.monotonic(), res.text.strip())) if self.worker.recorder is not None: self.worker.recorder.said(ts, speaker, name, channel, res.text, ev.audio, SR) self.send({"type": "final", "speaker": speaker, "utt": ev.utt, "text": res.text, @@ -499,6 +516,8 @@ class Session: rec["text"] += text while len(self.tts_recorded) > 50: # never reported as played: not kept forever self.tts_recorded.pop(next(iter(self.tts_recorded))) + if self.voice_channel and text.strip(): + self.lines.append((time.monotonic(), text.strip())) tts.feed(req, text, more) def _on_tts_cancel(self, msg: dict) -> None: @@ -523,6 +542,7 @@ class Session: if self.voice_channel: debug_log(f"session {self.sid} left voice channel {self.voice_channel}", "voice_worker") self.voice_channel = None + self.lines.clear() def _on_voice_join(self, msg: dict) -> None: got = self._strings(msg, "channel") diff --git a/src/voice_worker/settings.py b/src/voice_worker/settings.py index 67f18a4..5edc6e8 100644 --- a/src/voice_worker/settings.py +++ b/src/voice_worker/settings.py @@ -13,6 +13,7 @@ class SttSettings: final_beam: int = 2 language: str | None = "ko" hotwords: tuple[str, ...] = () + context_chars: int = 200 def update(self, changes: dict) -> None: """Apply ``changes``; raises ValueError and changes nothing if any value is bad.""" @@ -52,6 +53,8 @@ def _check(key: str, value): if value is not None and (not isinstance(value, str) or not 2 <= len(value) <= 8): raise ValueError("language must be a language code or null") return value + if key == "context_chars": + return _int_in(key, value, 0, 500) if key == "hotwords": if (not isinstance(value, (list, tuple)) or len(value) > 50 or not all(isinstance(w, str) and 0 < len(w) <= 64 for w in value)): diff --git a/src/voice_worker/stt_whisper.py b/src/voice_worker/stt_whisper.py index 2b1ad67..5d383b5 100644 --- a/src/voice_worker/stt_whisper.py +++ b/src/voice_worker/stt_whisper.py @@ -73,6 +73,25 @@ def choose_with_hint(hint_text: str, hint_lp: float, plain_text: str, plain_lp: return plain_text +def choose_with_context(heard: str, prompted: str, hotwords: tuple[str, ...]) -> str: + """Pick between the decode without the conversation context (``heard``, already through the hint + choice) and the one that had it as Whisper's previous-text prompt (``prompted``). + + The context fixes words of the topic the model misheard, but on noise or unsure audio it can write + words of the prompt, so the decode without it decides, as for the hint: nothing heard stays + nothing, and the prompted text is used only when it stays close to what was heard and holds every + hotword exactly as often. + """ + if not heard or not prompted: + return heard + h, p = _letters(heard), _letters(prompted) + if _edit_distance(p, h) > len(h) // 2: + return heard + if any(_mentions(prompted, w) != _mentions(heard, w) for w in hotwords): + return heard + return prompted + + def add_cuda_dll_dirs() -> None: """On Windows ctranslate2 loads cuBLAS/cuDNN from the PATH; the pip wheels put them in nvidia/*/bin.""" if not hasattr(os, "add_dll_directory"): @@ -104,8 +123,25 @@ class WhisperTranscriber: def warm_up(self) -> None: self.transcribe([np.zeros(SR, dtype=np.float32)], DecodeOptions(beam=1, language="en", hotwords=())) - def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions) -> list[str]: - """Clips arrive loudness-equalised by the session.""" + def transcribe(self, clips: list[np.ndarray], opts: DecodeOptions, + contexts: list[str] | None = None) -> list[str]: + """Clips arrive loudness-equalised by the session. ``contexts``: per clip, the recent + conversation to prompt with ("" for none); see ``choose_with_context``.""" + if contexts and any(contexts): + # Prompts of different lengths cannot share one generate call (the start token must sit + # at the same position): one clip at a time. The encoder costs the same per clip anyway. + return [self._with_context(c, opts, ctx) for c, ctx in zip(clips, contexts)] + enc, langs = self._encode(clips, opts) + return self._choose(enc, langs, opts, None) + + def _with_context(self, clip: np.ndarray, opts: DecodeOptions, context: str) -> str: + enc, langs = self._encode([clip], opts) + heard = self._choose(enc, langs, opts, None)[0] + if not context: + return heard + return choose_with_context(heard, self._choose(enc, langs, opts, [context])[0], opts.hotwords) + + def _encode(self, clips: list[np.ndarray], opts: DecodeOptions): m = self.model n_frames = m.feature_extractor.nb_max_frames feats = [] @@ -118,16 +154,22 @@ class WhisperTranscriber: langs = [opts.language] * len(clips) else: langs = [r[0][0][2:-2] for r in m.model.detect_language(enc)] - plain = self._decode(enc, langs, opts.beam, None) + return enc, langs + + def _choose(self, enc, langs: list[str], opts: DecodeOptions, contexts: list[str] | None) -> list[str]: + plain = self._decode(enc, langs, opts.beam, None, contexts) if not opts.hotwords: return [text if keep_transcript(text, lp) else "" for text, lp in plain] - hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords)) + hinted = self._decode(enc, langs, opts.beam, " ".join(opts.hotwords), contexts) return [choose_with_hint(ht, hlp, pt, plp, opts.hotwords) for (ht, hlp), (pt, plp) in zip(hinted, plain)] - def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None) -> list[tuple[str, float]]: + def _decode(self, enc, langs: list[str], beam: int, hotwords: str | None, + contexts: list[str] | None = None) -> list[tuple[str, float]]: m = self.model - prompts = [m.get_prompt(self._tokenizer(lang), [], without_timestamps=True, hotwords=hotwords) - for lang in langs] + prompts = [m.get_prompt(self._tokenizer(lang), + self._tokenizer(lang).encode(" " + ctx.strip()) if ctx else [], + without_timestamps=True, hotwords=hotwords) + for lang, ctx in zip(langs, contexts or [""] * len(langs))] results = m.model.generate(enc, prompts, beam_size=beam, max_length=224, suppress_blank=True, suppress_tokens=[-1], return_scores=True) # ``scores`` is the sequence log-probability divided by its length: the mean per token. diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index 329edb0..dae77cb 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -80,6 +80,7 @@ STT settings (all optional in `hello` and `config`): | `end_silence_ms` | 300 | Trailing VAD silence that ends an utterance. Range 100-2000. | | `final_beam` | 2 | Beam size for finals (partials always use 1). Range 1-5. | | `language` | `"ko"` | Whisper language code, or `null` to detect per utterance. | +| `context_chars` | 200 | Finals are also decoded with the recent conversation as Whisper's previous-text prompt, at most this many characters (Conversation context). `0` turns it off. Range 0-500. | | `hotwords` | `[]` | Words (wake words, names) passed to Whisper as a hint. The hint alone made Whisper write the wake word on laughter, so with hotwords every clip is also decoded without them and that plain decode decides (see Hallucination gate). | ### Utterance rules @@ -152,6 +153,35 @@ text is empty: `no_speech_prob` is not used: large-v3-turbo reports 0.0 for silence and noise alike. +### Conversation context + +People talk about one thing for a while, and the words of that topic ("디코", "화면 공유", "계산기") +were the ones misheard (live 2026-11-19): Whisper heard a casual, unclear "화면 공유" as "화면공이" +and "계산기" as "계산지", and five models (large-v3-turbo, large-v3 and three Korean fine-tunes) made +the same mistakes. Whisper was trained to continue from the text before a clip, so a final is also +decoded with the recent conversation as its previous-text prompt: + +- The context is what was said (the session's non-empty finals, everyone's) and what Jarvis said + (the text of every `tts` request) in the voice channel Jarvis is in, over the last 180 s, joined + by spaces, at most `context_chars` characters from the end. Joining another channel or leaving + clears it. It lives in memory only, is never logged and goes nowhere but the model's prompt. +- Partials have none (they stay one cheap decode). +- The clip is decoded as before (with the hotword choice above) and again with the context (the + same choice, each decode prompted). The prompted text is used only when the one without context + was kept, it stays within half that text's letters of it, and it holds every hotword exactly as + often; otherwise the text without context is used. On noise or unsure audio the prompt makes + Whisper write words of the conversation; this rule keeps that out, as for the hint. +- Prompts of different lengths cannot share one decoder call, so finals with a context are decoded + one clip at a time (the encoder costs the same per clip). + +Measured (2026-11-19, the game PC's RTX 5070 Ti with Strata loaded, large-v3-turbo int8_float16, +beam 2, the 100 recordings of the user's speech from 2026-11-16 to 19, 16 of them with the user's own +text): character error rate 12.0% without and 7.5% with the context (wrong sentences 10 to 8), no more +empty results (17 of 100 both ways), final decode 134 ms to 182 ms on average (p95 170 to 253 ms). +On the 108 noise clips decoded straight (no VAD), as many came out with text with the context as +without it (16 each); without the rule above it was 22 with turbo, and with large-v3 63, which wrote +whole sentences of the prompt. Evidence and the model comparison: `docs/v2/stt-accuracy-plan.md`. + Measured (2026-11-11, RTX 5050) with 54 real noise recordings from ESC-50 (typing, mouse, breathing, coughing, laughing, crying, vacuum, ...), each at its own level and at -25 dBFS: of the 108 noise-only clips streamed through the worker with the wake word hint on, 103 never reached the model (the VAD diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index af6ed7e..6a18419 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -303,7 +303,7 @@ class SlowTranscriber(LevelTranscriber): super().__init__() self.delay = delay - def transcribe(self, clips, opts): + def transcribe(self, clips, opts, contexts=None): import time as _t _t.sleep(self.delay) return super().transcribe(clips, opts) diff --git a/tests/test_voice_worker_gate.py b/tests/test_voice_worker_gate.py index dc94e43..7419be1 100644 --- a/tests/test_voice_worker_gate.py +++ b/tests/test_voice_worker_gate.py @@ -96,3 +96,39 @@ def test_wake_word_matching_ignores_case_spacing_and_punctuation(): def test_spacing_and_punctuation_differences_do_not_change_the_choice(): assert choose_with_hint("자비스 오늘 서울 날씨 어때", -0.1, "자비스, 오늘 서울 날씨 어때?", -0.1, HINT) \ == "자비스 오늘 서울 날씨 어때" + + +# The recent conversation as Whisper's previous-text prompt (voice_worker.spec.md, "Conversation +# context"). Live 2026-11-19 the prompt fixed words of the topic ("화면공이" -> "화면 공유", "계산지" -> +# "계산기"); on noise it wrote words of the prompt. The decode without it decides, as with the hint. +from voice_worker.stt_whisper import choose_with_context # noqa: E402 + + +@pytest.mark.unit +def test_the_context_may_fix_words_of_what_was_said(): + assert choose_with_context("내 화면공이 안 보여?", "내 화면 공유 안 보여?", HINT) == "내 화면 공유 안 보여?" + + +@pytest.mark.unit +def test_nothing_heard_without_the_context_stays_nothing(): + # Noise: the prompted decode wrote words of the conversation. + assert choose_with_context("", "화면에 디스코드", HINT) == "" + + +@pytest.mark.unit +def test_a_prompted_decode_far_from_what_was_heard_is_not_used(): + heard = "가상화면을 못 봐" + copied = "네, 현재 화면에 디스코드 채팅 창과 여러 프로그램이 정상적으로 보입니다." + assert choose_with_context(heard, copied, HINT) == heard + + +@pytest.mark.unit +def test_the_context_never_changes_how_many_wake_words_were_heard(): + assert choose_with_context("라비스 지금 영상 재생되고 있어?", "자비스 지금 영상 재생되고 있어?", HINT) \ + == "라비스 지금 영상 재생되고 있어?" + assert choose_with_context("자비스 날씨 알려 줘", "날씨 알려 줘", HINT) == "자비스 날씨 알려 줘" + + +@pytest.mark.unit +def test_an_empty_prompted_decode_keeps_what_was_heard(): + assert choose_with_context("네.", "", HINT) == "네." diff --git a/tests/test_voice_worker_scheduler.py b/tests/test_voice_worker_scheduler.py index 1cc674f..9f86d01 100644 --- a/tests/test_voice_worker_scheduler.py +++ b/tests/test_voice_worker_scheduler.py @@ -25,7 +25,7 @@ class FakeTranscriber: self.gate.set() self.fail_on_beam = fail_on_beam - def transcribe(self, clips, opts): + def transcribe(self, clips, opts, contexts=None): self.entered.set() self.gate.wait(5) self.calls.append((opts, [len(c) for c in clips])) @@ -218,12 +218,45 @@ class StepTranscriber: def step(self): self._steps.release() - def transcribe(self, clips, opts): + def transcribe(self, clips, opts, contexts=None): self.calls.append([len(c) for c in clips]) self._steps.acquire(timeout=5) return ["" for _ in clips] +@pytest.mark.unit +def test_each_final_reaches_the_model_with_its_own_context(): + class Ctx: + def __init__(self): + self.calls = [] + + def transcribe(self, clips, opts, contexts=None): + self.calls.append(list(contexts or [])) + return [f"n{len(c)}" for c in clips] + + eng = Ctx() + + async def body(): + s = SttScheduler(eng) + s.start() + got = [] + done = asyncio.Event() + + def cb(r): + got.append(r.text) + if len(got) == 3: + done.set() + s.submit_final("a", clip(100), FINAL, cb, context="화면 공유") + s.submit_final("b", clip(200), FINAL, cb) + s.submit_partial("c", clip(300), PARTIAL, cb) + await asyncio.wait_for(done.wait(), 5) + await s.stop() + + asyncio.run(body()) + assert eng.calls[0] == ["화면 공유", ""] + assert eng.calls[1] == [""], "partials go without context" + + @pytest.mark.unit def test_partial_batches_are_kept_small_so_finals_can_cut_in(): t = StepTranscriber() diff --git a/tests/test_voice_worker_server.py b/tests/test_voice_worker_server.py index c8917ec..9cf3064 100644 --- a/tests/test_voice_worker_server.py +++ b/tests/test_voice_worker_server.py @@ -32,7 +32,7 @@ class LevelTranscriber: def __init__(self): self.calls = [] - def transcribe(self, clips, opts): + def transcribe(self, clips, opts, contexts=None): self.calls.append((opts, len(clips))) return [f"level {np.argmax(np.abs(np.fft.rfft(c))) * 16000 / len(c) / 1000:.1f}" for c in clips] @@ -336,7 +336,7 @@ def test_48k_stereo_input_reaches_the_model_at_16k(): lengths = [] orig = t.transcribe - def spy(clips, opts): + def spy(clips, opts, contexts=None): lengths.extend(len(c) for c in clips) return orig(clips, opts) @@ -496,3 +496,47 @@ def test_old_recordings_are_also_removed_while_the_worker_keeps_running(tmp_path r.said(later.timestamp(), "1", "A", "vc", "나중", np.zeros(1600)) r.close() assert sorted(p.name for p in tmp_path.iterdir()) == [later.date().isoformat()] + + +class ContextTranscriber(LevelTranscriber): + def __init__(self): + super().__init__() + self.contexts = [] + + def transcribe(self, clips, opts, contexts=None): + if opts.beam > 1: # finals + self.contexts += list(contexts or [""] * len(clips)) + return super().transcribe(clips, opts) + + +async def _said(ws, speaker, level): + await stream(ws, speaker, level, 600) + await ws.send(json.dumps({"type": "speaking_end", "speaker": speaker})) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "final" and m["speaker"] == speaker) + + +@pytest.mark.unit +def test_each_final_is_transcribed_with_the_recent_conversation_of_the_channel(): + """voice_worker.spec.md, "Conversation context": what was said and answered just before, in the + channel Jarvis is in, is the model's prompt for the next final.""" + t = ContextTranscriber() + + async def body(url): + async with connect(url) as ws: + await hello(ws, stt={"partials": False}) + await ws.send(json.dumps({"type": "voice_join", "channel": "vc1"})) + await _said(ws, "100", 0.9) + await ws.send(json.dumps({"type": "tts", "id": "r1", "text": "디스코드 화면 공유가 보입니다.", + "more": False})) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "tts_done") + await _said(ws, "200", 0.6) + await ws.send(json.dumps({"type": "voice_join", "channel": "vc2"})) # another channel + await _said(ws, "100", 0.9) + await ws.send(json.dumps({"type": "config", "context_chars": 0})) + await _said(ws, "100", 0.6) + + asyncio.run(with_worker(body, transcriber=t, tts=FakeTts())) + assert t.contexts[0] == "" + assert t.contexts[1] == "level 0.9 디스코드 화면 공유가 보입니다." + assert t.contexts[2] == "", "a new channel starts without the old channel's conversation" + assert t.contexts[3] == "", "context_chars 0 turns it off"