Files
javis_bot/tests/test_voice_worker_scheduler.py
javis-bot 628d89c4f1 feat(voice_worker): finals are also decoded with the recent conversation as Whisper's prompt
The words of the topic were the ones misheard live (화면공이 for 화면 공유, 계산지 for 계산기), and
five models made the same mistakes. Each final is decoded as before and again with the last 180 s
of the channel's conversation (everyone's finals and Jarvis's speech, at most context_chars=200
characters) as the previous-text prompt; the prompted text is used only when the plain one was kept,
stays within half its letters and keeps every hotword count, as for the hint.

Measured on the game PC (RTX 5070 Ti, 100 recordings of the user, 16 with their own text): CER
12.0% -> 7.5%, empty results unchanged, noise clips with text unchanged (16/108), final decode
p95 170 -> 253 ms. Finals with a context are decoded one clip at a time (ctranslate2 needs the
start token at one position in a batch).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 19:38:10 +09:00

316 lines
9.1 KiB
Python

"""Voice worker: the shared-model STT scheduler (batching, finals first, partial coalescing)."""
from __future__ import annotations
import asyncio
import threading
import numpy as np
import pytest
from voice_worker.scheduler import DecodeOptions, SttScheduler
FINAL = DecodeOptions(beam=2, language="ko", hotwords=())
PARTIAL = DecodeOptions(beam=1, language="ko", hotwords=())
class FakeTranscriber:
"""Names each clip by its length; the first call can be held to build up a queue."""
def __init__(self, hold_first: bool = False, fail_on_beam: int | None = None):
self.calls: list[tuple[DecodeOptions, list[int]]] = []
self.gate = threading.Event()
self.entered = threading.Event()
if not hold_first:
self.gate.set()
self.fail_on_beam = fail_on_beam
def transcribe(self, clips, opts, contexts=None):
self.entered.set()
self.gate.wait(5)
self.calls.append((opts, [len(c) for c in clips]))
if opts.beam == self.fail_on_beam:
raise RuntimeError("boom")
return [f"n{len(c)}" for c in clips]
def clip(n: int) -> np.ndarray:
return np.zeros(n, dtype=np.float32)
async def _run(body):
return await asyncio.wait_for(body(), 10)
def run(body):
return asyncio.run(_run(body))
async def _wait_entered(t: FakeTranscriber):
while not t.entered.is_set():
await asyncio.sleep(0.005)
@pytest.mark.unit
def test_finals_queued_while_busy_go_in_one_batch_before_partials():
t = FakeTranscriber(hold_first=True)
async def body():
s = SttScheduler(t, max_partial_batch=4)
s.start()
got = []
done = asyncio.Event()
def cb(name):
def _cb(res):
got.append((name, res.text))
if len(got) == 5:
done.set()
return _cb
s.submit_partial(("a", "1"), clip(10), PARTIAL, cb("p-first"))
await _wait_entered(t)
s.submit_partial(("a", "2"), clip(20), PARTIAL, cb("p2"))
s.submit_final(("a", "3"), clip(30), FINAL, cb("f3"))
s.submit_final(("a", "4"), clip(40), FINAL, cb("f4"))
s.submit_partial(("a", "5"), clip(50), PARTIAL, cb("p5"))
t.gate.set()
await done.wait()
await s.stop()
return got
got = run(body)
assert [c[1] for c in t.calls] == [[10], [30, 40], [20, 50]]
assert [c[0].beam for c in t.calls] == [1, 2, 1]
assert got == [("p-first", "n10"), ("f3", "n30"), ("f4", "n40"), ("p2", "n20"), ("p5", "n50")]
@pytest.mark.unit
def test_a_newer_partial_replaces_the_waiting_one_for_the_same_speaker():
t = FakeTranscriber(hold_first=True)
async def body():
s = SttScheduler(t)
s.start()
got = []
done = asyncio.Event()
s.submit_final(("a", "x"), clip(5), FINAL, lambda r: None)
await _wait_entered(t)
s.submit_partial(("a", "1"), clip(10), PARTIAL, lambda r: got.append(r.text))
s.submit_partial(("a", "1"), clip(20), PARTIAL, lambda r: (got.append(r.text), done.set()))
t.gate.set()
await done.wait()
await asyncio.sleep(0.05)
await s.stop()
return got
assert run(body) == ["n20"]
assert [c[1] for c in t.calls] == [[5], [20]]
@pytest.mark.unit
def test_dropped_partials_are_never_transcribed():
t = FakeTranscriber(hold_first=True)
async def body():
s = SttScheduler(t)
s.start()
done = asyncio.Event()
got = []
s.submit_final(("a", "x"), clip(5), FINAL, lambda r: None)
await _wait_entered(t)
s.submit_partial(("a", "1"), clip(10), PARTIAL, lambda r: got.append(r.text))
s.submit_partial(("b", "1"), clip(20), PARTIAL, lambda r: (got.append(r.text), done.set()))
s.drop_partials(lambda key: key[0] == "a")
t.gate.set()
await done.wait()
await s.stop()
return got
assert run(body) == ["n20"]
@pytest.mark.unit
def test_different_decode_settings_are_not_mixed_in_one_batch():
t = FakeTranscriber(hold_first=True)
other = DecodeOptions(beam=2, language=None, hotwords=("자비스",))
async def body():
s = SttScheduler(t)
s.start()
n = []
done = asyncio.Event()
def cb(r):
n.append(r)
if len(n) == 4:
done.set()
s.submit_final(("a", "x"), clip(1), FINAL, cb)
await _wait_entered(t)
s.submit_final(("a", "1"), clip(10), FINAL, cb)
s.submit_final(("b", "1"), clip(20), other, cb)
s.submit_final(("a", "2"), clip(30), FINAL, cb)
t.gate.set()
await done.wait()
await s.stop()
run(body)
assert sorted(c[1] for c in t.calls[1:]) == [[10, 30], [20]]
@pytest.mark.unit
def test_batches_are_capped():
t = FakeTranscriber(hold_first=True)
async def body():
s = SttScheduler(t, max_batch=3)
s.start()
n = []
done = asyncio.Event()
def cb(r):
n.append(r)
if len(n) == 8:
done.set()
s.submit_final(("a", "x"), clip(1), FINAL, cb)
await _wait_entered(t)
for i in range(7):
s.submit_final(("a", str(i)), clip(10 + i), FINAL, cb)
t.gate.set()
await done.wait()
await s.stop()
run(body)
assert [len(c[1]) for c in t.calls] == [1, 3, 3, 1]
@pytest.mark.unit
def test_a_failing_batch_reports_the_error_and_the_scheduler_keeps_going():
t = FakeTranscriber(fail_on_beam=2)
async def body():
s = SttScheduler(t)
s.start()
out = []
done = asyncio.Event()
s.submit_final(("a", "1"), clip(10), FINAL, out.append)
s.submit_partial(("a", "2"), clip(20), PARTIAL, lambda r: (out.append(r), done.set()))
await done.wait()
await s.stop()
return out
out = run(body)
assert out[0].text == "" and out[0].error
assert out[1].text == "n20" and out[1].error is None
assert out[0].stt_ms >= 0
class StepTranscriber:
"""Every call waits for one ``step()``, so the test decides when each batch finishes."""
def __init__(self):
self.calls: list[list[int]] = []
self._steps = threading.Semaphore(0)
def step(self):
self._steps.release()
def transcribe(self, clips, opts, contexts=None):
self.calls.append([len(c) for c in clips])
self._steps.acquire(timeout=5)
return ["" for _ in clips]
@pytest.mark.unit
def test_each_final_reaches_the_model_with_its_own_context():
class Ctx:
def __init__(self):
self.calls = []
def transcribe(self, clips, opts, contexts=None):
self.calls.append(list(contexts or []))
return [f"n{len(c)}" for c in clips]
eng = Ctx()
async def body():
s = SttScheduler(eng)
s.start()
got = []
done = asyncio.Event()
def cb(r):
got.append(r.text)
if len(got) == 3:
done.set()
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
s.submit_final("b", clip(200), FINAL, cb)
s.submit_partial("c", clip(300), PARTIAL, cb)
await asyncio.wait_for(done.wait(), 5)
await s.stop()
asyncio.run(body())
assert eng.calls[0] == ["화면 공유", ""]
assert eng.calls[1] == [""], "partials go without context"
@pytest.mark.unit
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
t = StepTranscriber()
async def until(n):
while len(t.calls) < n:
await asyncio.sleep(0.005)
async def body():
s = SttScheduler(t, max_partial_batch=2)
s.start()
for i in range(5):
s.submit_partial(("a", str(i)), clip(10 + i), PARTIAL, lambda r: None)
await until(1) # first partial batch is running
s.submit_final(("a", "late"), clip(99), FINAL, lambda r: None)
t.step()
await until(2)
t.step()
await until(3)
t.step()
await until(4)
t.step()
await asyncio.sleep(0.02)
await s.stop()
run(body)
assert t.calls == [[10, 11], [99], [12, 13], [14]]
@pytest.mark.unit
def test_held_partials_wait_while_finals_still_go_and_run_once_released():
"""voice_worker.spec.md, "Scheduling": in a game, partial passes wait while an answer is being
written; finals never wait."""
t = FakeTranscriber()
held = {"on": True}
async def body():
s = SttScheduler(t, partials_held=lambda: held["on"])
s.start()
got = []
s.submit_partial("a", clip(10), PARTIAL, lambda r: got.append(("partial", r.text)))
s.submit_final("b", clip(20), FINAL, lambda r: got.append(("final", r.text)))
await asyncio.sleep(0.2)
before = list(got)
held["on"] = False
s.wake()
for _ in range(100):
if len(got) == 2:
break
await asyncio.sleep(0.01)
await s.stop()
return before, got
before, got = run(body)
assert before == [("final", "n20")], "the final went; the partial waited"
assert got == [("final", "n20"), ("partial", "n10")]