The words of the topic were the ones misheard live (화면공이 for 화면 공유, 계산지 for 계산기), and five models made the same mistakes. Each final is decoded as before and again with the last 180 s of the channel's conversation (everyone's finals and Jarvis's speech, at most context_chars=200 characters) as the previous-text prompt; the prompted text is used only when the plain one was kept, stays within half its letters and keeps every hotword count, as for the hint. Measured on the game PC (RTX 5070 Ti, 100 recordings of the user, 16 with their own text): CER 12.0% -> 7.5%, empty results unchanged, noise clips with text unchanged (16/108), final decode p95 170 -> 253 ms. Finals with a context are decoded one clip at a time (ctranslate2 needs the start token at one position in a batch). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
316 lines
9.1 KiB
Python
316 lines
9.1 KiB
Python
"""Voice worker: the shared-model STT scheduler (batching, finals first, partial coalescing)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import threading
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from voice_worker.scheduler import DecodeOptions, SttScheduler
|
|
|
|
FINAL = DecodeOptions(beam=2, language="ko", hotwords=())
|
|
PARTIAL = DecodeOptions(beam=1, language="ko", hotwords=())
|
|
|
|
|
|
class FakeTranscriber:
|
|
"""Names each clip by its length; the first call can be held to build up a queue."""
|
|
|
|
def __init__(self, hold_first: bool = False, fail_on_beam: int | None = None):
|
|
self.calls: list[tuple[DecodeOptions, list[int]]] = []
|
|
self.gate = threading.Event()
|
|
self.entered = threading.Event()
|
|
if not hold_first:
|
|
self.gate.set()
|
|
self.fail_on_beam = fail_on_beam
|
|
|
|
def transcribe(self, clips, opts, contexts=None):
|
|
self.entered.set()
|
|
self.gate.wait(5)
|
|
self.calls.append((opts, [len(c) for c in clips]))
|
|
if opts.beam == self.fail_on_beam:
|
|
raise RuntimeError("boom")
|
|
return [f"n{len(c)}" for c in clips]
|
|
|
|
|
|
def clip(n: int) -> np.ndarray:
|
|
return np.zeros(n, dtype=np.float32)
|
|
|
|
|
|
async def _run(body):
|
|
return await asyncio.wait_for(body(), 10)
|
|
|
|
|
|
def run(body):
|
|
return asyncio.run(_run(body))
|
|
|
|
|
|
async def _wait_entered(t: FakeTranscriber):
|
|
while not t.entered.is_set():
|
|
await asyncio.sleep(0.005)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_finals_queued_while_busy_go_in_one_batch_before_partials():
|
|
t = FakeTranscriber(hold_first=True)
|
|
|
|
async def body():
|
|
s = SttScheduler(t, max_partial_batch=4)
|
|
s.start()
|
|
got = []
|
|
done = asyncio.Event()
|
|
|
|
def cb(name):
|
|
def _cb(res):
|
|
got.append((name, res.text))
|
|
if len(got) == 5:
|
|
done.set()
|
|
return _cb
|
|
|
|
s.submit_partial(("a", "1"), clip(10), PARTIAL, cb("p-first"))
|
|
await _wait_entered(t)
|
|
s.submit_partial(("a", "2"), clip(20), PARTIAL, cb("p2"))
|
|
s.submit_final(("a", "3"), clip(30), FINAL, cb("f3"))
|
|
s.submit_final(("a", "4"), clip(40), FINAL, cb("f4"))
|
|
s.submit_partial(("a", "5"), clip(50), PARTIAL, cb("p5"))
|
|
t.gate.set()
|
|
await done.wait()
|
|
await s.stop()
|
|
return got
|
|
|
|
got = run(body)
|
|
assert [c[1] for c in t.calls] == [[10], [30, 40], [20, 50]]
|
|
assert [c[0].beam for c in t.calls] == [1, 2, 1]
|
|
assert got == [("p-first", "n10"), ("f3", "n30"), ("f4", "n40"), ("p2", "n20"), ("p5", "n50")]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_newer_partial_replaces_the_waiting_one_for_the_same_speaker():
|
|
t = FakeTranscriber(hold_first=True)
|
|
|
|
async def body():
|
|
s = SttScheduler(t)
|
|
s.start()
|
|
got = []
|
|
done = asyncio.Event()
|
|
s.submit_final(("a", "x"), clip(5), FINAL, lambda r: None)
|
|
await _wait_entered(t)
|
|
s.submit_partial(("a", "1"), clip(10), PARTIAL, lambda r: got.append(r.text))
|
|
s.submit_partial(("a", "1"), clip(20), PARTIAL, lambda r: (got.append(r.text), done.set()))
|
|
t.gate.set()
|
|
await done.wait()
|
|
await asyncio.sleep(0.05)
|
|
await s.stop()
|
|
return got
|
|
|
|
assert run(body) == ["n20"]
|
|
assert [c[1] for c in t.calls] == [[5], [20]]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_dropped_partials_are_never_transcribed():
|
|
t = FakeTranscriber(hold_first=True)
|
|
|
|
async def body():
|
|
s = SttScheduler(t)
|
|
s.start()
|
|
done = asyncio.Event()
|
|
got = []
|
|
s.submit_final(("a", "x"), clip(5), FINAL, lambda r: None)
|
|
await _wait_entered(t)
|
|
s.submit_partial(("a", "1"), clip(10), PARTIAL, lambda r: got.append(r.text))
|
|
s.submit_partial(("b", "1"), clip(20), PARTIAL, lambda r: (got.append(r.text), done.set()))
|
|
s.drop_partials(lambda key: key[0] == "a")
|
|
t.gate.set()
|
|
await done.wait()
|
|
await s.stop()
|
|
return got
|
|
|
|
assert run(body) == ["n20"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_different_decode_settings_are_not_mixed_in_one_batch():
|
|
t = FakeTranscriber(hold_first=True)
|
|
other = DecodeOptions(beam=2, language=None, hotwords=("자비스",))
|
|
|
|
async def body():
|
|
s = SttScheduler(t)
|
|
s.start()
|
|
n = []
|
|
done = asyncio.Event()
|
|
|
|
def cb(r):
|
|
n.append(r)
|
|
if len(n) == 4:
|
|
done.set()
|
|
|
|
s.submit_final(("a", "x"), clip(1), FINAL, cb)
|
|
await _wait_entered(t)
|
|
s.submit_final(("a", "1"), clip(10), FINAL, cb)
|
|
s.submit_final(("b", "1"), clip(20), other, cb)
|
|
s.submit_final(("a", "2"), clip(30), FINAL, cb)
|
|
t.gate.set()
|
|
await done.wait()
|
|
await s.stop()
|
|
|
|
run(body)
|
|
assert sorted(c[1] for c in t.calls[1:]) == [[10, 30], [20]]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_batches_are_capped():
|
|
t = FakeTranscriber(hold_first=True)
|
|
|
|
async def body():
|
|
s = SttScheduler(t, max_batch=3)
|
|
s.start()
|
|
n = []
|
|
done = asyncio.Event()
|
|
|
|
def cb(r):
|
|
n.append(r)
|
|
if len(n) == 8:
|
|
done.set()
|
|
|
|
s.submit_final(("a", "x"), clip(1), FINAL, cb)
|
|
await _wait_entered(t)
|
|
for i in range(7):
|
|
s.submit_final(("a", str(i)), clip(10 + i), FINAL, cb)
|
|
t.gate.set()
|
|
await done.wait()
|
|
await s.stop()
|
|
|
|
run(body)
|
|
assert [len(c[1]) for c in t.calls] == [1, 3, 3, 1]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_failing_batch_reports_the_error_and_the_scheduler_keeps_going():
|
|
t = FakeTranscriber(fail_on_beam=2)
|
|
|
|
async def body():
|
|
s = SttScheduler(t)
|
|
s.start()
|
|
out = []
|
|
done = asyncio.Event()
|
|
s.submit_final(("a", "1"), clip(10), FINAL, out.append)
|
|
s.submit_partial(("a", "2"), clip(20), PARTIAL, lambda r: (out.append(r), done.set()))
|
|
await done.wait()
|
|
await s.stop()
|
|
return out
|
|
|
|
out = run(body)
|
|
assert out[0].text == "" and out[0].error
|
|
assert out[1].text == "n20" and out[1].error is None
|
|
assert out[0].stt_ms >= 0
|
|
|
|
|
|
|
|
class StepTranscriber:
|
|
"""Every call waits for one ``step()``, so the test decides when each batch finishes."""
|
|
|
|
def __init__(self):
|
|
self.calls: list[list[int]] = []
|
|
self._steps = threading.Semaphore(0)
|
|
|
|
def step(self):
|
|
self._steps.release()
|
|
|
|
def transcribe(self, clips, opts, contexts=None):
|
|
self.calls.append([len(c) for c in clips])
|
|
self._steps.acquire(timeout=5)
|
|
return ["" for _ in clips]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_each_final_reaches_the_model_with_its_own_context():
|
|
class Ctx:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
def transcribe(self, clips, opts, contexts=None):
|
|
self.calls.append(list(contexts or []))
|
|
return [f"n{len(c)}" for c in clips]
|
|
|
|
eng = Ctx()
|
|
|
|
async def body():
|
|
s = SttScheduler(eng)
|
|
s.start()
|
|
got = []
|
|
done = asyncio.Event()
|
|
|
|
def cb(r):
|
|
got.append(r.text)
|
|
if len(got) == 3:
|
|
done.set()
|
|
s.submit_final("a", clip(100), FINAL, cb, context="화면 공유")
|
|
s.submit_final("b", clip(200), FINAL, cb)
|
|
s.submit_partial("c", clip(300), PARTIAL, cb)
|
|
await asyncio.wait_for(done.wait(), 5)
|
|
await s.stop()
|
|
|
|
asyncio.run(body())
|
|
assert eng.calls[0] == ["화면 공유", ""]
|
|
assert eng.calls[1] == [""], "partials go without context"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_partial_batches_are_kept_small_so_finals_can_cut_in():
|
|
t = StepTranscriber()
|
|
|
|
async def until(n):
|
|
while len(t.calls) < n:
|
|
await asyncio.sleep(0.005)
|
|
|
|
async def body():
|
|
s = SttScheduler(t, max_partial_batch=2)
|
|
s.start()
|
|
for i in range(5):
|
|
s.submit_partial(("a", str(i)), clip(10 + i), PARTIAL, lambda r: None)
|
|
await until(1) # first partial batch is running
|
|
s.submit_final(("a", "late"), clip(99), FINAL, lambda r: None)
|
|
t.step()
|
|
await until(2)
|
|
t.step()
|
|
await until(3)
|
|
t.step()
|
|
await until(4)
|
|
t.step()
|
|
await asyncio.sleep(0.02)
|
|
await s.stop()
|
|
|
|
run(body)
|
|
assert t.calls == [[10, 11], [99], [12, 13], [14]]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_held_partials_wait_while_finals_still_go_and_run_once_released():
|
|
"""voice_worker.spec.md, "Scheduling": in a game, partial passes wait while an answer is being
|
|
written; finals never wait."""
|
|
t = FakeTranscriber()
|
|
held = {"on": True}
|
|
|
|
async def body():
|
|
s = SttScheduler(t, partials_held=lambda: held["on"])
|
|
s.start()
|
|
got = []
|
|
s.submit_partial("a", clip(10), PARTIAL, lambda r: got.append(("partial", r.text)))
|
|
s.submit_final("b", clip(20), FINAL, lambda r: got.append(("final", r.text)))
|
|
await asyncio.sleep(0.2)
|
|
before = list(got)
|
|
held["on"] = False
|
|
s.wake()
|
|
for _ in range(100):
|
|
if len(got) == 2:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
await s.stop()
|
|
return before, got
|
|
|
|
before, got = run(body)
|
|
assert before == [("final", "n20")], "the final went; the partial waited"
|
|
assert got == [("final", "n20"), ("partial", "n10")]
|