From 3d58cf8ab1ca3363a9d3fce518d7f5273108a901 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sat, 14 Nov 2026 10:58:11 +0900 Subject: [PATCH] feat(voice): in a game, partial transcription waits while an answer is written - the app's ready message says whether the game stage runs (llm ready game=true); the worker then holds partial passes while a reply is being written and runs them again, the newest audio per speaker, when no answer is; finals never wait; without a game nothing changes - M0 gamemode_tune decision: the game placement answered 1.7 times faster with partials held (64.9 against 37.5 tok/s with a partial pass every 2 s) Co-Authored-By: Claude Opus 5.5 --- app/src/services.rs | 20 ++++++++------- app/src/worker_client.rs | 11 +++++++- app/tests/worker_client.rs | 11 ++++++++ src/voice_worker/scheduler.py | 11 ++++++-- src/voice_worker/server.py | 27 ++++++++++++++++---- src/voice_worker/voice_worker.spec.md | 13 +++++++--- tests/test_voice_worker_brain.py | 36 +++++++++++++++++++++++++++ tests/test_voice_worker_scheduler.py | 29 +++++++++++++++++++++ 8 files changed, 138 insertions(+), 20 deletions(-) diff --git a/app/src/services.rs b/app/src/services.rs index 6303973..922a60c 100644 --- a/app/src/services.rs +++ b/app/src/services.rs @@ -214,7 +214,8 @@ pub async fn start_services(data_dir: &Path, llm: Arc) -> Result) -> u64; - fn llm_state(&self, state: &str) -> bool; + /// `ready` (with whether a game runs) or `paused`. + fn llm_state(&self, state: &str, game: bool) -> bool; async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool; /// The STT worker's model is loaded (on the graphics card). fn stt_on_card(&self) -> bool; @@ -225,8 +226,8 @@ impl LlmBrain for WorkerClient { fn llm_switching(&self, notice: Option<&str>) -> u64 { WorkerClient::llm_switching(self, notice) } - fn llm_state(&self, state: &str) -> bool { - WorkerClient::llm_state(self, state) + fn llm_state(&self, state: &str, game: bool) -> bool { + if state == "ready" { WorkerClient::llm_ready(self, game) } else { WorkerClient::llm_state(self, state) } } async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool { WorkerClient::wait_llm_idle(self, id, t).await @@ -305,7 +306,8 @@ async fn perf_loop(settings: crate::settings::Settings, llm: Arc, brain: Ar back } }; - brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }); + // In the game stage partial passes wait while an answer is written (M0 gamemode_tune). + brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }, ended == Stage::Gaming); } } @@ -350,8 +352,8 @@ mod perf_tests { self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default())); 1 } - fn llm_state(&self, state: &str) -> bool { - self.0.lock().unwrap().push(state.into()); + fn llm_state(&self, state: &str, game: bool) -> bool { + self.0.lock().unwrap().push(if game { format!("{state} (game)") } else { state.into() }); true } async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool { @@ -389,8 +391,8 @@ mod perf_tests { panic!("never {want}: {:?}", brain.0.lock().unwrap()); } }; - until("ready").await; - assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready"]); + until("ready (game)").await; + assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready (game)"]); assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l)); assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming")); // The game ends (no wait set): back to the normal stage. @@ -506,7 +508,7 @@ mod perf_tests { self.0.lock().unwrap().push(("switching".into(), std::time::Instant::now())); 1 } - fn llm_state(&self, state: &str) -> bool { + fn llm_state(&self, state: &str, _game: bool) -> bool { self.0.lock().unwrap().push((state.into(), std::time::Instant::now())); true } diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index d6d60be..591fd2b 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -379,7 +379,16 @@ impl WorkerClient { /// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing). pub fn llm_state(&self, state: &str) -> bool { - let m = json!({"type": "llm", "state": state}); + self.set_llm(json!({"type": "llm", "state": state})) + } + + /// Tell the brain the LLM is ready, and whether a game runs (then partial passes wait while an + /// answer is written). + pub fn llm_ready(&self, game: bool) -> bool { + self.set_llm(json!({"type": "llm", "state": "ready", "game": game})) + } + + fn set_llm(&self, m: Value) -> bool { let mut llm = self.inner.llm.lock().unwrap(); *llm = Some(m.clone()); self.send_json(m) diff --git a/app/tests/worker_client.rs b/app/tests/worker_client.rs index 318c40d..96c6e94 100644 --- a/app/tests/worker_client.rs +++ b/app/tests/worker_client.rs @@ -492,3 +492,14 @@ async fn an_llm_state_set_while_the_worker_connects_again_is_the_last_it_gets() c.close(); } } + +#[tokio::test] +async fn ready_tells_the_worker_whether_a_game_runs() { + let w = FakeWorker::start(0).await; + let (c, _rx) = client(&w.url(), json!({})); + c.start(); + c.wait_ready(Duration::from_secs(2)).await.unwrap(); + assert!(c.llm_ready(true)); + until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready", "game": true})))).await; + c.close(); +} diff --git a/src/voice_worker/scheduler.py b/src/voice_worker/scheduler.py index 7204f6a..6174d69 100644 --- a/src/voice_worker/scheduler.py +++ b/src/voice_worker/scheduler.py @@ -47,8 +47,11 @@ class _Job: class SttScheduler: - def __init__(self, transcriber: Transcriber, max_batch: int = 16, max_partial_batch: int = 2): + def __init__(self, transcriber: Transcriber, max_batch: int = 16, max_partial_batch: int = 2, + partials_held: Callable[[], bool] = lambda: False): self.transcriber = transcriber + # While this says so (an answer is written in a game), partial passes wait; finals never do. + self.partials_held = partials_held self.max_batch = max_batch # Whisper's encoder always runs on a 30 s window, so its cost grows with every clip in a batch # and batching saves little. A small partial batch bounds how long a final can wait behind one. @@ -80,6 +83,10 @@ class SttScheduler: self._partials[key] = _Job(key, audio, opts, callback) self._wake.set() + def wake(self) -> None: + """Look again (partials are no longer held).""" + self._wake.set() + def drop_partials(self, predicate: Callable[[Hashable], bool]) -> None: for key in [k for k in self._partials if predicate(k)]: del self._partials[key] @@ -87,7 +94,7 @@ class SttScheduler: def _take(self) -> tuple[str, list[_Job]]: if self._finals: kind, pool = "final", self._finals - elif self._partials: + elif self._partials and not self.partials_held(): kind, pool = "partial", list(self._partials.values()) else: return "", [] diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index a67f2e8..a63ff91 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -49,6 +49,9 @@ class _LlmPaused(Exception): class VoiceWorker: + def _partials_held(self) -> bool: + return self.llm_game and self.answering > 0 + def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream], tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05, history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0, @@ -80,6 +83,10 @@ class VoiceWorker: self.llm_active = 0 self.llm_idle_to = None # the session to tell once nothing runs self.llm_switch_id = None # the switch to name in llm_idle + # In a game (the app says so with `ready`), partial passes wait while an answer is written + # (M0 gamemode_tune: the LLM answers 1.7 times faster); finals never wait. + self.llm_game = False + self.answering = 0 self.chat_timeout_s = chat_timeout_s self.delivery_timeout_s = delivery_timeout_s # a chat answer the app never reports: not posted # chat_action ids are unique across connections and worker runs, so a late answer from an @@ -105,7 +112,7 @@ class VoiceWorker: if n: debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker") if self.transcriber is not None: - self.scheduler = SttScheduler(self.transcriber) + self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held) self.scheduler.start() if self.tts_engine is not None: self.tts = TtsService(self.tts_engine) @@ -541,13 +548,17 @@ class Session: asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early)) def _on_llm(self, msg: dict) -> None: - state, notice, sid = msg.get("state"), msg.get("notice"), msg.get("id") + state, notice, sid, game = msg.get("state"), msg.get("notice"), msg.get("id"), msg.get("game", False) if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)) \ - or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))): - self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice and an optional integer id") + or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))) or not isinstance(game, bool): + self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice, integer id and boolean game") return w = self.worker w.llm_state = state + if state == "ready": + w.llm_game = game + if w.scheduler: + w.scheduler.wake() debug_log(f"llm {state}", "llm") if state == "ready": w.llm_notice, w.llm_idle_to = None, None @@ -735,7 +746,13 @@ class Session: if self.chat_channels else None try: async with self._llm(rid if speak else None): - answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat) + w.answering += 1 + try: + answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat) + finally: + w.answering -= 1 + if not w.answering and w.scheduler: + w.scheduler.wake() # held partials go on except Exception as e: # the brain must never take the session down debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm") answer = None diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index ff7a8a0..73b6604 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -125,7 +125,8 @@ wait seconds behind it; two clips keep that wait to about one clip's time. With captions fall behind and only the newest audio per speaker is transcribed. Batches are grouped by decode settings (beam, language, hotwords). A partial whose utterance has -already ended, or that is older than the last partial sent, is dropped. +already ended, or that is older than the last partial sent, is dropped. In a game, partials wait +while an answer is being written ("LLM switches"); finals never wait. ### Hallucination gate @@ -356,7 +357,7 @@ The app restarts the LLM with another placement when a game starts or ends (app/ ``` -> {"type": "llm", "state": "switching", "id": 7, "notice": "잠깐만요, 게임 모드로 바꾸는 중이에요"} // id, notice optional <- {"type": "llm_idle", "id": 7} // nothing that uses the LLM runs any more (id: the switch's, when it had one) --> {"type": "llm", "state": "ready"} // the LLM answers again +-> {"type": "llm", "state": "ready", "game": true} // the LLM answers again; game optional (default false) -> {"type": "llm", "state": "paused"} // the LLM is stopped (a game runs) ``` @@ -369,7 +370,13 @@ The app restarts the LLM with another placement when a game starts or ends (app/ then fails without calling the brain (the LLM may still be away, or a measuring server stand in). - On `paused` brain work answers nothing at once (`reply_text` `failed`, judgements failed) until `ready`; what waited for a switch stops waiting the same way. -- Any other `state`, a notice that is not a string or an id that is not an integer is `bad_message`. +- `game` on `ready` says the LLM runs in the game placement: from then on, while a `reply` is being + written, partial passes wait (finals go on at once, "Scheduling"); when no answer is being written + they go on, the newest audio per speaker. `ready` without it (or false) ends that. Without a game + partials never wait (M0 `gamemode_tune`: with partials held the game placement answered 1.7 + times faster, 64.9 against 37.5 tok/s with a partial pass every 2 s). +- Any other `state`, a notice that is not a string, an id that is not an integer or a `game` that is + not a boolean is `bad_message`. - The app sends its latest `llm` message again first on every connection, so a worker that missed one (a dropped connection, a restarted worker) catches up. diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index eaddf12..acec860 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -1254,3 +1254,39 @@ def test_a_request_that_waits_too_long_for_a_switch_fails_without_the_brain(stor seen = asyncio.run(with_worker(body, store=store, brain=brain)) reply = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"][0] assert reply["failed"] and brain.calls == [], "the LLM may be away: the brain is not called" + + +@pytest.mark.unit +def test_in_a_game_no_partials_are_transcribed_while_an_answer_is_written(store): + """voice_worker.spec.md, "LLM switches": `game` on ready holds partial passes while the brain + writes an answer (M0 gamemode_tune); without a game they go on.""" + import threading + from tests.test_voice_worker_server import stream + + for game in (True, False): + release = threading.Event() + + def slow(user, text): + release.wait(10) + return f"answer for {text}" + + async def body(url): + async with connect(url) as ws: + await hello(ws, stt={"partial_interval_ms": 100}) + await send(ws, type="llm", state="ready", game=game) + await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False) + await asyncio.sleep(0.1) + await stream(ws, "200", 0.9, 800, realtime=True) + seen = await drain(ws, 0.8) + release.set() + seen += await drain(ws, 0.5) + return seen + + seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow))) + partials = [m for m in seen if isinstance(m, dict) and m.get("type") == "partial"] + replies = [i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "reply_text"] + first_partial = next((i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "partial"), None) + if game: + assert first_partial is None or (replies and first_partial > replies[0]), "no partial while answering" + else: + assert partials and replies and first_partial < replies[0], "partials go on without a game" diff --git a/tests/test_voice_worker_scheduler.py b/tests/test_voice_worker_scheduler.py index 71e85c3..1cc674f 100644 --- a/tests/test_voice_worker_scheduler.py +++ b/tests/test_voice_worker_scheduler.py @@ -251,3 +251,32 @@ def test_partial_batches_are_kept_small_so_finals_can_cut_in(): run(body) assert t.calls == [[10, 11], [99], [12, 13], [14]] + + +@pytest.mark.unit +def test_held_partials_wait_while_finals_still_go_and_run_once_released(): + """voice_worker.spec.md, "Scheduling": in a game, partial passes wait while an answer is being + written; finals never wait.""" + t = FakeTranscriber() + held = {"on": True} + + async def body(): + s = SttScheduler(t, partials_held=lambda: held["on"]) + s.start() + got = [] + s.submit_partial("a", clip(10), PARTIAL, lambda r: got.append(("partial", r.text))) + s.submit_final("b", clip(20), FINAL, lambda r: got.append(("final", r.text))) + await asyncio.sleep(0.2) + before = list(got) + held["on"] = False + s.wake() + for _ in range(100): + if len(got) == 2: + break + await asyncio.sleep(0.01) + await s.stop() + return before, got + + before, got = run(body) + assert before == [("final", "n20")], "the final went; the partial waited" + assert got == [("final", "n20"), ("partial", "n10")]