feat(voice): in a game, partial transcription waits while an answer is written

- the app's ready message says whether the game stage runs (llm ready game=true); the worker then
  holds partial passes while a reply is being written and runs them again, the newest audio per
  speaker, when no answer is; finals never wait; without a game nothing changes
- M0 gamemode_tune decision: the game placement answered 1.7 times faster with partials held
  (64.9 against 37.5 tok/s with a partial pass every 2 s)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-14 10:58:11 +09:00
parent 5316f4e9b0
commit 3d58cf8ab1
8 changed files with 138 additions and 20 deletions

View File

@@ -214,7 +214,8 @@ pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services,
pub trait LlmBrain: Send + Sync {
/// Announce a switch; its number for `wait_llm_idle`.
fn llm_switching(&self, notice: Option<&str>) -> u64;
fn llm_state(&self, state: &str) -> bool;
/// `ready` (with whether a game runs) or `paused`.
fn llm_state(&self, state: &str, game: bool) -> bool;
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool;
/// The STT worker's model is loaded (on the graphics card).
fn stt_on_card(&self) -> bool;
@@ -225,8 +226,8 @@ impl LlmBrain for WorkerClient {
fn llm_switching(&self, notice: Option<&str>) -> u64 {
WorkerClient::llm_switching(self, notice)
}
fn llm_state(&self, state: &str) -> bool {
WorkerClient::llm_state(self, state)
fn llm_state(&self, state: &str, game: bool) -> bool {
if state == "ready" { WorkerClient::llm_ready(self, game) } else { WorkerClient::llm_state(self, state) }
}
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool {
WorkerClient::wait_llm_idle(self, id, t).await
@@ -305,7 +306,8 @@ async fn perf_loop(settings: crate::settings::Settings, llm: Arc<Llm>, brain: Ar
back
}
};
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" });
// In the game stage partial passes wait while an answer is written (M0 gamemode_tune).
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }, ended == Stage::Gaming);
}
}
@@ -350,8 +352,8 @@ mod perf_tests {
self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default()));
1
}
fn llm_state(&self, state: &str) -> bool {
self.0.lock().unwrap().push(state.into());
fn llm_state(&self, state: &str, game: bool) -> bool {
self.0.lock().unwrap().push(if game { format!("{state} (game)") } else { state.into() });
true
}
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
@@ -389,8 +391,8 @@ mod perf_tests {
panic!("never {want}: {:?}", brain.0.lock().unwrap());
}
};
until("ready").await;
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready"]);
until("ready (game)").await;
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready (game)"]);
assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
// The game ends (no wait set): back to the normal stage.
@@ -506,7 +508,7 @@ mod perf_tests {
self.0.lock().unwrap().push(("switching".into(), std::time::Instant::now()));
1
}
fn llm_state(&self, state: &str) -> bool {
fn llm_state(&self, state: &str, _game: bool) -> bool {
self.0.lock().unwrap().push((state.into(), std::time::Instant::now()));
true
}

View File

@@ -379,7 +379,16 @@ impl WorkerClient {
/// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing).
pub fn llm_state(&self, state: &str) -> bool {
let m = json!({"type": "llm", "state": state});
self.set_llm(json!({"type": "llm", "state": state}))
}
/// Tell the brain the LLM is ready, and whether a game runs (then partial passes wait while an
/// answer is written).
pub fn llm_ready(&self, game: bool) -> bool {
self.set_llm(json!({"type": "llm", "state": "ready", "game": game}))
}
fn set_llm(&self, m: Value) -> bool {
let mut llm = self.inner.llm.lock().unwrap();
*llm = Some(m.clone());
self.send_json(m)

View File

@@ -492,3 +492,14 @@ async fn an_llm_state_set_while_the_worker_connects_again_is_the_last_it_gets()
c.close();
}
}
#[tokio::test]
async fn ready_tells_the_worker_whether_a_game_runs() {
let w = FakeWorker::start(0).await;
let (c, _rx) = client(&w.url(), json!({}));
c.start();
c.wait_ready(Duration::from_secs(2)).await.unwrap();
assert!(c.llm_ready(true));
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready", "game": true})))).await;
c.close();
}

View File

@@ -47,8 +47,11 @@ class _Job:
class SttScheduler:
def __init__(self, transcriber: Transcriber, max_batch: int = 16, max_partial_batch: int = 2):
def __init__(self, transcriber: Transcriber, max_batch: int = 16, max_partial_batch: int = 2,
partials_held: Callable[[], bool] = lambda: False):
self.transcriber = transcriber
# While this says so (an answer is written in a game), partial passes wait; finals never do.
self.partials_held = partials_held
self.max_batch = max_batch
# Whisper's encoder always runs on a 30 s window, so its cost grows with every clip in a batch
# and batching saves little. A small partial batch bounds how long a final can wait behind one.
@@ -80,6 +83,10 @@ class SttScheduler:
self._partials[key] = _Job(key, audio, opts, callback)
self._wake.set()
def wake(self) -> None:
"""Look again (partials are no longer held)."""
self._wake.set()
def drop_partials(self, predicate: Callable[[Hashable], bool]) -> None:
for key in [k for k in self._partials if predicate(k)]:
del self._partials[key]
@@ -87,7 +94,7 @@ class SttScheduler:
def _take(self) -> tuple[str, list[_Job]]:
if self._finals:
kind, pool = "final", self._finals
elif self._partials:
elif self._partials and not self.partials_held():
kind, pool = "partial", list(self._partials.values())
else:
return "", []

View File

@@ -49,6 +49,9 @@ class _LlmPaused(Exception):
class VoiceWorker:
def _partials_held(self) -> bool:
return self.llm_game and self.answering > 0
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0,
@@ -80,6 +83,10 @@ class VoiceWorker:
self.llm_active = 0
self.llm_idle_to = None # the session to tell once nothing runs
self.llm_switch_id = None # the switch to name in llm_idle
# In a game (the app says so with `ready`), partial passes wait while an answer is written
# (M0 gamemode_tune: the LLM answers 1.7 times faster); finals never wait.
self.llm_game = False
self.answering = 0
self.chat_timeout_s = chat_timeout_s
self.delivery_timeout_s = delivery_timeout_s # a chat answer the app never reports: not posted
# chat_action ids are unique across connections and worker runs, so a late answer from an
@@ -105,7 +112,7 @@ class VoiceWorker:
if n:
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
if self.transcriber is not None:
self.scheduler = SttScheduler(self.transcriber)
self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held)
self.scheduler.start()
if self.tts_engine is not None:
self.tts = TtsService(self.tts_engine)
@@ -541,13 +548,17 @@ class Session:
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early))
def _on_llm(self, msg: dict) -> None:
state, notice, sid = msg.get("state"), msg.get("notice"), msg.get("id")
state, notice, sid, game = msg.get("state"), msg.get("notice"), msg.get("id"), msg.get("game", False)
if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)) \
or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))):
self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice and an optional integer id")
or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))) or not isinstance(game, bool):
self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice, integer id and boolean game")
return
w = self.worker
w.llm_state = state
if state == "ready":
w.llm_game = game
if w.scheduler:
w.scheduler.wake()
debug_log(f"llm {state}", "llm")
if state == "ready":
w.llm_notice, w.llm_idle_to = None, None
@@ -735,7 +746,13 @@ class Session:
if self.chat_channels else None
try:
async with self._llm(rid if speak else None):
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
w.answering += 1
try:
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
finally:
w.answering -= 1
if not w.answering and w.scheduler:
w.scheduler.wake() # held partials go on
except Exception as e: # the brain must never take the session down
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
answer = None

View File

@@ -125,7 +125,8 @@ wait seconds behind it; two clips keep that wait to about one clip's time. With
captions fall behind and only the newest audio per speaker is transcribed.
Batches are grouped by decode settings (beam, language, hotwords). A partial whose utterance has
already ended, or that is older than the last partial sent, is dropped.
already ended, or that is older than the last partial sent, is dropped. In a game, partials wait
while an answer is being written ("LLM switches"); finals never wait.
### Hallucination gate
@@ -356,7 +357,7 @@ The app restarts the LLM with another placement when a game starts or ends (app/
```
-> {"type": "llm", "state": "switching", "id": 7, "notice": "잠깐만요, 게임 모드로 바꾸는 중이에요"} // id, notice optional
<- {"type": "llm_idle", "id": 7} // nothing that uses the LLM runs any more (id: the switch's, when it had one)
-> {"type": "llm", "state": "ready"} // the LLM answers again
-> {"type": "llm", "state": "ready", "game": true} // the LLM answers again; game optional (default false)
-> {"type": "llm", "state": "paused"} // the LLM is stopped (a game runs)
```
@@ -369,7 +370,13 @@ The app restarts the LLM with another placement when a game starts or ends (app/
then fails without calling the brain (the LLM may still be away, or a measuring server stand in).
- On `paused` brain work answers nothing at once (`reply_text` `failed`, judgements failed) until
`ready`; what waited for a switch stops waiting the same way.
- Any other `state`, a notice that is not a string or an id that is not an integer is `bad_message`.
- `game` on `ready` says the LLM runs in the game placement: from then on, while a `reply` is being
written, partial passes wait (finals go on at once, "Scheduling"); when no answer is being written
they go on, the newest audio per speaker. `ready` without it (or false) ends that. Without a game
partials never wait (M0 `gamemode_tune`: with partials held the game placement answered 1.7
times faster, 64.9 against 37.5 tok/s with a partial pass every 2 s).
- Any other `state`, a notice that is not a string, an id that is not an integer or a `game` that is
not a boolean is `bad_message`.
- The app sends its latest `llm` message again first on every connection, so a worker that missed
one (a dropped connection, a restarted worker) catches up.

View File

@@ -1254,3 +1254,39 @@ def test_a_request_that_waits_too_long_for_a_switch_fails_without_the_brain(stor
seen = asyncio.run(with_worker(body, store=store, brain=brain))
reply = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"][0]
assert reply["failed"] and brain.calls == [], "the LLM may be away: the brain is not called"
@pytest.mark.unit
def test_in_a_game_no_partials_are_transcribed_while_an_answer_is_written(store):
"""voice_worker.spec.md, "LLM switches": `game` on ready holds partial passes while the brain
writes an answer (M0 gamemode_tune); without a game they go on."""
import threading
from tests.test_voice_worker_server import stream
for game in (True, False):
release = threading.Event()
def slow(user, text):
release.wait(10)
return f"answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partial_interval_ms": 100})
await send(ws, type="llm", state="ready", game=game)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await asyncio.sleep(0.1)
await stream(ws, "200", 0.9, 800, realtime=True)
seen = await drain(ws, 0.8)
release.set()
seen += await drain(ws, 0.5)
return seen
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow)))
partials = [m for m in seen if isinstance(m, dict) and m.get("type") == "partial"]
replies = [i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "reply_text"]
first_partial = next((i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "partial"), None)
if game:
assert first_partial is None or (replies and first_partial > replies[0]), "no partial while answering"
else:
assert partials and replies and first_partial < replies[0], "partials go on without a game"

View File

@@ -251,3 +251,32 @@ def test_partial_batches_are_kept_small_so_finals_can_cut_in():
run(body)
assert t.calls == [[10, 11], [99], [12, 13], [14]]
@pytest.mark.unit
def test_held_partials_wait_while_finals_still_go_and_run_once_released():
"""voice_worker.spec.md, "Scheduling": in a game, partial passes wait while an answer is being
written; finals never wait."""
t = FakeTranscriber()
held = {"on": True}
async def body():
s = SttScheduler(t, partials_held=lambda: held["on"])
s.start()
got = []
s.submit_partial("a", clip(10), PARTIAL, lambda r: got.append(("partial", r.text)))
s.submit_final("b", clip(20), FINAL, lambda r: got.append(("final", r.text)))
await asyncio.sleep(0.2)
before = list(got)
held["on"] = False
s.wake()
for _ in range(100):
if len(got) == 2:
break
await asyncio.sleep(0.01)
await s.stop()
return before, got
before, got = run(body)
assert before == [("final", "n20")], "the final went; the partial waited"
assert got == [("final", "n20"), ("partial", "n10")]