feat(voice): in a game, partial transcription waits while an answer is written
- the app's ready message says whether the game stage runs (llm ready game=true); the worker then holds partial passes while a reply is being written and runs them again, the newest audio per speaker, when no answer is; finals never wait; without a game nothing changes - M0 gamemode_tune decision: the game placement answered 1.7 times faster with partials held (64.9 against 37.5 tok/s with a partial pass every 2 s) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -214,7 +214,8 @@ pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services,
|
||||
pub trait LlmBrain: Send + Sync {
|
||||
/// Announce a switch; its number for `wait_llm_idle`.
|
||||
fn llm_switching(&self, notice: Option<&str>) -> u64;
|
||||
fn llm_state(&self, state: &str) -> bool;
|
||||
/// `ready` (with whether a game runs) or `paused`.
|
||||
fn llm_state(&self, state: &str, game: bool) -> bool;
|
||||
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool;
|
||||
/// The STT worker's model is loaded (on the graphics card).
|
||||
fn stt_on_card(&self) -> bool;
|
||||
@@ -225,8 +226,8 @@ impl LlmBrain for WorkerClient {
|
||||
fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
||||
WorkerClient::llm_switching(self, notice)
|
||||
}
|
||||
fn llm_state(&self, state: &str) -> bool {
|
||||
WorkerClient::llm_state(self, state)
|
||||
fn llm_state(&self, state: &str, game: bool) -> bool {
|
||||
if state == "ready" { WorkerClient::llm_ready(self, game) } else { WorkerClient::llm_state(self, state) }
|
||||
}
|
||||
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool {
|
||||
WorkerClient::wait_llm_idle(self, id, t).await
|
||||
@@ -305,7 +306,8 @@ async fn perf_loop(settings: crate::settings::Settings, llm: Arc<Llm>, brain: Ar
|
||||
back
|
||||
}
|
||||
};
|
||||
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" });
|
||||
// In the game stage partial passes wait while an answer is written (M0 gamemode_tune).
|
||||
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }, ended == Stage::Gaming);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,8 +352,8 @@ mod perf_tests {
|
||||
self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default()));
|
||||
1
|
||||
}
|
||||
fn llm_state(&self, state: &str) -> bool {
|
||||
self.0.lock().unwrap().push(state.into());
|
||||
fn llm_state(&self, state: &str, game: bool) -> bool {
|
||||
self.0.lock().unwrap().push(if game { format!("{state} (game)") } else { state.into() });
|
||||
true
|
||||
}
|
||||
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
|
||||
@@ -389,8 +391,8 @@ mod perf_tests {
|
||||
panic!("never {want}: {:?}", brain.0.lock().unwrap());
|
||||
}
|
||||
};
|
||||
until("ready").await;
|
||||
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready"]);
|
||||
until("ready (game)").await;
|
||||
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready (game)"]);
|
||||
assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l));
|
||||
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
|
||||
// The game ends (no wait set): back to the normal stage.
|
||||
@@ -506,7 +508,7 @@ mod perf_tests {
|
||||
self.0.lock().unwrap().push(("switching".into(), std::time::Instant::now()));
|
||||
1
|
||||
}
|
||||
fn llm_state(&self, state: &str) -> bool {
|
||||
fn llm_state(&self, state: &str, _game: bool) -> bool {
|
||||
self.0.lock().unwrap().push((state.into(), std::time::Instant::now()));
|
||||
true
|
||||
}
|
||||
|
||||
@@ -379,7 +379,16 @@ impl WorkerClient {
|
||||
|
||||
/// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing).
|
||||
pub fn llm_state(&self, state: &str) -> bool {
|
||||
let m = json!({"type": "llm", "state": state});
|
||||
self.set_llm(json!({"type": "llm", "state": state}))
|
||||
}
|
||||
|
||||
/// Tell the brain the LLM is ready, and whether a game runs (then partial passes wait while an
|
||||
/// answer is written).
|
||||
pub fn llm_ready(&self, game: bool) -> bool {
|
||||
self.set_llm(json!({"type": "llm", "state": "ready", "game": game}))
|
||||
}
|
||||
|
||||
fn set_llm(&self, m: Value) -> bool {
|
||||
let mut llm = self.inner.llm.lock().unwrap();
|
||||
*llm = Some(m.clone());
|
||||
self.send_json(m)
|
||||
|
||||
@@ -492,3 +492,14 @@ async fn an_llm_state_set_while_the_worker_connects_again_is_the_last_it_gets()
|
||||
c.close();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_tells_the_worker_whether_a_game_runs() {
|
||||
let w = FakeWorker::start(0).await;
|
||||
let (c, _rx) = client(&w.url(), json!({}));
|
||||
c.start();
|
||||
c.wait_ready(Duration::from_secs(2)).await.unwrap();
|
||||
assert!(c.llm_ready(true));
|
||||
until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready", "game": true})))).await;
|
||||
c.close();
|
||||
}
|
||||
|
||||
@@ -47,8 +47,11 @@ class _Job:
|
||||
|
||||
|
||||
class SttScheduler:
|
||||
def __init__(self, transcriber: Transcriber, max_batch: int = 16, max_partial_batch: int = 2):
|
||||
def __init__(self, transcriber: Transcriber, max_batch: int = 16, max_partial_batch: int = 2,
|
||||
partials_held: Callable[[], bool] = lambda: False):
|
||||
self.transcriber = transcriber
|
||||
# While this says so (an answer is written in a game), partial passes wait; finals never do.
|
||||
self.partials_held = partials_held
|
||||
self.max_batch = max_batch
|
||||
# Whisper's encoder always runs on a 30 s window, so its cost grows with every clip in a batch
|
||||
# and batching saves little. A small partial batch bounds how long a final can wait behind one.
|
||||
@@ -80,6 +83,10 @@ class SttScheduler:
|
||||
self._partials[key] = _Job(key, audio, opts, callback)
|
||||
self._wake.set()
|
||||
|
||||
def wake(self) -> None:
|
||||
"""Look again (partials are no longer held)."""
|
||||
self._wake.set()
|
||||
|
||||
def drop_partials(self, predicate: Callable[[Hashable], bool]) -> None:
|
||||
for key in [k for k in self._partials if predicate(k)]:
|
||||
del self._partials[key]
|
||||
@@ -87,7 +94,7 @@ class SttScheduler:
|
||||
def _take(self) -> tuple[str, list[_Job]]:
|
||||
if self._finals:
|
||||
kind, pool = "final", self._finals
|
||||
elif self._partials:
|
||||
elif self._partials and not self.partials_held():
|
||||
kind, pool = "partial", list(self._partials.values())
|
||||
else:
|
||||
return "", []
|
||||
|
||||
@@ -49,6 +49,9 @@ class _LlmPaused(Exception):
|
||||
|
||||
|
||||
class VoiceWorker:
|
||||
def _partials_held(self) -> bool:
|
||||
return self.llm_game and self.answering > 0
|
||||
|
||||
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
|
||||
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
|
||||
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0,
|
||||
@@ -80,6 +83,10 @@ class VoiceWorker:
|
||||
self.llm_active = 0
|
||||
self.llm_idle_to = None # the session to tell once nothing runs
|
||||
self.llm_switch_id = None # the switch to name in llm_idle
|
||||
# In a game (the app says so with `ready`), partial passes wait while an answer is written
|
||||
# (M0 gamemode_tune: the LLM answers 1.7 times faster); finals never wait.
|
||||
self.llm_game = False
|
||||
self.answering = 0
|
||||
self.chat_timeout_s = chat_timeout_s
|
||||
self.delivery_timeout_s = delivery_timeout_s # a chat answer the app never reports: not posted
|
||||
# chat_action ids are unique across connections and worker runs, so a late answer from an
|
||||
@@ -105,7 +112,7 @@ class VoiceWorker:
|
||||
if n:
|
||||
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
|
||||
if self.transcriber is not None:
|
||||
self.scheduler = SttScheduler(self.transcriber)
|
||||
self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held)
|
||||
self.scheduler.start()
|
||||
if self.tts_engine is not None:
|
||||
self.tts = TtsService(self.tts_engine)
|
||||
@@ -541,13 +548,17 @@ class Session:
|
||||
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early))
|
||||
|
||||
def _on_llm(self, msg: dict) -> None:
|
||||
state, notice, sid = msg.get("state"), msg.get("notice"), msg.get("id")
|
||||
state, notice, sid, game = msg.get("state"), msg.get("notice"), msg.get("id"), msg.get("game", False)
|
||||
if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)) \
|
||||
or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))):
|
||||
self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice and an optional integer id")
|
||||
or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))) or not isinstance(game, bool):
|
||||
self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice, integer id and boolean game")
|
||||
return
|
||||
w = self.worker
|
||||
w.llm_state = state
|
||||
if state == "ready":
|
||||
w.llm_game = game
|
||||
if w.scheduler:
|
||||
w.scheduler.wake()
|
||||
debug_log(f"llm {state}", "llm")
|
||||
if state == "ready":
|
||||
w.llm_notice, w.llm_idle_to = None, None
|
||||
@@ -735,7 +746,13 @@ class Session:
|
||||
if self.chat_channels else None
|
||||
try:
|
||||
async with self._llm(rid if speak else None):
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
w.scheduler.wake() # held partials go on
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = None
|
||||
|
||||
@@ -125,7 +125,8 @@ wait seconds behind it; two clips keep that wait to about one clip's time. With
|
||||
captions fall behind and only the newest audio per speaker is transcribed.
|
||||
|
||||
Batches are grouped by decode settings (beam, language, hotwords). A partial whose utterance has
|
||||
already ended, or that is older than the last partial sent, is dropped.
|
||||
already ended, or that is older than the last partial sent, is dropped. In a game, partials wait
|
||||
while an answer is being written ("LLM switches"); finals never wait.
|
||||
|
||||
### Hallucination gate
|
||||
|
||||
@@ -356,7 +357,7 @@ The app restarts the LLM with another placement when a game starts or ends (app/
|
||||
```
|
||||
-> {"type": "llm", "state": "switching", "id": 7, "notice": "잠깐만요, 게임 모드로 바꾸는 중이에요"} // id, notice optional
|
||||
<- {"type": "llm_idle", "id": 7} // nothing that uses the LLM runs any more (id: the switch's, when it had one)
|
||||
-> {"type": "llm", "state": "ready"} // the LLM answers again
|
||||
-> {"type": "llm", "state": "ready", "game": true} // the LLM answers again; game optional (default false)
|
||||
-> {"type": "llm", "state": "paused"} // the LLM is stopped (a game runs)
|
||||
```
|
||||
|
||||
@@ -369,7 +370,13 @@ The app restarts the LLM with another placement when a game starts or ends (app/
|
||||
then fails without calling the brain (the LLM may still be away, or a measuring server stand in).
|
||||
- On `paused` brain work answers nothing at once (`reply_text` `failed`, judgements failed) until
|
||||
`ready`; what waited for a switch stops waiting the same way.
|
||||
- Any other `state`, a notice that is not a string or an id that is not an integer is `bad_message`.
|
||||
- `game` on `ready` says the LLM runs in the game placement: from then on, while a `reply` is being
|
||||
written, partial passes wait (finals go on at once, "Scheduling"); when no answer is being written
|
||||
they go on, the newest audio per speaker. `ready` without it (or false) ends that. Without a game
|
||||
partials never wait (M0 `gamemode_tune`: with partials held the game placement answered 1.7
|
||||
times faster, 64.9 against 37.5 tok/s with a partial pass every 2 s).
|
||||
- Any other `state`, a notice that is not a string, an id that is not an integer or a `game` that is
|
||||
not a boolean is `bad_message`.
|
||||
- The app sends its latest `llm` message again first on every connection, so a worker that missed
|
||||
one (a dropped connection, a restarted worker) catches up.
|
||||
|
||||
|
||||
@@ -1254,3 +1254,39 @@ def test_a_request_that_waits_too_long_for_a_switch_fails_without_the_brain(stor
|
||||
seen = asyncio.run(with_worker(body, store=store, brain=brain))
|
||||
reply = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"][0]
|
||||
assert reply["failed"] and brain.calls == [], "the LLM may be away: the brain is not called"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_in_a_game_no_partials_are_transcribed_while_an_answer_is_written(store):
|
||||
"""voice_worker.spec.md, "LLM switches": `game` on ready holds partial passes while the brain
|
||||
writes an answer (M0 gamemode_tune); without a game they go on."""
|
||||
import threading
|
||||
from tests.test_voice_worker_server import stream
|
||||
|
||||
for game in (True, False):
|
||||
release = threading.Event()
|
||||
|
||||
def slow(user, text):
|
||||
release.wait(10)
|
||||
return f"answer for {text}"
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partial_interval_ms": 100})
|
||||
await send(ws, type="llm", state="ready", game=game)
|
||||
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
||||
await asyncio.sleep(0.1)
|
||||
await stream(ws, "200", 0.9, 800, realtime=True)
|
||||
seen = await drain(ws, 0.8)
|
||||
release.set()
|
||||
seen += await drain(ws, 0.5)
|
||||
return seen
|
||||
|
||||
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow)))
|
||||
partials = [m for m in seen if isinstance(m, dict) and m.get("type") == "partial"]
|
||||
replies = [i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "reply_text"]
|
||||
first_partial = next((i for i, m in enumerate(seen) if isinstance(m, dict) and m.get("type") == "partial"), None)
|
||||
if game:
|
||||
assert first_partial is None or (replies and first_partial > replies[0]), "no partial while answering"
|
||||
else:
|
||||
assert partials and replies and first_partial < replies[0], "partials go on without a game"
|
||||
|
||||
@@ -251,3 +251,32 @@ def test_partial_batches_are_kept_small_so_finals_can_cut_in():
|
||||
|
||||
run(body)
|
||||
assert t.calls == [[10, 11], [99], [12, 13], [14]]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_held_partials_wait_while_finals_still_go_and_run_once_released():
|
||||
"""voice_worker.spec.md, "Scheduling": in a game, partial passes wait while an answer is being
|
||||
written; finals never wait."""
|
||||
t = FakeTranscriber()
|
||||
held = {"on": True}
|
||||
|
||||
async def body():
|
||||
s = SttScheduler(t, partials_held=lambda: held["on"])
|
||||
s.start()
|
||||
got = []
|
||||
s.submit_partial("a", clip(10), PARTIAL, lambda r: got.append(("partial", r.text)))
|
||||
s.submit_final("b", clip(20), FINAL, lambda r: got.append(("final", r.text)))
|
||||
await asyncio.sleep(0.2)
|
||||
before = list(got)
|
||||
held["on"] = False
|
||||
s.wake()
|
||||
for _ in range(100):
|
||||
if len(got) == 2:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
await s.stop()
|
||||
return before, got
|
||||
|
||||
before, got = run(body)
|
||||
assert before == [("final", "n20")], "the final went; the partial waited"
|
||||
assert got == [("final", "n20"), ("partial", "n10")]
|
||||
|
||||
Reference in New Issue
Block a user