From aa6d950af6e7cc5f9d125680e92469fe8c58d9f3 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sat, 14 Nov 2026 10:34:24 +0900 Subject: [PATCH] fix(app): performance stage switches survive failures, reconnects and late signals - the normal stage reads other programs' memory afresh (the card after the LLM stopped, less the STT worker's model) and keeps it; the game stage counts from the latest normal reading, so the game is not counted against its own reserve - a restart that fails brings back the stage that ran before and tells the brain ready; the failed stage is tried again after 60 s, three times at most; when nothing comes back the brain is told paused so requests fail at once - the worker client keeps the latest llm state and sends it first on every connection - switches are numbered: llm switching carries an id, llm_idle names it, and a late idle of an earlier switch no longer lets a new one through - a request that waited 180 s for a switch fails without calling the brain - the dashboard shows the stage that runs also on a fallback placement without experts Co-Authored-By: Claude Opus 5.5 --- app/src/admin.spec.md | 28 ++++-- app/src/perf.rs | 57 ++++++++++++ app/src/placement.rs | 4 +- app/src/services.rs | 123 +++++++++++++++++++------- app/src/strata.rs | 101 +++++++++++++++++---- app/src/worker_client.rs | 49 +++++++--- app/tests/worker_client.rs | 39 ++++++-- src/voice_worker/server.py | 19 ++-- src/voice_worker/voice_worker.spec.md | 10 ++- tests/test_voice_worker_brain.py | 31 +++++++ 10 files changed, 374 insertions(+), 87 deletions(-) diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 37eb9f5..d59a435 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -353,15 +353,23 @@ between the normal placement and the game placement as games come and go (not wi - A change: the brain is told the LLM switches, with a notice for the first spoken request that waits ("잠깐만요, 게임 모드로 바꾸는 중이에요", or "잠깐만요, 평소 모드로 돌아가는 중이에요" on the way back; `voice_worker.spec.md`, "LLM switches"); the app waits until the brain has written the - answers in progress (at most 2 minutes), restarts the LLM in the new stage and tells the brain - it is ready (or paused). A restart that fails leaves the LLM `failed` on the dashboard and the - brain ready (its calls then fail); the stage is not tried again until the wanted stage changes. + answers in progress (at most 2 minutes; the switch is numbered, so an idle report for an earlier + switch that came late does not count), restarts the LLM in the new stage and tells the brain it + is ready (or paused). The worker client keeps that latest state and sends it again on every + connection. A restart that fails brings back the stage that ran before and tells the brain + ready; the failed stage is tried again after 60 s, three times at most, then not until the + wanted stage changes. When the stage before does not come back either, the LLM shows `failed` + and the brain is told paused (requests fail at once instead of waiting). - The game stage: context 16,384 (or the settings' smaller one), one request at a time, half the engine's CPU threads (read from its log at the normal start, else a quarter of the logical CPUs), and the game placement (`placement.rs`: the reserve beside more than 2 GiB of experts, else - 2 GiB, else 0 GiB, else pause), measured once at that context like the normal one. Every - placement of a run counts other programs' memory as it was when the services started (before - the LLM and the STT worker), not from a card that then also holds the worker and the game. + 2 GiB, else 0 GiB, else pause), measured once at that context like the normal one. Other + programs' memory is what the card holds after the LLM stopped, less the STT worker's model while + it is loaded (1300 MiB); the normal stage reads it afresh each time (programs opened since + count) and keeps it; the game stage takes the one kept at the latest normal placement, as the + card then also holds the game, which its reserve is for (a game already running when the + services start is counted in it, which only leaves the LLM less). The dashboard shows the stage + that runs, also on a fallback placement without experts. - Pause: the LLM stops (`llm` off, "게임 중 일시정지"); the brain answers nothing until the stage changes. - The foreground program is a game, in this order: in `notGames`: no; in `games`: yes @@ -598,8 +606,12 @@ a path with non-ASCII letters) or from a Battle.net game's install folder (a tes turns game mode on with its name and off once it ends. Performance stages (on every system, with a fake Strata and card): a game moves the LLM to the game placement (measured once at 16K and one request, 2 GiB on the game PC's numbers, half the 16 engine threads) and back after the wait, pause stops -it, a card with no room for a game pauses, and the stage loop tells the brain switching (with the -notice), waits, restarts and tells it ready, also on the way back. On every system: game detection's rules +it, a card with no room for a game pauses, the normal stage counts a program opened since and the +game stage does not count the game twice, a fallback game placement still shows as the game +stage, and the stage loop tells the brain switching (with the notice), waits, restarts and tells +it ready, also on the way back; a game stage that cannot start goes back to normal each time and +is tried three times. The worker client ignores a late idle of an earlier switch and sends the +latest LLM state first when a worker connects again. On every system: game detection's rules (a borderless game, a maximised program, a browser's full-screen video, exclusive full screen, launcher folders and their own windows, Battle.net games by install folder and their uninstall entries, non-ASCII paths, the person's lists first, staying on until the game ends, diff --git a/app/src/perf.rs b/app/src/perf.rs index b5604bf..4f768bb 100644 --- a/app/src/perf.rs +++ b/app/src/perf.rs @@ -47,6 +47,43 @@ impl Perf { } } +/// How often a stage that failed to start is tried again (then not until the wanted stage changes). +pub const MAX_TRIES: u32 = 3; + +/// The limited retry of a stage whose restart failed: after a failure it waits `after`, and after +/// `MAX_TRIES` failures of the same stage it stops until another stage is wanted. +#[derive(Debug)] +pub struct Retry { + after: Duration, + stage: Option, + tries: u32, + next: Option, +} + +impl Retry { + pub fn new(after: Duration) -> Self { + Self { after, stage: None, tries: 0, next: None } + } + + /// Whether switching to `want` may be tried now. + pub fn allowed(&self, want: Stage, now: Instant) -> bool { + self.stage != Some(want) || (self.tries < MAX_TRIES && self.next.is_none_or(|n| now >= n)) + } + + pub fn failed(&mut self, want: Stage, now: Instant) { + if self.stage == Some(want) { + self.tries += 1; + } else { + (self.stage, self.tries) = (Some(want), 1); + } + self.next = Some(now + self.after); + } + + pub fn succeeded(&mut self) { + (self.stage, self.tries, self.next) = (None, 0, None); + } +} + #[cfg(test)] mod tests { use super::*; @@ -69,6 +106,26 @@ mod tests { assert_eq!(p.target(false, s(500)), Stage::Normal); } + #[test] + fn a_failed_stage_is_tried_again_after_a_wait_three_times_at_most() { + let t = Instant::now(); + let s = |n: u64| t + Duration::from_secs(n); + let mut r = Retry::new(Duration::from_secs(60)); + assert!(r.allowed(Stage::Gaming, s(0))); + r.failed(Stage::Gaming, s(0)); + assert!(!r.allowed(Stage::Gaming, s(59)), "waits"); + assert!(r.allowed(Stage::Normal, s(1)), "another stage may be tried at once"); + assert!(r.allowed(Stage::Gaming, s(60))); + r.failed(Stage::Gaming, s(60)); + r.failed(Stage::Gaming, s(120)); + assert!(!r.allowed(Stage::Gaming, s(1000)), "three failures: no more tries"); + // Another stage wanted (and failing) starts its own count; a success clears it. + r.failed(Stage::Normal, s(1000)); + assert!(r.allowed(Stage::Gaming, s(1000))); + r.succeeded(); + assert!(r.allowed(Stage::Normal, s(1000))); + } + #[test] fn with_pause_chosen_a_game_pauses_the_llm() { let t = Instant::now(); diff --git a/app/src/placement.rs b/app/src/placement.rs index edb8662..bab6a7b 100644 --- a/app/src/placement.rs +++ b/app/src/placement.rs @@ -10,7 +10,9 @@ use serde::{Deserialize, Serialize}; pub const MARGIN_MIB: i64 = 768; /// The STT worker's share: its model (1300 MiB) and what ten speakers at once add (1440 MiB), /// measured on the game PC (M0 `stt`). The services start the LLM before the worker, so both count. -pub const STT_MIB: i64 = 1300 + 1440; +pub const STT_MIB: i64 = STT_MODEL_MIB + 1440; +/// The STT worker's model alone (M0 `stt`). +pub const STT_MODEL_MIB: i64 = 1300; /// The smallest expert budget in a game: it holds the long-prompt staging (about 1.1 GiB) first, /// so 2 GiB is the least that loads experts, and it was 1.5-2.1 times faster than none in a real /// game (user decision, 2026-11-10). diff --git a/app/src/services.rs b/app/src/services.rs index 5874410..abec559 100644 --- a/app/src/services.rs +++ b/app/src/services.rs @@ -203,7 +203,7 @@ pub async fn start_services(data_dir: &Path, llm: Arc) -> Result) -> Result) -> bool; - async fn wait_llm_idle(&self, t: std::time::Duration) -> bool; + /// Announce a switch; its number for `wait_llm_idle`. + fn llm_switching(&self, notice: Option<&str>) -> u64; + fn llm_state(&self, state: &str) -> bool; + async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool; + /// The STT worker's model is loaded (on the graphics card). + fn stt_on_card(&self) -> bool; } #[async_trait::async_trait] impl LlmBrain for WorkerClient { - fn llm_state(&self, state: &str, notice: Option<&str>) -> bool { - WorkerClient::llm_state(self, state, notice) + fn llm_switching(&self, notice: Option<&str>) -> u64 { + WorkerClient::llm_switching(self, notice) } - async fn wait_llm_idle(&self, t: std::time::Duration) -> bool { - WorkerClient::wait_llm_idle(self, t).await + fn llm_state(&self, state: &str) -> bool { + WorkerClient::llm_state(self, state) + } + async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool { + WorkerClient::wait_llm_idle(self, id, t).await + } + fn stt_on_card(&self) -> bool { + self.hello().is_some_and(|h| h.stt_ready) } } @@ -234,45 +244,63 @@ fn switch_notice(stage: crate::perf::Stage) -> Option<&'static str> { } } +/// How long after a failed switch the stage is tried again (`perf::Retry`). +const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(60); + /// The performance stages (admin.spec.md, "Performance stages"): every second the stage wanted /// from game mode (`perf.rs`); a change tells the brain the LLM switches, waits until the answers /// in progress are written (at most 2 minutes), restarts the LLM in that stage (or stops it to -/// pause) and tells the brain it is ready (or paused). With `llm.url` nothing moves. -async fn perf_loop(settings: crate::settings::Settings, llm: Arc, worker: Arc, game: Arc bool + Send + Sync>) { - use crate::perf::{Perf, Stage}; +/// pause) and tells the brain it is ready (or paused). A restart that fails brings back the stage +/// that ran before; the failed one is tried again after `retry_after`, three times at most. With +/// `llm.url` nothing moves. +async fn perf_loop(settings: crate::settings::Settings, llm: Arc, brain: Arc, + game: Arc bool + Send + Sync>, retry_after: std::time::Duration) { + use crate::perf::{Perf, Retry, Stage}; if settings.llm.url.is_some() { return; } let mut perf = Perf::new(std::time::Duration::from_secs(settings.game.revert_seconds.into()), settings.game.pause_in_game); - let mut applied = Stage::Normal; + let mut retry = Retry::new(retry_after); + // The stage that runs (a game stage that had to pause counts as the game stage, so it is not + // tried again while the game runs). + let mut actual = Stage::Normal; loop { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - let want = perf.target(game(), std::time::Instant::now()); - if want == applied { + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let now = std::time::Instant::now(); + let want = perf.target(game(), now); + if want == actual || !retry.allowed(want, now) { continue; } crate::info_log!("perf", "performance stage: {want:?}"); - worker.llm_state("switching", switch_notice(want)); - if !worker.wait_llm_idle(std::time::Duration::from_secs(120)).await { + let id = brain.llm_switching(switch_notice(want)); + if !brain.wait_llm_idle(id, std::time::Duration::from_secs(120)).await { crate::warn_log!("perf", "Warning: the brain did not finish its answers in 2 minutes; switching anyway"); } - // Wanted is what counts as applied: a game stage that ends paused is not tried again. - applied = want; - match llm.switch(&settings, want).await { - Ok(Stage::Paused) => { - worker.llm_state("paused", None); - } - Ok(_) => { - worker.llm_state("ready", None); + let ended = match llm.switch(&settings, want, brain.stt_on_card()).await { + Ok(ended) => { + retry.succeeded(); + actual = want; + ended } Err(e) => { - crate::error_log!("perf", "the LLM did not come back in the {want:?} stage: {e}"); - worker.llm_state("ready", None); + retry.failed(want, now); + crate::error_log!("perf", "the LLM did not start in the {want:?} stage: {e}; going back to {actual:?}"); + match llm.switch(&settings, actual, brain.stt_on_card()).await { + Ok(ended) => ended, + Err(e) => { + crate::error_log!("perf", "the LLM did not come back in the {actual:?} stage either: {e}"); + // Nothing answers: requests fail at once instead of waiting. + actual = Stage::Paused; + Stage::Paused + } + } } - } + }; + brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }); } } + /// The secret values the services get: the bot token and the MCP servers' environment values. pub fn secrets_in_use(token: &str, mcp_env: &Value) -> Vec { let mut v = vec![token.to_owned()]; @@ -309,14 +337,21 @@ mod perf_tests { #[async_trait::async_trait] impl LlmBrain for FakeBrain { - fn llm_state(&self, state: &str, notice: Option<&str>) -> bool { - self.0.lock().unwrap().push(format!("{state}{}", notice.map(|n| format!(": {n}")).unwrap_or_default())); + fn llm_switching(&self, notice: Option<&str>) -> u64 { + self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default())); + 1 + } + fn llm_state(&self, state: &str) -> bool { + self.0.lock().unwrap().push(state.into()); true } - async fn wait_llm_idle(&self, _t: Duration) -> bool { + async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool { self.0.lock().unwrap().push("waited".into()); true } + fn stt_on_card(&self) -> bool { + false + } } #[cfg(unix)] @@ -330,7 +365,7 @@ mod perf_tests { let brain = Arc::new(FakeBrain::default()); let game = Arc::new(AtomicBool::new(false)); let g = game.clone(); - let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)))); + let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), RETRY_AFTER)); std::fs::remove_file(l.data.join("calls")).unwrap(); game.store(true, Ordering::SeqCst); let until = |want: &'static str| { @@ -358,6 +393,32 @@ mod perf_tests { assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal")); task.abort(); } + + #[cfg(unix)] + #[tokio::test] + async fn a_game_stage_that_fails_to_start_goes_back_and_is_tried_three_times() { + let l = layout("perf-loop-fail"); + let port = server_after_launch(&l, "list").await; + // This Strata cannot start at the game's context. + let log = l.strata_home().join("x.engine.log"); + let exe = crate::strata::tests::fake_strata(&l, &format!( + "case \"$*\" in *'--context 16384'*) echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'", + l.data.join("gone").display(), log.display())); + let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0))); + let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap(); + llm.start(&s).await.unwrap(); + let brain = Arc::new(FakeBrain::default()); + let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300))); + tokio::time::sleep(Duration::from_secs(8)).await; + let got = brain.0.lock().unwrap().clone(); + // Each try: switching, waited, the game stage fails, the normal stage comes back, ready. + assert_eq!(got.iter().filter(|m| m.starts_with("switching")).count(), 3, "{got:?}"); + assert_eq!(got.iter().filter(|m| m.as_str() == "ready").count(), 3, "{got:?}"); + assert!(!got.iter().any(|m| m == "paused")); + assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal")); + assert_eq!(llm.status().state, crate::strata::LlmState::Ready, "the normal stage runs again"); + task.abort(); + } } #[cfg(test)] diff --git a/app/src/strata.rs b/app/src/strata.rs index fcfdd2f..a813614 100644 --- a/app/src/strata.rs +++ b/app/src/strata.rs @@ -252,10 +252,14 @@ pub struct Llm { stop_timeout: Duration, /// The graphics card's memory (none: Strata places the model itself). gpu: Option, - /// What other programs held when the services started (before the LLM and the STT worker): - /// every later placement of this run counts from it, not from a card that then also holds the - /// STT worker and a game. + /// What other programs held at the latest normal placement (no game then): the game stage + /// counts other programs from it, as the card then also holds the game, which its reserve is for. baseline: Mutex>, + /// The STT worker's model is on the card (true at switches while the worker runs), so it is not + /// taken for another program's memory. + stt_on_card: AtomicBool, + /// The graphics memory a game gets in the placement being started. + room: Mutex>, /// The engine's CPU threads in the normal stage; a game gets half. threads: Mutex>, /// The stage it runs in, and in a game the graphics memory the game gets. @@ -279,7 +283,7 @@ impl Llm { pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self { Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }), active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0), - stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), threads: Mutex::new(None), + stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), stt_on_card: AtomicBool::new(false), room: Mutex::new(None), threads: Mutex::new(None), stage: Mutex::new((Stage::Normal, None)) } } @@ -313,6 +317,7 @@ impl Llm { /// the card. async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str, stage: Stage) -> Result>, String> { let c = conf(settings, stage); + *self.room.lock().unwrap() = None; let workers = |sets: &mut Vec| if stage == Stage::Gaming { let half = self.threads.lock().unwrap().map(|n| n / 2) .unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get() as u32 / 4).unwrap_or(1)); @@ -332,7 +337,15 @@ impl Llm { return Ok(Some(sets)); } let Some(card) = self.read_gpu().await else { return Ok(Some(vec![])) }; - let others = *self.baseline.lock().unwrap().get_or_insert(card.used_mib); + // Other programs: what the card holds without this LLM and without the STT worker's model. + // The normal stage reads it afresh (programs opened since count); the game stage takes it + // from the latest normal placement, as the card then also holds the game. + let stt_model = if self.stt_on_card.load(Ordering::SeqCst) { crate::placement::STT_MODEL_MIB } else { 0 }; + let now_others = (card.used_mib - stt_model).max(0); + let others = match stage { + Stage::Gaming => *self.baseline.lock().unwrap().get_or_insert(now_others), + _ => *self.baseline.lock().unwrap().insert(now_others), + }; let key = format!("{}/{}", c.context, c.parallel); let file = self.probe_file(); let mut all: serde_json::Map = std::fs::read_to_string(&file).ok() @@ -368,7 +381,7 @@ impl Llm { crate::info_log!("llm", "placement: no room for a game beside the model; paused while the game runs"); return Ok(None); } - *self.stage.lock().unwrap() = (Stage::Gaming, Some(g.game_room_mib)); + *self.room.lock().unwrap() = Some(g.game_room_mib); g.expert_gib } _ => crate::placement::normal(card.total_mib, others, &probe, settings.game.other_room_mib()), @@ -418,16 +431,19 @@ impl Llm { // A new run: other programs' memory is read afresh. *self.baseline.lock().unwrap() = None; *self.stage.lock().unwrap() = (Stage::Normal, None); + self.stt_on_card.store(false, Ordering::SeqCst); self.start_stage(settings, Stage::Normal).await.map(|r| r.expect("the normal stage never pauses")) } /// Move the running LLM to `stage` (`perf.rs`): a restart with that stage's placement, or a stop /// when it pauses (also a game stage with no room for the game). Returns the stage it ended in. /// With `llm.url` nothing moves. - pub async fn switch(&self, settings: &Settings, stage: Stage) -> Result { + /// `stt_on_card`: the STT worker's model is loaded on the card. + pub async fn switch(&self, settings: &Settings, stage: Stage, stt_on_card: bool) -> Result { if settings.llm.url.is_some() { return Ok(Stage::Normal); } + self.stt_on_card.store(stt_on_card, Ordering::SeqCst); if stage != Stage::Paused && self.start_stage(settings, stage).await?.is_some() { return Ok(stage); } @@ -447,11 +463,9 @@ impl Llm { *self.threads.lock().unwrap() = Some(n); } } + // What runs now, also on a fallback placement (no experts), with the game's room if known. if let Ok(Some(_)) = &result { - let mut st = self.stage.lock().unwrap(); - if stage == Stage::Normal { - *st = (Stage::Normal, None); - } + *self.stage.lock().unwrap() = (stage, self.room.lock().unwrap().take()); } match &result { Ok(None) => {} @@ -955,7 +969,7 @@ pub(crate) mod tests { llm.start(&s).await.unwrap(); assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null})); std::fs::remove_file(l.data.join("calls")).unwrap(); - assert_eq!(tokio::time::timeout(Duration::from_secs(30), llm.switch(&s, Stage::Gaming)).await.unwrap().unwrap(), Stage::Gaming); + assert_eq!(tokio::time::timeout(Duration::from_secs(30), llm.switch(&s, Stage::Gaming, false)).await.unwrap().unwrap(), Stage::Gaming); let g = launches(&l); // Measured once at the game's context and one request, then the 2 GiB minimum (8 GB does not // fit on 16 GB) with half the engine's 16 CPU threads. @@ -967,17 +981,74 @@ pub(crate) mod tests { assert_eq!(llm.status().state, LlmState::Ready); // Back to normal: the run's first reading of other programs, the normal placement again. std::fs::remove_file(l.data.join("calls")).unwrap(); - assert_eq!(llm.switch(&s, Stage::Normal).await.unwrap(), Stage::Normal); + assert_eq!(llm.switch(&s, Stage::Normal, false).await.unwrap(), Stage::Normal); let n = launches(&l); assert_eq!(n.len(), 1, "{n:?}"); assert!(n[0].contains("--context 32768") && n[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", n[0]); assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null})); // Pause: the LLM stops. - assert_eq!(llm.switch(&s, Stage::Paused).await.unwrap(), Stage::Paused); + assert_eq!(llm.switch(&s, Stage::Paused, false).await.unwrap(), Stage::Paused); assert_eq!(llm.status().state, LlmState::Off); assert_eq!(llm.stage_view()["stage"], json!("paused")); } + /// A card whose other programs hold `others` MiB (changeable), plus the STT model while + /// `stt` is set, plus 9874 MiB while a Strata server runs. + #[cfg(unix)] + fn live_card(l: &Layout, others: Arc, stt: bool) -> GpuReader { + let calls = l.data.join("calls"); + Arc::new(move || { + let c = std::fs::read_to_string(&calls).unwrap_or_default(); + let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family")); + let used = others.load(Ordering::SeqCst) + if stt { 1300 } else { 0 } + if running { 9874 } else { 0 }; + Some(Gpu { id: "GPU-test".into(), used_mib: used, total_mib: 16303 }) + }) + } + + #[cfg(unix)] + #[tokio::test] + async fn the_normal_placement_reads_other_programs_afresh_and_a_game_is_not_counted_twice() { + let l = layout("llm-stage-fresh"); + let port = server_after_launch(&l, "list").await; + let others = Arc::new(std::sync::atomic::AtomicI64::new(1460)); + // At the start the STT worker has not loaded; at every switch it has. + let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card(&l, others.clone(), false)); + let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); + llm.start(&s).await.unwrap(); + let llm = llm.with_gpu(live_card(&l, others.clone(), true)); + // A game starts and holds 2500 MiB: the game stage counts other programs as before the game. + others.store(1460 + 2500, Ordering::SeqCst); + llm.switch(&s, Stage::Gaming, true).await.unwrap(); + assert!(launches(&l).last().unwrap().contains("--set expert-vram=2 "), "{:?}", launches(&l)); + // The game ends but a video player now holds 1000 MiB more: the normal stage reads that. + others.store(1460 + 1000, Ordering::SeqCst); + llm.switch(&s, Stage::Normal, true).await.unwrap(); + let expected = crate::placement::normal(16303, 1460 + 1000, &crate::placement::Probe { context: 32768, parallel: 1, + gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }, 1024); + assert!(expected < 1.5); + assert!(launches(&l).last().unwrap().ends_with(&format!("--set expert-vram={expected} --set prefill=2048")), "{:?}", launches(&l)); + } + + #[cfg(unix)] + #[tokio::test] + async fn a_game_stage_on_a_fallback_placement_still_shows_as_the_game_stage() { + // The game's measurement fails (the card cannot be read the second time): no experts, but + // the game stage is what runs. + let l = layout("llm-stage-fallback"); + let port = server_after_launch(&l, "list").await; + let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)); + let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); + llm.start(&s).await.unwrap(); + let reads = Arc::new(std::sync::atomic::AtomicU32::new(0)); + let r2 = reads.clone(); + let llm = llm.with_gpu(Arc::new(move || { + (r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 }) + })); + assert_eq!(llm.switch(&s, Stage::Gaming, false).await.unwrap(), Stage::Gaming); + assert!(launches(&l).last().unwrap().contains("--set expert-vram=0 "), "{:?}", launches(&l)); + assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": null})); + } + #[cfg(unix)] #[tokio::test] async fn a_game_with_no_room_beside_the_model_pauses_the_llm() { @@ -992,7 +1063,7 @@ pub(crate) mod tests { })); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); - assert_eq!(llm.switch(&s, Stage::Gaming).await.unwrap(), Stage::Paused); + assert_eq!(llm.switch(&s, Stage::Gaming, false).await.unwrap(), Stage::Paused); assert_eq!(llm.status().state, LlmState::Off); assert_eq!(llm.stage_view(), json!({"stage": "paused", "gameRoomMib": null})); let c = std::fs::read_to_string(l.data.join("calls")).unwrap(); diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index 2a1bb83..8fb1ef3 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -156,8 +156,13 @@ struct Inner { events: mpsc::UnboundedSender, ready: watch::Sender>, wake: Notify, - /// The brain finished the answers it was giving when the app said the LLM switches. - llm_idle: Notify, + /// The last switch the brain reported idle for (`llm_idle` names it). + llm_idle: watch::Sender, + /// The LLM state the brain should have: sent again on every connection, so a worker that + /// missed it (a dropped connection, a restarted worker) catches up. + llm: Mutex>, + /// The number of the latest switch. + llm_switches: std::sync::atomic::AtomicU64, } #[derive(Clone)] @@ -176,7 +181,9 @@ impl WorkerClient { events, ready: watch::channel(None).0, wake: Notify::new(), - llm_idle: Notify::new(), + llm_idle: watch::channel(0).0, + llm: Mutex::new(None), + llm_switches: std::sync::atomic::AtomicU64::new(0), }); (Self { inner }, rx) } @@ -354,20 +361,32 @@ impl WorkerClient { self.send_json(json!({"type": "addressed", "id": id, "user": user.to_string(), "text": text})) } - /// Tell the brain the LLM's state (voice_worker.spec.md, "LLM switches"): `switching` (start no - /// new LLM work, say `notice` once to the first spoken request that waits, report `llm_idle`), - /// `ready` (answer what waited) or `paused` (answer nothing until ready). - pub fn llm_state(&self, state: &str, notice: Option<&str>) -> bool { - let mut m = json!({"type": "llm", "state": state}); + /// Tell the brain the LLM switches (voice_worker.spec.md, "LLM switches"): it starts no new + /// LLM work, says `notice` once to the first spoken request that waits, and reports idle for + /// this switch, whose number is returned for `wait_llm_idle`. + pub fn llm_switching(&self, notice: Option<&str>) -> u64 { + let id = self.inner.llm_switches.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + let mut m = json!({"type": "llm", "state": "switching", "id": id}); if let Some(n) = notice { m["notice"] = json!(n); } + *self.inner.llm.lock().unwrap() = Some(m.clone()); + self.send_json(m); + id + } + + /// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing). + pub fn llm_state(&self, state: &str) -> bool { + let m = json!({"type": "llm", "state": state}); + *self.inner.llm.lock().unwrap() = Some(m.clone()); self.send_json(m) } - /// Wait until the brain reports `llm_idle` (also one that came before the wait), at most `t`. - pub async fn wait_llm_idle(&self, t: Duration) -> bool { - tokio::time::timeout(t, self.inner.llm_idle.notified()).await.is_ok() + /// Wait until the brain reports idle for switch `id` (also when it came before the wait); a + /// report for an earlier switch does not count. At most `t`. + pub async fn wait_llm_idle(&self, id: u64, t: Duration) -> bool { + let mut rx = self.inner.llm_idle.subscribe(); + tokio::time::timeout(t, rx.wait_for(|got| *got >= id)).await.is_ok_and(|r| r.is_ok()) } /// Tell the brain a call is being said (a partial transcript), so it gets ready to answer it. @@ -501,6 +520,10 @@ async fn connect_once(inner: &Arc) -> bool { if st.closed { return true; } + // The LLM state the brain should have, before anything else it is asked. + if let Some(m) = inner.llm.lock().unwrap().clone() { + let _ = tx.send(Message::text(m.to_string())); + } st.conn = Some(tx); } debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready); @@ -544,7 +567,9 @@ fn on_text(inner: &Inner, t: &str) { "final" => serde_json::from_value(v).ok().map(WorkerEvent::Final), "chat_action" => serde_json::from_value(v).ok().map(WorkerEvent::ChatAction), "llm_idle" => { - inner.llm_idle.notify_one(); + if let Some(id) = v["id"].as_u64() { + inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false }); + } return; } "reply_text" => { diff --git a/app/tests/worker_client.rs b/app/tests/worker_client.rs index 6858c23..97bc4c6 100644 --- a/app/tests/worker_client.rs +++ b/app/tests/worker_client.rs @@ -416,19 +416,42 @@ async fn a_chat_reply_is_not_spoken_and_its_text_comes_back() { } #[tokio::test] -async fn an_llm_switch_is_announced_and_the_client_waits_until_the_brain_is_idle() { +async fn an_llm_switch_is_announced_and_the_client_waits_until_the_brain_is_idle_for_that_switch() { let w = FakeWorker::start(0).await; let (c, _rx) = client(&w.url(), json!({})); c.start(); c.wait_ready(Duration::from_secs(2)).await.unwrap(); - assert!(c.llm_state("switching", Some("잠깐만요"))); - until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "switching", "notice": "잠깐만요"})))).await; + let first = c.llm_switching(Some("잠깐만요")); + until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "switching", "id": first, "notice": "잠깐만요"})))).await; // Nothing idle yet: the wait gives up after its time. - assert!(!c.wait_llm_idle(Duration::from_millis(100)).await); - // The worker reports idle (before or while the app waits): the wait ends. - w.send(Message::text(json!({"type": "llm_idle"}).to_string())); - assert!(c.wait_llm_idle(Duration::from_secs(2)).await); - assert!(c.llm_state("ready", None)); + assert!(!c.wait_llm_idle(first, Duration::from_millis(100)).await); + // The idle for this switch comes late, after the app gave up on it, and a new switch starts: + // that late idle must not let the new switch through. + let second = c.llm_switching(None); + w.send(Message::text(json!({"type": "llm_idle", "id": first}).to_string())); + assert!(!c.wait_llm_idle(second, Duration::from_millis(200)).await, "a late idle of an earlier switch"); + w.send(Message::text(json!({"type": "llm_idle", "id": second}).to_string())); + assert!(c.wait_llm_idle(second, Duration::from_secs(2)).await); + assert!(c.llm_state("ready")); until(|| w.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "ready"})))).await; c.close(); } + +#[tokio::test] +async fn a_worker_that_missed_the_llm_state_gets_it_when_it_connects_again() { + let w = FakeWorker::start(0).await; + let (c, mut rx) = client(&w.url(), json!({})); + c.start(); + c.wait_ready(Duration::from_secs(2)).await.unwrap(); + c.llm_switching(None); + // The worker goes away; the app sets the LLM paused meanwhile (that message is lost). + let port = w.kill(); + next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await; + assert!(!c.llm_state("paused"), "not sent: no connection"); + let w2 = FakeWorker::start(port).await; + until(|| w2.got().iter().any(|g| *g == Got::Json(json!({"type": "llm", "state": "paused"})))).await; + let got = w2.got(); + assert!(matches!(&got[0], Got::Json(v) if v["type"] == "hello")); + assert_eq!(got[1], Got::Json(json!({"type": "llm", "state": "paused"})), "first after the hello"); + c.close(); +} diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index 9195575..a67f2e8 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -79,6 +79,7 @@ class VoiceWorker: self.llm_noticed = False self.llm_active = 0 self.llm_idle_to = None # the session to tell once nothing runs + self.llm_switch_id = None # the switch to name in llm_idle self.chat_timeout_s = chat_timeout_s self.delivery_timeout_s = delivery_timeout_s # a chat answer the app never reports: not posted # chat_action ids are unique across connections and worker runs, so a late answer from an @@ -540,9 +541,10 @@ class Session: asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early)) def _on_llm(self, msg: dict) -> None: - state, notice = msg.get("state"), msg.get("notice") - if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)): - self.error("bad_message", "llm needs state switching|ready|paused and an optional string notice") + state, notice, sid = msg.get("state"), msg.get("notice"), msg.get("id") + if state not in ("switching", "ready", "paused") or not (notice is None or isinstance(notice, str)) \ + or not (sid is None or (isinstance(sid, int) and not isinstance(sid, bool))): + self.error("bad_message", "llm needs state switching|ready|paused, an optional string notice and an optional integer id") return w = self.worker w.llm_state = state @@ -554,7 +556,7 @@ class Session: w.llm_ready.clear() if state == "switching": w.llm_notice, w.llm_noticed = notice or None, False - w.llm_idle_to = self + w.llm_idle_to, w.llm_switch_id = self, sid self._llm_check_idle() else: w.llm_ready.set() # paused: waiting requests stop waiting and answer nothing @@ -563,13 +565,13 @@ class Session: w = self.worker if w.llm_state == "switching" and w.llm_active == 0 and w.llm_idle_to is not None: to, w.llm_idle_to = w.llm_idle_to, None - to.send({"type": "llm_idle"}) + to.send({"type": "llm_idle"} if w.llm_switch_id is None else {"type": "llm_idle", "id": w.llm_switch_id}) @contextlib.asynccontextmanager async def _llm(self, notice_id: str | None = None): """Brain work that uses the LLM: it waits while the LLM switches (the first spoken request - that waits hears the notice once, under its own id), fails while paused, and counts as - running so a switch waits for it.""" + that waits hears the notice once, under its own id), fails while paused or after waiting + `LLM_WAIT_S`, and counts as running so a switch waits for it.""" w = self.worker while w.llm_state != "ready": if w.llm_state == "paused": @@ -581,7 +583,8 @@ class Session: try: await asyncio.wait_for(w.llm_ready.wait(), LLM_WAIT_S) except asyncio.TimeoutError: - break # go on: the call fails on its own if the LLM is still away + # The LLM is still away (or a measuring server stands in): fail, never call it. + raise _LlmPaused() from None w.llm_active += 1 try: yield diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index 0b22be7..ff7a8a0 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -354,8 +354,8 @@ The app restarts the LLM with another placement when a game starts or ends (app/ "Performance stages"). It tells the worker first and waits for the brain to be idle: ``` --> {"type": "llm", "state": "switching", "notice": "잠깐만요, 게임 모드로 바꾸는 중이에요"} // notice optional -<- {"type": "llm_idle"} // nothing that uses the LLM runs any more +-> {"type": "llm", "state": "switching", "id": 7, "notice": "잠깐만요, 게임 모드로 바꾸는 중이에요"} // id, notice optional +<- {"type": "llm_idle", "id": 7} // nothing that uses the LLM runs any more (id: the switch's, when it had one) -> {"type": "llm", "state": "ready"} // the LLM answers again -> {"type": "llm", "state": "paused"} // the LLM is stopped (a game runs) ``` @@ -366,10 +366,12 @@ The app restarts the LLM with another placement when a game starts or ends (app/ - The first spoken `reply` that waits during one switch hears the notice under its own id (its answer follows in the same speech when ready); later ones wait without it. A notice ends with a newline, so it is spoken at once and never joined to the answer. A request waits at most 180 s, - then goes on (its LLM call fails on its own if the LLM is still away). + then fails without calling the brain (the LLM may still be away, or a measuring server stand in). - On `paused` brain work answers nothing at once (`reply_text` `failed`, judgements failed) until `ready`; what waited for a switch stops waiting the same way. -- Any other `state`, or a notice that is not a string, is `bad_message`. +- Any other `state`, a notice that is not a string or an id that is not an integer is `bad_message`. +- The app sends its latest `llm` message again first on every connection, so a worker that missed + one (a dropped connection, a restarted worker) catches up. ### Errors diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index 1e579c0..eaddf12 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -1223,3 +1223,34 @@ def test_an_unknown_llm_state_is_refused(store): seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain())) assert [m for m in seen if isinstance(m, dict) and m.get("type") == "error"][0]["code"] == "bad_message" + + +@pytest.mark.unit +def test_llm_idle_names_the_switch_it_answers(store): + async def body(url): + async with connect(url) as ws: + await hello(ws) + await send(ws, type="llm", state="switching", id=7) + return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle") + + seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain())) + assert [m for m in seen if isinstance(m, dict) and m.get("type") == "llm_idle"] == [{"type": "llm_idle", "id": 7}] + + +@pytest.mark.unit +def test_a_request_that_waits_too_long_for_a_switch_fails_without_the_brain(store, monkeypatch): + import voice_worker.server as server + monkeypatch.setattr(server, "LLM_WAIT_S", 0.3) + + async def body(url): + async with connect(url) as ws: + await hello(ws) + await send(ws, type="llm", state="switching", id=1) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "llm_idle") + await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False) + return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") + + brain = FakeBrain() + seen = asyncio.run(with_worker(body, store=store, brain=brain)) + reply = [m for m in seen if isinstance(m, dict) and m.get("type") == "reply_text"][0] + assert reply["failed"] and brain.calls == [], "the LLM may be away: the brain is not called"