diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index d59a435..239903d 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -355,21 +355,26 @@ between the normal placement and the game placement as games come and go (not wi back; `voice_worker.spec.md`, "LLM switches"); the app waits until the brain has written the answers in progress (at most 2 minutes; the switch is numbered, so an idle report for an earlier switch that came late does not count), restarts the LLM in the new stage and tells the brain it - is ready (or paused). The worker client keeps that latest state and sends it again on every - connection. A restart that fails brings back the stage that ran before and tells the brain - ready; the failed stage is tried again after 60 s, three times at most, then not until the - wanted stage changes. When the stage before does not come back either, the LLM shows `failed` + is ready (or paused). The worker client keeps that latest state and sends it again first on + every connection; keeping, sending and a connection becoming ready share one lock, so a state set + while a worker connects is never lost. A restart that fails brings back the stage that ran + before and tells the brain ready; the failed stage is tried again 60 s after that is over, three + times at most, and the count starts over whenever another stage is wanted (also when the game + ends and the normal stage already runs), so the next game tries again. When the stage before does not come back either, the LLM shows `failed` and the brain is told paused (requests fail at once instead of waiting). - The game stage: context 16,384 (or the settings' smaller one), one request at a time, half the engine's CPU threads (read from its log at the normal start, else a quarter of the logical CPUs), and the game placement (`placement.rs`: the reserve beside more than 2 GiB of experts, else 2 GiB, else 0 GiB, else pause), measured once at that context like the normal one. Other programs' memory is what the card holds after the LLM stopped, less the STT worker's model while - it is loaded (1300 MiB); the normal stage reads it afresh each time (programs opened since - count) and keeps it; the game stage takes the one kept at the latest normal placement, as the - card then also holds the game, which its reserve is for (a game already running when the - services start is counted in it, which only leaves the LLM less). The dashboard shows the stage - that runs, also on a fallback placement without experts. + it is loaded (1300 MiB), read afresh at every placement. A normal placement made with no game + running keeps its reading. The game stage counts other programs as that kept reading and leaves + a reserve of at least what has grown since (the game and any program opened after that + reading), so nothing is counted twice and nothing that grew is missed. When a game already ran + as the services started (game detection looks once before the LLM starts), there is no such + reading: the game stage then counts what the card holds now, the game included, and leaves only + the room for other programs. The dashboard shows the stage that runs, also on a fallback + placement without experts. - Pause: the LLM stops (`llm` off, "게임 중 일시정지"); the brain answers nothing until the stage changes. - The foreground program is a game, in this order: in `notGames`: no; in `games`: yes @@ -606,11 +611,13 @@ a path with non-ASCII letters) or from a Battle.net game's install folder (a tes turns game mode on with its name and off once it ends. Performance stages (on every system, with a fake Strata and card): a game moves the LLM to the game placement (measured once at 16K and one request, 2 GiB on the game PC's numbers, half the 16 engine threads) and back after the wait, pause stops -it, a card with no room for a game pauses, the normal stage counts a program opened since and the -game stage does not count the game twice, a fallback game placement still shows as the game +it, a card with no room for a game pauses, the normal stage counts a program opened since, the game +stage does not count the game twice and counts what grew since the normal reading, a game running +as the services start does not pause the LLM, a fallback game placement still shows as the game stage, and the stage loop tells the brain switching (with the notice), waits, restarts and tells it ready, also on the way back; a game stage that cannot start goes back to normal each time and -is tried three times. The worker client ignores a late idle of an earlier switch and sends the +is tried three times; after that a new game tries again, and the next try waits from when the +failed switch was over. The worker client ignores a late idle of an earlier switch and sends the latest LLM state first when a worker connects again. On every system: game detection's rules (a borderless game, a maximised program, a browser's full-screen video, exclusive full screen, launcher folders and their own windows, Battle.net games by install folder and their uninstall diff --git a/app/src/game.rs b/app/src/game.rs index 1fa5e15..dd6b66f 100644 --- a/app/src/game.rs +++ b/app/src/game.rs @@ -205,8 +205,14 @@ pub struct GameWatch { } impl GameWatch { + /// Start watching; the first look happens before this returns, so a game already running is + /// known at once (`on`). pub fn start(settings: GameSettings) -> Self { let (state, stop) = (Arc::new(Mutex::new(GameState::default())), Arc::new(AtomicBool::new(false))); + if cfg!(windows) && settings.detect { + let snap = probe::Probe::new().snapshot(); + state.lock().unwrap().update(&snap, &settings, probe::alive); + } let thread = (cfg!(windows) && settings.detect).then(|| { let (state, stop) = (state.clone(), stop.clone()); std::thread::spawn(move || { @@ -254,6 +260,13 @@ impl GameWatch { } } +impl Drop for GameWatch { + /// A start that fails part way drops the watch: its thread ends too. + fn drop(&mut self) { + self.stop(); + } +} + #[cfg(not(windows))] mod probe { use super::Snapshot; diff --git a/app/src/services.rs b/app/src/services.rs index abec559..bc756d6 100644 --- a/app/src/services.rs +++ b/app/src/services.rs @@ -119,7 +119,9 @@ async fn start_browser(data_dir: &Path, settings: &crate::settings::Settings) -> /// before the brain can. A failure stops what already started. pub async fn start_services(data_dir: &Path, llm: Arc) -> Result> { let settings = load_settings(&data_dir.join("config").join("settings.json"))?; - let game_settings = settings.game.clone(); + // Game detection first: a game already running is not taken for other programs' memory. + let game = Arc::new(crate::game::GameWatch::start(settings.game.clone())); + llm.game_at_start(game.on()); let perf_settings = settings.clone(); // The stored token (the setup wizard keeps it encrypted, admin.spec.md), else the environment. let secrets = crate::secrets::Secrets::new(&data_dir.join("config").join("secrets.dat")); @@ -201,7 +203,6 @@ pub async fn start_services(data_dir: &Path, llm: Arc) -> Result, brain: Ar // The stage that runs (a game stage that had to pause counts as the game stage, so it is not // tried again while the game runs). let mut actual = Stage::Normal; + let mut last_want = Stage::Normal; loop { tokio::time::sleep(std::time::Duration::from_millis(500)).await; let now = std::time::Instant::now(); let want = perf.target(game(), now); + // Another stage wanted (also back to the one that runs): earlier failures no longer count. + if want != last_want { + retry.succeeded(); + last_want = want; + } if want == actual || !retry.allowed(want, now) { continue; } @@ -283,9 +290,8 @@ async fn perf_loop(settings: crate::settings::Settings, llm: Arc, brain: Ar ended } Err(e) => { - retry.failed(want, now); crate::error_log!("perf", "the LLM did not start in the {want:?} stage: {e}; going back to {actual:?}"); - match llm.switch(&settings, actual, brain.stt_on_card()).await { + let back = match llm.switch(&settings, actual, brain.stt_on_card()).await { Ok(ended) => ended, Err(e) => { crate::error_log!("perf", "the LLM did not come back in the {actual:?} stage either: {e}"); @@ -293,7 +299,10 @@ async fn perf_loop(settings: crate::settings::Settings, llm: Arc, brain: Ar actual = Stage::Paused; Stage::Paused } - } + }; + // The wait counts from when that is over, not from when the switch began. + retry.failed(want, std::time::Instant::now()); + back } }; brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }); @@ -394,6 +403,80 @@ mod perf_tests { task.abort(); } + /// A Strata that cannot start at the game's context (taking `fail_secs` to fail). + #[cfg(unix)] + fn strata_failing_in_games(l: &crate::install::Layout, fail_secs: f64) -> std::path::PathBuf { + let log = l.strata_home().join("x.engine.log"); + crate::strata::tests::fake_strata(l, &format!( + "case \"$*\" in *'--context 16384'*) sleep {fail_secs}; echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'", + l.data.join("gone").display(), log.display())) + } + + #[cfg(unix)] + #[tokio::test] + async fn a_new_game_tries_the_game_stage_again_after_three_failures_with_the_last_one() { + let l = layout("perf-loop-new-game"); + let port = server_after_launch(&l, "list").await; + let llm = Arc::new(Llm::with_exe(l.clone(), strata_failing_in_games(&l, 0.0)).with_gpu(card(&l, "GPU-test", 0))); + let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap(); + llm.start(&s).await.unwrap(); + let brain = Arc::new(FakeBrain::default()); + let game = Arc::new(AtomicBool::new(true)); + let g = game.clone(); + let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), Duration::from_millis(200))); + let switchings = |b: &FakeBrain| b.0.lock().unwrap().iter().filter(|m| m.starts_with("switching")).count(); + tokio::time::sleep(Duration::from_secs(6)).await; + assert_eq!(switchings(&brain), 3, "three tries"); + // The game ends (the normal stage runs already) and a new one starts: tried again. + game.store(false, Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(1)).await; + game.store(true, Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(switchings(&brain) > 3, "{:?}", brain.0.lock().unwrap()); + task.abort(); + } + + #[cfg(unix)] + #[tokio::test] + async fn the_wait_before_the_next_try_counts_from_when_the_failed_switch_is_over() { + // The failing start takes 1.5 s, longer than the 1 s wait: the next try still waits 1 s + // after the normal stage came back. + let l = layout("perf-loop-wait"); + let port = server_after_launch(&l, "list").await; + let llm = Arc::new(Llm::with_exe(l.clone(), strata_failing_in_games(&l, 1.5)).with_gpu(card(&l, "GPU-test", 0))); + let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap(); + llm.start(&s).await.unwrap(); + let brain = Arc::new(TimedBrain::default()); + let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_secs(1))); + tokio::time::sleep(Duration::from_secs(7)).await; + task.abort(); + let got = brain.0.lock().unwrap().clone(); + let at = |what: &str, n: usize| got.iter().filter(|(m, _)| m.starts_with(what)).nth(n).map(|(_, t)| *t).expect(what); + let gap = at("switching", 1).duration_since(at("ready", 0)); + assert!(gap >= Duration::from_secs(1), "the next try came {gap:?} after the normal stage was back"); + } + + #[derive(Default)] + struct TimedBrain(Mutex>); + + #[async_trait::async_trait] + impl LlmBrain for TimedBrain { + fn llm_switching(&self, _notice: Option<&str>) -> u64 { + self.0.lock().unwrap().push(("switching".into(), std::time::Instant::now())); + 1 + } + fn llm_state(&self, state: &str) -> bool { + self.0.lock().unwrap().push((state.into(), std::time::Instant::now())); + true + } + async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool { + true + } + fn stt_on_card(&self) -> bool { + false + } + } + #[cfg(unix)] #[tokio::test] async fn a_game_stage_that_fails_to_start_goes_back_and_is_tried_three_times() { diff --git a/app/src/strata.rs b/app/src/strata.rs index a813614..a2ae83e 100644 --- a/app/src/strata.rs +++ b/app/src/strata.rs @@ -252,9 +252,12 @@ pub struct Llm { stop_timeout: Duration, /// The graphics card's memory (none: Strata places the model itself). gpu: Option, - /// What other programs held at the latest normal placement (no game then): the game stage - /// counts other programs from it, as the card then also holds the game, which its reserve is for. + /// What other programs held at the latest normal placement made with no game running: the + /// game stage counts other programs from it (the card then also holds the game, which its + /// reserve is for). None when the services started with a game running. baseline: Mutex>, + /// A game ran when the services started (set before `start`). + game_at_start: AtomicBool, /// The STT worker's model is on the card (true at switches while the worker runs), so it is not /// taken for another program's memory. stt_on_card: AtomicBool, @@ -283,10 +286,16 @@ impl Llm { pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self { Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }), active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0), - stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), stt_on_card: AtomicBool::new(false), room: Mutex::new(None), threads: Mutex::new(None), + stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), game_at_start: AtomicBool::new(false), stt_on_card: AtomicBool::new(false), room: Mutex::new(None), threads: Mutex::new(None), stage: Mutex::new((Stage::Normal, None)) } } + /// Whether a game runs as the services start: the start's reading of the card then includes it + /// and is not kept as other programs' memory. + pub fn game_at_start(&self, on: bool) { + self.game_at_start.store(on, Ordering::SeqCst); + } + /// `{stage, gameRoomMib}` for the dashboard. pub fn stage_view(&self) -> Value { let (stage, room) = *self.stage.lock().unwrap(); @@ -342,10 +351,12 @@ impl Llm { // from the latest normal placement, as the card then also holds the game. let stt_model = if self.stt_on_card.load(Ordering::SeqCst) { crate::placement::STT_MODEL_MIB } else { 0 }; let now_others = (card.used_mib - stt_model).max(0); - let others = match stage { - Stage::Gaming => *self.baseline.lock().unwrap().get_or_insert(now_others), - _ => *self.baseline.lock().unwrap().insert(now_others), - }; + // A normal placement keeps its reading, unless a game ran at the start (its memory would be + // taken for other programs'; a later normal placement comes only after the game ended). + if stage != Stage::Gaming && !self.game_at_start.swap(false, Ordering::SeqCst) { + *self.baseline.lock().unwrap() = Some(now_others); + } + let baseline = *self.baseline.lock().unwrap(); let key = format!("{}/{}", c.context, c.parallel); let file = self.probe_file(); let mut all: serde_json::Map = std::fs::read_to_string(&file).ok() @@ -374,9 +385,14 @@ impl Llm { p } }; - let expert = match stage { - Stage::Gaming => { - let g = crate::placement::game(card.total_mib, others, &probe, settings.game.reserve_mib()); + let mut others = now_others; + let expert = match (stage, baseline) { + // With a reading from before the game: other programs as then, and the reserve at least + // what has grown since (the game, and any program opened after that reading). + (Stage::Gaming, Some(base)) => { + others = base; + let reserve = settings.game.reserve_mib().max(now_others - base); + let g = crate::placement::game(card.total_mib, base, &probe, reserve); if g.paused { crate::info_log!("llm", "placement: no room for a game beside the model; paused while the game runs"); return Ok(None); @@ -384,7 +400,9 @@ impl Llm { *self.room.lock().unwrap() = Some(g.game_room_mib); g.expert_gib } - _ => crate::placement::normal(card.total_mib, others, &probe, settings.game.other_room_mib()), + // A game that already ran at the start is in what the card holds now: no reserve on + // top, only the room for other programs. + _ => crate::placement::normal(card.total_mib, now_others, &probe, settings.game.other_room_mib()), }; crate::info_log!("llm", "placement ({stage:?}): experts {expert} GiB (card {} MiB, held by others {others} MiB, model without experts {} MiB)", card.total_mib, probe.model_mib); @@ -1029,6 +1047,53 @@ pub(crate) mod tests { assert!(launches(&l).last().unwrap().ends_with(&format!("--set expert-vram={expected} --set prefill=2048")), "{:?}", launches(&l)); } + /// `live_card` on a card of `total` MiB. + #[cfg(unix)] + fn live_card_of(l: &Layout, others: Arc, total: i64) -> GpuReader { + let calls = l.data.join("calls"); + Arc::new(move || { + let c = std::fs::read_to_string(&calls).unwrap_or_default(); + let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family")); + Some(Gpu { id: "GPU-test".into(), used_mib: others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: total }) + }) + } + + #[cfg(unix)] + #[tokio::test] + async fn a_game_running_when_the_services_start_is_not_counted_twice() { + // A game holds 2500 MiB already: it is in what the card holds, so no reserve on top. + let l = layout("llm-stage-game-at-start"); + let port = server_after_launch(&l, "list").await; + let others = Arc::new(std::sync::atomic::AtomicI64::new(1460 + 2500)); + let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 16303)); + llm.game_at_start(true); + let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); + llm.start(&s).await.unwrap(); + assert_eq!(llm.switch(&s, Stage::Gaming, false).await.unwrap(), Stage::Gaming, "not paused"); + let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }; + let expected = crate::placement::normal(16303, 1460 + 2500, &probe16, 1024); + assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l)); + } + + #[cfg(unix)] + #[tokio::test] + async fn the_game_stage_reads_the_card_again_and_counts_what_grew_since_the_normal_reading() { + // Normal reading: 1460 MiB. Then a program takes 4000 MiB and the game 4500 MiB: together + // more than the 8 GB reserve, so the placement leaves that much, not just 8 GB. + let l = layout("llm-stage-grew"); + let port = server_after_launch(&l, "list").await; + let others = Arc::new(std::sync::atomic::AtomicI64::new(1460)); + let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 24576)); + let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); + llm.start(&s).await.unwrap(); + others.store(1460 + 4000 + 4500, Ordering::SeqCst); + llm.switch(&s, Stage::Gaming, false).await.unwrap(); + let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }; + let expected = crate::placement::game(24576, 1460, &probe16, 8500).expert_gib; + assert_ne!(expected, crate::placement::game(24576, 1460, &probe16, 8192).expert_gib, "the test tells them apart"); + assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l)); + } + #[cfg(unix)] #[tokio::test] async fn a_game_stage_on_a_fallback_placement_still_shows_as_the_game_stage() { diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index 8fb1ef3..d6d60be 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -370,7 +370,9 @@ impl WorkerClient { if let Some(n) = notice { m["notice"] = json!(n); } - *self.inner.llm.lock().unwrap() = Some(m.clone()); + // Kept and sent under one lock, so a connection being set up sends this or a later state. + let mut llm = self.inner.llm.lock().unwrap(); + *llm = Some(m.clone()); self.send_json(m); id } @@ -378,7 +380,8 @@ impl WorkerClient { /// Tell the brain the LLM is `ready` (answer what waited) or `paused` (answer nothing). pub fn llm_state(&self, state: &str) -> bool { let m = json!({"type": "llm", "state": state}); - *self.inner.llm.lock().unwrap() = Some(m.clone()); + let mut llm = self.inner.llm.lock().unwrap(); + *llm = Some(m.clone()); self.send_json(m) } @@ -516,18 +519,20 @@ async fn connect_once(inner: &Arc) -> bool { let (tx, mut rx) = mpsc::unbounded_channel::(); { + // Locks in the order llm, then state (as `llm_state` takes them): the latest LLM state goes + // first, and the connection is ready for sending before an LLM state set meanwhile is sent. + let llm = inner.llm.lock().unwrap(); let mut st = inner.state.lock().unwrap(); if st.closed { return true; } - // The LLM state the brain should have, before anything else it is asked. - if let Some(m) = inner.llm.lock().unwrap().clone() { + if let Some(m) = llm.clone() { let _ = tx.send(Message::text(m.to_string())); } st.conn = Some(tx); + inner.ready.send_replace(Some(hello_ok.clone())); } debug_log!("worker", "voice worker ready stt={} tts={}", hello_ok.stt_ready, hello_ok.tts_ready); - inner.ready.send_replace(Some(hello_ok.clone())); let _ = inner.events.send(WorkerEvent::Ready(hello_ok)); loop { diff --git a/app/tests/worker_client.rs b/app/tests/worker_client.rs index 97bc4c6..318c40d 100644 --- a/app/tests/worker_client.rs +++ b/app/tests/worker_client.rs @@ -455,3 +455,40 @@ async fn a_worker_that_missed_the_llm_state_gets_it_when_it_connects_again() { assert_eq!(got[1], Got::Json(json!({"type": "llm", "state": "paused"})), "first after the hello"); c.close(); } + +#[tokio::test] +async fn an_llm_state_set_while_the_worker_connects_again_is_the_last_it_gets() { + // The app sets "ready" over and over while a new worker connects: whatever the order, the + // last LLM message the worker gets is "ready", never the earlier "switching" alone. + for round in 0..20 { + let w = FakeWorker::start(0).await; + let (c, mut rx) = client(&w.url(), json!({})); + c.start(); + c.wait_ready(Duration::from_secs(2)).await.unwrap(); + c.llm_switching(None); + let port = w.kill(); + next_matching(&mut rx, |e| *e == WorkerEvent::Disconnected).await; + let w2 = FakeWorker::start(port).await; + // "ready" is set from the moment the new worker has the hello until the client is ready. + let received = w2.received.clone(); + let c2 = c.clone(); + let setter = tokio::task::spawn_blocking(move || { + while received.lock().unwrap().is_empty() { + std::thread::yield_now(); + } + while !c2.is_ready() { + c2.llm_state("ready"); + } + c2.llm_state("ready"); + }); + next_matching(&mut rx, |e| matches!(e, WorkerEvent::Ready(_))).await; + setter.await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + let last = w2.got().into_iter().rev().find_map(|g| match g { + Got::Json(v) if v["type"] == "llm" => Some(v["state"].clone()), + _ => None, + }); + assert_eq!(last, Some(json!("ready")), "round {round}"); + c.close(); + } +}