//! The LLM the brain answers with (`app/src/admin.spec.md`, "LLM"): the app starts the installed //! Strata from the data folder with the M0 kit's choices, waits until the model answers, and stops //! it with the services. Strata downloads the model (about 23 GB) on its first start; the dashboard //! shows how far it got. With `llm.url` the app uses that server instead and starts nothing. use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use serde::Serialize; use serde_json::{json, Value}; use crate::install::{Ctx, Layout}; use crate::perf::Stage; use crate::settings::Settings; /// The one model v2 runs (M0: Qwen3.6-35B-A3B UD-Q4_K_XL; docs/v2/roadmap.md). pub const FAMILY: &str = "qwen36"; pub const MODEL: &str = "ud-q4_k_xl"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum LlmState { Off, Starting, Ready, Failed, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct LlmStatus { pub state: LlmState, /// Strata's own server (false: `llm.url`). pub managed: bool, /// How far Strata's current line says it got while starting (a percentage, or the model /// download's `X of Y GB`); none on a line without progress. pub percent: Option, /// Strata's last line while starting, or why it failed. pub note: Option, } /// Requests answered at once: one is faster on a 16 GB card (M0), two only when a chat channel /// answers every message (so its answers do not wait behind the voice answers). pub fn parallel(settings: &Settings) -> u32 { if settings.chat_channels.iter().any(|c| c.answer_all) { 2 } else { 1 } } /// `strata launch` for the services: the M0 kit's normal placement, models in `models/llm`, this /// computer only. No pictures yet (nothing in v2 sends one; reading them takes graphics memory). /// `--restart` replaces a server started with other choices; the same choices are reused. pub fn launch_args(layout: &Layout, settings: &Settings, conf: &Conf, sets: &[String]) -> Vec { let models = layout.data.join("models").join("llm").to_string_lossy().into_owned(); let mut a: Vec = ["launch", "--family", FAMILY, "--model", MODEL, "--data-dir"].map(String::from).into(); a.push(models); for (k, v) in [("--context", conf.context.to_string()), ("--vision", conf.vision.into()), ("--parallel", conf.parallel.to_string()), ("--access", "local".into()), ("--port", settings.llm.port.to_string())] { a.push(k.into()); a.push(v); } a.extend(["--no-api-key", "--restart", "--yes", "--no-tui"].map(String::from)); for s in sets { a.push("--set".into()); a.push(s.clone()); } a } /// The context and requests at once of a stage: the settings' in the normal stage; in a game 16K /// (or less when the settings ask for less) and one request at a time. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Conf { pub context: u32, pub parallel: u32, /// `--vision`: `off`, `gpu` or `cpu` (`vision`). pub vision: &'static str, } pub fn conf(settings: &Settings, stage: Stage) -> Conf { let vision = vision(stage, crate::screen::available(settings)); match stage { Stage::Gaming => Conf { context: crate::placement::GAME_CONTEXT.min(settings.llm.context), parallel: 1, vision }, _ => Conf { context: settings.llm.context, parallel: parallel(settings), vision }, } } /// Where the LLM reads pictures (`--vision`): nowhere unless screen viewing is available, on the /// card normally and on the CPU in a game (M0's game placement). pub fn vision(stage: Stage, screen: bool) -> &'static str { match (screen, stage) { (false, _) => "off", (true, Stage::Gaming) => "cpu", (true, _) => "gpu", } } /// Where the probe of a placement is kept in `config/llm-probe.json`: pictures on the card take /// memory of their own, so they are measured apart. pub fn probe_key(c: &Conf) -> String { if c.vision == "gpu" { format!("{}/{}/vision", c.context, c.parallel) } else { format!("{}/{}", c.context, c.parallel) } } /// The engine's CPU threads from its log ("... on 16 CPU threads ..."). fn cpu_threads(log: &str) -> Option { log.lines().rev().find_map(|l| { let i = l.find(" CPU threads")?; l[..i].rsplit(' ').next()?.parse().ok() }) } /// The engine settings for an expert budget (`placement.rs`) and the M0 kit's prompt chunk. fn placement_sets(expert_gib: f64) -> Vec { vec![format!("expert-vram={expert_gib}"), "prefill=2048".into()] } /// The first NVIDIA card: its UUID and graphics memory in MiB. #[derive(Debug, Clone, PartialEq)] pub struct Gpu { pub id: String, pub used_mib: i64, pub total_mib: i64, } /// Reads the card (`nvidia-smi`); None without one. pub type GpuReader = Arc Option + Send + Sync>; fn nvidia_memory() -> Option { let mut cmd = std::process::Command::new("nvidia-smi"); cmd.args(["--query-gpu=uuid,memory.used,memory.total", "--format=csv,noheader,nounits"]); #[cfg(windows)] { use std::os::windows::process::CommandExt; cmd.creation_flags(0x0800_0000); } let out = cmd.output().ok()?; let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned(); let mut f = line.split(',').map(str::trim); Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()? }) } /// The newest engine log under Strata's folder, its end only (the placement lines of the last start). fn engine_log(home: &Path) -> String { fn walk(dir: &Path, depth: u32, best: &mut Option<(std::time::SystemTime, std::path::PathBuf)>) { let Ok(entries) = std::fs::read_dir(dir) else { return }; for e in entries.flatten() { let p = e.path(); if p.is_dir() && depth > 0 { walk(&p, depth - 1, best); } else if p.to_string_lossy().ends_with(".engine.log") { if let Ok(t) = e.metadata().and_then(|m| m.modified()) { if best.as_ref().is_none_or(|(b, _)| t > *b) { *best = Some((t, p)); } } } } } let mut best = None; walk(home, 5, &mut best); let Some((_, p)) = best else { return String::new() }; let bytes = std::fs::read(p).unwrap_or_default(); String::from_utf8_lossy(&bytes[bytes.len().saturating_sub(400_000)..]).into_owned() } /// How far a line says Strata got: its last `NN%` or `NN.N%`, else `X of Y GB` (or MB), which is /// how Strata 0.1.59 reports the model download. pub fn percent(line: &str) -> Option { percent_sign(line).or_else(|| size_progress(line)) } fn size_progress(line: &str) -> Option { let words: Vec<&str> = line.split_whitespace().collect(); words.windows(4).rev().find_map(|w| { let (done, total) = (w[0].parse::().ok()?, w[2].parse::().ok()?); (w[1] == "of" && matches!(w[3].trim_end_matches([',', '.', ')']), "GB" | "MB") && total > 0.0 && done <= total) .then(|| (done / total * 1000.0).round() / 10.0) }) } fn percent_sign(line: &str) -> Option { let b = line.as_bytes(); let mut found = None; for (i, _) in line.match_indices('%') { let mut s = i; while s > 0 && (b[s - 1].is_ascii_digit() || b[s - 1] == b'.') { s -= 1; } if let Ok(n) = line[s..i].parse::() { if (0.0..=100.0).contains(&n) { found = Some(n); } } } found } /// Wait until `base` (`.../v1`) lists a model; returns its id. Stops early on `cancel`. pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> Result { let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().map_err(|e| e.to_string())?; let until = Instant::now() + timeout; loop { if cancel.load(Ordering::Relaxed) { return Err("cancelled".into()); } if let Ok(r) = client.get(format!("{base}/models")).send().await { if let Ok(v) = r.text().await.map(|t| serde_json::from_str::(&t).unwrap_or_default()) { if let Some(id) = v["data"][0]["id"].as_str() { return Ok(id.to_owned()); } } } if Instant::now() >= until { return Err(format!("대화 모델이 {}초 안에 준비되지 않았습니다 ({base})", timeout.as_secs())); } tokio::time::sleep(Duration::from_millis(500)).await; } } /// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an /// error when the model gives none. async fn warm_up(base: &str) -> Result<(), String> { let client = reqwest::Client::builder().timeout(Duration::from_secs(60)).build().map_err(|e| e.to_string())?; let model = match client.get(format!("{base}/models")).send().await { Ok(r) => r.text().await.ok().and_then(|t| serde_json::from_str::(&t).ok()) .and_then(|v| v["data"][0]["id"].as_str().map(str::to_owned)), Err(_) => None, }; let body = json!({"model": model, "messages": [{"role": "user", "content": "안녕"}], "max_tokens": 8}); let r = client.post(format!("{base}/chat/completions")).header("content-type", "application/json") .body(body.to_string()).send().await.map_err(|e| e.to_string())?; if !r.status().is_success() { return Err(format!("the model answered {}", r.status())); } let v: Value = serde_json::from_str(&r.text().await.map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; v["choices"][0]["message"].is_object().then_some(()).ok_or_else(|| "the model gave no answer".into()) } /// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the /// memory database inside the data folder, the tools of the features that are off and the enabled /// MCP servers (without their environment values, `features.rs`). The app owns this file and writes /// it on every start. pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings) -> Result { let file = data.join("config").join("brain.json"); let cfg = json!({ "ollama_base_url": base_url, "ollama_chat_model": model, "db_path": data.join("memory").join("jarvis.db").to_string_lossy(), "disabled_tools": crate::features::disabled_tools(settings), "mcps": crate::features::brain_mcps(&settings.mcp_servers), "_config_version": 1, }); let mut text = serde_json::to_string_pretty(&cfg).unwrap(); text.push('\n'); crate::datadir::write_atomic(&file, text.as_bytes()).map_err(|e| e.to_string())?; Ok(file) } /// One LLM for the services: Strata from the layout, or the server at `llm.url`. pub struct Llm { layout: Layout, /// The program run as Strata (the installed one; a script in tests). exe: std::path::PathBuf, status: Mutex, active: Mutex>>, cancel: Arc, /// The port the managed server was started on (to confirm it is gone after a stop). port: Mutex, stop_timeout: Duration, /// The graphics card's memory (none: Strata places the model itself). gpu: Option, /// What other programs held at the latest normal placement made with no game running: the /// game stage counts other programs from it (the card then also holds the game, which its /// reserve is for). None when the services started with a game running. baseline: Mutex>, /// A game runs (or may run) as the LLM is placed: set before `start` and at every `switch`. /// Only a normal placement made while none runs keeps its reading. game_running: AtomicBool, /// The STT worker's model is on the card (true at switches while the worker runs), so it is not /// taken for another program's memory. stt_on_card: AtomicBool, /// The graphics memory a game gets in the placement being started. room: Mutex>, /// The engine's CPU threads in the normal stage; a game gets half. threads: Mutex>, /// The stage it runs in, and in a game the graphics memory the game gets. stage: Mutex<(Stage, Option)>, } /// How long the model may take to answer after `strata launch` returned (it loads in about 15 s /// on the game PC; the first start may also tune). const READY_TIMEOUT: Duration = Duration::from_secs(30 * 60); /// A server the app does not start must already answer. const URL_TIMEOUT: Duration = Duration::from_secs(30); impl Llm { pub fn new(data: &Path) -> Self { // Only the data folder matters here (the Python sources are the install's business). let layout = Layout { data: data.to_path_buf(), src: std::path::PathBuf::new() }; let exe = layout.strata_exe(); Self::with_exe(layout, exe).with_gpu(Arc::new(nvidia_memory)) } pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self { Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }), active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0), stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), game_running: AtomicBool::new(false), stt_on_card: AtomicBool::new(false), room: Mutex::new(None), threads: Mutex::new(None), stage: Mutex::new((Stage::Normal, None)) } } /// Whether a game runs (also in the background) as the services start: the start's reading of /// the card then includes it and is not kept as other programs' memory. pub fn game_at_start(&self, on: bool) { self.game_running.store(on, Ordering::SeqCst); } /// `{stage, gameRoomMib}` for the dashboard. pub fn stage_view(&self) -> Value { let (stage, room) = *self.stage.lock().unwrap(); json!({"stage": stage, "gameRoomMib": room}) } /// Place the model by the card's memory (`placement.rs`), read with `gpu`. pub fn with_gpu(mut self, gpu: GpuReader) -> Self { self.gpu = Some(gpu); self } async fn read_gpu(&self) -> Option { let gpu = self.gpu.clone()?; tokio::task::spawn_blocking(move || gpu()).await.ok().flatten() } fn probe_file(&self) -> std::path::PathBuf { self.layout.data.join("config").join("llm-probe.json") } /// The expert budget's engine settings (none without a card): the normal placement from what /// the card holds now and this PC's measurement without experts at this context and number of /// requests at once, which the first start makes once per card (a start without experts, then /// the real start replaces it). A server left from an earlier run is stopped first, so it is not /// counted as another program's memory (a stop that is not confirmed reads nothing: no /// experts). A measurement that fails is not kept: the model then starts without experts on /// the card. async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str, stage: Stage) -> Result>, String> { let c = conf(settings, stage); *self.room.lock().unwrap() = None; let workers = |sets: &mut Vec| if stage == Stage::Gaming { let half = self.threads.lock().unwrap().map(|n| n / 2) .unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get() as u32 / 4).unwrap_or(1)); sets.push(format!("pool-workers={}", half.max(1))); }; if self.gpu.is_none() { let mut sets = vec![]; workers(&mut sets); return Ok(Some(sets)); } // Whatever a server left from an earlier run does (answers, loads, stays silent), it holds // the card: stop it first and read the card only once the port is confirmed closed. if let Err(e) = self.stop_server_on(settings.llm.port).await { crate::warn_log!("llm", "Warning: a server left on the port did not stop ({e}); starting without experts on the card"); let mut sets = placement_sets(0.0); workers(&mut sets); return Ok(Some(sets)); } let Some(card) = self.read_gpu().await else { return Ok(Some(vec![])) }; // Other programs: what the card holds without this LLM and without the STT worker's model. // The normal stage reads it afresh (programs opened since count); the game stage takes it // from the latest normal placement, as the card then also holds the game. let stt_model = if self.stt_on_card.load(Ordering::SeqCst) { crate::placement::STT_MODEL_MIB } else { 0 }; let now_others = (card.used_mib - stt_model).max(0); // Only a normal placement made while no game runs keeps its reading (a game's memory would be // taken for other programs'; also on the way back from a failed game stage). if stage != Stage::Gaming && !self.game_running.load(Ordering::SeqCst) { *self.baseline.lock().unwrap() = Some(now_others); } let baseline = *self.baseline.lock().unwrap(); let key = probe_key(&c); let file = self.probe_file(); let mut all: serde_json::Map = std::fs::read_to_string(&file).ok() .and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default(); let known = all.get(&key).and_then(|v| serde_json::from_value::(v.clone()).ok()) .filter(|p| p.gpu == card.id); let probe = match known { Some(p) => p, None => { crate::info_log!("llm", "measuring the model without experts once (context {}, {} at once)", c.context, c.parallel); self.launch(settings, ctx, base, &c, &placement_sets(0.0)).await?; let warmed = warm_up(base).await; let after = self.read_gpu().await.filter(|g| g.id == card.id); let model_mib = after.map(|g| g.used_mib - card.used_mib).unwrap_or(0); if let Err(e) = warmed.and_then(|()| if model_mib > 0 { Ok(()) } else { Err("the card's memory could not be read".into()) }) { crate::warn_log!("llm", "Warning: the measurement failed ({e}); starting without experts on the card"); let mut sets = placement_sets(0.0); workers(&mut sets); return Ok(Some(sets)); } let p = crate::placement::Probe { context: c.context, parallel: c.parallel, gpu: card.id.clone(), model_mib, staging_gib: crate::placement::staging_gib(&engine_log(&self.layout.strata_home())) }; all.insert(key, json!(p)); crate::datadir::write_atomic(&file, Value::Object(all).to_string().as_bytes()) .map_err(|e| format!("배치 측정값을 저장하지 못했습니다: {e}"))?; p } }; let mut others = now_others; let expert = match (stage, baseline) { // With a reading from before the game: other programs as then, and the reserve at least // what has grown since (the game, and any program opened after that reading). (Stage::Gaming, Some(base)) => { others = base; let reserve = settings.game.reserve_mib().max(now_others - base); let g = crate::placement::game(card.total_mib, base, &probe, reserve); if g.paused { crate::info_log!("llm", "placement: no room for a game beside the model; paused while the game runs"); return Ok(None); } *self.room.lock().unwrap() = Some(g.game_room_mib); g.expert_gib } // A game that already ran at the start is in what the card holds now: no reserve on // top, only the room for other programs. _ => crate::placement::normal(card.total_mib, now_others, &probe, settings.game.other_room_mib()), }; crate::info_log!("llm", "placement ({stage:?}): experts {expert} GiB (card {} MiB, held by others {others} MiB, model without experts {} MiB)", card.total_mib, probe.model_mib); let mut sets = placement_sets(expert); workers(&mut sets); Ok(Some(sets)) } /// `strata launch` with `sets`, then wait until the model answers. async fn launch(&self, settings: &Settings, ctx: &Ctx, base: &str, c: &Conf, sets: &[String]) -> Result { let args = launch_args(&self.layout, settings, c, sets); let refs: Vec<&str> = args.iter().map(String::as_str).collect(); ctx.run_in(&self.exe, &refs, Some(&self.layout.strata_home())).await?; wait_ready(base, READY_TIMEOUT, &self.cancel).await } /// How long `strata launch --stop` may take before it is ended and the stop reported as failed. pub fn with_stop_timeout(mut self, t: Duration) -> Self { self.stop_timeout = t; self } pub fn status(&self) -> LlmStatus { let mut s = self.status.lock().unwrap().clone(); if let Some(ctx) = self.active.lock().unwrap().as_ref() { s.note = ctx.last_note(); // From the current line only: a percentage from an earlier stage would mislead. s.percent = s.note.as_deref().and_then(percent); } s } fn set(&self, state: LlmState, note: Option) { let mut s = self.status.lock().unwrap(); s.state = state; if state != LlmState::Starting { s.note = note; } } /// Start (or reuse) the LLM; returns its `/v1` address and model id. /// A cancel since the last `arm` (or before the first start) ends it before it runs anything. pub async fn start(&self, settings: &Settings) -> Result<(String, String), String> { // A new run: other programs' memory is read afresh. *self.baseline.lock().unwrap() = None; *self.stage.lock().unwrap() = (Stage::Normal, None); self.stt_on_card.store(false, Ordering::SeqCst); self.start_stage(settings, Stage::Normal).await.map(|r| r.expect("the normal stage never pauses")) } /// Move the running LLM to `stage` (`perf.rs`): a restart with that stage's placement, or a stop /// when it pauses (also a game stage with no room for the game). Returns the stage it ended in. /// With `llm.url` nothing moves. /// `stt_on_card`: the STT worker's model is loaded on the card; `game_running`: a game runs now. pub async fn switch(&self, settings: &Settings, stage: Stage, stt_on_card: bool, game_running: bool) -> Result { if settings.llm.url.is_some() { return Ok(Stage::Normal); } self.stt_on_card.store(stt_on_card, Ordering::SeqCst); self.game_running.store(game_running, Ordering::SeqCst); if stage != Stage::Paused && self.start_stage(settings, stage).await?.is_some() { return Ok(stage); } *self.port.lock().unwrap() = settings.llm.port; self.stop().await?; *self.stage.lock().unwrap() = (Stage::Paused, None); self.set(LlmState::Off, Some("게임 중 일시정지".into())); Ok(Stage::Paused) } async fn start_stage(&self, settings: &Settings, stage: Stage) -> Result, String> { let managed = settings.llm.url.is_none(); *self.status.lock().unwrap() = LlmStatus { state: LlmState::Starting, managed, percent: None, note: None }; let result = self.start_inner(settings, stage).await; if stage == Stage::Normal && result.as_ref().is_ok_and(Option::is_some) && self.gpu.is_some() { if let Some(n) = cpu_threads(&engine_log(&self.layout.strata_home())) { *self.threads.lock().unwrap() = Some(n); } } // What runs now, also on a fallback placement (no experts), with the game's room if known. if let Ok(Some(_)) = &result { *self.stage.lock().unwrap() = (stage, self.room.lock().unwrap().take()); } match &result { Ok(None) => {} Ok(Some((base, model))) => { crate::info_log!("llm", "ready: {base} model={model}"); self.set(LlmState::Ready, None); } Err(e) if e == "cancelled" => self.set(LlmState::Off, None), Err(e) => { crate::error_log!("llm", "did not start: {e}"); self.set(LlmState::Failed, Some(e.clone())); } } result } /// None: the stage pauses (nothing started). async fn start_inner(&self, settings: &Settings, stage: Stage) -> Result, String> { if self.cancel.load(Ordering::SeqCst) { return Err("cancelled".into()); } if let Some(url) = &settings.llm.url { let model = wait_ready(url, URL_TIMEOUT, &self.cancel).await?; return Ok(Some((url.clone(), model))); } if !self.exe.exists() { return Err("Strata가 설치되어 있지 않습니다. 마법사의 모델 단계에서 설치해 주세요".into()); } let ctx = Arc::new(Ctx::standalone(self.layout.clone(), "llm", self.cancel.clone())); *self.active.lock().unwrap() = Some(ctx.clone()); *self.port.lock().unwrap() = settings.llm.port; let base = format!("http://127.0.0.1:{}/v1", settings.llm.port); let c = conf(settings, stage); let ready = match self.placement(settings, &ctx, &base, stage).await { Ok(Some(sets)) => self.launch(settings, &ctx, &base, &c, &sets).await.map(Some), Ok(None) => Ok(None), Err(e) => Err(e), }; *self.active.lock().unwrap() = None; match ready { Ok(Some(model)) => Ok(Some((base, model))), Ok(None) => Ok(None), // Nothing half-started stays on the graphics card; if it may have, say so. Err(e) => match self.stop_server().await { Ok(()) => Err(e), Err(stop) if e == "cancelled" => Err(stop), Err(stop) => Err(format!("{e} ({stop})")), }, } } /// Make the next start cancellable afresh: called when a start is accepted, never by `start` /// itself, so a stop between accepting and running the start still cancels it. pub fn arm(&self) { self.cancel.store(false, Ordering::SeqCst); } /// Stop a start in progress (the download resumes on the next start). pub fn cancel(&self) { self.cancel.store(true, Ordering::SeqCst); } /// Stop the server this app started (nothing for `llm.url`). Off only once the server no longer /// answers; otherwise `failed` with the reason, which is also returned. pub async fn stop(&self) -> Result<(), String> { let managed = self.status.lock().unwrap().managed; let result = if managed { self.stop_server().await } else { Ok(()) }; match &result { Ok(()) => self.set(LlmState::Off, None), Err(e) => { crate::error_log!("llm", "{e}"); self.set(LlmState::Failed, Some(e.clone())); } } result } /// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that /// fails while nothing answers (nothing was serving) counts as stopped. async fn stop_server(&self) -> Result<(), String> { let port = *self.port.lock().unwrap(); self.stop_server_on(port).await } /// `stop_server` confirming on `port` (0: nothing was started, nothing to confirm). async fn stop_server_on(&self, port: u16) -> Result<(), String> { if !self.exe.exists() { return Ok(()); } // The timeout ends the command and everything it started, like an install cancel. let cancel = Arc::new(AtomicBool::new(false)); let ctx = Ctx::standalone(self.layout.clone(), "llm", cancel.clone()); let home = self.layout.strata_home(); let timer = { let (cancel, t) = (cancel.clone(), self.stop_timeout); tokio::spawn(async move { tokio::time::sleep(t).await; cancel.store(true, Ordering::SeqCst); }) }; let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await; timer.abort(); let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e }); let base = format!("http://127.0.0.1:{port}/v1"); let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await; match (ran, gone) { (Ok(()), true) => { crate::info_log!("llm", "stopped"); Ok(()) } (Err(e), true) => { crate::warn_log!("llm", "stop command failed but nothing answers: {e}"); Ok(()) } (Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 닫히지 않았습니다")), (Err(e), false) => Err(format!("Strata를 끄지 못했습니다: {e}")), } } } /// Whether `base` is gone within `timeout`: only a refused connection counts. A timeout, an error /// status (a server still loading answers 503) or an answer it cannot read is not a confirmed stop, /// so it keeps checking until the deadline. async fn wait_gone(base: &str, timeout: Duration) -> bool { // Windows answers a connection to a closed local port only after about 2 s of retries, so the // connect wait is long enough for that refusal to arrive as a refusal, not as a timeout. let Ok(client) = reqwest::Client::builder().connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(12)).build() else { return false }; let until = Instant::now() + timeout; loop { if let Err(e) = client.get(format!("{base}/models")).send().await { if e.is_connect() && !e.is_timeout() { return true; } } if Instant::now() >= until { return false; } tokio::time::sleep(Duration::from_millis(300)).await; } } #[cfg(test)] pub(crate) mod tests { use super::*; use crate::datadir::temp_dir; use crate::settings::{parse_settings, ChatChannelSettings}; pub(crate) fn layout(tag: &str) -> Layout { Layout { data: temp_dir(tag), src: std::path::PathBuf::from("src") } } #[test] fn the_launch_keeps_the_model_in_the_data_folder_and_follows_the_settings() { let l = layout("llm1"); let s = parse_settings(&json!({"llm": {"context": 16384, "port": 8099}})).unwrap(); let a = launch_args(&l, &s, &conf(&s, Stage::Normal), &[]); let after = |k: &str| a[a.iter().position(|x| x == k).unwrap() + 1].clone(); assert!(Path::new(&after("--data-dir")).starts_with(&l.data)); assert_eq!((after("--context"), after("--port"), after("--parallel")), ("16384".into(), "8099".into(), "1".into())); assert_eq!(after("--access"), "local"); assert!(a.iter().any(|x| x == "--no-tui") && a.iter().any(|x| x == "--yes")); } #[test] fn pictures_are_read_only_with_screen_viewing_and_measured_apart_on_the_card() { assert_eq!(vision(Stage::Normal, false), "off"); assert_eq!(vision(Stage::Gaming, false), "off"); assert_eq!(vision(Stage::Normal, true), "gpu"); assert_eq!(vision(Stage::Gaming, true), "cpu"); let s = Settings::default(); let c = conf(&s, Stage::Normal); assert_eq!(c.vision, "off", "nobody is registered for PC control by default"); let a = launch_args(&layout("llm-vision"), &s, &Conf { vision: "gpu", ..c }, &[]); assert_eq!(a[a.iter().position(|x| x == "--vision").unwrap() + 1], "gpu"); assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "off" }), "32768/1"); assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "cpu" }), "32768/1"); assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "gpu" }), "32768/1/vision"); } #[test] fn a_chat_channel_answering_everything_gets_a_second_request_slot() { let mut s = Settings::default(); assert_eq!(parallel(&s), 1); s.chat_channels.push(ChatChannelSettings { id: 1, name: String::new(), description: String::new(), read: true, send: true, edit: true, react: true, delete: false, pin: false, answer_all: true, sync_discord: true }); assert_eq!(parallel(&s), 2); } #[test] fn the_progress_is_the_last_percentage_on_a_line() { assert_eq!(percent("Downloading 3 of 5: 41.5% (9.6 GB of 23.1 GB)"), Some(41.5)); assert_eq!(percent("50% done, now 75%"), Some(75.0)); assert_eq!(percent("Loading the model… 15 s"), None); // Strata 0.1.59's model download prints sizes, not a percentage. assert_eq!(percent("Downloading the model files: 2.4 of 22.9 GB"), Some(10.5)); assert_eq!(percent("Downloading the model files: 22.9 of 22.9 GB"), Some(100.0)); } #[test] fn the_brain_config_points_at_the_llm_and_keeps_memory_in_the_data_folder() { let l = layout("llm2"); let s = parse_settings(&json!({"features": {"weather": false}, "mcpServers": [{"name": "brave", "command": "npx", "env": ["BRAVE_API_KEY"]}]})).unwrap(); let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s).unwrap(); let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap(); assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1"); assert_eq!(v["ollama_chat_model"], "m"); assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data)); assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s))); assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers)); } #[cfg(unix)] /// Default settings on a port nothing listens on (8080 may be taken on a developer's PC). fn unused_port_settings() -> Settings { let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); parse_settings(&json!({"llm": {"port": port}})).unwrap() } /// A tiny `/v1/models` server answering with `id`, after `delay` (before that it lists nothing). async fn models_server(id: &'static str, delay: Duration) -> u16 { serve(id, delay, None, Arc::new(Mutex::new("list"))).await } #[cfg(unix)] /// The same; once the file `gone` exists (a fake Strata's `--stop` makes it) it stops listening, /// so a connection is refused. async fn models_server_until(id: &'static str, delay: Duration, gone: Option) -> u16 { serve(id, delay, gone, Arc::new(Mutex::new("list"))).await } /// `mode`: "list" answers, "hang" keeps the connection and says nothing, "503" is a server still /// loading. pub(crate) async fn serve(id: &'static str, delay: Duration, gone: Option, mode: Arc>) -> u16 { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let opened = Instant::now(); tokio::spawn(async move { let mut listener = listener; while let Ok((mut s, _)) = listener.accept().await { if gone.as_ref().is_some_and(|g| g.exists()) { // The listener closes: later connections are refused, until a launch serves // again (removes `gone`). drop(s); drop(listener); let g = gone.clone().unwrap(); while g.exists() { tokio::time::sleep(Duration::from_millis(50)).await; } listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap(); // A new server: it answers (a left one loaded or stayed silent). let mut m = mode.lock().unwrap(); if matches!(*m, "503" | "hang") { *m = "list"; } continue; } let mut buf = [0u8; 1024]; let n = s.read(&mut buf).await.unwrap_or(0); let post = buf[..n].starts_with(b"POST"); let m = *mode.lock().unwrap(); let (status, body) = match m { "hang" => { tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(60)).await; drop(s); }); continue; } "503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()), "nochat" if post => ("500 Internal Server Error", r#"{"error":"broken"}"#.to_owned()), _ if post => ("200 OK", r#"{"choices":[{"message":{"content":"안녕하세요"}}]}"#.to_owned()), _ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)), _ => ("200 OK", "{}".to_owned()), }; let _ = s.write_all(format!("HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).as_bytes()).await; } }); port } #[tokio::test] async fn a_given_server_is_used_without_starting_strata() { let port = models_server("given-model", Duration::ZERO).await; let l = layout("llm3"); let llm = Llm::new(&l.data); let s = parse_settings(&json!({"llm": {"url": format!("http://127.0.0.1:{port}/v1")}})).unwrap(); let (base, model) = llm.start(&s).await.unwrap(); assert_eq!((base.as_str(), model.as_str()), (format!("http://127.0.0.1:{port}/v1").as_str(), "given-model")); assert_eq!(llm.status().state, LlmState::Ready); assert!(!llm.status().managed); } #[tokio::test] async fn without_strata_installed_the_start_fails_with_a_reason() { let llm = Llm::new(&layout("llm4").data); let e = llm.start(&Settings::default()).await.unwrap_err(); assert!(e.contains("Strata"), "{e}"); assert_eq!(llm.status().state, LlmState::Failed); assert_eq!(llm.status().note.as_deref(), Some(e.as_str())); } /// A fake `strata`: `launch --stop` records itself and runs `stop_body` (by default it stops the /// server: the file `gone` appears); `launch ...` runs `launch_body`. #[cfg(unix)] pub(crate) fn fake_strata(l: &Layout, launch_body: &str) -> std::path::PathBuf { fake_strata_stop(l, launch_body, &format!("touch '{}'", l.data.join("gone").display())) } #[cfg(unix)] pub(crate) fn fake_strata_stop(l: &Layout, launch_body: &str, stop_body: &str) -> std::path::PathBuf { use std::os::unix::fs::PermissionsExt; let exe = l.data.join("fake-strata"); std::fs::create_dir_all(l.strata_home()).unwrap(); let log = l.data.join("calls"); std::fs::write(&exe, format!("#!/bin/sh\necho \"$@\" >> '{}'\nif [ \"$2\" = --stop ]; then {stop_body}; exit $?; fi\n{launch_body}\n", log.display())).unwrap(); std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); exe } #[cfg(unix)] #[tokio::test] async fn strata_starts_reports_its_download_waits_for_the_model_and_stops() { let l = layout("llm5"); let port = models_server_until("qwen-test", Duration::from_millis(300), Some(l.data.join("gone"))).await; let exe = fake_strata(&l, "echo 'Downloading the engine: 80%'; sleep 1; echo 'Downloading the model files: 2.4 of 22.9 GB'; sleep 1; echo 'Starting the engine'; sleep 1; echo 'Strata is serving.'"); let llm = Arc::new(Llm::with_exe(l.clone(), exe)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); let l2 = llm.clone(); let s2 = s.clone(); let run = tokio::spawn(async move { l2.start(&s2).await }); let mut seen = None; for _ in 0..40 { tokio::time::sleep(Duration::from_millis(50)).await; let st = llm.status(); if st.percent.is_some() { seen = Some(st); break; } } let st = seen.expect("progress while starting"); assert_eq!((st.state, st.percent), (LlmState::Starting, Some(80.0))); // The bar follows the current line: the model download's sizes, then nothing (no stale 80 %). let mut later = vec![]; for _ in 0..60 { tokio::time::sleep(Duration::from_millis(50)).await; let st = llm.status(); if st.state != LlmState::Starting { break; } later.push((st.note.clone().unwrap_or_default(), st.percent)); } assert!(later.iter().any(|(n, p)| n.contains("2.4 of 22.9") && *p == Some(10.5)), "{later:?}"); assert!(later.iter().any(|(n, p)| n.contains("Starting the engine") && p.is_none()), "{later:?}"); let (base, model) = run.await.unwrap().unwrap(); assert_eq!((base, model.as_str()), (format!("http://127.0.0.1:{port}/v1"), "qwen-test")); assert_eq!(llm.status().state, LlmState::Ready); llm.stop().await.unwrap(); let calls = std::fs::read_to_string(l.data.join("calls")).unwrap(); assert!(calls.lines().next().unwrap().starts_with("launch --family"), "{calls}"); assert_eq!(calls.lines().last(), Some("launch --stop")); assert_eq!(llm.status().state, LlmState::Off); } #[cfg(unix)] #[tokio::test] async fn with_a_graphics_card_the_first_start_measures_the_llm_without_experts_then_places_them() { let l = layout("llm-place"); let port = server_after_launch(&l, "list").await; // The engine log, as Strata 0.1.59's qwen36 engine writes it. let log = l.strata_home().join("x.engine.log"); let exe = fake_strata(&l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 0 of 256 per layer in VRAM, bigger prompt chunks streamed (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'", l.data.join("gone").display(), log.display())); // The game PC in M0 (16303 MiB card, 1460 MiB held by other programs); the LLM without // experts held 9874 MiB. let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap(); let calls = std::fs::read_to_string(l.data.join("calls")).unwrap(); let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect(); assert_eq!(launches.len(), 2, "{calls}"); assert!(launches[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", launches[0]); // The M0 kit's normal placement on that PC: 1.5 GiB of experts. assert!(launches[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[1]); let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap(); assert_eq!(probe["32768/1"], json!({"context": 32768, "parallel": 1, "gpu": "GPU-test", "modelMib": 9874, "stagingGib": 1.1})); llm.stop().await.unwrap(); // The next start reuses the measurement: one launch, placed at once. std::fs::remove_file(l.data.join("calls")).unwrap(); tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap(); let calls = std::fs::read_to_string(l.data.join("calls")).unwrap(); let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect(); assert_eq!(launches.len(), 1, "{calls}"); assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]); } /// A fake card for `fake_placed`: 16303 MiB, 1460 MiB held by other programs (and by any /// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a /// Strata server runs (the calls file shows the last launch and stop). #[cfg(unix)] pub(crate) fn card(l: &Layout, id: &'static str, leftover: i64) -> GpuReader { let calls = l.data.join("calls"); Arc::new(move || { let c = std::fs::read_to_string(&calls).unwrap_or_default(); let stopped = c.contains("launch --stop"); let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family")); let used = 1460 + if running { 9874 } else if stopped { 0 } else { leftover }; Some(Gpu { id: id.into(), used_mib: used, total_mib: 16303 }) }) } /// A fake Strata that serves on `port` again (removes `gone`) and writes the engine's staging line. #[cfg(unix)] pub(crate) fn fake_placed(l: &Layout) -> std::path::PathBuf { let log = l.strata_home().join("x.engine.log"); fake_strata(l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 5 of 256 per layer in VRAM, the rest of a batch on 16 CPU threads (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'", l.data.join("gone").display(), log.display())) } /// A fake server on a free port that answers only once a launch serves (removes `gone`). #[cfg(unix)] pub(crate) async fn server_after_launch(l: &Layout, mode: &'static str) -> u16 { std::fs::create_dir_all(&l.data).unwrap(); std::fs::write(l.data.join("gone"), "").unwrap(); serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await } pub(crate) fn launches(l: &Layout) -> Vec { std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().lines() .filter(|c| c.starts_with("launch --family")).map(str::to_owned).collect() } #[cfg(unix)] #[tokio::test] async fn a_failed_measurement_is_not_kept_and_the_model_starts_without_experts() { // The card stops answering after the first read. let l = layout("llm-place-fail"); let port = server_after_launch(&l, "list").await; let reads = Arc::new(std::sync::atomic::AtomicU32::new(0)); let r2 = reads.clone(); let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || { (r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 }) })); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap(); let l2 = launches(&l); assert_eq!(l2.len(), 2, "{l2:?}"); assert!(l2[1].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[1]); assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept"); llm.stop().await.unwrap(); // The model does not answer the short question: not kept either. let l = layout("llm-place-nochat"); let port = server_after_launch(&l, "nochat").await; let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap(); assert!(launches(&l)[1].ends_with("--set expert-vram=0 --set prefill=2048")); assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept"); } #[cfg(unix)] #[tokio::test] async fn the_measurement_is_kept_per_requests_at_once_and_per_card() { let l = layout("llm-place-keys"); let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-a", 0)); let port = server_after_launch(&l, "list").await; let one = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&one).await.unwrap(); assert_eq!(launches(&l).len(), 2, "measured once"); llm.stop().await.unwrap(); // A chat channel that answers everything: two requests at once, measured again. std::fs::remove_file(l.data.join("calls")).unwrap(); let two = parse_settings(&json!({"llm": {"port": port}, "chatChannels": [{"id": "123456789012345678", "answer": "everything"}]})).unwrap(); assert_eq!(parallel(&two), 2); llm.start(&two).await.unwrap(); assert_eq!(launches(&l).len(), 2, "measured for two at once"); assert!(launches(&l)[0].contains("--parallel 2")); llm.stop().await.unwrap(); // Another card (same size): measured again. std::fs::remove_file(l.data.join("calls")).unwrap(); let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-b", 0)); llm.start(&one).await.unwrap(); assert_eq!(launches(&l).len(), 2, "measured for the new card"); let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap(); assert_eq!((probe["32768/1"]["gpu"].clone(), probe["32768/2"]["gpu"].clone()), (json!("GPU-b"), json!("GPU-a"))); llm.stop().await.unwrap(); } #[cfg(unix)] #[tokio::test] async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() { // Left over answering, loading (503) or silent: always stopped first, holding 9874 MiB until then. for mode in ["list", "503", "hang"] { let l = layout(&format!("llm-place-left-{mode}")); let port = serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await; let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap(); let calls = std::fs::read_to_string(l.data.join("calls")).unwrap(); assert!(calls.lines().next() == Some("launch --stop"), "{mode}: stopped first: {calls}"); let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap(); assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "{mode}: the left server is not counted"); assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{mode}: {:?}", launches(&l)); llm.stop().await.unwrap(); } } #[cfg(unix)] #[tokio::test] async fn a_game_moves_the_llm_to_the_game_placement_and_back() { let l = layout("llm-stage"); let port = server_after_launch(&l, "list").await; let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null})); std::fs::remove_file(l.data.join("calls")).unwrap(); assert_eq!(tokio::time::timeout(Duration::from_secs(30), llm.switch(&s, Stage::Gaming, false, true)).await.unwrap().unwrap(), Stage::Gaming); let g = launches(&l); // Measured once at the game's context and one request, then the 2 GiB minimum (8 GB does not // fit on 16 GB) with half the engine's 16 CPU threads. assert_eq!(g.len(), 2, "{g:?}"); assert!(g[0].contains("--context 16384") && g[0].contains("--parallel 1") && g[0].contains("--set expert-vram=0"), "{}", g[0]); assert!(g[1].ends_with("--set expert-vram=2 --set prefill=2048 --set pool-workers=8"), "{}", g[1]); let room = 16303 - 1460 - 9874 - crate::placement::STT_MIB - crate::placement::MARGIN_MIB - 921; assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": room})); assert_eq!(llm.status().state, LlmState::Ready); // Back to normal: the run's first reading of other programs, the normal placement again. std::fs::remove_file(l.data.join("calls")).unwrap(); assert_eq!(llm.switch(&s, Stage::Normal, false, false).await.unwrap(), Stage::Normal); let n = launches(&l); assert_eq!(n.len(), 1, "{n:?}"); assert!(n[0].contains("--context 32768") && n[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", n[0]); assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null})); // Pause: the LLM stops. assert_eq!(llm.switch(&s, Stage::Paused, false, true).await.unwrap(), Stage::Paused); assert_eq!(llm.status().state, LlmState::Off); assert_eq!(llm.stage_view()["stage"], json!("paused")); } /// A card whose other programs hold `others` MiB (changeable), plus the STT model while /// `stt` is set, plus 9874 MiB while a Strata server runs. #[cfg(unix)] fn live_card(l: &Layout, others: Arc, stt: bool) -> GpuReader { let calls = l.data.join("calls"); Arc::new(move || { let c = std::fs::read_to_string(&calls).unwrap_or_default(); let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family")); let used = others.load(Ordering::SeqCst) + if stt { 1300 } else { 0 } + if running { 9874 } else { 0 }; Some(Gpu { id: "GPU-test".into(), used_mib: used, total_mib: 16303 }) }) } #[cfg(unix)] #[tokio::test] async fn the_normal_placement_reads_other_programs_afresh_and_a_game_is_not_counted_twice() { let l = layout("llm-stage-fresh"); let port = server_after_launch(&l, "list").await; let others = Arc::new(std::sync::atomic::AtomicI64::new(1460)); // At the start the STT worker has not loaded; at every switch it has. let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card(&l, others.clone(), false)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); let llm = llm.with_gpu(live_card(&l, others.clone(), true)); // A game starts and holds 2500 MiB: the game stage counts other programs as before the game. others.store(1460 + 2500, Ordering::SeqCst); llm.switch(&s, Stage::Gaming, true, true).await.unwrap(); assert!(launches(&l).last().unwrap().contains("--set expert-vram=2 "), "{:?}", launches(&l)); // The game ends but a video player now holds 1000 MiB more: the normal stage reads that. others.store(1460 + 1000, Ordering::SeqCst); llm.switch(&s, Stage::Normal, true, false).await.unwrap(); let expected = crate::placement::normal(16303, 1460 + 1000, &crate::placement::Probe { context: 32768, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }, 1024); assert!(expected < 1.5); assert!(launches(&l).last().unwrap().ends_with(&format!("--set expert-vram={expected} --set prefill=2048")), "{:?}", launches(&l)); } /// `live_card` on a card of `total` MiB. #[cfg(unix)] fn live_card_of(l: &Layout, others: Arc, total: i64) -> GpuReader { let calls = l.data.join("calls"); Arc::new(move || { let c = std::fs::read_to_string(&calls).unwrap_or_default(); let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family")); Some(Gpu { id: "GPU-test".into(), used_mib: others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: total }) }) } #[cfg(unix)] #[tokio::test] async fn a_game_running_when_the_services_start_is_not_counted_twice() { // A game holds 2500 MiB already: it is in what the card holds, so no reserve on top. let l = layout("llm-stage-game-at-start"); let port = server_after_launch(&l, "list").await; let others = Arc::new(std::sync::atomic::AtomicI64::new(1460 + 2500)); let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 16303)); llm.game_at_start(true); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); assert_eq!(llm.switch(&s, Stage::Gaming, false, true).await.unwrap(), Stage::Gaming, "not paused"); let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }; let expected = crate::placement::normal(16303, 1460 + 2500, &probe16, 1024); assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l)); } #[cfg(unix)] #[tokio::test] async fn the_game_stage_reads_the_card_again_and_counts_what_grew_since_the_normal_reading() { // Normal reading: 1460 MiB. Then a program takes 4000 MiB and the game 4500 MiB: together // more than the 8 GB reserve, so the placement leaves that much, not just 8 GB. let l = layout("llm-stage-grew"); let port = server_after_launch(&l, "list").await; let others = Arc::new(std::sync::atomic::AtomicI64::new(1460)); let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 24576)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); others.store(1460 + 4000 + 4500, Ordering::SeqCst); llm.switch(&s, Stage::Gaming, false, true).await.unwrap(); let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }; let expected = crate::placement::game(24576, 1460, &probe16, 8500).expert_gib; assert_ne!(expected, crate::placement::game(24576, 1460, &probe16, 8192).expert_gib, "the test tells them apart"); assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l)); } #[cfg(unix)] #[tokio::test] async fn a_game_stage_on_a_fallback_placement_still_shows_as_the_game_stage() { // The game's measurement fails (the card cannot be read the second time): no experts, but // the game stage is what runs. let l = layout("llm-stage-fallback"); let port = server_after_launch(&l, "list").await; let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); let reads = Arc::new(std::sync::atomic::AtomicU32::new(0)); let r2 = reads.clone(); let llm = llm.with_gpu(Arc::new(move || { (r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 }) })); assert_eq!(llm.switch(&s, Stage::Gaming, false, true).await.unwrap(), Stage::Gaming); assert!(launches(&l).last().unwrap().contains("--set expert-vram=0 "), "{:?}", launches(&l)); assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": null})); } #[cfg(unix)] #[tokio::test] async fn a_game_with_no_room_beside_the_model_pauses_the_llm() { let l = layout("llm-stage-full"); let port = server_after_launch(&l, "list").await; let calls = l.data.join("calls"); // A 12 GB card: the model fits, but not with any room for a game. let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || { let c = std::fs::read_to_string(&calls).unwrap_or_default(); let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family")); Some(Gpu { id: "GPU-small".into(), used_mib: 1460 + if running { 9874 } else { 0 }, total_mib: 12288 }) })); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); llm.start(&s).await.unwrap(); assert_eq!(llm.switch(&s, Stage::Gaming, false, true).await.unwrap(), Stage::Paused); assert_eq!(llm.status().state, LlmState::Off); assert_eq!(llm.stage_view(), json!({"stage": "paused", "gameRoomMib": null})); let c = std::fs::read_to_string(l.data.join("calls")).unwrap(); assert_eq!(c.lines().last(), Some("launch --stop")); } #[cfg(unix)] #[tokio::test] async fn a_left_server_that_does_not_stop_means_no_experts_and_nothing_measured() { // The stop command fails and the server keeps answering: the card is not read as it is. let l = layout("llm-place-stuck"); let port = models_server_until("qwen-test", Duration::from_millis(0), None).await; let log = l.strata_home().join("x.engine.log"); let exe = fake_strata_stop(&l, &format!("echo 'strata-q36: (1.10 GiB staging)' >> '{}'; echo 'Strata is serving.'", log.display()), "exit 3"); let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 9874)).with_stop_timeout(Duration::from_secs(1)); let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap(); let l2 = launches(&l); assert_eq!(l2.len(), 1, "{l2:?}"); assert!(l2[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[0]); assert!(!l.data.join("config").join("llm-probe.json").exists()); } #[cfg(unix)] #[tokio::test] async fn a_cancel_during_the_download_stops_it_and_clears_the_server() { let l = layout("llm6"); let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30"); let llm = Arc::new(Llm::with_exe(l.clone(), exe)); let l2 = llm.clone(); let run = tokio::spawn(async move { l2.start(&unused_port_settings()).await }); tokio::time::sleep(Duration::from_millis(400)).await; let t = Instant::now(); llm.cancel(); assert_eq!(run.await.unwrap().unwrap_err(), "cancelled"); assert!(t.elapsed() < Duration::from_secs(5)); assert_eq!(llm.status().state, LlmState::Off); let calls = std::fs::read_to_string(l.data.join("calls")).unwrap(); assert_eq!(calls.lines().last(), Some("launch --stop")); } #[cfg(unix)] #[tokio::test] async fn a_failed_launch_shows_strata_s_last_line() { let l = layout("llm7"); let exe = fake_strata(&l, "echo 'Not enough disk space for the model' >&2; exit 4"); let llm = Llm::with_exe(l, exe); let e = llm.start(&unused_port_settings()).await.unwrap_err(); assert!(e.contains("Not enough disk space"), "{e}"); assert_eq!(llm.status().state, LlmState::Failed); } #[cfg(unix)] #[tokio::test] async fn a_server_still_answering_after_a_failed_stop_is_reported_not_marked_off() { let l = layout("llm8"); let port = models_server("qwen-test", Duration::ZERO).await; let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "echo 'could not stop the engine' >&2; exit 3"); let llm = Llm::with_exe(l, exe); llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap(); let e = llm.stop().await.unwrap_err(); assert!(e.contains("could not stop the engine"), "{e}"); assert_eq!(llm.status().state, LlmState::Failed); assert_eq!(llm.status().note.as_deref(), Some(e.as_str())); } #[cfg(unix)] #[tokio::test] async fn a_stop_that_hangs_is_given_up_on_and_reported() { let l = layout("llm9"); let port = models_server("qwen-test", Duration::ZERO).await; let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "sleep 30"); let llm = Llm::with_exe(l, exe).with_stop_timeout(Duration::from_millis(500)); llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap(); let t = Instant::now(); let e = llm.stop().await.unwrap_err(); assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed()); assert!(e.contains("Strata"), "{e}"); assert_eq!(llm.status().state, LlmState::Failed); } #[cfg(unix)] #[tokio::test] async fn a_stop_command_that_fails_while_nothing_answers_counts_as_stopped() { let l = layout("llm10"); let port = models_server_until("qwen-test", Duration::ZERO, Some(l.data.join("gone"))).await; let stop = format!("touch '{}'; exit 1", l.data.join("gone").display()); let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", &stop); let llm = Llm::with_exe(l, exe); llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap(); llm.stop().await.unwrap(); assert_eq!(llm.status().state, LlmState::Off); } /// A failed stop while the server is still there but not listing a model (holding connections /// silently, or a 503 while loading) is not a confirmed stop: the LLM and, through the /// controller, the services stay failed. #[cfg(unix)] #[tokio::test] async fn a_failed_stop_with_a_silent_or_loading_server_is_not_counted_as_stopped() { for m in ["hang", "503"] { let l = layout(&format!("llm11{m}")); let mode = Arc::new(Mutex::new("list")); let port = serve("qwen-test", Duration::ZERO, None, mode.clone()).await; let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "exit 3"); let llm = Arc::new(Llm::with_exe(l, exe)); let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: parse_settings(&json!({"llm": {"port": port}})).unwrap() }); c.start().await.unwrap(); *mode.lock().unwrap() = m; let e = c.stop().await.unwrap_err(); assert!(e.contains("Strata"), "{m}: {e}"); assert_eq!(c.state(), crate::control::State::Failed, "{m}"); assert_eq!(llm.status().state, LlmState::Failed, "{m}"); } } /// A stop right after a background start must cancel the LLM's start before it runs anything /// (no download), even though the start task runs after the stop. #[cfg(unix)] #[tokio::test] async fn a_stop_right_after_a_background_start_runs_no_launch() { let l = layout("llm12"); let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30"); let llm = Arc::new(Llm::with_exe(l.clone(), exe)); let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: unused_port_settings() }); c.start_in_background().unwrap(); c.stop().await.unwrap(); let t = Instant::now(); while c.state() != crate::control::State::Stopped && t.elapsed() < Duration::from_secs(5) { tokio::time::sleep(Duration::from_millis(20)).await; } assert_eq!(c.state(), crate::control::State::Stopped); let calls = std::fs::read_to_string(l.data.join("calls")).unwrap_or_default(); assert!(!calls.contains("--family"), "no launch ran: {calls}"); // A later start is not cancelled by the earlier stop. c.start_in_background().unwrap(); tokio::time::sleep(Duration::from_millis(400)).await; assert!(std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().contains("--family")); c.stop().await.unwrap(); } #[cfg(unix)] /// The services as the controller sees them, reduced to the LLM. struct LlmHost { llm: Arc, settings: Settings, } #[cfg(unix)] struct LlmRunning(Arc); #[cfg(unix)] #[async_trait::async_trait] impl crate::control::Running for LlmRunning { async fn stop(&self) -> Result<(), String> { self.0.stop().await } fn status(&self, _: Instant) -> Value { json!({"worker": null, "voiceMembers": [], "missed": []}) } } #[cfg(unix)] #[async_trait::async_trait] impl crate::control::Host for LlmHost { async fn start(&self) -> Result<(Arc, tokio::sync::mpsc::UnboundedReceiver), String> { self.llm.start(&self.settings).await?; Ok((Arc::new(LlmRunning(self.llm.clone())), tokio::sync::mpsc::unbounded_channel().1)) } fn secrets(&self) -> Vec { vec![] } fn prepare_start(&self) { self.llm.arm(); } fn cancel_start(&self) { self.llm.cancel(); } } }