app/src/pc.rs runs every pc_action from the worker (admin.spec.md, "PC control"): one agent-desktop 1.2.0 run per command with a 15 s limit and its state in the data folder, never --force or raw input, typed text only through stdin. Each action is decided by permissions::decide for its asker (not registered, never, ask without consent), then refused during the emergency stop, while an anti-cheat game runs (Vanguard, EAC, BattlEye, FACEIT, GameGuard, XIGNCODE3 and the person's list), on an administrator window and in a password field. Results carry the window and its elements read again (200 at most, cut to 120 chars), agent-desktop's error codes become Korean reasons, and each action is recorded as pc.<action> with the program's file name only. The emergency stop is a global hotkey (pcControl.stopHotkey, default Ctrl+Shift+F12) on its own message loop; it kills the running command and blocks everything but reading until pressed again or lifted on the PC control screen. features.pcControl (on by default, nothing happens until someone is registered) and the pinned exe hash decide whether the brain gets the tool. The installer ships agent-desktop.exe from the pinned release archive, both hashes checked. Tested with a fake agent-desktop (342 lib tests pass) and cargo check for x86_64-pc-windows-gnu; not yet run against real agent-desktop. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1407 lines
72 KiB
Rust
1407 lines
72 KiB
Rust
//! The LLM the brain answers with (`app/src/admin.spec.md`, "LLM"): the app starts the installed
|
|
//! Strata from the data folder with the M0 kit's choices, waits until the model answers, and stops
|
|
//! it with the services. Strata downloads the model (about 23 GB) on its first start; the dashboard
|
|
//! shows how far it got. With `llm.url` the app uses that server instead and starts nothing.
|
|
|
|
use std::path::Path;
|
|
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use serde::Serialize;
|
|
use serde_json::{json, Value};
|
|
|
|
use crate::install::{Ctx, Layout};
|
|
use crate::perf::Stage;
|
|
use crate::settings::Settings;
|
|
|
|
/// The one model v2 runs (M0: Qwen3.6-35B-A3B UD-Q4_K_XL; docs/v2/roadmap.md).
|
|
pub const FAMILY: &str = "qwen36";
|
|
pub const MODEL: &str = "ud-q4_k_xl";
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub enum LlmState {
|
|
Off,
|
|
Starting,
|
|
Ready,
|
|
Failed,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct LlmStatus {
|
|
pub state: LlmState,
|
|
/// Strata's own server (false: `llm.url`).
|
|
pub managed: bool,
|
|
/// How far Strata's current line says it got while starting (a percentage, or the model
|
|
/// download's `X of Y GB`); none on a line without progress.
|
|
pub percent: Option<f64>,
|
|
/// Strata's last line while starting, or why it failed.
|
|
pub note: Option<String>,
|
|
}
|
|
|
|
/// `strata launch` for the services: the M0 kit's normal placement, models in `models/llm`, this
|
|
/// computer only. No pictures yet (nothing in v2 sends one; reading them takes graphics memory).
|
|
/// `--restart` replaces a server started with other choices; the same choices are reused.
|
|
pub fn launch_args(layout: &Layout, settings: &Settings, conf: &Conf, sets: &[String]) -> Vec<String> {
|
|
let models = layout.data.join("models").join("llm").to_string_lossy().into_owned();
|
|
let mut a: Vec<String> = ["launch", "--family", FAMILY, "--model", MODEL, "--data-dir"].map(String::from).into();
|
|
a.push(models);
|
|
for (k, v) in [("--context", conf.context.to_string()), ("--vision", conf.vision.into()),
|
|
("--parallel", conf.parallel.to_string()), ("--access", "local".into()),
|
|
("--port", settings.llm.port.to_string())] {
|
|
a.push(k.into());
|
|
a.push(v);
|
|
}
|
|
a.extend(["--no-api-key", "--restart", "--yes", "--no-tui"].map(String::from));
|
|
for s in sets {
|
|
a.push("--set".into());
|
|
a.push(s.clone());
|
|
}
|
|
a
|
|
}
|
|
|
|
/// The context and requests at once of a stage: the settings' context in the normal stage, in a
|
|
/// game 16K (or less when the settings ask for less); always one request at a time (faster on a
|
|
/// 16 GB card, M0; chat answers take turns with the voice answers).
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct Conf {
|
|
pub context: u32,
|
|
pub parallel: u32,
|
|
/// `--vision`: `off`, `gpu` or `cpu` (`vision`).
|
|
pub vision: &'static str,
|
|
}
|
|
|
|
pub fn conf(settings: &Settings, stage: Stage) -> Conf {
|
|
let vision = vision(stage, crate::screen::available(settings));
|
|
match stage {
|
|
Stage::Gaming => Conf { context: crate::placement::GAME_CONTEXT.min(settings.llm.context), parallel: 1, vision },
|
|
_ => Conf { context: settings.llm.context, parallel: 1, vision },
|
|
}
|
|
}
|
|
|
|
/// Where the LLM reads pictures (`--vision`): nowhere unless screen viewing is available, on the
|
|
/// card normally and on the CPU in a game (M0's game placement).
|
|
pub fn vision(stage: Stage, screen: bool) -> &'static str {
|
|
match (screen, stage) {
|
|
(false, _) => "off",
|
|
(true, Stage::Gaming) => "cpu",
|
|
(true, _) => "gpu",
|
|
}
|
|
}
|
|
|
|
/// Where the probe of a placement is kept in `config/llm-probe.json`: pictures on the card take
|
|
/// memory of their own, so they are measured apart.
|
|
pub fn probe_key(c: &Conf) -> String {
|
|
if c.vision == "gpu" { format!("{}/{}/vision", c.context, c.parallel) } else { format!("{}/{}", c.context, c.parallel) }
|
|
}
|
|
|
|
/// The engine's CPU threads from its log ("... on 16 CPU threads ...").
|
|
fn cpu_threads(log: &str) -> Option<u32> {
|
|
log.lines().rev().find_map(|l| {
|
|
let i = l.find(" CPU threads")?;
|
|
l[..i].rsplit(' ').next()?.parse().ok()
|
|
})
|
|
}
|
|
|
|
/// The engine settings for an expert budget (`placement.rs`) and the M0 kit's prompt chunk.
|
|
fn placement_sets(expert_gib: f64) -> Vec<String> {
|
|
vec![format!("expert-vram={expert_gib}"), "prefill=2048".into()]
|
|
}
|
|
|
|
/// The first NVIDIA card: its UUID and graphics memory in MiB.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct Gpu {
|
|
pub id: String,
|
|
pub used_mib: i64,
|
|
pub total_mib: i64,
|
|
/// What is left for programs (`memory.free`): the total less what is used and what the driver reserves.
|
|
pub free_mib: i64,
|
|
}
|
|
|
|
/// Reads the card (`nvidia-smi`); None without one.
|
|
pub type GpuReader = Arc<dyn Fn() -> Option<Gpu> + Send + Sync>;
|
|
|
|
fn nvidia_memory() -> Option<Gpu> {
|
|
let mut cmd = std::process::Command::new("nvidia-smi");
|
|
cmd.args(["--query-gpu=uuid,memory.used,memory.total,memory.free", "--format=csv,noheader,nounits"]);
|
|
#[cfg(windows)]
|
|
{
|
|
use std::os::windows::process::CommandExt;
|
|
cmd.creation_flags(0x0800_0000);
|
|
}
|
|
let out = cmd.output().ok()?;
|
|
let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned();
|
|
let mut f = line.split(',').map(str::trim);
|
|
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()?,
|
|
free_mib: f.next()?.parse().ok()? })
|
|
}
|
|
|
|
/// The newest engine log under Strata's folder, its end only (the placement lines of the last start).
|
|
fn engine_log(home: &Path) -> String {
|
|
fn walk(dir: &Path, depth: u32, best: &mut Option<(std::time::SystemTime, std::path::PathBuf)>) {
|
|
let Ok(entries) = std::fs::read_dir(dir) else { return };
|
|
for e in entries.flatten() {
|
|
let p = e.path();
|
|
if p.is_dir() && depth > 0 {
|
|
walk(&p, depth - 1, best);
|
|
} else if p.to_string_lossy().ends_with(".engine.log") {
|
|
if let Ok(t) = e.metadata().and_then(|m| m.modified()) {
|
|
if best.as_ref().is_none_or(|(b, _)| t > *b) {
|
|
*best = Some((t, p));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let mut best = None;
|
|
walk(home, 5, &mut best);
|
|
let Some((_, p)) = best else { return String::new() };
|
|
let bytes = std::fs::read(p).unwrap_or_default();
|
|
String::from_utf8_lossy(&bytes[bytes.len().saturating_sub(400_000)..]).into_owned()
|
|
}
|
|
|
|
/// How far a line says Strata got: its last `NN%` or `NN.N%`, else `X of Y GB` (or MB), which is
|
|
/// how Strata 0.1.59 reports the model download.
|
|
pub fn percent(line: &str) -> Option<f64> {
|
|
percent_sign(line).or_else(|| size_progress(line))
|
|
}
|
|
|
|
fn size_progress(line: &str) -> Option<f64> {
|
|
let words: Vec<&str> = line.split_whitespace().collect();
|
|
words.windows(4).rev().find_map(|w| {
|
|
let (done, total) = (w[0].parse::<f64>().ok()?, w[2].parse::<f64>().ok()?);
|
|
(w[1] == "of" && matches!(w[3].trim_end_matches([',', '.', ')']), "GB" | "MB") && total > 0.0 && done <= total)
|
|
.then(|| (done / total * 1000.0).round() / 10.0)
|
|
})
|
|
}
|
|
|
|
fn percent_sign(line: &str) -> Option<f64> {
|
|
let b = line.as_bytes();
|
|
let mut found = None;
|
|
for (i, _) in line.match_indices('%') {
|
|
let mut s = i;
|
|
while s > 0 && (b[s - 1].is_ascii_digit() || b[s - 1] == b'.') {
|
|
s -= 1;
|
|
}
|
|
if let Ok(n) = line[s..i].parse::<f64>() {
|
|
if (0.0..=100.0).contains(&n) {
|
|
found = Some(n);
|
|
}
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
/// Wait until `base` (`.../v1`) lists a model; returns its id. Stops early on `cancel`.
|
|
pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> Result<String, String> {
|
|
let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().map_err(|e| e.to_string())?;
|
|
let until = Instant::now() + timeout;
|
|
loop {
|
|
if cancel.load(Ordering::Relaxed) {
|
|
return Err("cancelled".into());
|
|
}
|
|
if let Ok(r) = client.get(format!("{base}/models")).send().await {
|
|
if let Ok(v) = r.text().await.map(|t| serde_json::from_str::<Value>(&t).unwrap_or_default()) {
|
|
if let Some(id) = v["data"][0]["id"].as_str() {
|
|
return Ok(id.to_owned());
|
|
}
|
|
}
|
|
}
|
|
if Instant::now() >= until {
|
|
return Err(format!("대화 모델이 {}초 안에 준비되지 않았습니다 ({base})", timeout.as_secs()));
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
}
|
|
}
|
|
|
|
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an
|
|
/// error when the model gives none.
|
|
async fn warm_up(base: &str) -> Result<(), String> {
|
|
let client = reqwest::Client::builder().timeout(Duration::from_secs(60)).build().map_err(|e| e.to_string())?;
|
|
let model = match client.get(format!("{base}/models")).send().await {
|
|
Ok(r) => r.text().await.ok().and_then(|t| serde_json::from_str::<Value>(&t).ok())
|
|
.and_then(|v| v["data"][0]["id"].as_str().map(str::to_owned)),
|
|
Err(_) => None,
|
|
};
|
|
let body = json!({"model": model, "messages": [{"role": "user", "content": "안녕"}], "max_tokens": 8});
|
|
let r = client.post(format!("{base}/chat/completions")).header("content-type", "application/json")
|
|
.body(body.to_string()).send().await.map_err(|e| e.to_string())?;
|
|
if !r.status().is_success() {
|
|
return Err(format!("the model answered {}", r.status()));
|
|
}
|
|
let v: Value = serde_json::from_str(&r.text().await.map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
|
|
v["choices"][0]["message"].is_object().then_some(()).ok_or_else(|| "the model gave no answer".into())
|
|
}
|
|
|
|
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the
|
|
/// memory database inside the data folder, the tools of the features that are off (`pc`: whether PC
|
|
/// control is available) and the enabled MCP servers (without their environment values,
|
|
/// `features.rs`). The app owns this file and writes it on every start.
|
|
pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings, pc: bool)
|
|
-> Result<std::path::PathBuf, String> {
|
|
let file = data.join("config").join("brain.json");
|
|
let cfg = json!({
|
|
"ollama_base_url": base_url,
|
|
"ollama_chat_model": model,
|
|
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
|
|
"disabled_tools": crate::features::disabled_tools(settings, pc),
|
|
"mcps": crate::features::brain_mcps(&settings.mcp_servers),
|
|
"location_name": settings.location,
|
|
"_config_version": 1,
|
|
});
|
|
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
|
|
text.push('\n');
|
|
crate::datadir::write_atomic(&file, text.as_bytes()).map_err(|e| e.to_string())?;
|
|
Ok(file)
|
|
}
|
|
|
|
/// One LLM for the services: Strata from the layout, or the server at `llm.url`.
|
|
pub struct Llm {
|
|
layout: Layout,
|
|
/// The program run as Strata (the installed one; a script in tests).
|
|
exe: std::path::PathBuf,
|
|
status: Mutex<LlmStatus>,
|
|
active: Mutex<Option<Arc<Ctx>>>,
|
|
cancel: Arc<AtomicBool>,
|
|
/// The port the managed server was started on (to confirm it is gone after a stop).
|
|
port: Mutex<u16>,
|
|
stop_timeout: Duration,
|
|
/// The graphics card's memory (none: Strata places the model itself).
|
|
gpu: Option<GpuReader>,
|
|
/// What other programs held at the latest normal placement made with no game running: the
|
|
/// game stage counts other programs from it (the card then also holds the game, which its
|
|
/// reserve is for). None when the services started with a game running.
|
|
baseline: Mutex<Option<i64>>,
|
|
/// A game runs (or may run) as the LLM is placed: set before `start` and at every `switch`.
|
|
/// Only a normal placement made while none runs keeps its reading.
|
|
game_running: AtomicBool,
|
|
/// What the voice worker's models (STT, TTS) hold on the card (set at switches while the worker
|
|
/// runs), so it is not taken for another program's memory.
|
|
worker_models_mib: AtomicI64,
|
|
/// The graphics memory a game gets in the placement being started.
|
|
room: Mutex<Option<i64>>,
|
|
/// The engine's CPU threads in the normal stage; a game gets half.
|
|
threads: Mutex<Option<u32>>,
|
|
/// The stage it runs in, and in a game the graphics memory the game gets.
|
|
stage: Mutex<(Stage, Option<i64>)>,
|
|
}
|
|
|
|
/// How long the model may take to answer after `strata launch` returned (it loads in about 15 s
|
|
/// on the game PC; the first start may also tune).
|
|
const READY_TIMEOUT: Duration = Duration::from_secs(30 * 60);
|
|
/// A server the app does not start must already answer.
|
|
const URL_TIMEOUT: Duration = Duration::from_secs(30);
|
|
|
|
impl Llm {
|
|
pub fn new(data: &Path) -> Self {
|
|
// Only the data folder matters here (the Python sources are the install's business).
|
|
let layout = Layout { data: data.to_path_buf(), src: std::path::PathBuf::new() };
|
|
let exe = layout.strata_exe();
|
|
Self::with_exe(layout, exe).with_gpu(Arc::new(nvidia_memory))
|
|
}
|
|
|
|
pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self {
|
|
Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }),
|
|
active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0),
|
|
stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), game_running: AtomicBool::new(false), worker_models_mib: AtomicI64::new(0), room: Mutex::new(None), threads: Mutex::new(None),
|
|
stage: Mutex::new((Stage::Normal, None)) }
|
|
}
|
|
|
|
/// Whether a game runs (also in the background) as the services start: the start's reading of
|
|
/// the card then includes it and is not kept as other programs' memory.
|
|
pub fn game_at_start(&self, on: bool) {
|
|
self.game_running.store(on, Ordering::SeqCst);
|
|
}
|
|
|
|
/// `{stage, gameRoomMib}` for the dashboard.
|
|
pub fn stage_view(&self) -> Value {
|
|
let (stage, room) = *self.stage.lock().unwrap();
|
|
json!({"stage": stage, "gameRoomMib": room})
|
|
}
|
|
|
|
/// Place the model by the card's memory (`placement.rs`), read with `gpu`.
|
|
pub fn with_gpu(mut self, gpu: GpuReader) -> Self {
|
|
self.gpu = Some(gpu);
|
|
self
|
|
}
|
|
|
|
async fn read_gpu(&self) -> Option<Gpu> {
|
|
let gpu = self.gpu.clone()?;
|
|
tokio::task::spawn_blocking(move || gpu()).await.ok().flatten()
|
|
}
|
|
|
|
fn probe_file(&self) -> std::path::PathBuf {
|
|
self.layout.data.join("config").join("llm-probe.json")
|
|
}
|
|
|
|
/// The expert budget's engine settings (none without a card): the normal placement from what
|
|
/// the card holds now and this PC's measurement without experts at this context and number of
|
|
/// requests at once, which the first start makes once per card (a start without experts, then
|
|
/// the real start replaces it). A server left from an earlier run is stopped first, so it is not
|
|
/// counted as another program's memory (a stop that is not confirmed reads nothing: no
|
|
/// experts). A measurement that fails is not kept: the model then starts without experts on
|
|
/// the card.
|
|
async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str, stage: Stage) -> Result<Option<Vec<String>>, String> {
|
|
let c = conf(settings, stage);
|
|
*self.room.lock().unwrap() = None;
|
|
let workers = |sets: &mut Vec<String>| if stage == Stage::Gaming {
|
|
let half = self.threads.lock().unwrap().map(|n| n / 2)
|
|
.unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get() as u32 / 4).unwrap_or(1));
|
|
sets.push(format!("pool-workers={}", half.max(1)));
|
|
};
|
|
if self.gpu.is_none() {
|
|
let mut sets = vec![];
|
|
workers(&mut sets);
|
|
return Ok(Some(sets));
|
|
}
|
|
// Whatever a server left from an earlier run does (answers, loads, stays silent), it holds
|
|
// the card: stop it first and read the card only once the port is confirmed closed.
|
|
if let Err(e) = self.stop_server_on(settings.llm.port).await {
|
|
crate::warn_log!("llm", "Warning: a server left on the port did not stop ({e}); starting without experts on the card");
|
|
let mut sets = placement_sets(0.0);
|
|
workers(&mut sets);
|
|
return Ok(Some(sets));
|
|
}
|
|
let Some(card) = self.read_gpu().await else { return Ok(Some(vec![])) };
|
|
// Other programs: what the card holds without this LLM and without the voice worker's models.
|
|
// The normal stage reads it afresh (programs opened since count); the game stage takes it
|
|
// from the latest normal placement, as the card then also holds the game.
|
|
let now_others = (card.used_mib - self.worker_models_mib.load(Ordering::SeqCst)).max(0);
|
|
// Only a normal placement made while no game runs keeps its reading (a game's memory would be
|
|
// taken for other programs'; also on the way back from a failed game stage).
|
|
if stage != Stage::Gaming && !self.game_running.load(Ordering::SeqCst) {
|
|
*self.baseline.lock().unwrap() = Some(now_others);
|
|
}
|
|
let baseline = *self.baseline.lock().unwrap();
|
|
let key = probe_key(&c);
|
|
let file = self.probe_file();
|
|
let mut all: serde_json::Map<String, Value> = std::fs::read_to_string(&file).ok()
|
|
.and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();
|
|
let known = all.get(&key).and_then(|v| serde_json::from_value::<crate::placement::Probe>(v.clone()).ok())
|
|
.filter(|p| p.gpu == card.id);
|
|
let probe = match known {
|
|
Some(p) => p,
|
|
None => {
|
|
crate::info_log!("llm", "measuring the model without experts once (context {}, {} at once)", c.context, c.parallel);
|
|
self.launch(settings, ctx, base, &c, &placement_sets(0.0)).await?;
|
|
let warmed = warm_up(base).await;
|
|
let after = self.read_gpu().await.filter(|g| g.id == card.id);
|
|
let model_mib = after.map(|g| g.used_mib - card.used_mib).unwrap_or(0);
|
|
if let Err(e) = warmed.and_then(|()| if model_mib > 0 { Ok(()) } else { Err("the card's memory could not be read".into()) }) {
|
|
crate::warn_log!("llm", "Warning: the measurement failed ({e}); starting without experts on the card");
|
|
let mut sets = placement_sets(0.0);
|
|
workers(&mut sets);
|
|
return Ok(Some(sets));
|
|
}
|
|
let p = crate::placement::Probe { context: c.context, parallel: c.parallel, gpu: card.id.clone(), model_mib,
|
|
staging_gib: crate::placement::staging_gib(&engine_log(&self.layout.strata_home())) };
|
|
all.insert(key, json!(p));
|
|
crate::datadir::write_atomic(&file, Value::Object(all).to_string().as_bytes())
|
|
.map_err(|e| format!("배치 측정값을 저장하지 못했습니다: {e}"))?;
|
|
p
|
|
}
|
|
};
|
|
let mut others = now_others;
|
|
let expert = match (stage, baseline) {
|
|
// With a reading from before the game: other programs as then, and the reserve at least
|
|
// what has grown since (the game, and any program opened after that reading).
|
|
(Stage::Gaming, Some(base)) => {
|
|
others = base;
|
|
let reserve = settings.game.reserve_mib().max(now_others - base);
|
|
let g = crate::placement::game(card.total_mib, base, &probe, reserve);
|
|
if g.paused {
|
|
crate::info_log!("llm", "placement: no room for a game beside the model; paused while the game runs");
|
|
return Ok(None);
|
|
}
|
|
*self.room.lock().unwrap() = Some(g.game_room_mib);
|
|
g.expert_gib
|
|
}
|
|
// A game that already ran at the start is in what the card holds now: no reserve on
|
|
// top, only the room for other programs.
|
|
_ => crate::placement::normal(card.total_mib, now_others, &probe, settings.game.other_room_mib()),
|
|
};
|
|
crate::info_log!("llm", "placement ({stage:?}): experts {expert} GiB (card {} MiB, held by others {others} MiB, model without experts {} MiB)",
|
|
card.total_mib, probe.model_mib);
|
|
let mut sets = placement_sets(expert);
|
|
workers(&mut sets);
|
|
Ok(Some(sets))
|
|
}
|
|
|
|
/// `strata launch` with `sets`, then wait until the model answers.
|
|
async fn launch(&self, settings: &Settings, ctx: &Ctx, base: &str, c: &Conf, sets: &[String]) -> Result<String, String> {
|
|
let args = launch_args(&self.layout, settings, c, sets);
|
|
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
|
|
ctx.run_in(&self.exe, &refs, Some(&self.layout.strata_home())).await?;
|
|
wait_ready(base, READY_TIMEOUT, &self.cancel).await
|
|
}
|
|
|
|
/// How long `strata launch --stop` may take before it is ended and the stop reported as failed.
|
|
pub fn with_stop_timeout(mut self, t: Duration) -> Self {
|
|
self.stop_timeout = t;
|
|
self
|
|
}
|
|
|
|
pub fn status(&self) -> LlmStatus {
|
|
let mut s = self.status.lock().unwrap().clone();
|
|
if let Some(ctx) = self.active.lock().unwrap().as_ref() {
|
|
s.note = ctx.last_note();
|
|
// From the current line only: a percentage from an earlier stage would mislead.
|
|
s.percent = s.note.as_deref().and_then(percent);
|
|
}
|
|
s
|
|
}
|
|
|
|
fn set(&self, state: LlmState, note: Option<String>) {
|
|
let mut s = self.status.lock().unwrap();
|
|
s.state = state;
|
|
if state != LlmState::Starting {
|
|
s.note = note;
|
|
}
|
|
}
|
|
|
|
/// Start (or reuse) the LLM; returns its `/v1` address and model id.
|
|
/// A cancel since the last `arm` (or before the first start) ends it before it runs anything.
|
|
pub async fn start(&self, settings: &Settings) -> Result<(String, String), String> {
|
|
// A new run: other programs' memory is read afresh.
|
|
*self.baseline.lock().unwrap() = None;
|
|
*self.stage.lock().unwrap() = (Stage::Normal, None);
|
|
self.worker_models_mib.store(0, Ordering::SeqCst);
|
|
self.start_stage(settings, Stage::Normal).await.map(|r| r.expect("the normal stage never pauses"))
|
|
}
|
|
|
|
/// Move the running LLM to `stage` (`perf.rs`): a restart with that stage's placement, or a stop
|
|
/// when it pauses (also a game stage with no room for the game). Returns the stage it ended in.
|
|
/// With `llm.url` nothing moves.
|
|
/// `worker_models_mib`: what the voice worker's loaded models hold on the card
|
|
/// (`placement::worker_models_mib`); `game_running`: a game runs now.
|
|
pub async fn switch(&self, settings: &Settings, stage: Stage, worker_models_mib: i64, game_running: bool) -> Result<Stage, String> {
|
|
if settings.llm.url.is_some() {
|
|
return Ok(Stage::Normal);
|
|
}
|
|
self.worker_models_mib.store(worker_models_mib, Ordering::SeqCst);
|
|
self.game_running.store(game_running, Ordering::SeqCst);
|
|
if stage != Stage::Paused && self.start_stage(settings, stage).await?.is_some() {
|
|
return Ok(stage);
|
|
}
|
|
*self.port.lock().unwrap() = settings.llm.port;
|
|
self.stop().await?;
|
|
*self.stage.lock().unwrap() = (Stage::Paused, None);
|
|
self.set(LlmState::Off, Some("게임 중 일시정지".into()));
|
|
Ok(Stage::Paused)
|
|
}
|
|
|
|
async fn start_stage(&self, settings: &Settings, stage: Stage) -> Result<Option<(String, String)>, String> {
|
|
let managed = settings.llm.url.is_none();
|
|
*self.status.lock().unwrap() = LlmStatus { state: LlmState::Starting, managed, percent: None, note: None };
|
|
let result = self.start_inner(settings, stage).await;
|
|
if stage == Stage::Normal && result.as_ref().is_ok_and(Option::is_some) && self.gpu.is_some() {
|
|
if let Some(n) = cpu_threads(&engine_log(&self.layout.strata_home())) {
|
|
*self.threads.lock().unwrap() = Some(n);
|
|
}
|
|
}
|
|
// What runs now, also on a fallback placement (no experts), with the game's room if known.
|
|
if let Ok(Some(_)) = &result {
|
|
*self.stage.lock().unwrap() = (stage, self.room.lock().unwrap().take());
|
|
}
|
|
match &result {
|
|
Ok(None) => {}
|
|
Ok(Some((base, model))) => {
|
|
crate::info_log!("llm", "ready: {base} model={model}");
|
|
// What the card has left for everything else once the model is in (the voice
|
|
// worker's models, a game): a slow answer later can be matched to it.
|
|
if let Some(g) = self.read_gpu().await {
|
|
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.free_mib, g.total_mib);
|
|
}
|
|
self.set(LlmState::Ready, None);
|
|
}
|
|
Err(e) if e == "cancelled" => self.set(LlmState::Off, None),
|
|
Err(e) => {
|
|
crate::error_log!("llm", "did not start: {e}");
|
|
self.set(LlmState::Failed, Some(e.clone()));
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// None: the stage pauses (nothing started).
|
|
async fn start_inner(&self, settings: &Settings, stage: Stage) -> Result<Option<(String, String)>, String> {
|
|
if self.cancel.load(Ordering::SeqCst) {
|
|
return Err("cancelled".into());
|
|
}
|
|
if let Some(url) = &settings.llm.url {
|
|
let model = wait_ready(url, URL_TIMEOUT, &self.cancel).await?;
|
|
return Ok(Some((url.clone(), model)));
|
|
}
|
|
if !self.exe.exists() {
|
|
return Err("Strata가 설치되어 있지 않습니다. 마법사의 모델 단계에서 설치해 주세요".into());
|
|
}
|
|
let ctx = Arc::new(Ctx::standalone(self.layout.clone(), "llm", self.cancel.clone()));
|
|
*self.active.lock().unwrap() = Some(ctx.clone());
|
|
*self.port.lock().unwrap() = settings.llm.port;
|
|
let base = format!("http://127.0.0.1:{}/v1", settings.llm.port);
|
|
let c = conf(settings, stage);
|
|
let ready = match self.placement(settings, &ctx, &base, stage).await {
|
|
Ok(Some(sets)) => self.launch(settings, &ctx, &base, &c, &sets).await.map(Some),
|
|
Ok(None) => Ok(None),
|
|
Err(e) => Err(e),
|
|
};
|
|
*self.active.lock().unwrap() = None;
|
|
match ready {
|
|
Ok(Some(model)) => Ok(Some((base, model))),
|
|
Ok(None) => Ok(None),
|
|
// Nothing half-started stays on the graphics card; if it may have, say so.
|
|
Err(e) => match self.stop_server().await {
|
|
Ok(()) => Err(e),
|
|
Err(stop) if e == "cancelled" => Err(stop),
|
|
Err(stop) => Err(format!("{e} ({stop})")),
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Make the next start cancellable afresh: called when a start is accepted, never by `start`
|
|
/// itself, so a stop between accepting and running the start still cancels it.
|
|
pub fn arm(&self) {
|
|
self.cancel.store(false, Ordering::SeqCst);
|
|
}
|
|
|
|
/// Stop a start in progress (the download resumes on the next start).
|
|
pub fn cancel(&self) {
|
|
self.cancel.store(true, Ordering::SeqCst);
|
|
}
|
|
|
|
/// Stop the server this app started (nothing for `llm.url`). Off only once the server no longer
|
|
/// answers; otherwise `failed` with the reason, which is also returned.
|
|
pub async fn stop(&self) -> Result<(), String> {
|
|
let managed = self.status.lock().unwrap().managed;
|
|
let result = if managed { self.stop_server().await } else { Ok(()) };
|
|
match &result {
|
|
Ok(()) => self.set(LlmState::Off, None),
|
|
Err(e) => {
|
|
crate::error_log!("llm", "{e}");
|
|
self.set(LlmState::Failed, Some(e.clone()));
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that
|
|
/// fails while nothing answers (nothing was serving) counts as stopped.
|
|
async fn stop_server(&self) -> Result<(), String> {
|
|
let port = *self.port.lock().unwrap();
|
|
self.stop_server_on(port).await
|
|
}
|
|
|
|
/// `stop_server` confirming on `port` (0: nothing was started, nothing to confirm).
|
|
async fn stop_server_on(&self, port: u16) -> Result<(), String> {
|
|
if !self.exe.exists() {
|
|
return Ok(());
|
|
}
|
|
// The timeout ends the command and everything it started, like an install cancel.
|
|
let cancel = Arc::new(AtomicBool::new(false));
|
|
let ctx = Ctx::standalone(self.layout.clone(), "llm", cancel.clone());
|
|
let home = self.layout.strata_home();
|
|
let timer = {
|
|
let (cancel, t) = (cancel.clone(), self.stop_timeout);
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(t).await;
|
|
cancel.store(true, Ordering::SeqCst);
|
|
})
|
|
};
|
|
let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await;
|
|
timer.abort();
|
|
let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e });
|
|
let base = format!("http://127.0.0.1:{port}/v1");
|
|
let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await;
|
|
match (ran, gone) {
|
|
(Ok(()), true) => {
|
|
crate::info_log!("llm", "stopped");
|
|
Ok(())
|
|
}
|
|
(Err(e), true) => {
|
|
crate::warn_log!("llm", "stop command failed but nothing answers: {e}");
|
|
Ok(())
|
|
}
|
|
(Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 닫히지 않았습니다")),
|
|
(Err(e), false) => Err(format!("Strata를 끄지 못했습니다: {e}")),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Whether `base` is gone within `timeout`: only a refused connection counts. A timeout, an error
|
|
/// status (a server still loading answers 503) or an answer it cannot read is not a confirmed stop,
|
|
/// so it keeps checking until the deadline.
|
|
async fn wait_gone(base: &str, timeout: Duration) -> bool {
|
|
// Windows answers a connection to a closed local port only after about 2 s of retries, so the
|
|
// connect wait is long enough for that refusal to arrive as a refusal, not as a timeout.
|
|
let Ok(client) = reqwest::Client::builder().connect_timeout(Duration::from_secs(10))
|
|
.timeout(Duration::from_secs(12)).build() else { return false };
|
|
let until = Instant::now() + timeout;
|
|
loop {
|
|
if let Err(e) = client.get(format!("{base}/models")).send().await {
|
|
if e.is_connect() && !e.is_timeout() {
|
|
return true;
|
|
}
|
|
}
|
|
if Instant::now() >= until {
|
|
return false;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod tests {
|
|
use super::*;
|
|
use crate::datadir::temp_dir;
|
|
use crate::settings::{parse_settings, ChatChannelSettings};
|
|
|
|
/// Other programs on the fake cards: the game PC's 1460 MiB in M0 less the TTS's share, which
|
|
/// the app keeps beside and M0 did not run, so M0's placements come out.
|
|
/// A card reading as `nvidia-smi` gives it, with nothing reserved by the driver.
|
|
pub(crate) fn gpu(id: &str, used_mib: i64, total_mib: i64) -> Gpu {
|
|
Gpu { id: id.into(), used_mib, total_mib, free_mib: total_mib - used_mib }
|
|
}
|
|
|
|
pub(crate) const OTHERS: i64 = 1460 - crate::placement::TTS_MIB;
|
|
|
|
pub(crate) fn layout(tag: &str) -> Layout {
|
|
Layout { data: temp_dir(tag), src: std::path::PathBuf::from("src") }
|
|
}
|
|
|
|
#[test]
|
|
fn the_launch_keeps_the_model_in_the_data_folder_and_follows_the_settings() {
|
|
let l = layout("llm1");
|
|
let s = parse_settings(&json!({"llm": {"context": 16384, "port": 8099}})).unwrap();
|
|
let a = launch_args(&l, &s, &conf(&s, Stage::Normal), &[]);
|
|
let after = |k: &str| a[a.iter().position(|x| x == k).unwrap() + 1].clone();
|
|
assert!(Path::new(&after("--data-dir")).starts_with(&l.data));
|
|
assert_eq!((after("--context"), after("--port"), after("--parallel")), ("16384".into(), "8099".into(), "1".into()));
|
|
assert_eq!(after("--access"), "local");
|
|
assert!(a.iter().any(|x| x == "--no-tui") && a.iter().any(|x| x == "--yes"));
|
|
}
|
|
|
|
#[test]
|
|
fn pictures_are_read_only_with_screen_viewing_and_measured_apart_on_the_card() {
|
|
assert_eq!(vision(Stage::Normal, false), "off");
|
|
assert_eq!(vision(Stage::Gaming, false), "off");
|
|
assert_eq!(vision(Stage::Normal, true), "gpu");
|
|
assert_eq!(vision(Stage::Gaming, true), "cpu");
|
|
let s = Settings::default();
|
|
let c = conf(&s, Stage::Normal);
|
|
assert_eq!(c.vision, "off", "nobody is registered for PC control by default");
|
|
let a = launch_args(&layout("llm-vision"), &s, &Conf { vision: "gpu", ..c }, &[]);
|
|
assert_eq!(a[a.iter().position(|x| x == "--vision").unwrap() + 1], "gpu");
|
|
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "off" }), "32768/1");
|
|
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "cpu" }), "32768/1");
|
|
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "gpu" }), "32768/1/vision");
|
|
}
|
|
|
|
#[test]
|
|
fn the_llm_takes_one_request_at_a_time_also_with_a_chat_channel_answering_everything() {
|
|
let mut s = Settings::default();
|
|
s.chat_channels.push(ChatChannelSettings { id: 1, name: String::new(), description: String::new(), read: true,
|
|
send: true, edit: true, react: true, delete: false, pin: false, answer_all: true, sync_discord: true });
|
|
let a = launch_args(&layout("llm-one"), &s, &conf(&s, Stage::Normal), &[]);
|
|
assert_eq!(a[a.iter().position(|x| x == "--parallel").unwrap() + 1], "1");
|
|
}
|
|
|
|
#[test]
|
|
fn the_progress_is_the_last_percentage_on_a_line() {
|
|
assert_eq!(percent("Downloading 3 of 5: 41.5% (9.6 GB of 23.1 GB)"), Some(41.5));
|
|
assert_eq!(percent("50% done, now 75%"), Some(75.0));
|
|
assert_eq!(percent("Loading the model… 15 s"), None);
|
|
// Strata 0.1.59's model download prints sizes, not a percentage.
|
|
assert_eq!(percent("Downloading the model files: 2.4 of 22.9 GB"), Some(10.5));
|
|
assert_eq!(percent("Downloading the model files: 22.9 of 22.9 GB"), Some(100.0));
|
|
}
|
|
|
|
#[test]
|
|
fn the_brain_config_points_at_the_llm_and_keeps_memory_in_the_data_folder() {
|
|
let l = layout("llm2");
|
|
let s = parse_settings(&json!({"features": {"weather": false},
|
|
"mcpServers": [{"name": "brave", "command": "npx", "env": ["BRAVE_API_KEY"]}]})).unwrap();
|
|
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s, false).unwrap();
|
|
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
|
|
assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1");
|
|
assert_eq!(v["ollama_chat_model"], "m");
|
|
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
|
|
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s, false)));
|
|
assert!(v["disabled_tools"].as_array().unwrap().contains(&json!("pcControl")));
|
|
assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers));
|
|
assert_eq!(v["location_name"], crate::settings::DEFAULT_LOCATION, "the people are in Korea by default");
|
|
let none = parse_settings(&json!({"brain": {"location": ""}})).unwrap();
|
|
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none, false).unwrap();
|
|
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
|
|
assert_eq!(v["location_name"], Value::Null);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
/// Default settings on a port nothing listens on (8080 may be taken on a developer's PC).
|
|
fn unused_port_settings() -> Settings {
|
|
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
|
|
parse_settings(&json!({"llm": {"port": port}})).unwrap()
|
|
}
|
|
|
|
/// A tiny `/v1/models` server answering with `id`, after `delay` (before that it lists nothing).
|
|
async fn models_server(id: &'static str, delay: Duration) -> u16 {
|
|
serve(id, delay, None, Arc::new(Mutex::new("list"))).await
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
/// The same; once the file `gone` exists (a fake Strata's `--stop` makes it) it stops listening,
|
|
/// so a connection is refused.
|
|
async fn models_server_until(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>) -> u16 {
|
|
serve(id, delay, gone, Arc::new(Mutex::new("list"))).await
|
|
}
|
|
|
|
/// `mode`: "list" answers, "hang" keeps the connection and says nothing, "503" is a server still
|
|
/// loading.
|
|
pub(crate) async fn serve(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>, mode: Arc<Mutex<&'static str>>) -> u16 {
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let opened = Instant::now();
|
|
tokio::spawn(async move {
|
|
let mut listener = listener;
|
|
while let Ok((mut s, _)) = listener.accept().await {
|
|
if gone.as_ref().is_some_and(|g| g.exists()) {
|
|
// The listener closes: later connections are refused, until a launch serves
|
|
// again (removes `gone`).
|
|
drop(s);
|
|
drop(listener);
|
|
let g = gone.clone().unwrap();
|
|
while g.exists() {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap();
|
|
// A new server: it answers (a left one loaded or stayed silent).
|
|
let mut m = mode.lock().unwrap();
|
|
if matches!(*m, "503" | "hang") {
|
|
*m = "list";
|
|
}
|
|
continue;
|
|
}
|
|
let mut buf = [0u8; 1024];
|
|
let n = s.read(&mut buf).await.unwrap_or(0);
|
|
let post = buf[..n].starts_with(b"POST");
|
|
let m = *mode.lock().unwrap();
|
|
let (status, body) = match m {
|
|
"hang" => {
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(Duration::from_secs(60)).await;
|
|
drop(s);
|
|
});
|
|
continue;
|
|
}
|
|
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
|
|
"nochat" if post => ("500 Internal Server Error", r#"{"error":"broken"}"#.to_owned()),
|
|
_ if post => ("200 OK", r#"{"choices":[{"message":{"content":"안녕하세요"}}]}"#.to_owned()),
|
|
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
|
|
_ => ("200 OK", "{}".to_owned()),
|
|
};
|
|
let _ = s.write_all(format!("HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
|
body.len()).as_bytes()).await;
|
|
}
|
|
});
|
|
port
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_given_server_is_used_without_starting_strata() {
|
|
let port = models_server("given-model", Duration::ZERO).await;
|
|
let l = layout("llm3");
|
|
let llm = Llm::new(&l.data);
|
|
let s = parse_settings(&json!({"llm": {"url": format!("http://127.0.0.1:{port}/v1")}})).unwrap();
|
|
let (base, model) = llm.start(&s).await.unwrap();
|
|
assert_eq!((base.as_str(), model.as_str()), (format!("http://127.0.0.1:{port}/v1").as_str(), "given-model"));
|
|
assert_eq!(llm.status().state, LlmState::Ready);
|
|
assert!(!llm.status().managed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn without_strata_installed_the_start_fails_with_a_reason() {
|
|
let llm = Llm::new(&layout("llm4").data);
|
|
let e = llm.start(&Settings::default()).await.unwrap_err();
|
|
assert!(e.contains("Strata"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
|
|
}
|
|
|
|
/// A fake `strata`: `launch --stop` records itself and runs `stop_body` (by default it stops the
|
|
/// server: the file `gone` appears); `launch ...` runs `launch_body`.
|
|
#[cfg(unix)]
|
|
pub(crate) fn fake_strata(l: &Layout, launch_body: &str) -> std::path::PathBuf {
|
|
fake_strata_stop(l, launch_body, &format!("touch '{}'", l.data.join("gone").display()))
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
pub(crate) fn fake_strata_stop(l: &Layout, launch_body: &str, stop_body: &str) -> std::path::PathBuf {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let exe = l.data.join("fake-strata");
|
|
std::fs::create_dir_all(l.strata_home()).unwrap();
|
|
let log = l.data.join("calls");
|
|
std::fs::write(&exe, format!("#!/bin/sh\necho \"$@\" >> '{}'\nif [ \"$2\" = --stop ]; then {stop_body}; exit $?; fi\n{launch_body}\n",
|
|
log.display())).unwrap();
|
|
std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();
|
|
exe
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn strata_starts_reports_its_download_waits_for_the_model_and_stops() {
|
|
let l = layout("llm5");
|
|
let port = models_server_until("qwen-test", Duration::from_millis(300), Some(l.data.join("gone"))).await;
|
|
let exe = fake_strata(&l, "echo 'Downloading the engine: 80%'; sleep 1; echo 'Downloading the model files: 2.4 of 22.9 GB'; sleep 1; echo 'Starting the engine'; sleep 1; echo 'Strata is serving.'");
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
let l2 = llm.clone();
|
|
let s2 = s.clone();
|
|
let run = tokio::spawn(async move { l2.start(&s2).await });
|
|
let mut seen = None;
|
|
for _ in 0..40 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
let st = llm.status();
|
|
if st.percent.is_some() {
|
|
seen = Some(st);
|
|
break;
|
|
}
|
|
}
|
|
let st = seen.expect("progress while starting");
|
|
assert_eq!((st.state, st.percent), (LlmState::Starting, Some(80.0)));
|
|
// The bar follows the current line: the model download's sizes, then nothing (no stale 80 %).
|
|
let mut later = vec![];
|
|
for _ in 0..60 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
let st = llm.status();
|
|
if st.state != LlmState::Starting { break; }
|
|
later.push((st.note.clone().unwrap_or_default(), st.percent));
|
|
}
|
|
assert!(later.iter().any(|(n, p)| n.contains("2.4 of 22.9") && *p == Some(10.5)), "{later:?}");
|
|
assert!(later.iter().any(|(n, p)| n.contains("Starting the engine") && p.is_none()), "{later:?}");
|
|
let (base, model) = run.await.unwrap().unwrap();
|
|
assert_eq!((base, model.as_str()), (format!("http://127.0.0.1:{port}/v1"), "qwen-test"));
|
|
assert_eq!(llm.status().state, LlmState::Ready);
|
|
llm.stop().await.unwrap();
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
assert!(calls.lines().next().unwrap().starts_with("launch --family"), "{calls}");
|
|
assert_eq!(calls.lines().last(), Some("launch --stop"));
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn with_a_graphics_card_the_first_start_measures_the_llm_without_experts_then_places_them() {
|
|
let l = layout("llm-place");
|
|
let port = server_after_launch(&l, "list").await;
|
|
// The engine log, as Strata 0.1.59's qwen36 engine writes it.
|
|
let log = l.strata_home().join("x.engine.log");
|
|
let exe = fake_strata(&l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 0 of 256 per layer in VRAM, bigger prompt chunks streamed (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
|
|
l.data.join("gone").display(), log.display()));
|
|
// The game PC in M0 (16303 MiB card, `OTHERS` held by other programs); the LLM without
|
|
// experts held 9874 MiB.
|
|
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect();
|
|
assert_eq!(launches.len(), 2, "{calls}");
|
|
assert!(launches[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", launches[0]);
|
|
// The M0 kit's normal placement on that PC: 1.5 GiB of experts.
|
|
assert!(launches[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[1]);
|
|
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
|
assert_eq!(probe["32768/1"], json!({"context": 32768, "parallel": 1, "gpu": "GPU-test", "modelMib": 9874, "stagingGib": 1.1}));
|
|
llm.stop().await.unwrap();
|
|
// The next start reuses the measurement: one launch, placed at once.
|
|
std::fs::remove_file(l.data.join("calls")).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect();
|
|
assert_eq!(launches.len(), 1, "{calls}");
|
|
assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_llm_log_tells_how_much_of_the_card_is_free_once_the_model_answers() {
|
|
crate::logs::install(crate::logs::LogHub::new(None));
|
|
let l = layout("llm-free");
|
|
let port = server_after_launch(&l, "list").await;
|
|
// The driver keeps some of the card for itself (`memory.reserved`): it is neither used nor free.
|
|
let inner = card(&l, "GPU-test", 0);
|
|
let reserved = 444;
|
|
let reader: GpuReader = Arc::new(move || inner().map(|g| Gpu { free_mib: g.free_mib - reserved, ..g }));
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(reader);
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
|
let free = 16303 - OTHERS - 9874 - reserved;
|
|
let lines = crate::logs::installed().unwrap()
|
|
.query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000);
|
|
assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))),
|
|
"{:?}", lines.iter().map(|x| &x.text).collect::<Vec<_>>());
|
|
llm.stop().await.unwrap();
|
|
}
|
|
|
|
/// A fake card for `fake_placed`: 16303 MiB, `OTHERS` held by other programs (and by any
|
|
/// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a
|
|
/// Strata server runs (the calls file shows the last launch and stop).
|
|
#[cfg(unix)]
|
|
pub(crate) fn card(l: &Layout, id: &'static str, leftover: i64) -> GpuReader {
|
|
let calls = l.data.join("calls");
|
|
Arc::new(move || {
|
|
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
|
let stopped = c.contains("launch --stop");
|
|
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
|
let used = OTHERS + if running { 9874 } else if stopped { 0 } else { leftover };
|
|
Some(gpu(id, used, 16303))
|
|
})
|
|
}
|
|
|
|
/// A fake Strata that serves on `port` again (removes `gone`) and writes the engine's staging line.
|
|
#[cfg(unix)]
|
|
pub(crate) fn fake_placed(l: &Layout) -> std::path::PathBuf {
|
|
let log = l.strata_home().join("x.engine.log");
|
|
fake_strata(l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 5 of 256 per layer in VRAM, the rest of a batch on 16 CPU threads (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
|
|
l.data.join("gone").display(), log.display()))
|
|
}
|
|
|
|
/// A fake server on a free port that answers only once a launch serves (removes `gone`).
|
|
#[cfg(unix)]
|
|
pub(crate) async fn server_after_launch(l: &Layout, mode: &'static str) -> u16 {
|
|
std::fs::create_dir_all(&l.data).unwrap();
|
|
std::fs::write(l.data.join("gone"), "").unwrap();
|
|
serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await
|
|
}
|
|
|
|
pub(crate) fn launches(l: &Layout) -> Vec<String> {
|
|
std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().lines()
|
|
.filter(|c| c.starts_with("launch --family")).map(str::to_owned).collect()
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_failed_measurement_is_not_kept_and_the_model_starts_without_experts() {
|
|
// The card stops answering after the first read.
|
|
let l = layout("llm-place-fail");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
|
let r2 = reads.clone();
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
|
|
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| gpu("GPU-test", OTHERS, 16303))
|
|
}));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
|
let l2 = launches(&l);
|
|
assert_eq!(l2.len(), 2, "{l2:?}");
|
|
assert!(l2[1].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[1]);
|
|
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
|
|
llm.stop().await.unwrap();
|
|
// The model does not answer the short question: not kept either.
|
|
let l = layout("llm-place-nochat");
|
|
let port = server_after_launch(&l, "nochat").await;
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
|
assert!(launches(&l)[1].ends_with("--set expert-vram=0 --set prefill=2048"));
|
|
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn the_measurement_is_kept_per_context_and_per_card() {
|
|
let l = layout("llm-place-keys");
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-a", 0));
|
|
let port = server_after_launch(&l, "list").await;
|
|
let one = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&one).await.unwrap();
|
|
assert_eq!(launches(&l).len(), 2, "measured once");
|
|
llm.stop().await.unwrap();
|
|
// Another context: measured again.
|
|
std::fs::remove_file(l.data.join("calls")).unwrap();
|
|
let short = parse_settings(&json!({"llm": {"port": port, "context": 16384}})).unwrap();
|
|
llm.start(&short).await.unwrap();
|
|
assert_eq!(launches(&l).len(), 2, "measured for the other context");
|
|
llm.stop().await.unwrap();
|
|
// Another card (same size): measured again.
|
|
std::fs::remove_file(l.data.join("calls")).unwrap();
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-b", 0));
|
|
llm.start(&one).await.unwrap();
|
|
assert_eq!(launches(&l).len(), 2, "measured for the new card");
|
|
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
|
assert_eq!((probe["32768/1"]["gpu"].clone(), probe["16384/1"]["gpu"].clone()), (json!("GPU-b"), json!("GPU-a")));
|
|
llm.stop().await.unwrap();
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() {
|
|
// Left over answering, loading (503) or silent: always stopped first, holding 9874 MiB until then.
|
|
for mode in ["list", "503", "hang"] {
|
|
let l = layout(&format!("llm-place-left-{mode}"));
|
|
let port = serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await;
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
assert!(calls.lines().next() == Some("launch --stop"), "{mode}: stopped first: {calls}");
|
|
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
|
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "{mode}: the left server is not counted");
|
|
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{mode}: {:?}", launches(&l));
|
|
llm.stop().await.unwrap();
|
|
}
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_game_moves_the_llm_to_the_game_placement_and_back() {
|
|
let l = layout("llm-stage");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null}));
|
|
std::fs::remove_file(l.data.join("calls")).unwrap();
|
|
assert_eq!(tokio::time::timeout(Duration::from_secs(30), llm.switch(&s, Stage::Gaming, 0, true)).await.unwrap().unwrap(), Stage::Gaming);
|
|
let g = launches(&l);
|
|
// Measured once at the game's context and one request, then the 2 GiB minimum (8 GB does not
|
|
// fit on 16 GB) with half the engine's 16 CPU threads.
|
|
assert_eq!(g.len(), 2, "{g:?}");
|
|
assert!(g[0].contains("--context 16384") && g[0].contains("--parallel 1") && g[0].contains("--set expert-vram=0"), "{}", g[0]);
|
|
assert!(g[1].ends_with("--set expert-vram=2 --set prefill=2048 --set pool-workers=8"), "{}", g[1]);
|
|
let room = 16303 - OTHERS - 9874 - crate::placement::STT_MIB - crate::placement::TTS_MIB - crate::placement::MARGIN_MIB - 921;
|
|
assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": room}));
|
|
assert_eq!(llm.status().state, LlmState::Ready);
|
|
// Back to normal: the run's first reading of other programs, the normal placement again.
|
|
std::fs::remove_file(l.data.join("calls")).unwrap();
|
|
assert_eq!(llm.switch(&s, Stage::Normal, 0, false).await.unwrap(), Stage::Normal);
|
|
let n = launches(&l);
|
|
assert_eq!(n.len(), 1, "{n:?}");
|
|
assert!(n[0].contains("--context 32768") && n[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", n[0]);
|
|
assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null}));
|
|
// Pause: the LLM stops.
|
|
assert_eq!(llm.switch(&s, Stage::Paused, 0, true).await.unwrap(), Stage::Paused);
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
assert_eq!(llm.stage_view()["stage"], json!("paused"));
|
|
}
|
|
|
|
/// A card whose other programs hold `others` MiB (changeable), plus the voice worker's models
|
|
/// (`worker` MiB), plus 9874 MiB while a Strata server runs.
|
|
#[cfg(unix)]
|
|
fn live_card(l: &Layout, others: Arc<std::sync::atomic::AtomicI64>, worker: i64) -> GpuReader {
|
|
let calls = l.data.join("calls");
|
|
Arc::new(move || {
|
|
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
|
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
|
let used = others.load(Ordering::SeqCst) + worker + if running { 9874 } else { 0 };
|
|
Some(gpu("GPU-test", used, 16303))
|
|
})
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn the_normal_placement_reads_other_programs_afresh_and_a_game_is_not_counted_twice() {
|
|
let l = layout("llm-stage-fresh");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let others = Arc::new(std::sync::atomic::AtomicI64::new(OTHERS));
|
|
// At the start the STT worker has not loaded; at every switch it has.
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card(&l, others.clone(), 0));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
let stt = crate::placement::worker_models_mib(true, false);
|
|
let llm = llm.with_gpu(live_card(&l, others.clone(), stt));
|
|
// A game starts and holds 2500 MiB: the game stage counts other programs as before the game.
|
|
others.store(OTHERS + 2500, Ordering::SeqCst);
|
|
llm.switch(&s, Stage::Gaming, stt, true).await.unwrap();
|
|
assert!(launches(&l).last().unwrap().contains("--set expert-vram=2 "), "{:?}", launches(&l));
|
|
// The game ends but a video player now holds 1000 MiB more: the normal stage reads that.
|
|
others.store(OTHERS + 1000, Ordering::SeqCst);
|
|
llm.switch(&s, Stage::Normal, stt, false).await.unwrap();
|
|
let expected = crate::placement::normal(16303, OTHERS + 1000, &crate::placement::Probe { context: 32768, parallel: 1,
|
|
gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }, 1024);
|
|
assert!(expected < 1.5);
|
|
assert!(launches(&l).last().unwrap().ends_with(&format!("--set expert-vram={expected} --set prefill=2048")), "{:?}", launches(&l));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn the_voice_models_loaded_after_the_start_change_nothing_on_the_way_back() {
|
|
// At the start neither the STT nor the TTS has loaded; at the switches both hold the card.
|
|
// With nothing else changed, the normal stage comes back with the placement it started with.
|
|
let l = layout("llm-stage-voice");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let others = Arc::new(std::sync::atomic::AtomicI64::new(460));
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card(&l, others.clone(), 0));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
let first = launches(&l).last().unwrap().clone();
|
|
let voice = crate::placement::worker_models_mib(true, true);
|
|
let held = crate::placement::STT_MODEL_MIB + crate::placement::TTS_MIB;
|
|
let llm = llm.with_gpu(live_card(&l, others.clone(), held));
|
|
llm.switch(&s, Stage::Gaming, voice, true).await.unwrap();
|
|
llm.switch(&s, Stage::Normal, voice, false).await.unwrap();
|
|
let back = launches(&l).last().unwrap().clone();
|
|
assert!(!first.contains("--set expert-vram=0 "), "the test has room for experts: {first}");
|
|
assert_eq!(back, first);
|
|
}
|
|
|
|
/// `live_card` on a card of `total` MiB.
|
|
#[cfg(unix)]
|
|
fn live_card_of(l: &Layout, others: Arc<std::sync::atomic::AtomicI64>, total: i64) -> GpuReader {
|
|
let calls = l.data.join("calls");
|
|
Arc::new(move || {
|
|
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
|
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
|
Some(gpu("GPU-test", others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total))
|
|
})
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_game_running_when_the_services_start_is_not_counted_twice() {
|
|
// A game holds 2500 MiB already: it is in what the card holds, so no reserve on top.
|
|
let l = layout("llm-stage-game-at-start");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let others = Arc::new(std::sync::atomic::AtomicI64::new(OTHERS + 2500));
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 16303));
|
|
llm.game_at_start(true);
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
assert_eq!(llm.switch(&s, Stage::Gaming, 0, true).await.unwrap(), Stage::Gaming, "not paused");
|
|
let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 };
|
|
let expected = crate::placement::normal(16303, OTHERS + 2500, &probe16, 1024);
|
|
assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn the_game_stage_reads_the_card_again_and_counts_what_grew_since_the_normal_reading() {
|
|
// Normal reading: OTHERS. Then a program takes 4000 MiB and the game 4500 MiB: together
|
|
// more than the 8 GB reserve, so the placement leaves that much, not just 8 GB.
|
|
let l = layout("llm-stage-grew");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let others = Arc::new(std::sync::atomic::AtomicI64::new(OTHERS));
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 24576));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
others.store(OTHERS + 4000 + 4500, Ordering::SeqCst);
|
|
llm.switch(&s, Stage::Gaming, 0, true).await.unwrap();
|
|
let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 };
|
|
let expected = crate::placement::game(24576, OTHERS, &probe16, 8500).expert_gib;
|
|
assert_ne!(expected, crate::placement::game(24576, OTHERS, &probe16, 8192).expert_gib, "the test tells them apart");
|
|
assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_game_stage_on_a_fallback_placement_still_shows_as_the_game_stage() {
|
|
// The game's measurement fails (the card cannot be read the second time): no experts, but
|
|
// the game stage is what runs.
|
|
let l = layout("llm-stage-fallback");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
|
let r2 = reads.clone();
|
|
let llm = llm.with_gpu(Arc::new(move || {
|
|
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| gpu("GPU-test", OTHERS, 16303))
|
|
}));
|
|
assert_eq!(llm.switch(&s, Stage::Gaming, 0, true).await.unwrap(), Stage::Gaming);
|
|
assert!(launches(&l).last().unwrap().contains("--set expert-vram=0 "), "{:?}", launches(&l));
|
|
assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": null}));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_game_with_no_room_beside_the_model_pauses_the_llm() {
|
|
let l = layout("llm-stage-full");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let calls = l.data.join("calls");
|
|
// A 12 GB card: the model fits, but not with any room for a game.
|
|
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
|
|
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
|
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
|
Some(gpu("GPU-small", OTHERS + if running { 9874 } else { 0 }, 12288))
|
|
}));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
assert_eq!(llm.switch(&s, Stage::Gaming, 0, true).await.unwrap(), Stage::Paused);
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
assert_eq!(llm.stage_view(), json!({"stage": "paused", "gameRoomMib": null}));
|
|
let c = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
assert_eq!(c.lines().last(), Some("launch --stop"));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_left_server_that_does_not_stop_means_no_experts_and_nothing_measured() {
|
|
// The stop command fails and the server keeps answering: the card is not read as it is.
|
|
let l = layout("llm-place-stuck");
|
|
let port = models_server_until("qwen-test", Duration::from_millis(0), None).await;
|
|
let log = l.strata_home().join("x.engine.log");
|
|
let exe = fake_strata_stop(&l, &format!("echo 'strata-q36: (1.10 GiB staging)' >> '{}'; echo 'Strata is serving.'", log.display()), "exit 3");
|
|
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 9874)).with_stop_timeout(Duration::from_secs(1));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
|
|
let l2 = launches(&l);
|
|
assert_eq!(l2.len(), 1, "{l2:?}");
|
|
assert!(l2[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[0]);
|
|
assert!(!l.data.join("config").join("llm-probe.json").exists());
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_cancel_during_the_download_stops_it_and_clears_the_server() {
|
|
let l = layout("llm6");
|
|
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
|
let l2 = llm.clone();
|
|
let run = tokio::spawn(async move { l2.start(&unused_port_settings()).await });
|
|
tokio::time::sleep(Duration::from_millis(400)).await;
|
|
let t = Instant::now();
|
|
llm.cancel();
|
|
assert_eq!(run.await.unwrap().unwrap_err(), "cancelled");
|
|
assert!(t.elapsed() < Duration::from_secs(5));
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
assert_eq!(calls.lines().last(), Some("launch --stop"));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_failed_launch_shows_strata_s_last_line() {
|
|
let l = layout("llm7");
|
|
let exe = fake_strata(&l, "echo 'Not enough disk space for the model' >&2; exit 4");
|
|
let llm = Llm::with_exe(l, exe);
|
|
let e = llm.start(&unused_port_settings()).await.unwrap_err();
|
|
assert!(e.contains("Not enough disk space"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_server_still_answering_after_a_failed_stop_is_reported_not_marked_off() {
|
|
let l = layout("llm8");
|
|
let port = models_server("qwen-test", Duration::ZERO).await;
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "echo 'could not stop the engine' >&2; exit 3");
|
|
let llm = Llm::with_exe(l, exe);
|
|
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
|
|
let e = llm.stop().await.unwrap_err();
|
|
assert!(e.contains("could not stop the engine"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_stop_that_hangs_is_given_up_on_and_reported() {
|
|
let l = layout("llm9");
|
|
let port = models_server("qwen-test", Duration::ZERO).await;
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "sleep 30");
|
|
let llm = Llm::with_exe(l, exe).with_stop_timeout(Duration::from_millis(500));
|
|
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
|
|
let t = Instant::now();
|
|
let e = llm.stop().await.unwrap_err();
|
|
assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed());
|
|
assert!(e.contains("Strata"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_stop_command_that_fails_while_nothing_answers_counts_as_stopped() {
|
|
let l = layout("llm10");
|
|
let port = models_server_until("qwen-test", Duration::ZERO, Some(l.data.join("gone"))).await;
|
|
let stop = format!("touch '{}'; exit 1", l.data.join("gone").display());
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", &stop);
|
|
let llm = Llm::with_exe(l, exe);
|
|
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
|
|
llm.stop().await.unwrap();
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
}
|
|
|
|
/// A failed stop while the server is still there but not listing a model (holding connections
|
|
/// silently, or a 503 while loading) is not a confirmed stop: the LLM and, through the
|
|
/// controller, the services stay failed.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_failed_stop_with_a_silent_or_loading_server_is_not_counted_as_stopped() {
|
|
for m in ["hang", "503"] {
|
|
let l = layout(&format!("llm11{m}"));
|
|
let mode = Arc::new(Mutex::new("list"));
|
|
let port = serve("qwen-test", Duration::ZERO, None, mode.clone()).await;
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "exit 3");
|
|
let llm = Arc::new(Llm::with_exe(l, exe));
|
|
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(),
|
|
settings: parse_settings(&json!({"llm": {"port": port}})).unwrap() });
|
|
c.start().await.unwrap();
|
|
*mode.lock().unwrap() = m;
|
|
let e = c.stop().await.unwrap_err();
|
|
assert!(e.contains("Strata"), "{m}: {e}");
|
|
assert_eq!(c.state(), crate::control::State::Failed, "{m}");
|
|
assert_eq!(llm.status().state, LlmState::Failed, "{m}");
|
|
}
|
|
}
|
|
|
|
/// A stop right after a background start must cancel the LLM's start before it runs anything
|
|
/// (no download), even though the start task runs after the stop.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_stop_right_after_a_background_start_runs_no_launch() {
|
|
let l = layout("llm12");
|
|
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
|
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: unused_port_settings() });
|
|
c.start_in_background().unwrap();
|
|
c.stop().await.unwrap();
|
|
let t = Instant::now();
|
|
while c.state() != crate::control::State::Stopped && t.elapsed() < Duration::from_secs(5) {
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
assert_eq!(c.state(), crate::control::State::Stopped);
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap_or_default();
|
|
assert!(!calls.contains("--family"), "no launch ran: {calls}");
|
|
// A later start is not cancelled by the earlier stop.
|
|
c.start_in_background().unwrap();
|
|
tokio::time::sleep(Duration::from_millis(400)).await;
|
|
assert!(std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().contains("--family"));
|
|
c.stop().await.unwrap();
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
/// The services as the controller sees them, reduced to the LLM.
|
|
struct LlmHost {
|
|
llm: Arc<Llm>,
|
|
settings: Settings,
|
|
}
|
|
#[cfg(unix)]
|
|
struct LlmRunning(Arc<Llm>);
|
|
#[cfg(unix)]
|
|
#[async_trait::async_trait]
|
|
impl crate::control::Running for LlmRunning {
|
|
async fn stop(&self) -> Result<(), String> {
|
|
self.0.stop().await
|
|
}
|
|
fn status(&self, _: Instant) -> Value {
|
|
json!({"worker": null, "voiceMembers": [], "missed": []})
|
|
}
|
|
}
|
|
#[cfg(unix)]
|
|
#[async_trait::async_trait]
|
|
impl crate::control::Host for LlmHost {
|
|
async fn start(&self) -> Result<(Arc<dyn crate::control::Running>, tokio::sync::mpsc::UnboundedReceiver<crate::discord::CoreEvent>), String> {
|
|
self.llm.start(&self.settings).await?;
|
|
Ok((Arc::new(LlmRunning(self.llm.clone())), tokio::sync::mpsc::unbounded_channel().1))
|
|
}
|
|
fn secrets(&self) -> Vec<String> {
|
|
vec![]
|
|
}
|
|
fn prepare_start(&self) {
|
|
self.llm.arm();
|
|
}
|
|
fn cancel_start(&self) {
|
|
self.llm.cancel();
|
|
}
|
|
}
|
|
}
|