Files
javis_bot/app/src/strata.rs
javis-bot cda8db7fce fix: play on YouTube in one step, Korea as the default location, searches in the user's language
- botBrowser `play`: searches YouTube for the words as said, opens the first video (not an ad,
  without the playlist) and starts it, in one call (an address given opens that page); `type`
  also takes `query`. Live 0.1.19: the model stopped on YouTube's home page or the results, a
  retry was refused as a third call, and the answer still said the video played.
- `brain.location` (기본 위치, default 대한민국) goes to the brain as `location_name`: the time line
  says it without GeoIP (no database on the game PC), and getWeather without a place uses it.
- Planner and webSearch keep search words in the user's language (never translated into
  English), so recommendations fit where the user is.

Evals (qwen3:8b, 3 runs): play ending on a playing video found with the words as said 0/6 -> 5/6;
Korean searches for Korean requests 6/9 -> 8/9.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 17:26:30 +09:00

1353 lines
69 KiB
Rust

//! The LLM the brain answers with (`app/src/admin.spec.md`, "LLM"): the app starts the installed
//! Strata from the data folder with the M0 kit's choices, waits until the model answers, and stops
//! it with the services. Strata downloads the model (about 23 GB) on its first start; the dashboard
//! shows how far it got. With `llm.url` the app uses that server instead and starts nothing.
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use serde::Serialize;
use serde_json::{json, Value};
use crate::install::{Ctx, Layout};
use crate::perf::Stage;
use crate::settings::Settings;
/// The one model v2 runs (M0: Qwen3.6-35B-A3B UD-Q4_K_XL; docs/v2/roadmap.md).
pub const FAMILY: &str = "qwen36";
pub const MODEL: &str = "ud-q4_k_xl";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum LlmState {
Off,
Starting,
Ready,
Failed,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct LlmStatus {
pub state: LlmState,
/// Strata's own server (false: `llm.url`).
pub managed: bool,
/// How far Strata's current line says it got while starting (a percentage, or the model
/// download's `X of Y GB`); none on a line without progress.
pub percent: Option<f64>,
/// Strata's last line while starting, or why it failed.
pub note: Option<String>,
}
/// Requests answered at once: one is faster on a 16 GB card (M0), two only when a chat channel
/// answers every message (so its answers do not wait behind the voice answers).
pub fn parallel(settings: &Settings) -> u32 {
if settings.chat_channels.iter().any(|c| c.answer_all) { 2 } else { 1 }
}
/// `strata launch` for the services: the M0 kit's normal placement, models in `models/llm`, this
/// computer only. No pictures yet (nothing in v2 sends one; reading them takes graphics memory).
/// `--restart` replaces a server started with other choices; the same choices are reused.
pub fn launch_args(layout: &Layout, settings: &Settings, conf: &Conf, sets: &[String]) -> Vec<String> {
let models = layout.data.join("models").join("llm").to_string_lossy().into_owned();
let mut a: Vec<String> = ["launch", "--family", FAMILY, "--model", MODEL, "--data-dir"].map(String::from).into();
a.push(models);
for (k, v) in [("--context", conf.context.to_string()), ("--vision", conf.vision.into()),
("--parallel", conf.parallel.to_string()), ("--access", "local".into()),
("--port", settings.llm.port.to_string())] {
a.push(k.into());
a.push(v);
}
a.extend(["--no-api-key", "--restart", "--yes", "--no-tui"].map(String::from));
for s in sets {
a.push("--set".into());
a.push(s.clone());
}
a
}
/// The context and requests at once of a stage: the settings' in the normal stage; in a game 16K
/// (or less when the settings ask for less) and one request at a time.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Conf {
pub context: u32,
pub parallel: u32,
/// `--vision`: `off`, `gpu` or `cpu` (`vision`).
pub vision: &'static str,
}
pub fn conf(settings: &Settings, stage: Stage) -> Conf {
let vision = vision(stage, crate::screen::available(settings));
match stage {
Stage::Gaming => Conf { context: crate::placement::GAME_CONTEXT.min(settings.llm.context), parallel: 1, vision },
_ => Conf { context: settings.llm.context, parallel: parallel(settings), vision },
}
}
/// Where the LLM reads pictures (`--vision`): nowhere unless screen viewing is available, on the
/// card normally and on the CPU in a game (M0's game placement).
pub fn vision(stage: Stage, screen: bool) -> &'static str {
match (screen, stage) {
(false, _) => "off",
(true, Stage::Gaming) => "cpu",
(true, _) => "gpu",
}
}
/// Where the probe of a placement is kept in `config/llm-probe.json`: pictures on the card take
/// memory of their own, so they are measured apart.
pub fn probe_key(c: &Conf) -> String {
if c.vision == "gpu" { format!("{}/{}/vision", c.context, c.parallel) } else { format!("{}/{}", c.context, c.parallel) }
}
/// The engine's CPU threads from its log ("... on 16 CPU threads ...").
fn cpu_threads(log: &str) -> Option<u32> {
log.lines().rev().find_map(|l| {
let i = l.find(" CPU threads")?;
l[..i].rsplit(' ').next()?.parse().ok()
})
}
/// The engine settings for an expert budget (`placement.rs`) and the M0 kit's prompt chunk.
fn placement_sets(expert_gib: f64) -> Vec<String> {
vec![format!("expert-vram={expert_gib}"), "prefill=2048".into()]
}
/// The first NVIDIA card: its UUID and graphics memory in MiB.
#[derive(Debug, Clone, PartialEq)]
pub struct Gpu {
pub id: String,
pub used_mib: i64,
pub total_mib: i64,
}
/// Reads the card (`nvidia-smi`); None without one.
pub type GpuReader = Arc<dyn Fn() -> Option<Gpu> + Send + Sync>;
fn nvidia_memory() -> Option<Gpu> {
let mut cmd = std::process::Command::new("nvidia-smi");
cmd.args(["--query-gpu=uuid,memory.used,memory.total", "--format=csv,noheader,nounits"]);
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
cmd.creation_flags(0x0800_0000);
}
let out = cmd.output().ok()?;
let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned();
let mut f = line.split(',').map(str::trim);
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()? })
}
/// The newest engine log under Strata's folder, its end only (the placement lines of the last start).
fn engine_log(home: &Path) -> String {
fn walk(dir: &Path, depth: u32, best: &mut Option<(std::time::SystemTime, std::path::PathBuf)>) {
let Ok(entries) = std::fs::read_dir(dir) else { return };
for e in entries.flatten() {
let p = e.path();
if p.is_dir() && depth > 0 {
walk(&p, depth - 1, best);
} else if p.to_string_lossy().ends_with(".engine.log") {
if let Ok(t) = e.metadata().and_then(|m| m.modified()) {
if best.as_ref().is_none_or(|(b, _)| t > *b) {
*best = Some((t, p));
}
}
}
}
}
let mut best = None;
walk(home, 5, &mut best);
let Some((_, p)) = best else { return String::new() };
let bytes = std::fs::read(p).unwrap_or_default();
String::from_utf8_lossy(&bytes[bytes.len().saturating_sub(400_000)..]).into_owned()
}
/// How far a line says Strata got: its last `NN%` or `NN.N%`, else `X of Y GB` (or MB), which is
/// how Strata 0.1.59 reports the model download.
pub fn percent(line: &str) -> Option<f64> {
percent_sign(line).or_else(|| size_progress(line))
}
fn size_progress(line: &str) -> Option<f64> {
let words: Vec<&str> = line.split_whitespace().collect();
words.windows(4).rev().find_map(|w| {
let (done, total) = (w[0].parse::<f64>().ok()?, w[2].parse::<f64>().ok()?);
(w[1] == "of" && matches!(w[3].trim_end_matches([',', '.', ')']), "GB" | "MB") && total > 0.0 && done <= total)
.then(|| (done / total * 1000.0).round() / 10.0)
})
}
fn percent_sign(line: &str) -> Option<f64> {
let b = line.as_bytes();
let mut found = None;
for (i, _) in line.match_indices('%') {
let mut s = i;
while s > 0 && (b[s - 1].is_ascii_digit() || b[s - 1] == b'.') {
s -= 1;
}
if let Ok(n) = line[s..i].parse::<f64>() {
if (0.0..=100.0).contains(&n) {
found = Some(n);
}
}
}
found
}
/// Wait until `base` (`.../v1`) lists a model; returns its id. Stops early on `cancel`.
pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> Result<String, String> {
let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().map_err(|e| e.to_string())?;
let until = Instant::now() + timeout;
loop {
if cancel.load(Ordering::Relaxed) {
return Err("cancelled".into());
}
if let Ok(r) = client.get(format!("{base}/models")).send().await {
if let Ok(v) = r.text().await.map(|t| serde_json::from_str::<Value>(&t).unwrap_or_default()) {
if let Some(id) = v["data"][0]["id"].as_str() {
return Ok(id.to_owned());
}
}
}
if Instant::now() >= until {
return Err(format!("대화 모델이 {}초 안에 준비되지 않았습니다 ({base})", timeout.as_secs()));
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an
/// error when the model gives none.
async fn warm_up(base: &str) -> Result<(), String> {
let client = reqwest::Client::builder().timeout(Duration::from_secs(60)).build().map_err(|e| e.to_string())?;
let model = match client.get(format!("{base}/models")).send().await {
Ok(r) => r.text().await.ok().and_then(|t| serde_json::from_str::<Value>(&t).ok())
.and_then(|v| v["data"][0]["id"].as_str().map(str::to_owned)),
Err(_) => None,
};
let body = json!({"model": model, "messages": [{"role": "user", "content": "안녕"}], "max_tokens": 8});
let r = client.post(format!("{base}/chat/completions")).header("content-type", "application/json")
.body(body.to_string()).send().await.map_err(|e| e.to_string())?;
if !r.status().is_success() {
return Err(format!("the model answered {}", r.status()));
}
let v: Value = serde_json::from_str(&r.text().await.map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
v["choices"][0]["message"].is_object().then_some(()).ok_or_else(|| "the model gave no answer".into())
}
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the
/// memory database inside the data folder, the tools of the features that are off and the enabled
/// MCP servers (without their environment values, `features.rs`). The app owns this file and writes
/// it on every start.
pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings)
-> Result<std::path::PathBuf, String> {
let file = data.join("config").join("brain.json");
let cfg = json!({
"ollama_base_url": base_url,
"ollama_chat_model": model,
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
"disabled_tools": crate::features::disabled_tools(settings),
"mcps": crate::features::brain_mcps(&settings.mcp_servers),
"location_name": settings.location,
"_config_version": 1,
});
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
text.push('\n');
crate::datadir::write_atomic(&file, text.as_bytes()).map_err(|e| e.to_string())?;
Ok(file)
}
/// One LLM for the services: Strata from the layout, or the server at `llm.url`.
pub struct Llm {
layout: Layout,
/// The program run as Strata (the installed one; a script in tests).
exe: std::path::PathBuf,
status: Mutex<LlmStatus>,
active: Mutex<Option<Arc<Ctx>>>,
cancel: Arc<AtomicBool>,
/// The port the managed server was started on (to confirm it is gone after a stop).
port: Mutex<u16>,
stop_timeout: Duration,
/// The graphics card's memory (none: Strata places the model itself).
gpu: Option<GpuReader>,
/// What other programs held at the latest normal placement made with no game running: the
/// game stage counts other programs from it (the card then also holds the game, which its
/// reserve is for). None when the services started with a game running.
baseline: Mutex<Option<i64>>,
/// A game runs (or may run) as the LLM is placed: set before `start` and at every `switch`.
/// Only a normal placement made while none runs keeps its reading.
game_running: AtomicBool,
/// The STT worker's model is on the card (true at switches while the worker runs), so it is not
/// taken for another program's memory.
stt_on_card: AtomicBool,
/// The graphics memory a game gets in the placement being started.
room: Mutex<Option<i64>>,
/// The engine's CPU threads in the normal stage; a game gets half.
threads: Mutex<Option<u32>>,
/// The stage it runs in, and in a game the graphics memory the game gets.
stage: Mutex<(Stage, Option<i64>)>,
}
/// How long the model may take to answer after `strata launch` returned (it loads in about 15 s
/// on the game PC; the first start may also tune).
const READY_TIMEOUT: Duration = Duration::from_secs(30 * 60);
/// A server the app does not start must already answer.
const URL_TIMEOUT: Duration = Duration::from_secs(30);
impl Llm {
pub fn new(data: &Path) -> Self {
// Only the data folder matters here (the Python sources are the install's business).
let layout = Layout { data: data.to_path_buf(), src: std::path::PathBuf::new() };
let exe = layout.strata_exe();
Self::with_exe(layout, exe).with_gpu(Arc::new(nvidia_memory))
}
pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self {
Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }),
active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0),
stop_timeout: Duration::from_secs(120), gpu: None, baseline: Mutex::new(None), game_running: AtomicBool::new(false), stt_on_card: AtomicBool::new(false), room: Mutex::new(None), threads: Mutex::new(None),
stage: Mutex::new((Stage::Normal, None)) }
}
/// Whether a game runs (also in the background) as the services start: the start's reading of
/// the card then includes it and is not kept as other programs' memory.
pub fn game_at_start(&self, on: bool) {
self.game_running.store(on, Ordering::SeqCst);
}
/// `{stage, gameRoomMib}` for the dashboard.
pub fn stage_view(&self) -> Value {
let (stage, room) = *self.stage.lock().unwrap();
json!({"stage": stage, "gameRoomMib": room})
}
/// Place the model by the card's memory (`placement.rs`), read with `gpu`.
pub fn with_gpu(mut self, gpu: GpuReader) -> Self {
self.gpu = Some(gpu);
self
}
async fn read_gpu(&self) -> Option<Gpu> {
let gpu = self.gpu.clone()?;
tokio::task::spawn_blocking(move || gpu()).await.ok().flatten()
}
fn probe_file(&self) -> std::path::PathBuf {
self.layout.data.join("config").join("llm-probe.json")
}
/// The expert budget's engine settings (none without a card): the normal placement from what
/// the card holds now and this PC's measurement without experts at this context and number of
/// requests at once, which the first start makes once per card (a start without experts, then
/// the real start replaces it). A server left from an earlier run is stopped first, so it is not
/// counted as another program's memory (a stop that is not confirmed reads nothing: no
/// experts). A measurement that fails is not kept: the model then starts without experts on
/// the card.
async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str, stage: Stage) -> Result<Option<Vec<String>>, String> {
let c = conf(settings, stage);
*self.room.lock().unwrap() = None;
let workers = |sets: &mut Vec<String>| if stage == Stage::Gaming {
let half = self.threads.lock().unwrap().map(|n| n / 2)
.unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get() as u32 / 4).unwrap_or(1));
sets.push(format!("pool-workers={}", half.max(1)));
};
if self.gpu.is_none() {
let mut sets = vec![];
workers(&mut sets);
return Ok(Some(sets));
}
// Whatever a server left from an earlier run does (answers, loads, stays silent), it holds
// the card: stop it first and read the card only once the port is confirmed closed.
if let Err(e) = self.stop_server_on(settings.llm.port).await {
crate::warn_log!("llm", "Warning: a server left on the port did not stop ({e}); starting without experts on the card");
let mut sets = placement_sets(0.0);
workers(&mut sets);
return Ok(Some(sets));
}
let Some(card) = self.read_gpu().await else { return Ok(Some(vec![])) };
// Other programs: what the card holds without this LLM and without the STT worker's model.
// The normal stage reads it afresh (programs opened since count); the game stage takes it
// from the latest normal placement, as the card then also holds the game.
let stt_model = if self.stt_on_card.load(Ordering::SeqCst) { crate::placement::STT_MODEL_MIB } else { 0 };
let now_others = (card.used_mib - stt_model).max(0);
// Only a normal placement made while no game runs keeps its reading (a game's memory would be
// taken for other programs'; also on the way back from a failed game stage).
if stage != Stage::Gaming && !self.game_running.load(Ordering::SeqCst) {
*self.baseline.lock().unwrap() = Some(now_others);
}
let baseline = *self.baseline.lock().unwrap();
let key = probe_key(&c);
let file = self.probe_file();
let mut all: serde_json::Map<String, Value> = std::fs::read_to_string(&file).ok()
.and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();
let known = all.get(&key).and_then(|v| serde_json::from_value::<crate::placement::Probe>(v.clone()).ok())
.filter(|p| p.gpu == card.id);
let probe = match known {
Some(p) => p,
None => {
crate::info_log!("llm", "measuring the model without experts once (context {}, {} at once)", c.context, c.parallel);
self.launch(settings, ctx, base, &c, &placement_sets(0.0)).await?;
let warmed = warm_up(base).await;
let after = self.read_gpu().await.filter(|g| g.id == card.id);
let model_mib = after.map(|g| g.used_mib - card.used_mib).unwrap_or(0);
if let Err(e) = warmed.and_then(|()| if model_mib > 0 { Ok(()) } else { Err("the card's memory could not be read".into()) }) {
crate::warn_log!("llm", "Warning: the measurement failed ({e}); starting without experts on the card");
let mut sets = placement_sets(0.0);
workers(&mut sets);
return Ok(Some(sets));
}
let p = crate::placement::Probe { context: c.context, parallel: c.parallel, gpu: card.id.clone(), model_mib,
staging_gib: crate::placement::staging_gib(&engine_log(&self.layout.strata_home())) };
all.insert(key, json!(p));
crate::datadir::write_atomic(&file, Value::Object(all).to_string().as_bytes())
.map_err(|e| format!("배치 측정값을 저장하지 못했습니다: {e}"))?;
p
}
};
let mut others = now_others;
let expert = match (stage, baseline) {
// With a reading from before the game: other programs as then, and the reserve at least
// what has grown since (the game, and any program opened after that reading).
(Stage::Gaming, Some(base)) => {
others = base;
let reserve = settings.game.reserve_mib().max(now_others - base);
let g = crate::placement::game(card.total_mib, base, &probe, reserve);
if g.paused {
crate::info_log!("llm", "placement: no room for a game beside the model; paused while the game runs");
return Ok(None);
}
*self.room.lock().unwrap() = Some(g.game_room_mib);
g.expert_gib
}
// A game that already ran at the start is in what the card holds now: no reserve on
// top, only the room for other programs.
_ => crate::placement::normal(card.total_mib, now_others, &probe, settings.game.other_room_mib()),
};
crate::info_log!("llm", "placement ({stage:?}): experts {expert} GiB (card {} MiB, held by others {others} MiB, model without experts {} MiB)",
card.total_mib, probe.model_mib);
let mut sets = placement_sets(expert);
workers(&mut sets);
Ok(Some(sets))
}
/// `strata launch` with `sets`, then wait until the model answers.
async fn launch(&self, settings: &Settings, ctx: &Ctx, base: &str, c: &Conf, sets: &[String]) -> Result<String, String> {
let args = launch_args(&self.layout, settings, c, sets);
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
ctx.run_in(&self.exe, &refs, Some(&self.layout.strata_home())).await?;
wait_ready(base, READY_TIMEOUT, &self.cancel).await
}
/// How long `strata launch --stop` may take before it is ended and the stop reported as failed.
pub fn with_stop_timeout(mut self, t: Duration) -> Self {
self.stop_timeout = t;
self
}
pub fn status(&self) -> LlmStatus {
let mut s = self.status.lock().unwrap().clone();
if let Some(ctx) = self.active.lock().unwrap().as_ref() {
s.note = ctx.last_note();
// From the current line only: a percentage from an earlier stage would mislead.
s.percent = s.note.as_deref().and_then(percent);
}
s
}
fn set(&self, state: LlmState, note: Option<String>) {
let mut s = self.status.lock().unwrap();
s.state = state;
if state != LlmState::Starting {
s.note = note;
}
}
/// Start (or reuse) the LLM; returns its `/v1` address and model id.
/// A cancel since the last `arm` (or before the first start) ends it before it runs anything.
pub async fn start(&self, settings: &Settings) -> Result<(String, String), String> {
// A new run: other programs' memory is read afresh.
*self.baseline.lock().unwrap() = None;
*self.stage.lock().unwrap() = (Stage::Normal, None);
self.stt_on_card.store(false, Ordering::SeqCst);
self.start_stage(settings, Stage::Normal).await.map(|r| r.expect("the normal stage never pauses"))
}
/// Move the running LLM to `stage` (`perf.rs`): a restart with that stage's placement, or a stop
/// when it pauses (also a game stage with no room for the game). Returns the stage it ended in.
/// With `llm.url` nothing moves.
/// `stt_on_card`: the STT worker's model is loaded on the card; `game_running`: a game runs now.
pub async fn switch(&self, settings: &Settings, stage: Stage, stt_on_card: bool, game_running: bool) -> Result<Stage, String> {
if settings.llm.url.is_some() {
return Ok(Stage::Normal);
}
self.stt_on_card.store(stt_on_card, Ordering::SeqCst);
self.game_running.store(game_running, Ordering::SeqCst);
if stage != Stage::Paused && self.start_stage(settings, stage).await?.is_some() {
return Ok(stage);
}
*self.port.lock().unwrap() = settings.llm.port;
self.stop().await?;
*self.stage.lock().unwrap() = (Stage::Paused, None);
self.set(LlmState::Off, Some("게임 중 일시정지".into()));
Ok(Stage::Paused)
}
async fn start_stage(&self, settings: &Settings, stage: Stage) -> Result<Option<(String, String)>, String> {
let managed = settings.llm.url.is_none();
*self.status.lock().unwrap() = LlmStatus { state: LlmState::Starting, managed, percent: None, note: None };
let result = self.start_inner(settings, stage).await;
if stage == Stage::Normal && result.as_ref().is_ok_and(Option::is_some) && self.gpu.is_some() {
if let Some(n) = cpu_threads(&engine_log(&self.layout.strata_home())) {
*self.threads.lock().unwrap() = Some(n);
}
}
// What runs now, also on a fallback placement (no experts), with the game's room if known.
if let Ok(Some(_)) = &result {
*self.stage.lock().unwrap() = (stage, self.room.lock().unwrap().take());
}
match &result {
Ok(None) => {}
Ok(Some((base, model))) => {
crate::info_log!("llm", "ready: {base} model={model}");
self.set(LlmState::Ready, None);
}
Err(e) if e == "cancelled" => self.set(LlmState::Off, None),
Err(e) => {
crate::error_log!("llm", "did not start: {e}");
self.set(LlmState::Failed, Some(e.clone()));
}
}
result
}
/// None: the stage pauses (nothing started).
async fn start_inner(&self, settings: &Settings, stage: Stage) -> Result<Option<(String, String)>, String> {
if self.cancel.load(Ordering::SeqCst) {
return Err("cancelled".into());
}
if let Some(url) = &settings.llm.url {
let model = wait_ready(url, URL_TIMEOUT, &self.cancel).await?;
return Ok(Some((url.clone(), model)));
}
if !self.exe.exists() {
return Err("Strata가 설치되어 있지 않습니다. 마법사의 모델 단계에서 설치해 주세요".into());
}
let ctx = Arc::new(Ctx::standalone(self.layout.clone(), "llm", self.cancel.clone()));
*self.active.lock().unwrap() = Some(ctx.clone());
*self.port.lock().unwrap() = settings.llm.port;
let base = format!("http://127.0.0.1:{}/v1", settings.llm.port);
let c = conf(settings, stage);
let ready = match self.placement(settings, &ctx, &base, stage).await {
Ok(Some(sets)) => self.launch(settings, &ctx, &base, &c, &sets).await.map(Some),
Ok(None) => Ok(None),
Err(e) => Err(e),
};
*self.active.lock().unwrap() = None;
match ready {
Ok(Some(model)) => Ok(Some((base, model))),
Ok(None) => Ok(None),
// Nothing half-started stays on the graphics card; if it may have, say so.
Err(e) => match self.stop_server().await {
Ok(()) => Err(e),
Err(stop) if e == "cancelled" => Err(stop),
Err(stop) => Err(format!("{e} ({stop})")),
},
}
}
/// Make the next start cancellable afresh: called when a start is accepted, never by `start`
/// itself, so a stop between accepting and running the start still cancels it.
pub fn arm(&self) {
self.cancel.store(false, Ordering::SeqCst);
}
/// Stop a start in progress (the download resumes on the next start).
pub fn cancel(&self) {
self.cancel.store(true, Ordering::SeqCst);
}
/// Stop the server this app started (nothing for `llm.url`). Off only once the server no longer
/// answers; otherwise `failed` with the reason, which is also returned.
pub async fn stop(&self) -> Result<(), String> {
let managed = self.status.lock().unwrap().managed;
let result = if managed { self.stop_server().await } else { Ok(()) };
match &result {
Ok(()) => self.set(LlmState::Off, None),
Err(e) => {
crate::error_log!("llm", "{e}");
self.set(LlmState::Failed, Some(e.clone()));
}
}
result
}
/// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that
/// fails while nothing answers (nothing was serving) counts as stopped.
async fn stop_server(&self) -> Result<(), String> {
let port = *self.port.lock().unwrap();
self.stop_server_on(port).await
}
/// `stop_server` confirming on `port` (0: nothing was started, nothing to confirm).
async fn stop_server_on(&self, port: u16) -> Result<(), String> {
if !self.exe.exists() {
return Ok(());
}
// The timeout ends the command and everything it started, like an install cancel.
let cancel = Arc::new(AtomicBool::new(false));
let ctx = Ctx::standalone(self.layout.clone(), "llm", cancel.clone());
let home = self.layout.strata_home();
let timer = {
let (cancel, t) = (cancel.clone(), self.stop_timeout);
tokio::spawn(async move {
tokio::time::sleep(t).await;
cancel.store(true, Ordering::SeqCst);
})
};
let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await;
timer.abort();
let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e });
let base = format!("http://127.0.0.1:{port}/v1");
let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await;
match (ran, gone) {
(Ok(()), true) => {
crate::info_log!("llm", "stopped");
Ok(())
}
(Err(e), true) => {
crate::warn_log!("llm", "stop command failed but nothing answers: {e}");
Ok(())
}
(Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 닫히지 않았습니다")),
(Err(e), false) => Err(format!("Strata를 끄지 못했습니다: {e}")),
}
}
}
/// Whether `base` is gone within `timeout`: only a refused connection counts. A timeout, an error
/// status (a server still loading answers 503) or an answer it cannot read is not a confirmed stop,
/// so it keeps checking until the deadline.
async fn wait_gone(base: &str, timeout: Duration) -> bool {
// Windows answers a connection to a closed local port only after about 2 s of retries, so the
// connect wait is long enough for that refusal to arrive as a refusal, not as a timeout.
let Ok(client) = reqwest::Client::builder().connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(12)).build() else { return false };
let until = Instant::now() + timeout;
loop {
if let Err(e) = client.get(format!("{base}/models")).send().await {
if e.is_connect() && !e.is_timeout() {
return true;
}
}
if Instant::now() >= until {
return false;
}
tokio::time::sleep(Duration::from_millis(300)).await;
}
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::datadir::temp_dir;
use crate::settings::{parse_settings, ChatChannelSettings};
pub(crate) fn layout(tag: &str) -> Layout {
Layout { data: temp_dir(tag), src: std::path::PathBuf::from("src") }
}
#[test]
fn the_launch_keeps_the_model_in_the_data_folder_and_follows_the_settings() {
let l = layout("llm1");
let s = parse_settings(&json!({"llm": {"context": 16384, "port": 8099}})).unwrap();
let a = launch_args(&l, &s, &conf(&s, Stage::Normal), &[]);
let after = |k: &str| a[a.iter().position(|x| x == k).unwrap() + 1].clone();
assert!(Path::new(&after("--data-dir")).starts_with(&l.data));
assert_eq!((after("--context"), after("--port"), after("--parallel")), ("16384".into(), "8099".into(), "1".into()));
assert_eq!(after("--access"), "local");
assert!(a.iter().any(|x| x == "--no-tui") && a.iter().any(|x| x == "--yes"));
}
#[test]
fn pictures_are_read_only_with_screen_viewing_and_measured_apart_on_the_card() {
assert_eq!(vision(Stage::Normal, false), "off");
assert_eq!(vision(Stage::Gaming, false), "off");
assert_eq!(vision(Stage::Normal, true), "gpu");
assert_eq!(vision(Stage::Gaming, true), "cpu");
let s = Settings::default();
let c = conf(&s, Stage::Normal);
assert_eq!(c.vision, "off", "nobody is registered for PC control by default");
let a = launch_args(&layout("llm-vision"), &s, &Conf { vision: "gpu", ..c }, &[]);
assert_eq!(a[a.iter().position(|x| x == "--vision").unwrap() + 1], "gpu");
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "off" }), "32768/1");
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "cpu" }), "32768/1");
assert_eq!(probe_key(&Conf { context: 32768, parallel: 1, vision: "gpu" }), "32768/1/vision");
}
#[test]
fn a_chat_channel_answering_everything_gets_a_second_request_slot() {
let mut s = Settings::default();
assert_eq!(parallel(&s), 1);
s.chat_channels.push(ChatChannelSettings { id: 1, name: String::new(), description: String::new(), read: true,
send: true, edit: true, react: true, delete: false, pin: false, answer_all: true, sync_discord: true });
assert_eq!(parallel(&s), 2);
}
#[test]
fn the_progress_is_the_last_percentage_on_a_line() {
assert_eq!(percent("Downloading 3 of 5: 41.5% (9.6 GB of 23.1 GB)"), Some(41.5));
assert_eq!(percent("50% done, now 75%"), Some(75.0));
assert_eq!(percent("Loading the model… 15 s"), None);
// Strata 0.1.59's model download prints sizes, not a percentage.
assert_eq!(percent("Downloading the model files: 2.4 of 22.9 GB"), Some(10.5));
assert_eq!(percent("Downloading the model files: 22.9 of 22.9 GB"), Some(100.0));
}
#[test]
fn the_brain_config_points_at_the_llm_and_keeps_memory_in_the_data_folder() {
let l = layout("llm2");
let s = parse_settings(&json!({"features": {"weather": false},
"mcpServers": [{"name": "brave", "command": "npx", "env": ["BRAVE_API_KEY"]}]})).unwrap();
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s).unwrap();
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1");
assert_eq!(v["ollama_chat_model"], "m");
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s)));
assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers));
assert_eq!(v["location_name"], crate::settings::DEFAULT_LOCATION, "the people are in Korea by default");
let none = parse_settings(&json!({"brain": {"location": ""}})).unwrap();
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none).unwrap();
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
assert_eq!(v["location_name"], Value::Null);
}
#[cfg(unix)]
/// Default settings on a port nothing listens on (8080 may be taken on a developer's PC).
fn unused_port_settings() -> Settings {
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
parse_settings(&json!({"llm": {"port": port}})).unwrap()
}
/// A tiny `/v1/models` server answering with `id`, after `delay` (before that it lists nothing).
async fn models_server(id: &'static str, delay: Duration) -> u16 {
serve(id, delay, None, Arc::new(Mutex::new("list"))).await
}
#[cfg(unix)]
/// The same; once the file `gone` exists (a fake Strata's `--stop` makes it) it stops listening,
/// so a connection is refused.
async fn models_server_until(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>) -> u16 {
serve(id, delay, gone, Arc::new(Mutex::new("list"))).await
}
/// `mode`: "list" answers, "hang" keeps the connection and says nothing, "503" is a server still
/// loading.
pub(crate) async fn serve(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>, mode: Arc<Mutex<&'static str>>) -> u16 {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let opened = Instant::now();
tokio::spawn(async move {
let mut listener = listener;
while let Ok((mut s, _)) = listener.accept().await {
if gone.as_ref().is_some_and(|g| g.exists()) {
// The listener closes: later connections are refused, until a launch serves
// again (removes `gone`).
drop(s);
drop(listener);
let g = gone.clone().unwrap();
while g.exists() {
tokio::time::sleep(Duration::from_millis(50)).await;
}
listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap();
// A new server: it answers (a left one loaded or stayed silent).
let mut m = mode.lock().unwrap();
if matches!(*m, "503" | "hang") {
*m = "list";
}
continue;
}
let mut buf = [0u8; 1024];
let n = s.read(&mut buf).await.unwrap_or(0);
let post = buf[..n].starts_with(b"POST");
let m = *mode.lock().unwrap();
let (status, body) = match m {
"hang" => {
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(60)).await;
drop(s);
});
continue;
}
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
"nochat" if post => ("500 Internal Server Error", r#"{"error":"broken"}"#.to_owned()),
_ if post => ("200 OK", r#"{"choices":[{"message":{"content":"안녕하세요"}}]}"#.to_owned()),
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
_ => ("200 OK", "{}".to_owned()),
};
let _ = s.write_all(format!("HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()).as_bytes()).await;
}
});
port
}
#[tokio::test]
async fn a_given_server_is_used_without_starting_strata() {
let port = models_server("given-model", Duration::ZERO).await;
let l = layout("llm3");
let llm = Llm::new(&l.data);
let s = parse_settings(&json!({"llm": {"url": format!("http://127.0.0.1:{port}/v1")}})).unwrap();
let (base, model) = llm.start(&s).await.unwrap();
assert_eq!((base.as_str(), model.as_str()), (format!("http://127.0.0.1:{port}/v1").as_str(), "given-model"));
assert_eq!(llm.status().state, LlmState::Ready);
assert!(!llm.status().managed);
}
#[tokio::test]
async fn without_strata_installed_the_start_fails_with_a_reason() {
let llm = Llm::new(&layout("llm4").data);
let e = llm.start(&Settings::default()).await.unwrap_err();
assert!(e.contains("Strata"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
}
/// A fake `strata`: `launch --stop` records itself and runs `stop_body` (by default it stops the
/// server: the file `gone` appears); `launch ...` runs `launch_body`.
#[cfg(unix)]
pub(crate) fn fake_strata(l: &Layout, launch_body: &str) -> std::path::PathBuf {
fake_strata_stop(l, launch_body, &format!("touch '{}'", l.data.join("gone").display()))
}
#[cfg(unix)]
pub(crate) fn fake_strata_stop(l: &Layout, launch_body: &str, stop_body: &str) -> std::path::PathBuf {
use std::os::unix::fs::PermissionsExt;
let exe = l.data.join("fake-strata");
std::fs::create_dir_all(l.strata_home()).unwrap();
let log = l.data.join("calls");
std::fs::write(&exe, format!("#!/bin/sh\necho \"$@\" >> '{}'\nif [ \"$2\" = --stop ]; then {stop_body}; exit $?; fi\n{launch_body}\n",
log.display())).unwrap();
std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();
exe
}
#[cfg(unix)]
#[tokio::test]
async fn strata_starts_reports_its_download_waits_for_the_model_and_stops() {
let l = layout("llm5");
let port = models_server_until("qwen-test", Duration::from_millis(300), Some(l.data.join("gone"))).await;
let exe = fake_strata(&l, "echo 'Downloading the engine: 80%'; sleep 1; echo 'Downloading the model files: 2.4 of 22.9 GB'; sleep 1; echo 'Starting the engine'; sleep 1; echo 'Strata is serving.'");
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
let l2 = llm.clone();
let s2 = s.clone();
let run = tokio::spawn(async move { l2.start(&s2).await });
let mut seen = None;
for _ in 0..40 {
tokio::time::sleep(Duration::from_millis(50)).await;
let st = llm.status();
if st.percent.is_some() {
seen = Some(st);
break;
}
}
let st = seen.expect("progress while starting");
assert_eq!((st.state, st.percent), (LlmState::Starting, Some(80.0)));
// The bar follows the current line: the model download's sizes, then nothing (no stale 80 %).
let mut later = vec![];
for _ in 0..60 {
tokio::time::sleep(Duration::from_millis(50)).await;
let st = llm.status();
if st.state != LlmState::Starting { break; }
later.push((st.note.clone().unwrap_or_default(), st.percent));
}
assert!(later.iter().any(|(n, p)| n.contains("2.4 of 22.9") && *p == Some(10.5)), "{later:?}");
assert!(later.iter().any(|(n, p)| n.contains("Starting the engine") && p.is_none()), "{later:?}");
let (base, model) = run.await.unwrap().unwrap();
assert_eq!((base, model.as_str()), (format!("http://127.0.0.1:{port}/v1"), "qwen-test"));
assert_eq!(llm.status().state, LlmState::Ready);
llm.stop().await.unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert!(calls.lines().next().unwrap().starts_with("launch --family"), "{calls}");
assert_eq!(calls.lines().last(), Some("launch --stop"));
assert_eq!(llm.status().state, LlmState::Off);
}
#[cfg(unix)]
#[tokio::test]
async fn with_a_graphics_card_the_first_start_measures_the_llm_without_experts_then_places_them() {
let l = layout("llm-place");
let port = server_after_launch(&l, "list").await;
// The engine log, as Strata 0.1.59's qwen36 engine writes it.
let log = l.strata_home().join("x.engine.log");
let exe = fake_strata(&l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 0 of 256 per layer in VRAM, bigger prompt chunks streamed (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()));
// The game PC in M0 (16303 MiB card, 1460 MiB held by other programs); the LLM without
// experts held 9874 MiB.
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect();
assert_eq!(launches.len(), 2, "{calls}");
assert!(launches[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", launches[0]);
// The M0 kit's normal placement on that PC: 1.5 GiB of experts.
assert!(launches[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[1]);
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!(probe["32768/1"], json!({"context": 32768, "parallel": 1, "gpu": "GPU-test", "modelMib": 9874, "stagingGib": 1.1}));
llm.stop().await.unwrap();
// The next start reuses the measurement: one launch, placed at once.
std::fs::remove_file(l.data.join("calls")).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect();
assert_eq!(launches.len(), 1, "{calls}");
assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]);
}
/// A fake card for `fake_placed`: 16303 MiB, 1460 MiB held by other programs (and by any
/// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a
/// Strata server runs (the calls file shows the last launch and stop).
#[cfg(unix)]
pub(crate) fn card(l: &Layout, id: &'static str, leftover: i64) -> GpuReader {
let calls = l.data.join("calls");
Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let stopped = c.contains("launch --stop");
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
let used = 1460 + if running { 9874 } else if stopped { 0 } else { leftover };
Some(Gpu { id: id.into(), used_mib: used, total_mib: 16303 })
})
}
/// A fake Strata that serves on `port` again (removes `gone`) and writes the engine's staging line.
#[cfg(unix)]
pub(crate) fn fake_placed(l: &Layout) -> std::path::PathBuf {
let log = l.strata_home().join("x.engine.log");
fake_strata(l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 5 of 256 per layer in VRAM, the rest of a batch on 16 CPU threads (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()))
}
/// A fake server on a free port that answers only once a launch serves (removes `gone`).
#[cfg(unix)]
pub(crate) async fn server_after_launch(l: &Layout, mode: &'static str) -> u16 {
std::fs::create_dir_all(&l.data).unwrap();
std::fs::write(l.data.join("gone"), "").unwrap();
serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await
}
pub(crate) fn launches(l: &Layout) -> Vec<String> {
std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().lines()
.filter(|c| c.starts_with("launch --family")).map(str::to_owned).collect()
}
#[cfg(unix)]
#[tokio::test]
async fn a_failed_measurement_is_not_kept_and_the_model_starts_without_experts() {
// The card stops answering after the first read.
let l = layout("llm-place-fail");
let port = server_after_launch(&l, "list").await;
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
let r2 = reads.clone();
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 })
}));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let l2 = launches(&l);
assert_eq!(l2.len(), 2, "{l2:?}");
assert!(l2[1].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[1]);
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
llm.stop().await.unwrap();
// The model does not answer the short question: not kept either.
let l = layout("llm-place-nochat");
let port = server_after_launch(&l, "nochat").await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
assert!(launches(&l)[1].ends_with("--set expert-vram=0 --set prefill=2048"));
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
}
#[cfg(unix)]
#[tokio::test]
async fn the_measurement_is_kept_per_requests_at_once_and_per_card() {
let l = layout("llm-place-keys");
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-a", 0));
let port = server_after_launch(&l, "list").await;
let one = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&one).await.unwrap();
assert_eq!(launches(&l).len(), 2, "measured once");
llm.stop().await.unwrap();
// A chat channel that answers everything: two requests at once, measured again.
std::fs::remove_file(l.data.join("calls")).unwrap();
let two = parse_settings(&json!({"llm": {"port": port}, "chatChannels": [{"id": "123456789012345678", "answer": "everything"}]})).unwrap();
assert_eq!(parallel(&two), 2);
llm.start(&two).await.unwrap();
assert_eq!(launches(&l).len(), 2, "measured for two at once");
assert!(launches(&l)[0].contains("--parallel 2"));
llm.stop().await.unwrap();
// Another card (same size): measured again.
std::fs::remove_file(l.data.join("calls")).unwrap();
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-b", 0));
llm.start(&one).await.unwrap();
assert_eq!(launches(&l).len(), 2, "measured for the new card");
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!((probe["32768/1"]["gpu"].clone(), probe["32768/2"]["gpu"].clone()), (json!("GPU-b"), json!("GPU-a")));
llm.stop().await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() {
// Left over answering, loading (503) or silent: always stopped first, holding 9874 MiB until then.
for mode in ["list", "503", "hang"] {
let l = layout(&format!("llm-place-left-{mode}"));
let port = serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert!(calls.lines().next() == Some("launch --stop"), "{mode}: stopped first: {calls}");
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "{mode}: the left server is not counted");
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{mode}: {:?}", launches(&l));
llm.stop().await.unwrap();
}
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_moves_the_llm_to_the_game_placement_and_back() {
let l = layout("llm-stage");
let port = server_after_launch(&l, "list").await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null}));
std::fs::remove_file(l.data.join("calls")).unwrap();
assert_eq!(tokio::time::timeout(Duration::from_secs(30), llm.switch(&s, Stage::Gaming, false, true)).await.unwrap().unwrap(), Stage::Gaming);
let g = launches(&l);
// Measured once at the game's context and one request, then the 2 GiB minimum (8 GB does not
// fit on 16 GB) with half the engine's 16 CPU threads.
assert_eq!(g.len(), 2, "{g:?}");
assert!(g[0].contains("--context 16384") && g[0].contains("--parallel 1") && g[0].contains("--set expert-vram=0"), "{}", g[0]);
assert!(g[1].ends_with("--set expert-vram=2 --set prefill=2048 --set pool-workers=8"), "{}", g[1]);
let room = 16303 - 1460 - 9874 - crate::placement::STT_MIB - crate::placement::MARGIN_MIB - 921;
assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": room}));
assert_eq!(llm.status().state, LlmState::Ready);
// Back to normal: the run's first reading of other programs, the normal placement again.
std::fs::remove_file(l.data.join("calls")).unwrap();
assert_eq!(llm.switch(&s, Stage::Normal, false, false).await.unwrap(), Stage::Normal);
let n = launches(&l);
assert_eq!(n.len(), 1, "{n:?}");
assert!(n[0].contains("--context 32768") && n[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", n[0]);
assert_eq!(llm.stage_view(), json!({"stage": "normal", "gameRoomMib": null}));
// Pause: the LLM stops.
assert_eq!(llm.switch(&s, Stage::Paused, false, true).await.unwrap(), Stage::Paused);
assert_eq!(llm.status().state, LlmState::Off);
assert_eq!(llm.stage_view()["stage"], json!("paused"));
}
/// A card whose other programs hold `others` MiB (changeable), plus the STT model while
/// `stt` is set, plus 9874 MiB while a Strata server runs.
#[cfg(unix)]
fn live_card(l: &Layout, others: Arc<std::sync::atomic::AtomicI64>, stt: bool) -> GpuReader {
let calls = l.data.join("calls");
Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
let used = others.load(Ordering::SeqCst) + if stt { 1300 } else { 0 } + if running { 9874 } else { 0 };
Some(Gpu { id: "GPU-test".into(), used_mib: used, total_mib: 16303 })
})
}
#[cfg(unix)]
#[tokio::test]
async fn the_normal_placement_reads_other_programs_afresh_and_a_game_is_not_counted_twice() {
let l = layout("llm-stage-fresh");
let port = server_after_launch(&l, "list").await;
let others = Arc::new(std::sync::atomic::AtomicI64::new(1460));
// At the start the STT worker has not loaded; at every switch it has.
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card(&l, others.clone(), false));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
let llm = llm.with_gpu(live_card(&l, others.clone(), true));
// A game starts and holds 2500 MiB: the game stage counts other programs as before the game.
others.store(1460 + 2500, Ordering::SeqCst);
llm.switch(&s, Stage::Gaming, true, true).await.unwrap();
assert!(launches(&l).last().unwrap().contains("--set expert-vram=2 "), "{:?}", launches(&l));
// The game ends but a video player now holds 1000 MiB more: the normal stage reads that.
others.store(1460 + 1000, Ordering::SeqCst);
llm.switch(&s, Stage::Normal, true, false).await.unwrap();
let expected = crate::placement::normal(16303, 1460 + 1000, &crate::placement::Probe { context: 32768, parallel: 1,
gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }, 1024);
assert!(expected < 1.5);
assert!(launches(&l).last().unwrap().ends_with(&format!("--set expert-vram={expected} --set prefill=2048")), "{:?}", launches(&l));
}
/// `live_card` on a card of `total` MiB.
#[cfg(unix)]
fn live_card_of(l: &Layout, others: Arc<std::sync::atomic::AtomicI64>, total: i64) -> GpuReader {
let calls = l.data.join("calls");
Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
Some(Gpu { id: "GPU-test".into(), used_mib: others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: total })
})
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_running_when_the_services_start_is_not_counted_twice() {
// A game holds 2500 MiB already: it is in what the card holds, so no reserve on top.
let l = layout("llm-stage-game-at-start");
let port = server_after_launch(&l, "list").await;
let others = Arc::new(std::sync::atomic::AtomicI64::new(1460 + 2500));
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 16303));
llm.game_at_start(true);
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
assert_eq!(llm.switch(&s, Stage::Gaming, false, true).await.unwrap(), Stage::Gaming, "not paused");
let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 };
let expected = crate::placement::normal(16303, 1460 + 2500, &probe16, 1024);
assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l));
}
#[cfg(unix)]
#[tokio::test]
async fn the_game_stage_reads_the_card_again_and_counts_what_grew_since_the_normal_reading() {
// Normal reading: 1460 MiB. Then a program takes 4000 MiB and the game 4500 MiB: together
// more than the 8 GB reserve, so the placement leaves that much, not just 8 GB.
let l = layout("llm-stage-grew");
let port = server_after_launch(&l, "list").await;
let others = Arc::new(std::sync::atomic::AtomicI64::new(1460));
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(live_card_of(&l, others.clone(), 24576));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
others.store(1460 + 4000 + 4500, Ordering::SeqCst);
llm.switch(&s, Stage::Gaming, false, true).await.unwrap();
let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 };
let expected = crate::placement::game(24576, 1460, &probe16, 8500).expert_gib;
assert_ne!(expected, crate::placement::game(24576, 1460, &probe16, 8192).expert_gib, "the test tells them apart");
assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={expected} ")), "{:?}", launches(&l));
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_stage_on_a_fallback_placement_still_shows_as_the_game_stage() {
// The game's measurement fails (the card cannot be read the second time): no experts, but
// the game stage is what runs.
let l = layout("llm-stage-fallback");
let port = server_after_launch(&l, "list").await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
let r2 = reads.clone();
let llm = llm.with_gpu(Arc::new(move || {
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 })
}));
assert_eq!(llm.switch(&s, Stage::Gaming, false, true).await.unwrap(), Stage::Gaming);
assert!(launches(&l).last().unwrap().contains("--set expert-vram=0 "), "{:?}", launches(&l));
assert_eq!(llm.stage_view(), json!({"stage": "gaming", "gameRoomMib": null}));
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_with_no_room_beside_the_model_pauses_the_llm() {
let l = layout("llm-stage-full");
let port = server_after_launch(&l, "list").await;
let calls = l.data.join("calls");
// A 12 GB card: the model fits, but not with any room for a game.
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
Some(Gpu { id: "GPU-small".into(), used_mib: 1460 + if running { 9874 } else { 0 }, total_mib: 12288 })
}));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
assert_eq!(llm.switch(&s, Stage::Gaming, false, true).await.unwrap(), Stage::Paused);
assert_eq!(llm.status().state, LlmState::Off);
assert_eq!(llm.stage_view(), json!({"stage": "paused", "gameRoomMib": null}));
let c = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert_eq!(c.lines().last(), Some("launch --stop"));
}
#[cfg(unix)]
#[tokio::test]
async fn a_left_server_that_does_not_stop_means_no_experts_and_nothing_measured() {
// The stop command fails and the server keeps answering: the card is not read as it is.
let l = layout("llm-place-stuck");
let port = models_server_until("qwen-test", Duration::from_millis(0), None).await;
let log = l.strata_home().join("x.engine.log");
let exe = fake_strata_stop(&l, &format!("echo 'strata-q36: (1.10 GiB staging)' >> '{}'; echo 'Strata is serving.'", log.display()), "exit 3");
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 9874)).with_stop_timeout(Duration::from_secs(1));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
let l2 = launches(&l);
assert_eq!(l2.len(), 1, "{l2:?}");
assert!(l2[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[0]);
assert!(!l.data.join("config").join("llm-probe.json").exists());
}
#[cfg(unix)]
#[tokio::test]
async fn a_cancel_during_the_download_stops_it_and_clears_the_server() {
let l = layout("llm6");
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
let l2 = llm.clone();
let run = tokio::spawn(async move { l2.start(&unused_port_settings()).await });
tokio::time::sleep(Duration::from_millis(400)).await;
let t = Instant::now();
llm.cancel();
assert_eq!(run.await.unwrap().unwrap_err(), "cancelled");
assert!(t.elapsed() < Duration::from_secs(5));
assert_eq!(llm.status().state, LlmState::Off);
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert_eq!(calls.lines().last(), Some("launch --stop"));
}
#[cfg(unix)]
#[tokio::test]
async fn a_failed_launch_shows_strata_s_last_line() {
let l = layout("llm7");
let exe = fake_strata(&l, "echo 'Not enough disk space for the model' >&2; exit 4");
let llm = Llm::with_exe(l, exe);
let e = llm.start(&unused_port_settings()).await.unwrap_err();
assert!(e.contains("Not enough disk space"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
}
#[cfg(unix)]
#[tokio::test]
async fn a_server_still_answering_after_a_failed_stop_is_reported_not_marked_off() {
let l = layout("llm8");
let port = models_server("qwen-test", Duration::ZERO).await;
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "echo 'could not stop the engine' >&2; exit 3");
let llm = Llm::with_exe(l, exe);
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
let e = llm.stop().await.unwrap_err();
assert!(e.contains("could not stop the engine"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
}
#[cfg(unix)]
#[tokio::test]
async fn a_stop_that_hangs_is_given_up_on_and_reported() {
let l = layout("llm9");
let port = models_server("qwen-test", Duration::ZERO).await;
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "sleep 30");
let llm = Llm::with_exe(l, exe).with_stop_timeout(Duration::from_millis(500));
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
let t = Instant::now();
let e = llm.stop().await.unwrap_err();
assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed());
assert!(e.contains("Strata"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
}
#[cfg(unix)]
#[tokio::test]
async fn a_stop_command_that_fails_while_nothing_answers_counts_as_stopped() {
let l = layout("llm10");
let port = models_server_until("qwen-test", Duration::ZERO, Some(l.data.join("gone"))).await;
let stop = format!("touch '{}'; exit 1", l.data.join("gone").display());
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", &stop);
let llm = Llm::with_exe(l, exe);
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
llm.stop().await.unwrap();
assert_eq!(llm.status().state, LlmState::Off);
}
/// A failed stop while the server is still there but not listing a model (holding connections
/// silently, or a 503 while loading) is not a confirmed stop: the LLM and, through the
/// controller, the services stay failed.
#[cfg(unix)]
#[tokio::test]
async fn a_failed_stop_with_a_silent_or_loading_server_is_not_counted_as_stopped() {
for m in ["hang", "503"] {
let l = layout(&format!("llm11{m}"));
let mode = Arc::new(Mutex::new("list"));
let port = serve("qwen-test", Duration::ZERO, None, mode.clone()).await;
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "exit 3");
let llm = Arc::new(Llm::with_exe(l, exe));
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(),
settings: parse_settings(&json!({"llm": {"port": port}})).unwrap() });
c.start().await.unwrap();
*mode.lock().unwrap() = m;
let e = c.stop().await.unwrap_err();
assert!(e.contains("Strata"), "{m}: {e}");
assert_eq!(c.state(), crate::control::State::Failed, "{m}");
assert_eq!(llm.status().state, LlmState::Failed, "{m}");
}
}
/// A stop right after a background start must cancel the LLM's start before it runs anything
/// (no download), even though the start task runs after the stop.
#[cfg(unix)]
#[tokio::test]
async fn a_stop_right_after_a_background_start_runs_no_launch() {
let l = layout("llm12");
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: unused_port_settings() });
c.start_in_background().unwrap();
c.stop().await.unwrap();
let t = Instant::now();
while c.state() != crate::control::State::Stopped && t.elapsed() < Duration::from_secs(5) {
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(c.state(), crate::control::State::Stopped);
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap_or_default();
assert!(!calls.contains("--family"), "no launch ran: {calls}");
// A later start is not cancelled by the earlier stop.
c.start_in_background().unwrap();
tokio::time::sleep(Duration::from_millis(400)).await;
assert!(std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().contains("--family"));
c.stop().await.unwrap();
}
#[cfg(unix)]
/// The services as the controller sees them, reduced to the LLM.
struct LlmHost {
llm: Arc<Llm>,
settings: Settings,
}
#[cfg(unix)]
struct LlmRunning(Arc<Llm>);
#[cfg(unix)]
#[async_trait::async_trait]
impl crate::control::Running for LlmRunning {
async fn stop(&self) -> Result<(), String> {
self.0.stop().await
}
fn status(&self, _: Instant) -> Value {
json!({"worker": null, "voiceMembers": [], "missed": []})
}
}
#[cfg(unix)]
#[async_trait::async_trait]
impl crate::control::Host for LlmHost {
async fn start(&self) -> Result<(Arc<dyn crate::control::Running>, tokio::sync::mpsc::UnboundedReceiver<crate::discord::CoreEvent>), String> {
self.llm.start(&self.settings).await?;
Ok((Arc::new(LlmRunning(self.llm.clone())), tokio::sync::mpsc::unbounded_channel().1))
}
fn secrets(&self) -> Vec<String> {
vec![]
}
fn prepare_start(&self) {
self.llm.arm();
}
fn cancel_start(&self) {
self.llm.cancel();
}
}
}