Files
javis_bot/app/src/strata.rs
javis-bot 3ef4281180 feat(app): features screen with tool switches and MCP servers; v2 drops host file and screen tools
- Settings `features` (weather, webSearch, meals) and `mcpServers` (name, command, args, env names,
  enabled), edited on the new 기능 screen. The tools of a feature that is off go into the brain's
  `disabled_tools`; only enabled MCP servers go into its `mcps`.
- MCP environment values (API keys) live only in the encrypted secrets (`mcp:<server>:<NAME>`),
  shown as stored or not, masked in logs, removed with their server, and reach the worker only in
  `JARVIS_MCP_ENV`, which the brain takes out of its environment before giving each server its own.
- `enabled_builtin_tools(cfg)` is the one list the router, toolSearchTool and the executor use, so
  a turned-off tool is neither offered nor run, also when the model names it.
- The v2 brain never offers v1's `localFiles` (home folder read/write/delete) and `screenshot`:
  anyone in the voice or chat channels could ask for them. Screen viewing comes in M7.
- The brain connects MCP servers on its own thread and logs each outcome without debug logging.
- Stored secrets are masked in the log hub as soon as the data folder is found.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 09:56:17 +09:00

684 lines
31 KiB
Rust

//! The LLM the brain answers with (`app/src/admin.spec.md`, "LLM"): the app starts the installed
//! Strata from the data folder with the M0 kit's choices, waits until the model answers, and stops
//! it with the services. Strata downloads the model (about 23 GB) on its first start; the dashboard
//! shows how far it got. With `llm.url` the app uses that server instead and starts nothing.
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use serde::Serialize;
use serde_json::{json, Value};
use crate::install::{Ctx, Layout};
use crate::settings::Settings;
/// The one model v2 runs (M0: Qwen3.6-35B-A3B UD-Q4_K_XL; docs/v2/roadmap.md).
pub const FAMILY: &str = "qwen36";
pub const MODEL: &str = "ud-q4_k_xl";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum LlmState {
Off,
Starting,
Ready,
Failed,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct LlmStatus {
pub state: LlmState,
/// Strata's own server (false: `llm.url`).
pub managed: bool,
/// How far Strata's current line says it got while starting (a percentage, or the model
/// download's `X of Y GB`); none on a line without progress.
pub percent: Option<f64>,
/// Strata's last line while starting, or why it failed.
pub note: Option<String>,
}
/// Requests answered at once: one is faster on a 16 GB card (M0), two only when a chat channel
/// answers every message (so its answers do not wait behind the voice answers).
pub fn parallel(settings: &Settings) -> u32 {
if settings.chat_channels.iter().any(|c| c.answer_all) { 2 } else { 1 }
}
/// `strata launch` for the services: the M0 kit's normal placement, models in `models/llm`, this
/// computer only. No pictures yet (nothing in v2 sends one; reading them takes graphics memory).
/// `--restart` replaces a server started with other choices; the same choices are reused.
pub fn launch_args(layout: &Layout, settings: &Settings) -> Vec<String> {
let models = layout.data.join("models").join("llm").to_string_lossy().into_owned();
let mut a: Vec<String> = ["launch", "--family", FAMILY, "--model", MODEL, "--data-dir"].map(String::from).into();
a.push(models);
for (k, v) in [("--context", settings.llm.context.to_string()), ("--vision", "off".into()),
("--parallel", parallel(settings).to_string()), ("--access", "local".into()),
("--port", settings.llm.port.to_string())] {
a.push(k.into());
a.push(v);
}
a.extend(["--no-api-key", "--restart", "--yes", "--no-tui"].map(String::from));
a
}
/// How far a line says Strata got: its last `NN%` or `NN.N%`, else `X of Y GB` (or MB), which is
/// how Strata 0.1.59 reports the model download.
pub fn percent(line: &str) -> Option<f64> {
percent_sign(line).or_else(|| size_progress(line))
}
fn size_progress(line: &str) -> Option<f64> {
let words: Vec<&str> = line.split_whitespace().collect();
words.windows(4).rev().find_map(|w| {
let (done, total) = (w[0].parse::<f64>().ok()?, w[2].parse::<f64>().ok()?);
(w[1] == "of" && matches!(w[3].trim_end_matches([',', '.', ')']), "GB" | "MB") && total > 0.0 && done <= total)
.then(|| (done / total * 1000.0).round() / 10.0)
})
}
fn percent_sign(line: &str) -> Option<f64> {
let b = line.as_bytes();
let mut found = None;
for (i, _) in line.match_indices('%') {
let mut s = i;
while s > 0 && (b[s - 1].is_ascii_digit() || b[s - 1] == b'.') {
s -= 1;
}
if let Ok(n) = line[s..i].parse::<f64>() {
if (0.0..=100.0).contains(&n) {
found = Some(n);
}
}
}
found
}
/// Wait until `base` (`.../v1`) lists a model; returns its id. Stops early on `cancel`.
pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> Result<String, String> {
let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().map_err(|e| e.to_string())?;
let until = Instant::now() + timeout;
loop {
if cancel.load(Ordering::Relaxed) {
return Err("cancelled".into());
}
if let Ok(r) = client.get(format!("{base}/models")).send().await {
if let Ok(v) = r.text().await.map(|t| serde_json::from_str::<Value>(&t).unwrap_or_default()) {
if let Some(id) = v["data"][0]["id"].as_str() {
return Ok(id.to_owned());
}
}
}
if Instant::now() >= until {
return Err(format!("대화 모델이 {}초 안에 준비되지 않았습니다 ({base})", timeout.as_secs()));
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the
/// memory database inside the data folder, the tools of the features that are off and the enabled
/// MCP servers (without their environment values, `features.rs`). The app owns this file and writes
/// it on every start.
pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings)
-> Result<std::path::PathBuf, String> {
let file = data.join("config").join("brain.json");
let cfg = json!({
"ollama_base_url": base_url,
"ollama_chat_model": model,
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
"disabled_tools": crate::features::disabled_tools(&settings.features),
"mcps": crate::features::brain_mcps(&settings.mcp_servers),
"_config_version": 1,
});
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
text.push('\n');
crate::datadir::write_atomic(&file, text.as_bytes()).map_err(|e| e.to_string())?;
Ok(file)
}
/// One LLM for the services: Strata from the layout, or the server at `llm.url`.
pub struct Llm {
layout: Layout,
/// The program run as Strata (the installed one; a script in tests).
exe: std::path::PathBuf,
status: Mutex<LlmStatus>,
active: Mutex<Option<Arc<Ctx>>>,
cancel: Arc<AtomicBool>,
/// The port the managed server was started on (to confirm it is gone after a stop).
port: Mutex<u16>,
stop_timeout: Duration,
}
/// How long the model may take to answer after `strata launch` returned (it loads in about 15 s
/// on the game PC; the first start may also tune).
const READY_TIMEOUT: Duration = Duration::from_secs(30 * 60);
/// A server the app does not start must already answer.
const URL_TIMEOUT: Duration = Duration::from_secs(30);
impl Llm {
pub fn new(data: &Path) -> Self {
// Only the data folder matters here (the Python sources are the install's business).
let layout = Layout { data: data.to_path_buf(), src: std::path::PathBuf::new() };
let exe = layout.strata_exe();
Self::with_exe(layout, exe)
}
pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self {
Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }),
active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0),
stop_timeout: Duration::from_secs(120) }
}
/// How long `strata launch --stop` may take before it is ended and the stop reported as failed.
pub fn with_stop_timeout(mut self, t: Duration) -> Self {
self.stop_timeout = t;
self
}
pub fn status(&self) -> LlmStatus {
let mut s = self.status.lock().unwrap().clone();
if let Some(ctx) = self.active.lock().unwrap().as_ref() {
s.note = ctx.last_note();
// From the current line only: a percentage from an earlier stage would mislead.
s.percent = s.note.as_deref().and_then(percent);
}
s
}
fn set(&self, state: LlmState, note: Option<String>) {
let mut s = self.status.lock().unwrap();
s.state = state;
if state != LlmState::Starting {
s.note = note;
}
}
/// Start (or reuse) the LLM; returns its `/v1` address and model id.
/// A cancel since the last `arm` (or before the first start) ends it before it runs anything.
pub async fn start(&self, settings: &Settings) -> Result<(String, String), String> {
let managed = settings.llm.url.is_none();
*self.status.lock().unwrap() = LlmStatus { state: LlmState::Starting, managed, percent: None, note: None };
let result = self.start_inner(settings).await;
match &result {
Ok((base, model)) => {
crate::info_log!("llm", "ready: {base} model={model}");
self.set(LlmState::Ready, None);
}
Err(e) if e == "cancelled" => self.set(LlmState::Off, None),
Err(e) => {
crate::error_log!("llm", "did not start: {e}");
self.set(LlmState::Failed, Some(e.clone()));
}
}
result
}
async fn start_inner(&self, settings: &Settings) -> Result<(String, String), String> {
if self.cancel.load(Ordering::SeqCst) {
return Err("cancelled".into());
}
if let Some(url) = &settings.llm.url {
let model = wait_ready(url, URL_TIMEOUT, &self.cancel).await?;
return Ok((url.clone(), model));
}
if !self.exe.exists() {
return Err("Strata가 설치되어 있지 않습니다. 마법사의 모델 단계에서 설치해 주세요".into());
}
let ctx = Arc::new(Ctx::standalone(self.layout.clone(), "llm", self.cancel.clone()));
*self.active.lock().unwrap() = Some(ctx.clone());
*self.port.lock().unwrap() = settings.llm.port;
let args = launch_args(&self.layout, settings);
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
let home = self.layout.strata_home();
let ran = ctx.run_in(&self.exe, &refs, Some(&home)).await;
let base = format!("http://127.0.0.1:{}/v1", settings.llm.port);
let ready = match ran {
Ok(()) => wait_ready(&base, READY_TIMEOUT, &self.cancel).await,
Err(e) => Err(e),
};
*self.active.lock().unwrap() = None;
match ready {
Ok(model) => Ok((base, model)),
// Nothing half-started stays on the graphics card; if it may have, say so.
Err(e) => match self.stop_server().await {
Ok(()) => Err(e),
Err(stop) if e == "cancelled" => Err(stop),
Err(stop) => Err(format!("{e} ({stop})")),
},
}
}
/// Make the next start cancellable afresh: called when a start is accepted, never by `start`
/// itself, so a stop between accepting and running the start still cancels it.
pub fn arm(&self) {
self.cancel.store(false, Ordering::SeqCst);
}
/// Stop a start in progress (the download resumes on the next start).
pub fn cancel(&self) {
self.cancel.store(true, Ordering::SeqCst);
}
/// Stop the server this app started (nothing for `llm.url`). Off only once the server no longer
/// answers; otherwise `failed` with the reason, which is also returned.
pub async fn stop(&self) -> Result<(), String> {
let managed = self.status.lock().unwrap().managed;
let result = if managed { self.stop_server().await } else { Ok(()) };
match &result {
Ok(()) => self.set(LlmState::Off, None),
Err(e) => {
crate::error_log!("llm", "{e}");
self.set(LlmState::Failed, Some(e.clone()));
}
}
result
}
/// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that
/// fails while nothing answers (nothing was serving) counts as stopped.
async fn stop_server(&self) -> Result<(), String> {
if !self.exe.exists() {
return Ok(());
}
// The timeout ends the command and everything it started, like an install cancel.
let cancel = Arc::new(AtomicBool::new(false));
let ctx = Ctx::standalone(self.layout.clone(), "llm", cancel.clone());
let home = self.layout.strata_home();
let timer = {
let (cancel, t) = (cancel.clone(), self.stop_timeout);
tokio::spawn(async move {
tokio::time::sleep(t).await;
cancel.store(true, Ordering::SeqCst);
})
};
let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await;
timer.abort();
let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e });
let port = *self.port.lock().unwrap();
let base = format!("http://127.0.0.1:{port}/v1");
let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await;
match (ran, gone) {
(Ok(()), true) => {
crate::info_log!("llm", "stopped");
Ok(())
}
(Err(e), true) => {
crate::warn_log!("llm", "stop command failed but nothing answers: {e}");
Ok(())
}
(Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 닫히지 않았습니다")),
(Err(e), false) => Err(format!("Strata를 끄지 못했습니다: {e}")),
}
}
}
/// Whether `base` is gone within `timeout`: only a refused connection counts. A timeout, an error
/// status (a server still loading answers 503) or an answer it cannot read is not a confirmed stop,
/// so it keeps checking until the deadline.
async fn wait_gone(base: &str, timeout: Duration) -> bool {
// Windows answers a connection to a closed local port only after about 2 s of retries, so the
// connect wait is long enough for that refusal to arrive as a refusal, not as a timeout.
let Ok(client) = reqwest::Client::builder().connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(12)).build() else { return false };
let until = Instant::now() + timeout;
loop {
if let Err(e) = client.get(format!("{base}/models")).send().await {
if e.is_connect() && !e.is_timeout() {
return true;
}
}
if Instant::now() >= until {
return false;
}
tokio::time::sleep(Duration::from_millis(300)).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::datadir::temp_dir;
use crate::settings::{parse_settings, ChatChannelSettings};
fn layout(tag: &str) -> Layout {
Layout { data: temp_dir(tag), src: std::path::PathBuf::from("src") }
}
#[test]
fn the_launch_keeps_the_model_in_the_data_folder_and_follows_the_settings() {
let l = layout("llm1");
let s = parse_settings(&json!({"llm": {"context": 16384, "port": 8099}})).unwrap();
let a = launch_args(&l, &s);
let after = |k: &str| a[a.iter().position(|x| x == k).unwrap() + 1].clone();
assert!(Path::new(&after("--data-dir")).starts_with(&l.data));
assert_eq!((after("--context"), after("--port"), after("--parallel")), ("16384".into(), "8099".into(), "1".into()));
assert_eq!(after("--access"), "local");
assert!(a.iter().any(|x| x == "--no-tui") && a.iter().any(|x| x == "--yes"));
}
#[test]
fn a_chat_channel_answering_everything_gets_a_second_request_slot() {
let mut s = Settings::default();
assert_eq!(parallel(&s), 1);
s.chat_channels.push(ChatChannelSettings { id: 1, name: String::new(), description: String::new(), read: true,
send: true, edit: true, react: true, delete: false, pin: false, answer_all: true, sync_discord: true });
assert_eq!(parallel(&s), 2);
}
#[test]
fn the_progress_is_the_last_percentage_on_a_line() {
assert_eq!(percent("Downloading 3 of 5: 41.5% (9.6 GB of 23.1 GB)"), Some(41.5));
assert_eq!(percent("50% done, now 75%"), Some(75.0));
assert_eq!(percent("Loading the model… 15 s"), None);
// Strata 0.1.59's model download prints sizes, not a percentage.
assert_eq!(percent("Downloading the model files: 2.4 of 22.9 GB"), Some(10.5));
assert_eq!(percent("Downloading the model files: 22.9 of 22.9 GB"), Some(100.0));
}
#[test]
fn the_brain_config_points_at_the_llm_and_keeps_memory_in_the_data_folder() {
let l = layout("llm2");
let s = parse_settings(&json!({"features": {"weather": false},
"mcpServers": [{"name": "brave", "command": "npx", "env": ["BRAVE_API_KEY"]}]})).unwrap();
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s).unwrap();
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1");
assert_eq!(v["ollama_chat_model"], "m");
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s.features)));
assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers));
}
#[cfg(unix)]
/// Default settings on a port nothing listens on (8080 may be taken on a developer's PC).
fn unused_port_settings() -> Settings {
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
parse_settings(&json!({"llm": {"port": port}})).unwrap()
}
/// A tiny `/v1/models` server answering with `id`, after `delay` (before that it lists nothing).
async fn models_server(id: &'static str, delay: Duration) -> u16 {
serve(id, delay, None, Arc::new(Mutex::new("list"))).await
}
#[cfg(unix)]
/// The same; once the file `gone` exists (a fake Strata's `--stop` makes it) it stops listening,
/// so a connection is refused.
async fn models_server_until(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>) -> u16 {
serve(id, delay, gone, Arc::new(Mutex::new("list"))).await
}
/// `mode`: "list" answers, "hang" keeps the connection and says nothing, "503" is a server still
/// loading.
async fn serve(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>, mode: Arc<Mutex<&'static str>>) -> u16 {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let opened = Instant::now();
tokio::spawn(async move {
while let Ok((mut s, _)) = listener.accept().await {
if gone.as_ref().is_some_and(|g| g.exists()) {
return; // the listener closes: later connections are refused
}
let mut buf = [0u8; 1024];
let _ = s.read(&mut buf).await;
let m = *mode.lock().unwrap();
let (status, body) = match m {
"hang" => {
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(60)).await;
drop(s);
});
continue;
}
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
_ => ("200 OK", "{}".to_owned()),
};
let _ = s.write_all(format!("HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()).as_bytes()).await;
}
});
port
}
#[tokio::test]
async fn a_given_server_is_used_without_starting_strata() {
let port = models_server("given-model", Duration::ZERO).await;
let l = layout("llm3");
let llm = Llm::new(&l.data);
let s = parse_settings(&json!({"llm": {"url": format!("http://127.0.0.1:{port}/v1")}})).unwrap();
let (base, model) = llm.start(&s).await.unwrap();
assert_eq!((base.as_str(), model.as_str()), (format!("http://127.0.0.1:{port}/v1").as_str(), "given-model"));
assert_eq!(llm.status().state, LlmState::Ready);
assert!(!llm.status().managed);
}
#[tokio::test]
async fn without_strata_installed_the_start_fails_with_a_reason() {
let llm = Llm::new(&layout("llm4").data);
let e = llm.start(&Settings::default()).await.unwrap_err();
assert!(e.contains("Strata"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
}
/// A fake `strata`: `launch --stop` records itself and runs `stop_body` (by default it stops the
/// server: the file `gone` appears); `launch ...` runs `launch_body`.
#[cfg(unix)]
fn fake_strata(l: &Layout, launch_body: &str) -> std::path::PathBuf {
fake_strata_stop(l, launch_body, &format!("touch '{}'", l.data.join("gone").display()))
}
#[cfg(unix)]
fn fake_strata_stop(l: &Layout, launch_body: &str, stop_body: &str) -> std::path::PathBuf {
use std::os::unix::fs::PermissionsExt;
let exe = l.data.join("fake-strata");
std::fs::create_dir_all(l.strata_home()).unwrap();
let log = l.data.join("calls");
std::fs::write(&exe, format!("#!/bin/sh\necho \"$@\" >> '{}'\nif [ \"$2\" = --stop ]; then {stop_body}; exit $?; fi\n{launch_body}\n",
log.display())).unwrap();
std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();
exe
}
#[cfg(unix)]
#[tokio::test]
async fn strata_starts_reports_its_download_waits_for_the_model_and_stops() {
let l = layout("llm5");
let port = models_server_until("qwen-test", Duration::from_millis(300), Some(l.data.join("gone"))).await;
let exe = fake_strata(&l, "echo 'Downloading the engine: 80%'; sleep 1; echo 'Downloading the model files: 2.4 of 22.9 GB'; sleep 1; echo 'Starting the engine'; sleep 1; echo 'Strata is serving.'");
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
let l2 = llm.clone();
let s2 = s.clone();
let run = tokio::spawn(async move { l2.start(&s2).await });
let mut seen = None;
for _ in 0..40 {
tokio::time::sleep(Duration::from_millis(50)).await;
let st = llm.status();
if st.percent.is_some() {
seen = Some(st);
break;
}
}
let st = seen.expect("progress while starting");
assert_eq!((st.state, st.percent), (LlmState::Starting, Some(80.0)));
// The bar follows the current line: the model download's sizes, then nothing (no stale 80 %).
let mut later = vec![];
for _ in 0..60 {
tokio::time::sleep(Duration::from_millis(50)).await;
let st = llm.status();
if st.state != LlmState::Starting { break; }
later.push((st.note.clone().unwrap_or_default(), st.percent));
}
assert!(later.iter().any(|(n, p)| n.contains("2.4 of 22.9") && *p == Some(10.5)), "{later:?}");
assert!(later.iter().any(|(n, p)| n.contains("Starting the engine") && p.is_none()), "{later:?}");
let (base, model) = run.await.unwrap().unwrap();
assert_eq!((base, model.as_str()), (format!("http://127.0.0.1:{port}/v1"), "qwen-test"));
assert_eq!(llm.status().state, LlmState::Ready);
llm.stop().await.unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert!(calls.lines().next().unwrap().starts_with("launch --family"), "{calls}");
assert_eq!(calls.lines().last(), Some("launch --stop"));
assert_eq!(llm.status().state, LlmState::Off);
}
#[cfg(unix)]
#[tokio::test]
async fn a_cancel_during_the_download_stops_it_and_clears_the_server() {
let l = layout("llm6");
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
let l2 = llm.clone();
let run = tokio::spawn(async move { l2.start(&unused_port_settings()).await });
tokio::time::sleep(Duration::from_millis(400)).await;
let t = Instant::now();
llm.cancel();
assert_eq!(run.await.unwrap().unwrap_err(), "cancelled");
assert!(t.elapsed() < Duration::from_secs(5));
assert_eq!(llm.status().state, LlmState::Off);
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert_eq!(calls.lines().last(), Some("launch --stop"));
}
#[cfg(unix)]
#[tokio::test]
async fn a_failed_launch_shows_strata_s_last_line() {
let l = layout("llm7");
let exe = fake_strata(&l, "echo 'Not enough disk space for the model' >&2; exit 4");
let llm = Llm::with_exe(l, exe);
let e = llm.start(&unused_port_settings()).await.unwrap_err();
assert!(e.contains("Not enough disk space"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
}
#[cfg(unix)]
#[tokio::test]
async fn a_server_still_answering_after_a_failed_stop_is_reported_not_marked_off() {
let l = layout("llm8");
let port = models_server("qwen-test", Duration::ZERO).await;
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "echo 'could not stop the engine' >&2; exit 3");
let llm = Llm::with_exe(l, exe);
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
let e = llm.stop().await.unwrap_err();
assert!(e.contains("could not stop the engine"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
}
#[cfg(unix)]
#[tokio::test]
async fn a_stop_that_hangs_is_given_up_on_and_reported() {
let l = layout("llm9");
let port = models_server("qwen-test", Duration::ZERO).await;
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "sleep 30");
let llm = Llm::with_exe(l, exe).with_stop_timeout(Duration::from_millis(500));
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
let t = Instant::now();
let e = llm.stop().await.unwrap_err();
assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed());
assert!(e.contains("Strata"), "{e}");
assert_eq!(llm.status().state, LlmState::Failed);
}
#[cfg(unix)]
#[tokio::test]
async fn a_stop_command_that_fails_while_nothing_answers_counts_as_stopped() {
let l = layout("llm10");
let port = models_server_until("qwen-test", Duration::ZERO, Some(l.data.join("gone"))).await;
let stop = format!("touch '{}'; exit 1", l.data.join("gone").display());
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", &stop);
let llm = Llm::with_exe(l, exe);
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
llm.stop().await.unwrap();
assert_eq!(llm.status().state, LlmState::Off);
}
/// A failed stop while the server is still there but not listing a model (holding connections
/// silently, or a 503 while loading) is not a confirmed stop: the LLM and, through the
/// controller, the services stay failed.
#[cfg(unix)]
#[tokio::test]
async fn a_failed_stop_with_a_silent_or_loading_server_is_not_counted_as_stopped() {
for m in ["hang", "503"] {
let l = layout(&format!("llm11{m}"));
let mode = Arc::new(Mutex::new("list"));
let port = serve("qwen-test", Duration::ZERO, None, mode.clone()).await;
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "exit 3");
let llm = Arc::new(Llm::with_exe(l, exe));
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(),
settings: parse_settings(&json!({"llm": {"port": port}})).unwrap() });
c.start().await.unwrap();
*mode.lock().unwrap() = m;
let e = c.stop().await.unwrap_err();
assert!(e.contains("Strata"), "{m}: {e}");
assert_eq!(c.state(), crate::control::State::Failed, "{m}");
assert_eq!(llm.status().state, LlmState::Failed, "{m}");
}
}
/// A stop right after a background start must cancel the LLM's start before it runs anything
/// (no download), even though the start task runs after the stop.
#[cfg(unix)]
#[tokio::test]
async fn a_stop_right_after_a_background_start_runs_no_launch() {
let l = layout("llm12");
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: unused_port_settings() });
c.start_in_background().unwrap();
c.stop().await.unwrap();
let t = Instant::now();
while c.state() != crate::control::State::Stopped && t.elapsed() < Duration::from_secs(5) {
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(c.state(), crate::control::State::Stopped);
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap_or_default();
assert!(!calls.contains("--family"), "no launch ran: {calls}");
// A later start is not cancelled by the earlier stop.
c.start_in_background().unwrap();
tokio::time::sleep(Duration::from_millis(400)).await;
assert!(std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().contains("--family"));
c.stop().await.unwrap();
}
#[cfg(unix)]
/// The services as the controller sees them, reduced to the LLM.
struct LlmHost {
llm: Arc<Llm>,
settings: Settings,
}
#[cfg(unix)]
struct LlmRunning(Arc<Llm>);
#[cfg(unix)]
#[async_trait::async_trait]
impl crate::control::Running for LlmRunning {
async fn stop(&self) -> Result<(), String> {
self.0.stop().await
}
fn status(&self, _: Instant) -> Value {
json!({"worker": null, "voiceMembers": [], "missed": []})
}
}
#[cfg(unix)]
#[async_trait::async_trait]
impl crate::control::Host for LlmHost {
async fn start(&self) -> Result<(Arc<dyn crate::control::Running>, tokio::sync::mpsc::UnboundedReceiver<crate::discord::CoreEvent>), String> {
self.llm.start(&self.settings).await?;
Ok((Arc::new(LlmRunning(self.llm.clone())), tokio::sync::mpsc::unbounded_channel().1))
}
fn secrets(&self) -> Vec<String> {
vec![]
}
fn prepare_start(&self) {
self.llm.arm();
}
fn cancel_start(&self) {
self.llm.cancel();
}
}
}