A stop right after a background start set the LLM's cancel flag, but the start task then cleared it in Llm::start and went on to download and load the model. The cancel is now armed in the controller's begin_start, under its lock (Host::prepare_start -> Llm::arm), and Llm::start never clears it; a start that finds it set returns before running anything. wait_gone counted a timeout, a 503 (a server still loading) or an unreadable answer as stopped, so a failed `launch --stop` could still mark a live engine off. Only a refused connection confirms the stop now; the rest keeps checking until the deadline and then reports failed. Tests: a stop right after a background start runs no launch (and the next start is not cancelled); a failed stop with a silent or 503 server leaves the LLM and the services failed through the controller. Fixed-port tests use an unused port. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
641 lines
28 KiB
Rust
641 lines
28 KiB
Rust
//! The LLM the brain answers with (`app/src/admin.spec.md`, "LLM"): the app starts the installed
|
|
//! Strata from the data folder with the M0 kit's choices, waits until the model answers, and stops
|
|
//! it with the services. Strata downloads the model (about 23 GB) on its first start; the dashboard
|
|
//! shows how far it got. With `llm.url` the app uses that server instead and starts nothing.
|
|
|
|
use std::path::Path;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use serde::Serialize;
|
|
use serde_json::{json, Value};
|
|
|
|
use crate::install::{Ctx, Layout};
|
|
use crate::settings::Settings;
|
|
|
|
/// The one model v2 runs (M0: Qwen3.6-35B-A3B UD-Q4_K_XL; docs/v2/roadmap.md).
|
|
pub const FAMILY: &str = "qwen36";
|
|
pub const MODEL: &str = "ud-q4_k_xl";
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub enum LlmState {
|
|
Off,
|
|
Starting,
|
|
Ready,
|
|
Failed,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct LlmStatus {
|
|
pub state: LlmState,
|
|
/// Strata's own server (false: `llm.url`).
|
|
pub managed: bool,
|
|
/// The last percentage Strata printed while starting (the model download).
|
|
pub percent: Option<f64>,
|
|
/// Strata's last line while starting, or why it failed.
|
|
pub note: Option<String>,
|
|
}
|
|
|
|
/// Requests answered at once: one is faster on a 16 GB card (M0), two only when a chat channel
|
|
/// answers every message (so its answers do not wait behind the voice answers).
|
|
pub fn parallel(settings: &Settings) -> u32 {
|
|
if settings.chat_channels.iter().any(|c| c.answer_all) { 2 } else { 1 }
|
|
}
|
|
|
|
/// `strata launch` for the services: the M0 kit's normal placement, models in `models/llm`, this
|
|
/// computer only. No pictures yet (nothing in v2 sends one; reading them takes graphics memory).
|
|
/// `--restart` replaces a server started with other choices; the same choices are reused.
|
|
pub fn launch_args(layout: &Layout, settings: &Settings) -> Vec<String> {
|
|
let models = layout.data.join("models").join("llm").to_string_lossy().into_owned();
|
|
let mut a: Vec<String> = ["launch", "--family", FAMILY, "--model", MODEL, "--data-dir"].map(String::from).into();
|
|
a.push(models);
|
|
for (k, v) in [("--context", settings.llm.context.to_string()), ("--vision", "off".into()),
|
|
("--parallel", parallel(settings).to_string()), ("--access", "local".into()),
|
|
("--port", settings.llm.port.to_string())] {
|
|
a.push(k.into());
|
|
a.push(v);
|
|
}
|
|
a.extend(["--no-api-key", "--restart", "--yes", "--no-tui"].map(String::from));
|
|
a
|
|
}
|
|
|
|
/// The last `NN%` or `NN.N%` in a line.
|
|
pub fn percent(line: &str) -> Option<f64> {
|
|
let b = line.as_bytes();
|
|
let mut found = None;
|
|
for (i, _) in line.match_indices('%') {
|
|
let mut s = i;
|
|
while s > 0 && (b[s - 1].is_ascii_digit() || b[s - 1] == b'.') {
|
|
s -= 1;
|
|
}
|
|
if let Ok(n) = line[s..i].parse::<f64>() {
|
|
if (0.0..=100.0).contains(&n) {
|
|
found = Some(n);
|
|
}
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
/// Wait until `base` (`.../v1`) lists a model; returns its id. Stops early on `cancel`.
|
|
pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> Result<String, String> {
|
|
let client = reqwest::Client::builder().timeout(Duration::from_secs(5)).build().map_err(|e| e.to_string())?;
|
|
let until = Instant::now() + timeout;
|
|
loop {
|
|
if cancel.load(Ordering::Relaxed) {
|
|
return Err("cancelled".into());
|
|
}
|
|
if let Ok(r) = client.get(format!("{base}/models")).send().await {
|
|
if let Ok(v) = r.text().await.map(|t| serde_json::from_str::<Value>(&t).unwrap_or_default()) {
|
|
if let Some(id) = v["data"][0]["id"].as_str() {
|
|
return Ok(id.to_owned());
|
|
}
|
|
}
|
|
}
|
|
if Instant::now() >= until {
|
|
return Err(format!("대화 모델이 {}초 안에 준비되지 않았습니다 ({base})", timeout.as_secs()));
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
}
|
|
}
|
|
|
|
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM and the
|
|
/// memory database inside the data folder. The app owns this file and writes it on every start.
|
|
pub fn write_brain_config(data: &Path, base_url: &str, model: &str) -> Result<std::path::PathBuf, String> {
|
|
let file = data.join("config").join("brain.json");
|
|
let cfg = json!({
|
|
"ollama_base_url": base_url,
|
|
"ollama_chat_model": model,
|
|
"db_path": data.join("memory").join("jarvis.db").to_string_lossy(),
|
|
"_config_version": 1,
|
|
});
|
|
let mut text = serde_json::to_string_pretty(&cfg).unwrap();
|
|
text.push('\n');
|
|
crate::datadir::write_atomic(&file, text.as_bytes()).map_err(|e| e.to_string())?;
|
|
Ok(file)
|
|
}
|
|
|
|
/// One LLM for the services: Strata from the layout, or the server at `llm.url`.
|
|
pub struct Llm {
|
|
layout: Layout,
|
|
/// The program run as Strata (the installed one; a script in tests).
|
|
exe: std::path::PathBuf,
|
|
status: Mutex<LlmStatus>,
|
|
active: Mutex<Option<Arc<Ctx>>>,
|
|
cancel: Arc<AtomicBool>,
|
|
/// The port the managed server was started on (to confirm it is gone after a stop).
|
|
port: Mutex<u16>,
|
|
stop_timeout: Duration,
|
|
}
|
|
|
|
/// How long the model may take to answer after `strata launch` returned (it loads in about 15 s
|
|
/// on the game PC; the first start may also tune).
|
|
const READY_TIMEOUT: Duration = Duration::from_secs(30 * 60);
|
|
/// A server the app does not start must already answer.
|
|
const URL_TIMEOUT: Duration = Duration::from_secs(30);
|
|
|
|
impl Llm {
|
|
pub fn new(data: &Path) -> Self {
|
|
// Only the data folder matters here (the Python sources are the install's business).
|
|
let layout = Layout { data: data.to_path_buf(), src: std::path::PathBuf::new() };
|
|
let exe = layout.strata_exe();
|
|
Self::with_exe(layout, exe)
|
|
}
|
|
|
|
pub fn with_exe(layout: Layout, exe: std::path::PathBuf) -> Self {
|
|
Self { layout, exe, status: Mutex::new(LlmStatus { state: LlmState::Off, managed: false, percent: None, note: None }),
|
|
active: Mutex::new(None), cancel: Arc::new(AtomicBool::new(false)), port: Mutex::new(0),
|
|
stop_timeout: Duration::from_secs(120) }
|
|
}
|
|
|
|
/// How long `strata launch --stop` may take before it is ended and the stop reported as failed.
|
|
pub fn with_stop_timeout(mut self, t: Duration) -> Self {
|
|
self.stop_timeout = t;
|
|
self
|
|
}
|
|
|
|
pub fn status(&self) -> LlmStatus {
|
|
let mut s = self.status.lock().unwrap().clone();
|
|
if let Some(ctx) = self.active.lock().unwrap().as_ref() {
|
|
s.note = ctx.last_note();
|
|
if let Some(p) = s.note.as_deref().and_then(percent) {
|
|
// Kept in the status, so the bar does not vanish on a line without one.
|
|
self.status.lock().unwrap().percent = Some(p);
|
|
}
|
|
s.percent = self.status.lock().unwrap().percent;
|
|
}
|
|
s
|
|
}
|
|
|
|
fn set(&self, state: LlmState, note: Option<String>) {
|
|
let mut s = self.status.lock().unwrap();
|
|
s.state = state;
|
|
if state != LlmState::Starting {
|
|
s.note = note;
|
|
}
|
|
}
|
|
|
|
/// Start (or reuse) the LLM; returns its `/v1` address and model id.
|
|
/// A cancel since the last `arm` (or before the first start) ends it before it runs anything.
|
|
pub async fn start(&self, settings: &Settings) -> Result<(String, String), String> {
|
|
let managed = settings.llm.url.is_none();
|
|
*self.status.lock().unwrap() = LlmStatus { state: LlmState::Starting, managed, percent: None, note: None };
|
|
let result = self.start_inner(settings).await;
|
|
match &result {
|
|
Ok((base, model)) => {
|
|
crate::info_log!("llm", "ready: {base} model={model}");
|
|
self.set(LlmState::Ready, None);
|
|
}
|
|
Err(e) if e == "cancelled" => self.set(LlmState::Off, None),
|
|
Err(e) => {
|
|
crate::error_log!("llm", "did not start: {e}");
|
|
self.set(LlmState::Failed, Some(e.clone()));
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
async fn start_inner(&self, settings: &Settings) -> Result<(String, String), String> {
|
|
if self.cancel.load(Ordering::SeqCst) {
|
|
return Err("cancelled".into());
|
|
}
|
|
if let Some(url) = &settings.llm.url {
|
|
let model = wait_ready(url, URL_TIMEOUT, &self.cancel).await?;
|
|
return Ok((url.clone(), model));
|
|
}
|
|
if !self.exe.exists() {
|
|
return Err("Strata가 설치되어 있지 않습니다. 마법사의 모델 단계에서 설치해 주세요".into());
|
|
}
|
|
let ctx = Arc::new(Ctx::standalone(self.layout.clone(), "llm", self.cancel.clone()));
|
|
*self.active.lock().unwrap() = Some(ctx.clone());
|
|
*self.port.lock().unwrap() = settings.llm.port;
|
|
let args = launch_args(&self.layout, settings);
|
|
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
|
|
let home = self.layout.strata_home();
|
|
let ran = ctx.run_in(&self.exe, &refs, Some(&home)).await;
|
|
let base = format!("http://127.0.0.1:{}/v1", settings.llm.port);
|
|
let ready = match ran {
|
|
Ok(()) => wait_ready(&base, READY_TIMEOUT, &self.cancel).await,
|
|
Err(e) => Err(e),
|
|
};
|
|
*self.active.lock().unwrap() = None;
|
|
match ready {
|
|
Ok(model) => Ok((base, model)),
|
|
// Nothing half-started stays on the graphics card; if it may have, say so.
|
|
Err(e) => match self.stop_server().await {
|
|
Ok(()) => Err(e),
|
|
Err(stop) if e == "cancelled" => Err(stop),
|
|
Err(stop) => Err(format!("{e} ({stop})")),
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Make the next start cancellable afresh: called when a start is accepted, never by `start`
|
|
/// itself, so a stop between accepting and running the start still cancels it.
|
|
pub fn arm(&self) {
|
|
self.cancel.store(false, Ordering::SeqCst);
|
|
}
|
|
|
|
/// Stop a start in progress (the download resumes on the next start).
|
|
pub fn cancel(&self) {
|
|
self.cancel.store(true, Ordering::SeqCst);
|
|
}
|
|
|
|
/// Stop the server this app started (nothing for `llm.url`). Off only once the server no longer
|
|
/// answers; otherwise `failed` with the reason, which is also returned.
|
|
pub async fn stop(&self) -> Result<(), String> {
|
|
let managed = self.status.lock().unwrap().managed;
|
|
let result = if managed { self.stop_server().await } else { Ok(()) };
|
|
match &result {
|
|
Ok(()) => self.set(LlmState::Off, None),
|
|
Err(e) => {
|
|
crate::error_log!("llm", "{e}");
|
|
self.set(LlmState::Failed, Some(e.clone()));
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
/// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that
|
|
/// fails while nothing answers (nothing was serving) counts as stopped.
|
|
async fn stop_server(&self) -> Result<(), String> {
|
|
if !self.exe.exists() {
|
|
return Ok(());
|
|
}
|
|
// The timeout ends the command and everything it started, like an install cancel.
|
|
let cancel = Arc::new(AtomicBool::new(false));
|
|
let ctx = Ctx::standalone(self.layout.clone(), "llm", cancel.clone());
|
|
let home = self.layout.strata_home();
|
|
let timer = {
|
|
let (cancel, t) = (cancel.clone(), self.stop_timeout);
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(t).await;
|
|
cancel.store(true, Ordering::SeqCst);
|
|
})
|
|
};
|
|
let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await;
|
|
timer.abort();
|
|
let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e });
|
|
let port = *self.port.lock().unwrap();
|
|
let base = format!("http://127.0.0.1:{port}/v1");
|
|
let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await;
|
|
match (ran, gone) {
|
|
(Ok(()), true) => {
|
|
crate::info_log!("llm", "stopped");
|
|
Ok(())
|
|
}
|
|
(Err(e), true) => {
|
|
crate::warn_log!("llm", "stop command failed but nothing answers: {e}");
|
|
Ok(())
|
|
}
|
|
(Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 닫히지 않았습니다")),
|
|
(Err(e), false) => Err(format!("Strata를 끄지 못했습니다: {e}")),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Whether `base` is gone within `timeout`: only a refused connection counts. A timeout, an error
|
|
/// status (a server still loading answers 503) or an answer it cannot read is not a confirmed stop,
|
|
/// so it keeps checking until the deadline.
|
|
async fn wait_gone(base: &str, timeout: Duration) -> bool {
|
|
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(2)).build() else { return false };
|
|
let until = Instant::now() + timeout;
|
|
loop {
|
|
if let Err(e) = client.get(format!("{base}/models")).send().await {
|
|
if e.is_connect() {
|
|
return true;
|
|
}
|
|
}
|
|
if Instant::now() >= until {
|
|
return false;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::datadir::temp_dir;
|
|
use crate::settings::{parse_settings, ChatChannelSettings};
|
|
|
|
fn layout(tag: &str) -> Layout {
|
|
Layout { data: temp_dir(tag), src: std::path::PathBuf::from("src") }
|
|
}
|
|
|
|
#[test]
|
|
fn the_launch_keeps_the_model_in_the_data_folder_and_follows_the_settings() {
|
|
let l = layout("llm1");
|
|
let s = parse_settings(&json!({"llm": {"context": 16384, "port": 8099}})).unwrap();
|
|
let a = launch_args(&l, &s);
|
|
let after = |k: &str| a[a.iter().position(|x| x == k).unwrap() + 1].clone();
|
|
assert!(Path::new(&after("--data-dir")).starts_with(&l.data));
|
|
assert_eq!((after("--context"), after("--port"), after("--parallel")), ("16384".into(), "8099".into(), "1".into()));
|
|
assert_eq!(after("--access"), "local");
|
|
assert!(a.iter().any(|x| x == "--no-tui") && a.iter().any(|x| x == "--yes"));
|
|
}
|
|
|
|
#[test]
|
|
fn a_chat_channel_answering_everything_gets_a_second_request_slot() {
|
|
let mut s = Settings::default();
|
|
assert_eq!(parallel(&s), 1);
|
|
s.chat_channels.push(ChatChannelSettings { id: 1, name: String::new(), description: String::new(), read: true,
|
|
send: true, edit: true, react: true, delete: false, pin: false, answer_all: true });
|
|
assert_eq!(parallel(&s), 2);
|
|
}
|
|
|
|
#[test]
|
|
fn the_progress_is_the_last_percentage_on_a_line() {
|
|
assert_eq!(percent("Downloading 3 of 5: 41.5% (9.6 GB of 23.1 GB)"), Some(41.5));
|
|
assert_eq!(percent("50% done, now 75%"), Some(75.0));
|
|
assert_eq!(percent("Loading the model… 15 s"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn the_brain_config_points_at_the_llm_and_keeps_memory_in_the_data_folder() {
|
|
let l = layout("llm2");
|
|
let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m").unwrap();
|
|
let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap();
|
|
assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1");
|
|
assert_eq!(v["ollama_chat_model"], "m");
|
|
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
|
|
}
|
|
|
|
/// Default settings on a port nothing listens on (8080 may be taken on a developer's PC).
|
|
fn unused_port_settings() -> Settings {
|
|
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
|
|
parse_settings(&json!({"llm": {"port": port}})).unwrap()
|
|
}
|
|
|
|
/// A tiny `/v1/models` server answering with `id`, after `delay` (before that it lists nothing).
|
|
async fn models_server(id: &'static str, delay: Duration) -> u16 {
|
|
serve(id, delay, None, Arc::new(Mutex::new("list"))).await
|
|
}
|
|
|
|
/// The same; once the file `gone` exists (a fake Strata's `--stop` makes it) it stops listening,
|
|
/// so a connection is refused.
|
|
async fn models_server_until(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>) -> u16 {
|
|
serve(id, delay, gone, Arc::new(Mutex::new("list"))).await
|
|
}
|
|
|
|
/// `mode`: "list" answers, "hang" keeps the connection and says nothing, "503" is a server still
|
|
/// loading.
|
|
async fn serve(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>, mode: Arc<Mutex<&'static str>>) -> u16 {
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let opened = Instant::now();
|
|
tokio::spawn(async move {
|
|
while let Ok((mut s, _)) = listener.accept().await {
|
|
if gone.as_ref().is_some_and(|g| g.exists()) {
|
|
return; // the listener closes: later connections are refused
|
|
}
|
|
let mut buf = [0u8; 1024];
|
|
let _ = s.read(&mut buf).await;
|
|
let m = *mode.lock().unwrap();
|
|
let (status, body) = match m {
|
|
"hang" => {
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(Duration::from_secs(60)).await;
|
|
drop(s);
|
|
});
|
|
continue;
|
|
}
|
|
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
|
|
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
|
|
_ => ("200 OK", "{}".to_owned()),
|
|
};
|
|
let _ = s.write_all(format!("HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
|
body.len()).as_bytes()).await;
|
|
}
|
|
});
|
|
port
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_given_server_is_used_without_starting_strata() {
|
|
let port = models_server("given-model", Duration::ZERO).await;
|
|
let l = layout("llm3");
|
|
let llm = Llm::new(&l.data);
|
|
let s = parse_settings(&json!({"llm": {"url": format!("http://127.0.0.1:{port}/v1")}})).unwrap();
|
|
let (base, model) = llm.start(&s).await.unwrap();
|
|
assert_eq!((base.as_str(), model.as_str()), (format!("http://127.0.0.1:{port}/v1").as_str(), "given-model"));
|
|
assert_eq!(llm.status().state, LlmState::Ready);
|
|
assert!(!llm.status().managed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn without_strata_installed_the_start_fails_with_a_reason() {
|
|
let llm = Llm::new(&layout("llm4").data);
|
|
let e = llm.start(&Settings::default()).await.unwrap_err();
|
|
assert!(e.contains("Strata"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
|
|
}
|
|
|
|
/// A fake `strata`: `launch --stop` records itself and runs `stop_body` (by default it stops the
|
|
/// server: the file `gone` appears); `launch ...` runs `launch_body`.
|
|
#[cfg(unix)]
|
|
fn fake_strata(l: &Layout, launch_body: &str) -> std::path::PathBuf {
|
|
fake_strata_stop(l, launch_body, &format!("touch '{}'", l.data.join("gone").display()))
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
fn fake_strata_stop(l: &Layout, launch_body: &str, stop_body: &str) -> std::path::PathBuf {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let exe = l.data.join("fake-strata");
|
|
std::fs::create_dir_all(l.strata_home()).unwrap();
|
|
let log = l.data.join("calls");
|
|
std::fs::write(&exe, format!("#!/bin/sh\necho \"$@\" >> '{}'\nif [ \"$2\" = --stop ]; then {stop_body}; exit $?; fi\n{launch_body}\n",
|
|
log.display())).unwrap();
|
|
std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();
|
|
exe
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn strata_starts_reports_its_download_waits_for_the_model_and_stops() {
|
|
let l = layout("llm5");
|
|
let port = models_server_until("qwen-test", Duration::from_millis(300), Some(l.data.join("gone"))).await;
|
|
let exe = fake_strata(&l, "echo 'Downloading: 12.5%'; sleep 1; echo 'Downloading: 100%'; echo 'Strata is serving.'");
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
|
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
|
let l2 = llm.clone();
|
|
let s2 = s.clone();
|
|
let run = tokio::spawn(async move { l2.start(&s2).await });
|
|
let mut seen = None;
|
|
for _ in 0..40 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
let st = llm.status();
|
|
if st.percent.is_some() {
|
|
seen = Some(st);
|
|
break;
|
|
}
|
|
}
|
|
let st = seen.expect("progress while starting");
|
|
assert_eq!((st.state, st.percent), (LlmState::Starting, Some(12.5)));
|
|
let (base, model) = run.await.unwrap().unwrap();
|
|
assert_eq!((base, model.as_str()), (format!("http://127.0.0.1:{port}/v1"), "qwen-test"));
|
|
assert_eq!(llm.status().state, LlmState::Ready);
|
|
llm.stop().await.unwrap();
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
assert!(calls.lines().next().unwrap().starts_with("launch --family"), "{calls}");
|
|
assert_eq!(calls.lines().last(), Some("launch --stop"));
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_cancel_during_the_download_stops_it_and_clears_the_server() {
|
|
let l = layout("llm6");
|
|
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
|
let l2 = llm.clone();
|
|
let run = tokio::spawn(async move { l2.start(&unused_port_settings()).await });
|
|
tokio::time::sleep(Duration::from_millis(400)).await;
|
|
let t = Instant::now();
|
|
llm.cancel();
|
|
assert_eq!(run.await.unwrap().unwrap_err(), "cancelled");
|
|
assert!(t.elapsed() < Duration::from_secs(5));
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
|
assert_eq!(calls.lines().last(), Some("launch --stop"));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_failed_launch_shows_strata_s_last_line() {
|
|
let l = layout("llm7");
|
|
let exe = fake_strata(&l, "echo 'Not enough disk space for the model' >&2; exit 4");
|
|
let llm = Llm::with_exe(l, exe);
|
|
let e = llm.start(&unused_port_settings()).await.unwrap_err();
|
|
assert!(e.contains("Not enough disk space"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_server_still_answering_after_a_failed_stop_is_reported_not_marked_off() {
|
|
let l = layout("llm8");
|
|
let port = models_server("qwen-test", Duration::ZERO).await;
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "echo 'could not stop the engine' >&2; exit 3");
|
|
let llm = Llm::with_exe(l, exe);
|
|
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
|
|
let e = llm.stop().await.unwrap_err();
|
|
assert!(e.contains("could not stop the engine"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
assert_eq!(llm.status().note.as_deref(), Some(e.as_str()));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_stop_that_hangs_is_given_up_on_and_reported() {
|
|
let l = layout("llm9");
|
|
let port = models_server("qwen-test", Duration::ZERO).await;
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "sleep 30");
|
|
let llm = Llm::with_exe(l, exe).with_stop_timeout(Duration::from_millis(500));
|
|
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
|
|
let t = Instant::now();
|
|
let e = llm.stop().await.unwrap_err();
|
|
assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed());
|
|
assert!(e.contains("Strata"), "{e}");
|
|
assert_eq!(llm.status().state, LlmState::Failed);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_stop_command_that_fails_while_nothing_answers_counts_as_stopped() {
|
|
let l = layout("llm10");
|
|
let port = models_server_until("qwen-test", Duration::ZERO, Some(l.data.join("gone"))).await;
|
|
let stop = format!("touch '{}'; exit 1", l.data.join("gone").display());
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", &stop);
|
|
let llm = Llm::with_exe(l, exe);
|
|
llm.start(&parse_settings(&json!({"llm": {"port": port}})).unwrap()).await.unwrap();
|
|
llm.stop().await.unwrap();
|
|
assert_eq!(llm.status().state, LlmState::Off);
|
|
}
|
|
|
|
/// A failed stop while the server is still there but not listing a model (holding connections
|
|
/// silently, or a 503 while loading) is not a confirmed stop: the LLM and, through the
|
|
/// controller, the services stay failed.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_failed_stop_with_a_silent_or_loading_server_is_not_counted_as_stopped() {
|
|
for m in ["hang", "503"] {
|
|
let l = layout(&format!("llm11{m}"));
|
|
let mode = Arc::new(Mutex::new("list"));
|
|
let port = serve("qwen-test", Duration::ZERO, None, mode.clone()).await;
|
|
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "exit 3");
|
|
let llm = Arc::new(Llm::with_exe(l, exe));
|
|
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(),
|
|
settings: parse_settings(&json!({"llm": {"port": port}})).unwrap() });
|
|
c.start().await.unwrap();
|
|
*mode.lock().unwrap() = m;
|
|
let e = c.stop().await.unwrap_err();
|
|
assert!(e.contains("Strata"), "{m}: {e}");
|
|
assert_eq!(c.state(), crate::control::State::Failed, "{m}");
|
|
assert_eq!(llm.status().state, LlmState::Failed, "{m}");
|
|
}
|
|
}
|
|
|
|
/// A stop right after a background start must cancel the LLM's start before it runs anything
|
|
/// (no download), even though the start task runs after the stop.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_stop_right_after_a_background_start_runs_no_launch() {
|
|
let l = layout("llm12");
|
|
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
|
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: unused_port_settings() });
|
|
c.start_in_background().unwrap();
|
|
c.stop().await.unwrap();
|
|
let t = Instant::now();
|
|
while c.state() != crate::control::State::Stopped && t.elapsed() < Duration::from_secs(5) {
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
assert_eq!(c.state(), crate::control::State::Stopped);
|
|
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap_or_default();
|
|
assert!(!calls.contains("--family"), "no launch ran: {calls}");
|
|
// A later start is not cancelled by the earlier stop.
|
|
c.start_in_background().unwrap();
|
|
tokio::time::sleep(Duration::from_millis(400)).await;
|
|
assert!(std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().contains("--family"));
|
|
c.stop().await.unwrap();
|
|
}
|
|
|
|
/// The services as the controller sees them, reduced to the LLM.
|
|
struct LlmHost {
|
|
llm: Arc<Llm>,
|
|
settings: Settings,
|
|
}
|
|
struct LlmRunning(Arc<Llm>);
|
|
#[async_trait::async_trait]
|
|
impl crate::control::Running for LlmRunning {
|
|
async fn stop(&self) -> Result<(), String> {
|
|
self.0.stop().await
|
|
}
|
|
fn status(&self, _: Instant) -> Value {
|
|
json!({"worker": null, "voiceMembers": [], "missed": []})
|
|
}
|
|
}
|
|
#[async_trait::async_trait]
|
|
impl crate::control::Host for LlmHost {
|
|
async fn start(&self) -> Result<(Arc<dyn crate::control::Running>, tokio::sync::mpsc::UnboundedReceiver<crate::discord::CoreEvent>), String> {
|
|
self.llm.start(&self.settings).await?;
|
|
Ok((Arc::new(LlmRunning(self.llm.clone())), tokio::sync::mpsc::unbounded_channel().1))
|
|
}
|
|
fn secrets(&self) -> Vec<String> {
|
|
vec![]
|
|
}
|
|
fn prepare_start(&self) {
|
|
self.llm.arm();
|
|
}
|
|
fn cancel_start(&self) {
|
|
self.llm.cancel();
|
|
}
|
|
}
|
|
}
|