fix(app): arm the LLM cancel when a start is accepted; confirm stops by refusal
A stop right after a background start set the LLM's cancel flag, but the start task then cleared it in Llm::start and went on to download and load the model. The cancel is now armed in the controller's begin_start, under its lock (Host::prepare_start -> Llm::arm), and Llm::start never clears it; a start that finds it set returns before running anything. wait_gone counted a timeout, a 503 (a server still loading) or an unreadable answer as stopped, so a failed `launch --stop` could still mark a live engine off. Only a refused connection confirms the stop now; the rest keeps checking until the deadline and then reports failed. Tests: a stop right after a background start runs no launch (and the next start is not cancelled); a failed stop with a silent or 503 server leaves the LLM and the services failed through the controller. Fixed-port tests use an unused port. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -187,10 +187,13 @@ start it first, so nothing answers before the brain can, and stop it last.
|
||||
on a 16 GB card, M0) and 2 when a chat channel answers every message, so its answers do not wait
|
||||
behind voice answers. Pictures are off until something in v2 sends one. Stopping the services
|
||||
runs `strata launch --stop` (ended after 2 minutes, with everything it started) and then checks
|
||||
that the port no longer lists a model; so does a failed or cancelled start, so nothing
|
||||
half-started keeps graphics memory. Stopped means nothing answers any more (a stop command that
|
||||
fails while nothing answers counts as stopped); a server still answering makes the LLM `failed`
|
||||
with the reason, and the stop or the start reports it. Not installed: the start fails and says to
|
||||
that the port refuses connections; so does a failed or cancelled start, so nothing half-started
|
||||
keeps graphics memory. Stopped means the port refuses connections (a stop command that fails
|
||||
while it does counts as stopped); a timeout, an error status (a server still loading answers
|
||||
503) or an answer that cannot be read is not a confirmed stop: the LLM is `failed` with the
|
||||
reason, and the stop or the start reports it. The start's cancel is armed when the controller
|
||||
accepts the start (`Host::prepare_start`), never by the start itself, so a stop between accepting
|
||||
and running the start cancels it before Strata runs anything. Not installed: the start fails and says to
|
||||
run the install.
|
||||
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
|
||||
settings file only) the app starts nothing; that server must list a model within 30 seconds.
|
||||
@@ -265,8 +268,9 @@ service starter, a stop while starting (cancelled, or finished and stopped again
|
||||
channel answering everything, the download percentage, the brain's settings file, a given server
|
||||
used without Strata, Strata missing, a fake Strata that reports progress, returns, serves and is
|
||||
stopped, is cancelled during its download and cleared, or fails with its last line, a stop that
|
||||
fails or hangs while the server still answers (failed, not off) and one that fails while nothing
|
||||
answers (off); a background start that is starting at once, so a stop right after it cancels it
|
||||
fails or hangs while the server still answers, holds connections silently or answers 503 (failed,
|
||||
not off, also through the controller) and one that fails while the port refuses connections (off);
|
||||
a stop right after a background start running no launch at all, and the next start not cancelled; a background start that is starting at once, so a stop right after it cancels it
|
||||
and a second start is refused, also through the admin commands; a failed stop reported as failed; the brain's
|
||||
config loaded without writing to the home folder, `tests/test_config_db_path.py`); the dashboard from made-up events (captions replaced and cleared, latency stages
|
||||
and medians, an answer matched to its asker's final, missed requests without text); the admin
|
||||
|
||||
@@ -38,6 +38,9 @@ pub trait Host: Send + Sync + 'static {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String>;
|
||||
/// Stored secrets, masked in error messages.
|
||||
fn secrets(&self) -> Vec<String>;
|
||||
/// Called when a start is accepted (state `starting`, under the controller's lock): a fresh
|
||||
/// cancel, so a stop from now on cancels this start and an earlier stop does not.
|
||||
fn prepare_start(&self) {}
|
||||
/// Stop a start in progress (the LLM's first start downloads the model); `start` then returns
|
||||
/// `Err("cancelled")` if it had not finished.
|
||||
fn cancel_start(&self) {}
|
||||
@@ -100,6 +103,7 @@ impl Controller {
|
||||
i.state = State::Starting;
|
||||
i.error = None;
|
||||
i.stop_asked = false;
|
||||
self.host.prepare_start();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -282,6 +286,10 @@ impl Host for ServicesHost {
|
||||
crate::secrets::Secrets::new(&self.data_dir.join("config").join("secrets.dat")).values()
|
||||
}
|
||||
|
||||
fn prepare_start(&self) {
|
||||
self.llm.arm();
|
||||
}
|
||||
|
||||
fn cancel_start(&self) {
|
||||
self.llm.cancel();
|
||||
}
|
||||
@@ -384,7 +392,6 @@ mod tests {
|
||||
impl Host for SlowHost {
|
||||
async fn start(&self) -> Result<(Arc<dyn Running>, mpsc::UnboundedReceiver<CoreEvent>), String> {
|
||||
self.counts.starts.fetch_add(1, Ordering::SeqCst);
|
||||
self.cancelled.store(false, Ordering::SeqCst); // as the LLM does: a cancel ends one start
|
||||
for _ in 0..50 {
|
||||
if self.honours_cancel && self.cancelled.load(Ordering::SeqCst) {
|
||||
return Err("cancelled".into());
|
||||
@@ -396,6 +403,9 @@ mod tests {
|
||||
fn secrets(&self) -> Vec<String> {
|
||||
vec![]
|
||||
}
|
||||
fn prepare_start(&self) {
|
||||
self.cancelled.store(false, Ordering::SeqCst); // as the LLM does: armed when a start is accepted
|
||||
}
|
||||
fn cancel_start(&self) {
|
||||
self.cancelled.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
@@ -179,8 +179,8 @@ impl Llm {
|
||||
}
|
||||
|
||||
/// Start (or reuse) the LLM; returns its `/v1` address and model id.
|
||||
/// A cancel since the last `arm` (or before the first start) ends it before it runs anything.
|
||||
pub async fn start(&self, settings: &Settings) -> Result<(String, String), String> {
|
||||
self.cancel.store(false, Ordering::SeqCst);
|
||||
let managed = settings.llm.url.is_none();
|
||||
*self.status.lock().unwrap() = LlmStatus { state: LlmState::Starting, managed, percent: None, note: None };
|
||||
let result = self.start_inner(settings).await;
|
||||
@@ -199,6 +199,9 @@ impl Llm {
|
||||
}
|
||||
|
||||
async fn start_inner(&self, settings: &Settings) -> Result<(String, String), String> {
|
||||
if self.cancel.load(Ordering::SeqCst) {
|
||||
return Err("cancelled".into());
|
||||
}
|
||||
if let Some(url) = &settings.llm.url {
|
||||
let model = wait_ready(url, URL_TIMEOUT, &self.cancel).await?;
|
||||
return Ok((url.clone(), model));
|
||||
@@ -230,6 +233,12 @@ impl Llm {
|
||||
}
|
||||
}
|
||||
|
||||
/// Make the next start cancellable afresh: called when a start is accepted, never by `start`
|
||||
/// itself, so a stop between accepting and running the start still cancels it.
|
||||
pub fn arm(&self) {
|
||||
self.cancel.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// Stop a start in progress (the download resumes on the next start).
|
||||
pub fn cancel(&self) {
|
||||
self.cancel.store(true, Ordering::SeqCst);
|
||||
@@ -282,24 +291,23 @@ impl Llm {
|
||||
crate::warn_log!("llm", "stop command failed but nothing answers: {e}");
|
||||
Ok(())
|
||||
}
|
||||
(Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 응답합니다")),
|
||||
(Ok(()), false) => Err(format!("Strata를 껐지만 {base}가 아직 닫히지 않았습니다")),
|
||||
(Err(e), false) => Err(format!("Strata를 끄지 못했습니다: {e}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `base` stops listing a model within `timeout` (a refused connection counts).
|
||||
/// Whether `base` is gone within `timeout`: only a refused connection counts. A timeout, an error
|
||||
/// status (a server still loading answers 503) or an answer it cannot read is not a confirmed stop,
|
||||
/// so it keeps checking until the deadline.
|
||||
async fn wait_gone(base: &str, timeout: Duration) -> bool {
|
||||
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(2)).build() else { return false };
|
||||
let until = Instant::now() + timeout;
|
||||
loop {
|
||||
let listed = match client.get(format!("{base}/models")).send().await {
|
||||
Ok(r) => r.text().await.ok().and_then(|t| serde_json::from_str::<Value>(&t).ok())
|
||||
.is_some_and(|v| v["data"][0]["id"].is_string()),
|
||||
Err(_) => false,
|
||||
};
|
||||
if !listed {
|
||||
return true;
|
||||
if let Err(e) = client.get(format!("{base}/models")).send().await {
|
||||
if e.is_connect() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if Instant::now() >= until {
|
||||
return false;
|
||||
@@ -356,24 +364,51 @@ mod tests {
|
||||
assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data));
|
||||
}
|
||||
|
||||
/// A tiny `/v1/models` server answering with `id`, after `delay`; it lists nothing once the file
|
||||
/// `gone` exists (a fake Strata's `--stop` makes it).
|
||||
async fn models_server(id: &'static str, delay: Duration) -> u16 {
|
||||
models_server_until(id, delay, None).await
|
||||
/// Default settings on a port nothing listens on (8080 may be taken on a developer's PC).
|
||||
fn unused_port_settings() -> Settings {
|
||||
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
|
||||
parse_settings(&json!({"llm": {"port": port}})).unwrap()
|
||||
}
|
||||
|
||||
/// A tiny `/v1/models` server answering with `id`, after `delay` (before that it lists nothing).
|
||||
async fn models_server(id: &'static str, delay: Duration) -> u16 {
|
||||
serve(id, delay, None, Arc::new(Mutex::new("list"))).await
|
||||
}
|
||||
|
||||
/// The same; once the file `gone` exists (a fake Strata's `--stop` makes it) it stops listening,
|
||||
/// so a connection is refused.
|
||||
async fn models_server_until(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>) -> u16 {
|
||||
serve(id, delay, gone, Arc::new(Mutex::new("list"))).await
|
||||
}
|
||||
|
||||
/// `mode`: "list" answers, "hang" keeps the connection and says nothing, "503" is a server still
|
||||
/// loading.
|
||||
async fn serve(id: &'static str, delay: Duration, gone: Option<std::path::PathBuf>, mode: Arc<Mutex<&'static str>>) -> u16 {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let opened = Instant::now();
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut s, _)) = listener.accept().await {
|
||||
if gone.as_ref().is_some_and(|g| g.exists()) {
|
||||
return; // the listener closes: later connections are refused
|
||||
}
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = s.read(&mut buf).await;
|
||||
let stopped = gone.as_ref().is_some_and(|g| g.exists());
|
||||
let body = if opened.elapsed() >= delay && !stopped { format!(r#"{{"data":[{{"id":"{id}"}}]}}"#) } else { "{}".into() };
|
||||
let _ = s.write_all(format!("HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
let m = *mode.lock().unwrap();
|
||||
let (status, body) = match m {
|
||||
"hang" => {
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
drop(s);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
|
||||
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
|
||||
_ => ("200 OK", "{}".to_owned()),
|
||||
};
|
||||
let _ = s.write_all(format!("HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()).as_bytes()).await;
|
||||
}
|
||||
});
|
||||
@@ -459,7 +494,7 @@ mod tests {
|
||||
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
||||
let l2 = llm.clone();
|
||||
let run = tokio::spawn(async move { l2.start(&Settings::default()).await });
|
||||
let run = tokio::spawn(async move { l2.start(&unused_port_settings()).await });
|
||||
tokio::time::sleep(Duration::from_millis(400)).await;
|
||||
let t = Instant::now();
|
||||
llm.cancel();
|
||||
@@ -476,7 +511,7 @@ mod tests {
|
||||
let l = layout("llm7");
|
||||
let exe = fake_strata(&l, "echo 'Not enough disk space for the model' >&2; exit 4");
|
||||
let llm = Llm::with_exe(l, exe);
|
||||
let e = llm.start(&Settings::default()).await.unwrap_err();
|
||||
let e = llm.start(&unused_port_settings()).await.unwrap_err();
|
||||
assert!(e.contains("Not enough disk space"), "{e}");
|
||||
assert_eq!(llm.status().state, LlmState::Failed);
|
||||
}
|
||||
@@ -522,4 +557,84 @@ mod tests {
|
||||
llm.stop().await.unwrap();
|
||||
assert_eq!(llm.status().state, LlmState::Off);
|
||||
}
|
||||
|
||||
/// A failed stop while the server is still there but not listing a model (holding connections
|
||||
/// silently, or a 503 while loading) is not a confirmed stop: the LLM and, through the
|
||||
/// controller, the services stay failed.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_failed_stop_with_a_silent_or_loading_server_is_not_counted_as_stopped() {
|
||||
for m in ["hang", "503"] {
|
||||
let l = layout(&format!("llm11{m}"));
|
||||
let mode = Arc::new(Mutex::new("list"));
|
||||
let port = serve("qwen-test", Duration::ZERO, None, mode.clone()).await;
|
||||
let exe = fake_strata_stop(&l, "echo 'Strata is serving.'", "exit 3");
|
||||
let llm = Arc::new(Llm::with_exe(l, exe));
|
||||
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(),
|
||||
settings: parse_settings(&json!({"llm": {"port": port}})).unwrap() });
|
||||
c.start().await.unwrap();
|
||||
*mode.lock().unwrap() = m;
|
||||
let e = c.stop().await.unwrap_err();
|
||||
assert!(e.contains("Strata"), "{m}: {e}");
|
||||
assert_eq!(c.state(), crate::control::State::Failed, "{m}");
|
||||
assert_eq!(llm.status().state, LlmState::Failed, "{m}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A stop right after a background start must cancel the LLM's start before it runs anything
|
||||
/// (no download), even though the start task runs after the stop.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_stop_right_after_a_background_start_runs_no_launch() {
|
||||
let l = layout("llm12");
|
||||
let exe = fake_strata(&l, "echo 'Downloading: 1%'; sleep 30");
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), exe));
|
||||
let c = crate::control::Controller::new(LlmHost { llm: llm.clone(), settings: unused_port_settings() });
|
||||
c.start_in_background().unwrap();
|
||||
c.stop().await.unwrap();
|
||||
let t = Instant::now();
|
||||
while c.state() != crate::control::State::Stopped && t.elapsed() < Duration::from_secs(5) {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(c.state(), crate::control::State::Stopped);
|
||||
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap_or_default();
|
||||
assert!(!calls.contains("--family"), "no launch ran: {calls}");
|
||||
// A later start is not cancelled by the earlier stop.
|
||||
c.start_in_background().unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(400)).await;
|
||||
assert!(std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().contains("--family"));
|
||||
c.stop().await.unwrap();
|
||||
}
|
||||
|
||||
/// The services as the controller sees them, reduced to the LLM.
|
||||
struct LlmHost {
|
||||
llm: Arc<Llm>,
|
||||
settings: Settings,
|
||||
}
|
||||
struct LlmRunning(Arc<Llm>);
|
||||
#[async_trait::async_trait]
|
||||
impl crate::control::Running for LlmRunning {
|
||||
async fn stop(&self) -> Result<(), String> {
|
||||
self.0.stop().await
|
||||
}
|
||||
fn status(&self, _: Instant) -> Value {
|
||||
json!({"worker": null, "voiceMembers": [], "missed": []})
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl crate::control::Host for LlmHost {
|
||||
async fn start(&self) -> Result<(Arc<dyn crate::control::Running>, tokio::sync::mpsc::UnboundedReceiver<crate::discord::CoreEvent>), String> {
|
||||
self.llm.start(&self.settings).await?;
|
||||
Ok((Arc::new(LlmRunning(self.llm.clone())), tokio::sync::mpsc::unbounded_channel().1))
|
||||
}
|
||||
fn secrets(&self) -> Vec<String> {
|
||||
vec![]
|
||||
}
|
||||
fn prepare_start(&self) {
|
||||
self.llm.arm();
|
||||
}
|
||||
fn cancel_start(&self) {
|
||||
self.llm.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user