fix(app): the LLM placement always stops a left server and confirms the port closed before reading the card

- strata launch --stop now always runs before the card is read and the start's own port must
  refuse connections; a left server that answers, still loads (503) or stays silent was skipped
  before and its memory counted as other programs'
- a stop that is not confirmed reads and keeps nothing: the model starts with expert-vram=0
- the stop confirms on the port it is given (stop_server_on), not on one remembered from an
  earlier start

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-14 10:06:53 +09:00
parent 1400a3ff97
commit b6bd1be6fb
3 changed files with 55 additions and 27 deletions

View File

@@ -391,9 +391,11 @@ start it first, so nothing answers before the brain can, and stop it last.
short answer, then the card's memory again and the staging from the engine log; kept in
`config/llm-probe.json` under `<context>/<parallel>`, and the real start then replaces that
server. A measurement whose answer or second reading fails (or shows nothing more held) is not
kept: the model starts with `expert-vram=0` and the next start measures again. A server of this
app's port that still answers before the start (left from a run that ended badly) is stopped
first, so it is not counted as other programs' memory. Without a card nothing is set. The game
kept: the model starts with `expert-vram=0` and the next start measures again. Before the card
is read, `strata launch --stop` always runs and the start's port must refuse connections, so a
server left from a run that ended badly (answering, still loading with 503, or silent) is never
counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and
the model starts with `expert-vram=0`. Without a card nothing is set. The game
placement (used by the switching that comes next) takes the reserve when it fits beside more
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from.
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the

View File

@@ -175,12 +175,6 @@ pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> R
}
}
/// Whether a server answers `/models` at `base` (within 2 s).
async fn answers(base: &str) -> bool {
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(2)).build() else { return false };
client.get(format!("{base}/models")).send().await.is_ok_and(|r| r.status().is_success())
}
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an
/// error when the model gives none.
async fn warm_up(base: &str) -> Result<(), String> {
@@ -275,15 +269,18 @@ impl Llm {
/// the card holds now and this PC's measurement without experts at this context and number of
/// requests at once, which the first start makes once per card (a start without experts, then
/// the real start replaces it). A server left from an earlier run is stopped first, so it is not
/// counted as another program's memory. A measurement that fails is not kept: the model then
/// starts without experts on the card.
/// counted as another program's memory (a stop that is not confirmed reads nothing: no
/// experts). A measurement that fails is not kept: the model then starts without experts on
/// the card.
async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str) -> Result<Vec<String>, String> {
if self.gpu.is_none() {
return Ok(vec![]);
}
if answers(base).await {
crate::info_log!("llm", "a server from an earlier run still answers; stopping it first");
self.stop_server().await?;
// Whatever a server left from an earlier run does (answers, loads, stays silent), it holds
// the card: stop it first and read the card only once the port is confirmed closed.
if let Err(e) = self.stop_server_on(settings.llm.port).await {
crate::warn_log!("llm", "Warning: a server left on the port did not stop ({e}); starting without experts on the card");
return Ok(placement_sets(0.0));
}
let Some(card) = self.read_gpu().await else { return Ok(vec![]) };
let (context, parallel) = (settings.llm.context, parallel(settings));
@@ -431,6 +428,12 @@ impl Llm {
/// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that
/// fails while nothing answers (nothing was serving) counts as stopped.
async fn stop_server(&self) -> Result<(), String> {
let port = *self.port.lock().unwrap();
self.stop_server_on(port).await
}
/// `stop_server` confirming on `port` (0: nothing was started, nothing to confirm).
async fn stop_server_on(&self, port: u16) -> Result<(), String> {
if !self.exe.exists() {
return Ok(());
}
@@ -448,7 +451,6 @@ impl Llm {
let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await;
timer.abort();
let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e });
let port = *self.port.lock().unwrap();
let base = format!("http://127.0.0.1:{port}/v1");
let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await;
match (ran, gone) {
@@ -582,6 +584,11 @@ mod tests {
tokio::time::sleep(Duration::from_millis(50)).await;
}
listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap();
// A new server: it answers (a left one loaded or stayed silent).
let mut m = mode.lock().unwrap();
if matches!(*m, "503" | "hang") {
*m = "list";
}
continue;
}
let mut buf = [0u8; 1024];
@@ -818,18 +825,37 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() {
let l = layout("llm-place-left");
// A server still answers on the port, holding 9874 MiB.
let port = models_server_until("qwen-test", Duration::from_millis(0), Some(l.data.join("gone"))).await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
// Left over answering, loading (503) or silent: always stopped first, holding 9874 MiB until then.
for mode in ["list", "503", "hang"] {
let l = layout(&format!("llm-place-left-{mode}"));
let port = serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert!(calls.lines().next() == Some("launch --stop"), "{mode}: stopped first: {calls}");
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "{mode}: the left server is not counted");
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{mode}: {:?}", launches(&l));
llm.stop().await.unwrap();
}
}
#[cfg(unix)]
#[tokio::test]
async fn a_left_server_that_does_not_stop_means_no_experts_and_nothing_measured() {
// The stop command fails and the server keeps answering: the card is not read as it is.
let l = layout("llm-place-stuck");
let port = models_server_until("qwen-test", Duration::from_millis(0), None).await;
let log = l.strata_home().join("x.engine.log");
let exe = fake_strata_stop(&l, &format!("echo 'strata-q36: (1.10 GiB staging)' >> '{}'; echo 'Strata is serving.'", log.display()), "exit 3");
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 9874)).with_stop_timeout(Duration::from_secs(1));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(30), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert!(calls.lines().next() == Some("launch --stop"), "stopped first: {calls}");
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "the left server is not counted");
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{:?}", launches(&l));
llm.stop().await.unwrap();
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
let l2 = launches(&l);
assert_eq!(l2.len(), 1, "{l2:?}");
assert!(l2[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[0]);
assert!(!l.data.join("config").join("llm-probe.json").exists());
}
#[cfg(unix)]

File diff suppressed because one or more lines are too long