fix(app): the LLM placement always stops a left server and confirms the port closed before reading the card
- strata launch --stop now always runs before the card is read and the start's own port must refuse connections; a left server that answers, still loads (503) or stays silent was skipped before and its memory counted as other programs' - a stop that is not confirmed reads and keeps nothing: the model starts with expert-vram=0 - the stop confirms on the port it is given (stop_server_on), not on one remembered from an earlier start Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -391,9 +391,11 @@ start it first, so nothing answers before the brain can, and stop it last.
|
||||
short answer, then the card's memory again and the staging from the engine log; kept in
|
||||
`config/llm-probe.json` under `<context>/<parallel>`, and the real start then replaces that
|
||||
server. A measurement whose answer or second reading fails (or shows nothing more held) is not
|
||||
kept: the model starts with `expert-vram=0` and the next start measures again. A server of this
|
||||
app's port that still answers before the start (left from a run that ended badly) is stopped
|
||||
first, so it is not counted as other programs' memory. Without a card nothing is set. The game
|
||||
kept: the model starts with `expert-vram=0` and the next start measures again. Before the card
|
||||
is read, `strata launch --stop` always runs and the start's port must refuse connections, so a
|
||||
server left from a run that ended badly (answering, still loading with 503, or silent) is never
|
||||
counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and
|
||||
the model starts with `expert-vram=0`. Without a card nothing is set. The game
|
||||
placement (used by the switching that comes next) takes the reserve when it fits beside more
|
||||
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from.
|
||||
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
|
||||
|
||||
@@ -175,12 +175,6 @@ pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> R
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a server answers `/models` at `base` (within 2 s).
|
||||
async fn answers(base: &str) -> bool {
|
||||
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(2)).build() else { return false };
|
||||
client.get(format!("{base}/models")).send().await.is_ok_and(|r| r.status().is_success())
|
||||
}
|
||||
|
||||
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an
|
||||
/// error when the model gives none.
|
||||
async fn warm_up(base: &str) -> Result<(), String> {
|
||||
@@ -275,15 +269,18 @@ impl Llm {
|
||||
/// the card holds now and this PC's measurement without experts at this context and number of
|
||||
/// requests at once, which the first start makes once per card (a start without experts, then
|
||||
/// the real start replaces it). A server left from an earlier run is stopped first, so it is not
|
||||
/// counted as another program's memory. A measurement that fails is not kept: the model then
|
||||
/// starts without experts on the card.
|
||||
/// counted as another program's memory (a stop that is not confirmed reads nothing: no
|
||||
/// experts). A measurement that fails is not kept: the model then starts without experts on
|
||||
/// the card.
|
||||
async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str) -> Result<Vec<String>, String> {
|
||||
if self.gpu.is_none() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
if answers(base).await {
|
||||
crate::info_log!("llm", "a server from an earlier run still answers; stopping it first");
|
||||
self.stop_server().await?;
|
||||
// Whatever a server left from an earlier run does (answers, loads, stays silent), it holds
|
||||
// the card: stop it first and read the card only once the port is confirmed closed.
|
||||
if let Err(e) = self.stop_server_on(settings.llm.port).await {
|
||||
crate::warn_log!("llm", "Warning: a server left on the port did not stop ({e}); starting without experts on the card");
|
||||
return Ok(placement_sets(0.0));
|
||||
}
|
||||
let Some(card) = self.read_gpu().await else { return Ok(vec![]) };
|
||||
let (context, parallel) = (settings.llm.context, parallel(settings));
|
||||
@@ -431,6 +428,12 @@ impl Llm {
|
||||
/// `strata launch --stop`, then confirm the port no longer lists a model. A stop command that
|
||||
/// fails while nothing answers (nothing was serving) counts as stopped.
|
||||
async fn stop_server(&self) -> Result<(), String> {
|
||||
let port = *self.port.lock().unwrap();
|
||||
self.stop_server_on(port).await
|
||||
}
|
||||
|
||||
/// `stop_server` confirming on `port` (0: nothing was started, nothing to confirm).
|
||||
async fn stop_server_on(&self, port: u16) -> Result<(), String> {
|
||||
if !self.exe.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -448,7 +451,6 @@ impl Llm {
|
||||
let ran = ctx.run_in(&self.exe, &["launch", "--stop"], Some(&home)).await;
|
||||
timer.abort();
|
||||
let ran = ran.map_err(|e| if e == "cancelled" { format!("{}초 안에 끝나지 않았습니다", self.stop_timeout.as_secs()) } else { e });
|
||||
let port = *self.port.lock().unwrap();
|
||||
let base = format!("http://127.0.0.1:{port}/v1");
|
||||
let gone = port == 0 || wait_gone(&base, if ran.is_ok() { Duration::from_secs(15) } else { Duration::from_secs(2) }).await;
|
||||
match (ran, gone) {
|
||||
@@ -582,6 +584,11 @@ mod tests {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap();
|
||||
// A new server: it answers (a left one loaded or stayed silent).
|
||||
let mut m = mode.lock().unwrap();
|
||||
if matches!(*m, "503" | "hang") {
|
||||
*m = "list";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let mut buf = [0u8; 1024];
|
||||
@@ -818,18 +825,37 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() {
|
||||
let l = layout("llm-place-left");
|
||||
// A server still answers on the port, holding 9874 MiB.
|
||||
let port = models_server_until("qwen-test", Duration::from_millis(0), Some(l.data.join("gone"))).await;
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
|
||||
// Left over answering, loading (503) or silent: always stopped first, holding 9874 MiB until then.
|
||||
for mode in ["list", "503", "hang"] {
|
||||
let l = layout(&format!("llm-place-left-{mode}"));
|
||||
let port = serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await;
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
||||
assert!(calls.lines().next() == Some("launch --stop"), "{mode}: stopped first: {calls}");
|
||||
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
||||
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "{mode}: the left server is not counted");
|
||||
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{mode}: {:?}", launches(&l));
|
||||
llm.stop().await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_left_server_that_does_not_stop_means_no_experts_and_nothing_measured() {
|
||||
// The stop command fails and the server keeps answering: the card is not read as it is.
|
||||
let l = layout("llm-place-stuck");
|
||||
let port = models_server_until("qwen-test", Duration::from_millis(0), None).await;
|
||||
let log = l.strata_home().join("x.engine.log");
|
||||
let exe = fake_strata_stop(&l, &format!("echo 'strata-q36: (1.10 GiB staging)' >> '{}'; echo 'Strata is serving.'", log.display()), "exit 3");
|
||||
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 9874)).with_stop_timeout(Duration::from_secs(1));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(30), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
||||
assert!(calls.lines().next() == Some("launch --stop"), "stopped first: {calls}");
|
||||
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
||||
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "the left server is not counted");
|
||||
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{:?}", launches(&l));
|
||||
llm.stop().await.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(40), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let l2 = launches(&l);
|
||||
assert_eq!(l2.len(), 1, "{l2:?}");
|
||||
assert!(l2[0].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[0]);
|
||||
assert!(!l.data.join("config").join("llm-probe.json").exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user