fix(app): the LLM placement measures only what it can trust and keeps a game the 0 GiB step

- a measurement whose short answer fails, or whose second card reading fails or shows nothing
  more held, is not kept: the model starts with expert-vram=0 and the next start measures again
- measurements are kept per context and requests at once (<context>/<parallel>) and per card
  (its NVIDIA UUID), not by the card's size
- a server on this app's port left from a run that ended badly is stopped before the card is
  read, so its memory is not counted as other programs'
- the game placement falls back to 0 GiB of experts before pausing when 2 GiB leaves a game
  nothing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-14 10:01:53 +09:00
parent af6f2c855d
commit 1400a3ff97
4 changed files with 230 additions and 59 deletions

View File

@@ -72,12 +72,20 @@ fn placement_sets(expert_gib: f64) -> Vec<String> {
vec![format!("expert-vram={expert_gib}"), "prefill=2048".into()]
}
/// Graphics memory `(used, total)` in MiB of the first NVIDIA card, from `nvidia-smi`; None without one.
pub type GpuReader = Arc<dyn Fn() -> Option<(i64, i64)> + Send + Sync>;
/// The first NVIDIA card: its UUID and graphics memory in MiB.
#[derive(Debug, Clone, PartialEq)]
pub struct Gpu {
pub id: String,
pub used_mib: i64,
pub total_mib: i64,
}
fn nvidia_memory() -> Option<(i64, i64)> {
/// Reads the card (`nvidia-smi`); None without one.
pub type GpuReader = Arc<dyn Fn() -> Option<Gpu> + Send + Sync>;
fn nvidia_memory() -> Option<Gpu> {
let mut cmd = std::process::Command::new("nvidia-smi");
cmd.args(["--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"]);
cmd.args(["--query-gpu=uuid,memory.used,memory.total", "--format=csv,noheader,nounits"]);
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
@@ -85,8 +93,8 @@ fn nvidia_memory() -> Option<(i64, i64)> {
}
let out = cmd.output().ok()?;
let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned();
let (used, total) = line.split_once(',')?;
Some((used.trim().parse().ok()?, total.trim().parse().ok()?))
let mut f = line.split(',').map(str::trim);
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()? })
}
/// The newest engine log under Strata's folder, its end only (the placement lines of the last start).
@@ -167,16 +175,29 @@ pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> R
}
}
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit).
async fn warm_up(base: &str) {
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(60)).build() else { return };
/// Whether a server answers `/models` at `base` (within 2 s).
async fn answers(base: &str) -> bool {
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(2)).build() else { return false };
client.get(format!("{base}/models")).send().await.is_ok_and(|r| r.status().is_success())
}
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an
/// error when the model gives none.
async fn warm_up(base: &str) -> Result<(), String> {
let client = reqwest::Client::builder().timeout(Duration::from_secs(60)).build().map_err(|e| e.to_string())?;
let model = match client.get(format!("{base}/models")).send().await {
Ok(r) => r.text().await.ok().and_then(|t| serde_json::from_str::<Value>(&t).ok())
.and_then(|v| v["data"][0]["id"].as_str().map(str::to_owned)),
Err(_) => None,
};
let body = json!({"model": model, "messages": [{"role": "user", "content": "안녕"}], "max_tokens": 8});
let _ = client.post(format!("{base}/chat/completions")).header("content-type", "application/json").body(body.to_string()).send().await;
let r = client.post(format!("{base}/chat/completions")).header("content-type", "application/json")
.body(body.to_string()).send().await.map_err(|e| e.to_string())?;
if !r.status().is_success() {
return Err(format!("the model answered {}", r.status()));
}
let v: Value = serde_json::from_str(&r.text().await.map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
v["choices"][0]["message"].is_object().then_some(()).ok_or_else(|| "the model gave no answer".into())
}
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the
@@ -241,7 +262,7 @@ impl Llm {
self
}
async fn read_gpu(&self) -> Option<(i64, i64)> {
async fn read_gpu(&self) -> Option<Gpu> {
let gpu = self.gpu.clone()?;
tokio::task::spawn_blocking(move || gpu()).await.ok().flatten()
}
@@ -251,34 +272,50 @@ impl Llm {
}
/// The expert budget's engine settings (none without a card): the normal placement from what
/// the card holds now and this PC's measurement without experts at this context, which the
/// first start makes once (a start without experts, then the real start replaces it).
/// the card holds now and this PC's measurement without experts at this context and number of
/// requests at once, which the first start makes once per card (a start without experts, then
/// the real start replaces it). A server left from an earlier run is stopped first, so it is not
/// counted as another program's memory. A measurement that fails is not kept: the model then
/// starts without experts on the card.
async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str) -> Result<Vec<String>, String> {
let Some((used, total)) = self.read_gpu().await else { return Ok(vec![]) };
let context = settings.llm.context;
if self.gpu.is_none() {
return Ok(vec![]);
}
if answers(base).await {
crate::info_log!("llm", "a server from an earlier run still answers; stopping it first");
self.stop_server().await?;
}
let Some(card) = self.read_gpu().await else { return Ok(vec![]) };
let (context, parallel) = (settings.llm.context, parallel(settings));
let key = format!("{context}/{parallel}");
let file = self.probe_file();
let mut all: serde_json::Map<String, Value> = std::fs::read_to_string(&file).ok()
.and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();
let known = all.get(&context.to_string()).and_then(|v| serde_json::from_value::<crate::placement::Probe>(v.clone()).ok())
.filter(|p| p.total_mib == total);
let known = all.get(&key).and_then(|v| serde_json::from_value::<crate::placement::Probe>(v.clone()).ok())
.filter(|p| p.gpu == card.id);
let probe = match known {
Some(p) => p,
None => {
crate::info_log!("llm", "measuring the model without experts once (context {context})");
crate::info_log!("llm", "measuring the model without experts once (context {context}, {parallel} at once)");
self.launch(settings, ctx, base, &placement_sets(0.0)).await?;
warm_up(base).await;
let after = self.read_gpu().await.map(|(u, _)| u).unwrap_or(used);
let p = crate::placement::Probe { context, total_mib: total, model_mib: (after - used).max(0),
let warmed = warm_up(base).await;
let after = self.read_gpu().await.filter(|g| g.id == card.id);
let model_mib = after.map(|g| g.used_mib - card.used_mib).unwrap_or(0);
if let Err(e) = warmed.and_then(|()| if model_mib > 0 { Ok(()) } else { Err("the card's memory could not be read".into()) }) {
crate::warn_log!("llm", "Warning: the measurement failed ({e}); starting without experts on the card");
return Ok(placement_sets(0.0));
}
let p = crate::placement::Probe { context, parallel, gpu: card.id.clone(), model_mib,
staging_gib: crate::placement::staging_gib(&engine_log(&self.layout.strata_home())) };
all.insert(context.to_string(), json!(p));
all.insert(key, json!(p));
crate::datadir::write_atomic(&file, Value::Object(all).to_string().as_bytes())
.map_err(|e| format!("배치 측정값을 저장하지 못했습니다: {e}"))?;
p
}
};
let expert = crate::placement::normal(total, used, &probe, settings.game.other_room_mib());
crate::info_log!("llm", "placement: experts {expert} GiB (card {total} MiB, held by others {used} MiB, model without experts {} MiB)",
probe.model_mib);
let expert = crate::placement::normal(card.total_mib, card.used_mib, &probe, settings.game.other_room_mib());
crate::info_log!("llm", "placement: experts {expert} GiB (card {} MiB, held by others {} MiB, model without experts {} MiB)",
card.total_mib, card.used_mib, probe.model_mib);
Ok(placement_sets(expert))
}
@@ -533,12 +570,23 @@ mod tests {
let port = listener.local_addr().unwrap().port();
let opened = Instant::now();
tokio::spawn(async move {
let mut listener = listener;
while let Ok((mut s, _)) = listener.accept().await {
if gone.as_ref().is_some_and(|g| g.exists()) {
return; // the listener closes: later connections are refused
// The listener closes: later connections are refused, until a launch serves
// again (removes `gone`).
drop(s);
drop(listener);
let g = gone.clone().unwrap();
while g.exists() {
tokio::time::sleep(Duration::from_millis(50)).await;
}
listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap();
continue;
}
let mut buf = [0u8; 1024];
let _ = s.read(&mut buf).await;
let n = s.read(&mut buf).await.unwrap_or(0);
let post = buf[..n].starts_with(b"POST");
let m = *mode.lock().unwrap();
let (status, body) = match m {
"hang" => {
@@ -549,6 +597,8 @@ mod tests {
continue;
}
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
"nochat" if post => ("500 Internal Server Error", r#"{"error":"broken"}"#.to_owned()),
_ if post => ("200 OK", r#"{"choices":[{"message":{"content":"안녕하세요"}}]}"#.to_owned()),
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
_ => ("200 OK", "{}".to_owned()),
};
@@ -645,19 +695,14 @@ mod tests {
#[tokio::test]
async fn with_a_graphics_card_the_first_start_measures_the_llm_without_experts_then_places_them() {
let l = layout("llm-place");
let port = models_server_until("qwen-test", Duration::from_millis(0), Some(l.data.join("gone"))).await;
let port = server_after_launch(&l, "list").await;
// The engine log, as Strata 0.1.59's qwen36 engine writes it.
let log = l.strata_home().join("x.engine.log");
let exe = fake_strata(&l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 0 of 256 per layer in VRAM, bigger prompt chunks streamed (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()));
// The game PC in M0 (16303 MiB card, 1460 MiB held by other programs); the LLM without
// experts held 9874 MiB.
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
let r2 = reads.clone();
let llm = Llm::with_exe(l.clone(), exe).with_gpu(Arc::new(move || {
let n = r2.fetch_add(1, Ordering::SeqCst);
Some((if n == 1 { 1460 + 9874 } else { 1460 }, 16303))
}));
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
@@ -667,20 +712,126 @@ mod tests {
// The M0 kit's normal placement on that PC: 1.5 GiB of experts.
assert!(launches[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[1]);
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!(probe["32768"], json!({"context": 32768, "totalMib": 16303, "modelMib": 9874, "stagingGib": 1.1}));
assert_eq!(probe["32768/1"], json!({"context": 32768, "parallel": 1, "gpu": "GPU-test", "modelMib": 9874, "stagingGib": 1.1}));
llm.stop().await.unwrap();
// The next start reuses the measurement: one launch, placed at once (a new server, as the
// fake one does not come back after a stop).
// The next start reuses the measurement: one launch, placed at once.
std::fs::remove_file(l.data.join("calls")).unwrap();
let port2 = models_server_until("qwen-test", Duration::from_millis(0), None).await;
let s2 = parse_settings(&json!({"llm": {"port": port2}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s2)).await.expect("starts").unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect();
assert_eq!(launches.len(), 1, "{calls}");
assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]);
}
/// A fake card for `fake_placed`: 16303 MiB, 1460 MiB held by other programs (and by any
/// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a
/// Strata server runs (the calls file shows the last launch and stop).
#[cfg(unix)]
fn card(l: &Layout, id: &'static str, leftover: i64) -> GpuReader {
let calls = l.data.join("calls");
Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let stopped = c.contains("launch --stop");
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
let used = 1460 + if running { 9874 } else if stopped { 0 } else { leftover };
Some(Gpu { id: id.into(), used_mib: used, total_mib: 16303 })
})
}
/// A fake Strata that serves on `port` again (removes `gone`) and writes the engine's staging line.
#[cfg(unix)]
fn fake_placed(l: &Layout) -> std::path::PathBuf {
let log = l.strata_home().join("x.engine.log");
fake_strata(l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()))
}
/// A fake server on a free port that answers only once a launch serves (removes `gone`).
#[cfg(unix)]
async fn server_after_launch(l: &Layout, mode: &'static str) -> u16 {
std::fs::create_dir_all(&l.data).unwrap();
std::fs::write(l.data.join("gone"), "").unwrap();
serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await
}
fn launches(l: &Layout) -> Vec<String> {
std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().lines()
.filter(|c| c.starts_with("launch --family")).map(str::to_owned).collect()
}
#[cfg(unix)]
#[tokio::test]
async fn a_failed_measurement_is_not_kept_and_the_model_starts_without_experts() {
// The card stops answering after the first read.
let l = layout("llm-place-fail");
let port = server_after_launch(&l, "list").await;
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
let r2 = reads.clone();
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 })
}));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let l2 = launches(&l);
assert_eq!(l2.len(), 2, "{l2:?}");
assert!(l2[1].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[1]);
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
llm.stop().await.unwrap();
// The model does not answer the short question: not kept either.
let l = layout("llm-place-nochat");
let port = server_after_launch(&l, "nochat").await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
assert!(launches(&l)[1].ends_with("--set expert-vram=0 --set prefill=2048"));
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
}
#[cfg(unix)]
#[tokio::test]
async fn the_measurement_is_kept_per_requests_at_once_and_per_card() {
let l = layout("llm-place-keys");
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-a", 0));
let port = server_after_launch(&l, "list").await;
let one = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&one).await.unwrap();
assert_eq!(launches(&l).len(), 2, "measured once");
llm.stop().await.unwrap();
// A chat channel that answers everything: two requests at once, measured again.
std::fs::remove_file(l.data.join("calls")).unwrap();
let two = parse_settings(&json!({"llm": {"port": port}, "chatChannels": [{"id": "123456789012345678", "answer": "everything"}]})).unwrap();
assert_eq!(parallel(&two), 2);
llm.start(&two).await.unwrap();
assert_eq!(launches(&l).len(), 2, "measured for two at once");
assert!(launches(&l)[0].contains("--parallel 2"));
llm.stop().await.unwrap();
// Another card (same size): measured again.
std::fs::remove_file(l.data.join("calls")).unwrap();
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-b", 0));
llm.start(&one).await.unwrap();
assert_eq!(launches(&l).len(), 2, "measured for the new card");
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!((probe["32768/1"]["gpu"].clone(), probe["32768/2"]["gpu"].clone()), (json!("GPU-b"), json!("GPU-a")));
llm.stop().await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() {
let l = layout("llm-place-left");
// A server still answers on the port, holding 9874 MiB.
let port = models_server_until("qwen-test", Duration::from_millis(0), Some(l.data.join("gone"))).await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(30), llm.start(&s)).await.expect("starts").unwrap();
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
assert!(calls.lines().next() == Some("launch --stop"), "stopped first: {calls}");
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "the left server is not counted");
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{:?}", launches(&l));
llm.stop().await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn a_cancel_during_the_download_stops_it_and_clears_the_server() {