fix(app): the LLM placement measures only what it can trust and keeps a game the 0 GiB step
- a measurement whose short answer fails, or whose second card reading fails or shows nothing more held, is not kept: the model starts with expert-vram=0 and the next start measures again - measurements are kept per context and requests at once (<context>/<parallel>) and per card (its NVIDIA UUID), not by the card's size - a server on this app's port left from a run that ended badly is stopped before the card is read, so its memory is not counted as other programs' - the game placement falls back to 0 GiB of experts before pausing when 2 GiB leaves a game nothing Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -386,10 +386,16 @@ start it first, so nothing answers before the brain can, and stop it last.
|
||||
experts, the STT worker's share (2740 MiB: its model and ten speakers at once, M0 `stt`), the
|
||||
engine's margin (768 MiB) and the room for programs opened later (`game.otherRoomGb`, default
|
||||
1 GB); the budget first holds the long-prompt staging, rounded down to 0.1 GiB, 0 when nothing is
|
||||
left. What the model holds without experts is measured once per context and card (the first-run
|
||||
self-check: a start with `expert-vram=0`, one short answer, then the card's memory again and the
|
||||
staging from the engine log), kept in `config/llm-probe.json`; the real start then replaces that
|
||||
server. Without a card nothing is set. The `llm` log tab gets the placement and what it came from.
|
||||
left. What the model holds without experts is measured once per context, number of requests at
|
||||
once and card (its NVIDIA UUID): the first-run self-check, a start with `expert-vram=0`, one
|
||||
short answer, then the card's memory again and the staging from the engine log; kept in
|
||||
`config/llm-probe.json` under `<context>/<parallel>`, and the real start then replaces that
|
||||
server. A measurement whose answer or second reading fails (or shows nothing more held) is not
|
||||
kept: the model starts with `expert-vram=0` and the next start measures again. A server of this
|
||||
app's port that still answers before the start (left from a run that ended badly) is stopped
|
||||
first, so it is not counted as other programs' memory. Without a card nothing is set. The game
|
||||
placement (used by the switching that comes next) takes the reserve when it fits beside more
|
||||
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from.
|
||||
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
|
||||
settings file only) the app starts nothing; that server must list a model within 30 seconds.
|
||||
- **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port`
|
||||
|
||||
@@ -19,12 +19,14 @@ pub const MIN_GAME_EXPERT_GIB: f64 = 2.0;
|
||||
pub const GAME_CONTEXT: u32 = 16384;
|
||||
|
||||
/// What one start without experts at `context` showed on this PC (the first-run self-check).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Probe {
|
||||
pub context: u32,
|
||||
/// The card's size when it was measured (another card means measuring again).
|
||||
pub total_mib: i64,
|
||||
/// Requests answered at once, which changes the engine's buffers.
|
||||
pub parallel: u32,
|
||||
/// The card measured (its NVIDIA UUID): another card means measuring again.
|
||||
pub gpu: String,
|
||||
/// What the LLM held without experts.
|
||||
pub model_mib: i64,
|
||||
/// The long-prompt staging the expert budget holds first (engine log, "x.xx GiB staging").
|
||||
@@ -58,16 +60,22 @@ pub fn normal(total_mib: i64, used_mib: i64, p: &Probe, other_room_mib: i64) ->
|
||||
}
|
||||
|
||||
/// The game placement: experts get what is left after the game's reserve, but never less than the
|
||||
/// smallest placement; then the game gets what that leaves (shown on the dashboard), and with no
|
||||
/// room at all the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
|
||||
/// smallest placement (2 GiB); then the game gets what that leaves (shown on the dashboard). When
|
||||
/// 2 GiB leaves a game nothing, no experts on the card (0 GiB) is the last step; with no room even
|
||||
/// then the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
|
||||
pub fn game(total_mib: i64, used_mib: i64, p: &Probe, reserve_mib: i64) -> GamePlacement {
|
||||
let expert = expert_budget_gib(total_mib, used_mib, p, reserve_mib);
|
||||
if expert >= MIN_GAME_EXPERT_GIB {
|
||||
return GamePlacement { expert_gib: expert, game_room_mib: reserve_mib, paused: false };
|
||||
}
|
||||
// Beside the model without experts (which already holds the staging): what 2 GiB adds.
|
||||
let free = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB;
|
||||
let extra = ((MIN_GAME_EXPERT_GIB - p.staging_gib) * 1024.0).max(0.0) as i64;
|
||||
let room = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB - extra;
|
||||
GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: room.max(0), paused: room <= 0 }
|
||||
if free - extra > 0 {
|
||||
return GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: free - extra, paused: false };
|
||||
}
|
||||
// The last step before pausing: no experts on the card (slower, but the game gets room).
|
||||
GamePlacement { expert_gib: 0.0, game_room_mib: free.max(0), paused: free <= 0 }
|
||||
}
|
||||
|
||||
/// The staging the engine log reports ("(1.10 GiB staging)"), else the usual 1.1 GiB.
|
||||
@@ -86,21 +94,23 @@ mod tests {
|
||||
/// included the STT model (1300 MiB), which the app starts after the LLM.
|
||||
const TOTAL: i64 = 16303;
|
||||
const OTHERS: i64 = 2760 - 1300;
|
||||
const PROBE32: Probe = Probe { context: 32768, total_mib: TOTAL, model_mib: 9874, staging_gib: 1.1 };
|
||||
fn probe32() -> Probe {
|
||||
Probe { context: 32768, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_normal_placement_is_the_one_m0_measured_with_room_for_the_stt() {
|
||||
// M0 got 1.5 GiB (the STT model counted as already held); the app counts it in the STT share.
|
||||
assert_eq!(normal(TOTAL, OTHERS, &PROBE32, 1024), 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS, &probe32(), 1024), 1.5);
|
||||
// More room for other programs, fewer experts; no room at all, none.
|
||||
assert!(normal(TOTAL, OTHERS, &PROBE32, 2048) < 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS + 8000, &PROBE32, 1024), 0.0);
|
||||
assert!(normal(TOTAL, OTHERS, &probe32(), 2048) < 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS + 8000, &probe32(), 1024), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_game_gets_its_reserve_or_what_the_smallest_placement_leaves_or_the_llm_pauses() {
|
||||
// 16K saves about 0.7 GB of conversation memory (M0).
|
||||
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..PROBE32 };
|
||||
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..probe32() };
|
||||
// 8 GB does not fit on 16 GB: the smallest placement, and the game gets what is left.
|
||||
let g = game(TOTAL, OTHERS, &probe16, 8192);
|
||||
assert_eq!((g.expert_gib, g.paused), (MIN_GAME_EXPERT_GIB, false));
|
||||
@@ -108,11 +118,15 @@ mod tests {
|
||||
assert_eq!(g.game_room_mib, expected);
|
||||
assert!(g.game_room_mib > 0 && g.game_room_mib < 8192);
|
||||
// A bigger card keeps the whole reserve, with more experts than the minimum.
|
||||
let big = game(24 * 1024, OTHERS, &Probe { total_mib: 24 * 1024, ..probe16 }, 8192);
|
||||
let big = game(24 * 1024, OTHERS, &probe16, 8192);
|
||||
assert_eq!(big.game_room_mib, 8192);
|
||||
assert!(big.expert_gib > MIN_GAME_EXPERT_GIB && !big.paused);
|
||||
// No room beside 2 GiB but some beside none: the 0 GiB placement, before pausing.
|
||||
let tight = game(TOTAL, 2960, &Probe { model_mib: 9124, ..probe16.clone() }, 8192);
|
||||
assert_eq!((tight.expert_gib, tight.paused), (0.0, false));
|
||||
assert_eq!(tight.game_room_mib, TOTAL - 2960 - 9124 - STT_MIB - MARGIN_MIB);
|
||||
// Other programs already hold the card: no room for a game, the LLM pauses in one.
|
||||
let full = game(TOTAL, OTHERS + 3000, &probe16, 8192);
|
||||
let full = game(TOTAL, OTHERS + 4000, &probe16, 8192);
|
||||
assert!(full.paused);
|
||||
assert_eq!(full.game_room_mib, 0);
|
||||
}
|
||||
|
||||
@@ -72,12 +72,20 @@ fn placement_sets(expert_gib: f64) -> Vec<String> {
|
||||
vec![format!("expert-vram={expert_gib}"), "prefill=2048".into()]
|
||||
}
|
||||
|
||||
/// Graphics memory `(used, total)` in MiB of the first NVIDIA card, from `nvidia-smi`; None without one.
|
||||
pub type GpuReader = Arc<dyn Fn() -> Option<(i64, i64)> + Send + Sync>;
|
||||
/// The first NVIDIA card: its UUID and graphics memory in MiB.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Gpu {
|
||||
pub id: String,
|
||||
pub used_mib: i64,
|
||||
pub total_mib: i64,
|
||||
}
|
||||
|
||||
fn nvidia_memory() -> Option<(i64, i64)> {
|
||||
/// Reads the card (`nvidia-smi`); None without one.
|
||||
pub type GpuReader = Arc<dyn Fn() -> Option<Gpu> + Send + Sync>;
|
||||
|
||||
fn nvidia_memory() -> Option<Gpu> {
|
||||
let mut cmd = std::process::Command::new("nvidia-smi");
|
||||
cmd.args(["--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"]);
|
||||
cmd.args(["--query-gpu=uuid,memory.used,memory.total", "--format=csv,noheader,nounits"]);
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::process::CommandExt;
|
||||
@@ -85,8 +93,8 @@ fn nvidia_memory() -> Option<(i64, i64)> {
|
||||
}
|
||||
let out = cmd.output().ok()?;
|
||||
let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned();
|
||||
let (used, total) = line.split_once(',')?;
|
||||
Some((used.trim().parse().ok()?, total.trim().parse().ok()?))
|
||||
let mut f = line.split(',').map(str::trim);
|
||||
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()? })
|
||||
}
|
||||
|
||||
/// The newest engine log under Strata's folder, its end only (the placement lines of the last start).
|
||||
@@ -167,16 +175,29 @@ pub async fn wait_ready(base: &str, timeout: Duration, cancel: &AtomicBool) -> R
|
||||
}
|
||||
}
|
||||
|
||||
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit).
|
||||
async fn warm_up(base: &str) {
|
||||
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(60)).build() else { return };
|
||||
/// Whether a server answers `/models` at `base` (within 2 s).
|
||||
async fn answers(base: &str) -> bool {
|
||||
let Ok(client) = reqwest::Client::builder().timeout(Duration::from_secs(2)).build() else { return false };
|
||||
client.get(format!("{base}/models")).send().await.is_ok_and(|r| r.status().is_success())
|
||||
}
|
||||
|
||||
/// One short answer, so the engine has made its buffers before its memory is read (M0 kit); an
|
||||
/// error when the model gives none.
|
||||
async fn warm_up(base: &str) -> Result<(), String> {
|
||||
let client = reqwest::Client::builder().timeout(Duration::from_secs(60)).build().map_err(|e| e.to_string())?;
|
||||
let model = match client.get(format!("{base}/models")).send().await {
|
||||
Ok(r) => r.text().await.ok().and_then(|t| serde_json::from_str::<Value>(&t).ok())
|
||||
.and_then(|v| v["data"][0]["id"].as_str().map(str::to_owned)),
|
||||
Err(_) => None,
|
||||
};
|
||||
let body = json!({"model": model, "messages": [{"role": "user", "content": "안녕"}], "max_tokens": 8});
|
||||
let _ = client.post(format!("{base}/chat/completions")).header("content-type", "application/json").body(body.to_string()).send().await;
|
||||
let r = client.post(format!("{base}/chat/completions")).header("content-type", "application/json")
|
||||
.body(body.to_string()).send().await.map_err(|e| e.to_string())?;
|
||||
if !r.status().is_success() {
|
||||
return Err(format!("the model answered {}", r.status()));
|
||||
}
|
||||
let v: Value = serde_json::from_str(&r.text().await.map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
|
||||
v["choices"][0]["message"].is_object().then_some(()).ok_or_else(|| "the model gave no answer".into())
|
||||
}
|
||||
|
||||
/// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the
|
||||
@@ -241,7 +262,7 @@ impl Llm {
|
||||
self
|
||||
}
|
||||
|
||||
async fn read_gpu(&self) -> Option<(i64, i64)> {
|
||||
async fn read_gpu(&self) -> Option<Gpu> {
|
||||
let gpu = self.gpu.clone()?;
|
||||
tokio::task::spawn_blocking(move || gpu()).await.ok().flatten()
|
||||
}
|
||||
@@ -251,34 +272,50 @@ impl Llm {
|
||||
}
|
||||
|
||||
/// The expert budget's engine settings (none without a card): the normal placement from what
|
||||
/// the card holds now and this PC's measurement without experts at this context, which the
|
||||
/// first start makes once (a start without experts, then the real start replaces it).
|
||||
/// the card holds now and this PC's measurement without experts at this context and number of
|
||||
/// requests at once, which the first start makes once per card (a start without experts, then
|
||||
/// the real start replaces it). A server left from an earlier run is stopped first, so it is not
|
||||
/// counted as another program's memory. A measurement that fails is not kept: the model then
|
||||
/// starts without experts on the card.
|
||||
async fn placement(&self, settings: &Settings, ctx: &Ctx, base: &str) -> Result<Vec<String>, String> {
|
||||
let Some((used, total)) = self.read_gpu().await else { return Ok(vec![]) };
|
||||
let context = settings.llm.context;
|
||||
if self.gpu.is_none() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
if answers(base).await {
|
||||
crate::info_log!("llm", "a server from an earlier run still answers; stopping it first");
|
||||
self.stop_server().await?;
|
||||
}
|
||||
let Some(card) = self.read_gpu().await else { return Ok(vec![]) };
|
||||
let (context, parallel) = (settings.llm.context, parallel(settings));
|
||||
let key = format!("{context}/{parallel}");
|
||||
let file = self.probe_file();
|
||||
let mut all: serde_json::Map<String, Value> = std::fs::read_to_string(&file).ok()
|
||||
.and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();
|
||||
let known = all.get(&context.to_string()).and_then(|v| serde_json::from_value::<crate::placement::Probe>(v.clone()).ok())
|
||||
.filter(|p| p.total_mib == total);
|
||||
let known = all.get(&key).and_then(|v| serde_json::from_value::<crate::placement::Probe>(v.clone()).ok())
|
||||
.filter(|p| p.gpu == card.id);
|
||||
let probe = match known {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
crate::info_log!("llm", "measuring the model without experts once (context {context})");
|
||||
crate::info_log!("llm", "measuring the model without experts once (context {context}, {parallel} at once)");
|
||||
self.launch(settings, ctx, base, &placement_sets(0.0)).await?;
|
||||
warm_up(base).await;
|
||||
let after = self.read_gpu().await.map(|(u, _)| u).unwrap_or(used);
|
||||
let p = crate::placement::Probe { context, total_mib: total, model_mib: (after - used).max(0),
|
||||
let warmed = warm_up(base).await;
|
||||
let after = self.read_gpu().await.filter(|g| g.id == card.id);
|
||||
let model_mib = after.map(|g| g.used_mib - card.used_mib).unwrap_or(0);
|
||||
if let Err(e) = warmed.and_then(|()| if model_mib > 0 { Ok(()) } else { Err("the card's memory could not be read".into()) }) {
|
||||
crate::warn_log!("llm", "Warning: the measurement failed ({e}); starting without experts on the card");
|
||||
return Ok(placement_sets(0.0));
|
||||
}
|
||||
let p = crate::placement::Probe { context, parallel, gpu: card.id.clone(), model_mib,
|
||||
staging_gib: crate::placement::staging_gib(&engine_log(&self.layout.strata_home())) };
|
||||
all.insert(context.to_string(), json!(p));
|
||||
all.insert(key, json!(p));
|
||||
crate::datadir::write_atomic(&file, Value::Object(all).to_string().as_bytes())
|
||||
.map_err(|e| format!("배치 측정값을 저장하지 못했습니다: {e}"))?;
|
||||
p
|
||||
}
|
||||
};
|
||||
let expert = crate::placement::normal(total, used, &probe, settings.game.other_room_mib());
|
||||
crate::info_log!("llm", "placement: experts {expert} GiB (card {total} MiB, held by others {used} MiB, model without experts {} MiB)",
|
||||
probe.model_mib);
|
||||
let expert = crate::placement::normal(card.total_mib, card.used_mib, &probe, settings.game.other_room_mib());
|
||||
crate::info_log!("llm", "placement: experts {expert} GiB (card {} MiB, held by others {} MiB, model without experts {} MiB)",
|
||||
card.total_mib, card.used_mib, probe.model_mib);
|
||||
Ok(placement_sets(expert))
|
||||
}
|
||||
|
||||
@@ -533,12 +570,23 @@ mod tests {
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let opened = Instant::now();
|
||||
tokio::spawn(async move {
|
||||
let mut listener = listener;
|
||||
while let Ok((mut s, _)) = listener.accept().await {
|
||||
if gone.as_ref().is_some_and(|g| g.exists()) {
|
||||
return; // the listener closes: later connections are refused
|
||||
// The listener closes: later connections are refused, until a launch serves
|
||||
// again (removes `gone`).
|
||||
drop(s);
|
||||
drop(listener);
|
||||
let g = gone.clone().unwrap();
|
||||
while g.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
listener = tokio::net::TcpListener::bind(("127.0.0.1", port)).await.unwrap();
|
||||
continue;
|
||||
}
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = s.read(&mut buf).await;
|
||||
let n = s.read(&mut buf).await.unwrap_or(0);
|
||||
let post = buf[..n].starts_with(b"POST");
|
||||
let m = *mode.lock().unwrap();
|
||||
let (status, body) = match m {
|
||||
"hang" => {
|
||||
@@ -549,6 +597,8 @@ mod tests {
|
||||
continue;
|
||||
}
|
||||
"503" => ("503 Service Unavailable", r#"{"error":"loading model"}"#.to_owned()),
|
||||
"nochat" if post => ("500 Internal Server Error", r#"{"error":"broken"}"#.to_owned()),
|
||||
_ if post => ("200 OK", r#"{"choices":[{"message":{"content":"안녕하세요"}}]}"#.to_owned()),
|
||||
_ if opened.elapsed() >= delay => ("200 OK", format!(r#"{{"data":[{{"id":"{id}"}}]}}"#)),
|
||||
_ => ("200 OK", "{}".to_owned()),
|
||||
};
|
||||
@@ -645,19 +695,14 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn with_a_graphics_card_the_first_start_measures_the_llm_without_experts_then_places_them() {
|
||||
let l = layout("llm-place");
|
||||
let port = models_server_until("qwen-test", Duration::from_millis(0), Some(l.data.join("gone"))).await;
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
// The engine log, as Strata 0.1.59's qwen36 engine writes it.
|
||||
let log = l.strata_home().join("x.engine.log");
|
||||
let exe = fake_strata(&l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: 0 of 256 per layer in VRAM, bigger prompt chunks streamed (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
|
||||
l.data.join("gone").display(), log.display()));
|
||||
// The game PC in M0 (16303 MiB card, 1460 MiB held by other programs); the LLM without
|
||||
// experts held 9874 MiB.
|
||||
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
||||
let r2 = reads.clone();
|
||||
let llm = Llm::with_exe(l.clone(), exe).with_gpu(Arc::new(move || {
|
||||
let n = r2.fetch_add(1, Ordering::SeqCst);
|
||||
Some((if n == 1 { 1460 + 9874 } else { 1460 }, 16303))
|
||||
}));
|
||||
let llm = Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
||||
@@ -667,20 +712,126 @@ mod tests {
|
||||
// The M0 kit's normal placement on that PC: 1.5 GiB of experts.
|
||||
assert!(launches[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[1]);
|
||||
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
||||
assert_eq!(probe["32768"], json!({"context": 32768, "totalMib": 16303, "modelMib": 9874, "stagingGib": 1.1}));
|
||||
assert_eq!(probe["32768/1"], json!({"context": 32768, "parallel": 1, "gpu": "GPU-test", "modelMib": 9874, "stagingGib": 1.1}));
|
||||
llm.stop().await.unwrap();
|
||||
// The next start reuses the measurement: one launch, placed at once (a new server, as the
|
||||
// fake one does not come back after a stop).
|
||||
// The next start reuses the measurement: one launch, placed at once.
|
||||
std::fs::remove_file(l.data.join("calls")).unwrap();
|
||||
let port2 = models_server_until("qwen-test", Duration::from_millis(0), None).await;
|
||||
let s2 = parse_settings(&json!({"llm": {"port": port2}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s2)).await.expect("starts").unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
||||
let launches: Vec<&str> = calls.lines().filter(|c| c.starts_with("launch --family")).collect();
|
||||
assert_eq!(launches.len(), 1, "{calls}");
|
||||
assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]);
|
||||
}
|
||||
|
||||
/// A fake card for `fake_placed`: 16303 MiB, 1460 MiB held by other programs (and by any
|
||||
/// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a
|
||||
/// Strata server runs (the calls file shows the last launch and stop).
|
||||
#[cfg(unix)]
|
||||
fn card(l: &Layout, id: &'static str, leftover: i64) -> GpuReader {
|
||||
let calls = l.data.join("calls");
|
||||
Arc::new(move || {
|
||||
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
||||
let stopped = c.contains("launch --stop");
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
let used = 1460 + if running { 9874 } else if stopped { 0 } else { leftover };
|
||||
Some(Gpu { id: id.into(), used_mib: used, total_mib: 16303 })
|
||||
})
|
||||
}
|
||||
|
||||
/// A fake Strata that serves on `port` again (removes `gone`) and writes the engine's staging line.
|
||||
#[cfg(unix)]
|
||||
fn fake_placed(l: &Layout) -> std::path::PathBuf {
|
||||
let log = l.strata_home().join("x.engine.log");
|
||||
fake_strata(l, &format!("rm -f '{}'; echo 'strata-q36: hybrid experts: (1.10 GiB staging); 7.1 s' >> '{}'; echo 'Strata is serving.'",
|
||||
l.data.join("gone").display(), log.display()))
|
||||
}
|
||||
|
||||
/// A fake server on a free port that answers only once a launch serves (removes `gone`).
|
||||
#[cfg(unix)]
|
||||
async fn server_after_launch(l: &Layout, mode: &'static str) -> u16 {
|
||||
std::fs::create_dir_all(&l.data).unwrap();
|
||||
std::fs::write(l.data.join("gone"), "").unwrap();
|
||||
serve("qwen-test", Duration::from_millis(0), Some(l.data.join("gone")), Arc::new(Mutex::new(mode))).await
|
||||
}
|
||||
|
||||
fn launches(l: &Layout) -> Vec<String> {
|
||||
std::fs::read_to_string(l.data.join("calls")).unwrap_or_default().lines()
|
||||
.filter(|c| c.starts_with("launch --family")).map(str::to_owned).collect()
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_failed_measurement_is_not_kept_and_the_model_starts_without_experts() {
|
||||
// The card stops answering after the first read.
|
||||
let l = layout("llm-place-fail");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
||||
let r2 = reads.clone();
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
|
||||
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: 1460, total_mib: 16303 })
|
||||
}));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let l2 = launches(&l);
|
||||
assert_eq!(l2.len(), 2, "{l2:?}");
|
||||
assert!(l2[1].ends_with("--set expert-vram=0 --set prefill=2048"), "{}", l2[1]);
|
||||
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
|
||||
llm.stop().await.unwrap();
|
||||
// The model does not answer the short question: not kept either.
|
||||
let l = layout("llm-place-nochat");
|
||||
let port = server_after_launch(&l, "nochat").await;
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
assert!(launches(&l)[1].ends_with("--set expert-vram=0 --set prefill=2048"));
|
||||
assert!(!l.data.join("config").join("llm-probe.json").exists(), "nothing kept");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn the_measurement_is_kept_per_requests_at_once_and_per_card() {
|
||||
let l = layout("llm-place-keys");
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-a", 0));
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let one = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
llm.start(&one).await.unwrap();
|
||||
assert_eq!(launches(&l).len(), 2, "measured once");
|
||||
llm.stop().await.unwrap();
|
||||
// A chat channel that answers everything: two requests at once, measured again.
|
||||
std::fs::remove_file(l.data.join("calls")).unwrap();
|
||||
let two = parse_settings(&json!({"llm": {"port": port}, "chatChannels": [{"id": "123456789012345678", "answer": "everything"}]})).unwrap();
|
||||
assert_eq!(parallel(&two), 2);
|
||||
llm.start(&two).await.unwrap();
|
||||
assert_eq!(launches(&l).len(), 2, "measured for two at once");
|
||||
assert!(launches(&l)[0].contains("--parallel 2"));
|
||||
llm.stop().await.unwrap();
|
||||
// Another card (same size): measured again.
|
||||
std::fs::remove_file(l.data.join("calls")).unwrap();
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-b", 0));
|
||||
llm.start(&one).await.unwrap();
|
||||
assert_eq!(launches(&l).len(), 2, "measured for the new card");
|
||||
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
||||
assert_eq!((probe["32768/1"]["gpu"].clone(), probe["32768/2"]["gpu"].clone()), (json!("GPU-b"), json!("GPU-a")));
|
||||
llm.stop().await.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_server_left_from_an_earlier_run_is_stopped_before_the_card_is_read() {
|
||||
let l = layout("llm-place-left");
|
||||
// A server still answers on the port, holding 9874 MiB.
|
||||
let port = models_server_until("qwen-test", Duration::from_millis(0), Some(l.data.join("gone"))).await;
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 9874));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(30), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let calls = std::fs::read_to_string(l.data.join("calls")).unwrap();
|
||||
assert!(calls.lines().next() == Some("launch --stop"), "stopped first: {calls}");
|
||||
let probe: Value = serde_json::from_str(&std::fs::read_to_string(l.data.join("config").join("llm-probe.json")).unwrap()).unwrap();
|
||||
assert_eq!(probe["32768/1"]["modelMib"], json!(9874), "the left server is not counted");
|
||||
assert!(launches(&l)[1].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{:?}", launches(&l));
|
||||
llm.stop().await.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_cancel_during_the_download_stops_it_and_clears_the_server() {
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user