Files
javis_bot/app/src/placement.rs
javis-bot 1400a3ff97 fix(app): the LLM placement measures only what it can trust and keeps a game the 0 GiB step
- a measurement whose short answer fails, or whose second card reading fails or shows nothing
  more held, is not kept: the model starts with expert-vram=0 and the next start measures again
- measurements are kept per context and requests at once (<context>/<parallel>) and per card
  (its NVIDIA UUID), not by the card's size
- a server on this app's port left from a run that ended badly is stopped before the card is
  read, so its memory is not counted as other programs'
- the game placement falls back to 0 GiB of experts before pausing when 2 GiB leaves a game
  nothing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 10:01:53 +09:00

141 lines
7.3 KiB
Rust

//! Where the LLM's routed experts go (`docs/v2/pc-features.md`, "PC마다 값을 계산"; admin.spec.md,
//! "LLM placement"): the one model is placed by the M0 kit's formula, so the speech-to-text worker
//! and other programs keep their room on the graphics card. Strata's automatic placement fills the
//! card with experts and left the STT no room (M0 `gamemode_stt`), and its qwen36 engine takes no
//! VRAM reserve, so the app sets `--expert-vram` itself.
use serde::{Deserialize, Serialize};
/// What Strata's qwen36 engine keeps free itself when it places experts (M0 kit).
pub const MARGIN_MIB: i64 = 768;
/// The STT worker's share: its model (1300 MiB) and what ten speakers at once add (1440 MiB),
/// measured on the game PC (M0 `stt`). The services start the LLM before the worker, so both count.
pub const STT_MIB: i64 = 1300 + 1440;
/// The smallest expert budget in a game: it holds the long-prompt staging (about 1.1 GiB) first,
/// so 2 GiB is the least that loads experts, and it was 1.5-2.1 times faster than none in a real
/// game (user decision, 2026-11-10).
pub const MIN_GAME_EXPERT_GIB: f64 = 2.0;
/// The context in a game (16K keeps room for the history, tools and results; 8K saved only 0.4 GB).
pub const GAME_CONTEXT: u32 = 16384;
/// What one start without experts at `context` showed on this PC (the first-run self-check).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Probe {
pub context: u32,
/// Requests answered at once, which changes the engine's buffers.
pub parallel: u32,
/// The card measured (its NVIDIA UUID): another card means measuring again.
pub gpu: String,
/// What the LLM held without experts.
pub model_mib: i64,
/// The long-prompt staging the expert budget holds first (engine log, "x.xx GiB staging").
pub staging_gib: f64,
}
/// The expert budget (GiB, down to 0.1) that leaves `room_mib` free besides what the card already
/// holds (`used_mib`), the LLM without experts, the STT's share and the engine's margin. The budget
/// first holds the staging, which the start without experts already has (M0 `expert_budget_gib`).
pub fn expert_budget_gib(total_mib: i64, used_mib: i64, p: &Probe, room_mib: i64) -> f64 {
let spare = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB - room_mib;
if spare <= 0 {
return 0.0;
}
((p.staging_gib * 1024.0 + spare as f64) / 1024.0 * 10.0).floor() / 10.0
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GamePlacement {
pub expert_gib: f64,
/// What a game really gets when the reserve does not fit beside the smallest placement.
pub game_room_mib: i64,
/// Not even the smallest placement leaves a game any room: the LLM pauses in a game.
pub paused: bool,
}
/// The normal placement: experts get what is left after the room for other programs opened later.
pub fn normal(total_mib: i64, used_mib: i64, p: &Probe, other_room_mib: i64) -> f64 {
expert_budget_gib(total_mib, used_mib, p, other_room_mib)
}
/// The game placement: experts get what is left after the game's reserve, but never less than the
/// smallest placement (2 GiB); then the game gets what that leaves (shown on the dashboard). When
/// 2 GiB leaves a game nothing, no experts on the card (0 GiB) is the last step; with no room even
/// then the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
pub fn game(total_mib: i64, used_mib: i64, p: &Probe, reserve_mib: i64) -> GamePlacement {
let expert = expert_budget_gib(total_mib, used_mib, p, reserve_mib);
if expert >= MIN_GAME_EXPERT_GIB {
return GamePlacement { expert_gib: expert, game_room_mib: reserve_mib, paused: false };
}
// Beside the model without experts (which already holds the staging): what 2 GiB adds.
let free = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB;
let extra = ((MIN_GAME_EXPERT_GIB - p.staging_gib) * 1024.0).max(0.0) as i64;
if free - extra > 0 {
return GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: free - extra, paused: false };
}
// The last step before pausing: no experts on the card (slower, but the game gets room).
GamePlacement { expert_gib: 0.0, game_room_mib: free.max(0), paused: free <= 0 }
}
/// The staging the engine log reports ("(1.10 GiB staging)"), else the usual 1.1 GiB.
pub fn staging_gib(lines: &str) -> f64 {
lines.lines().rev().find_map(|l| {
let i = l.find(" GiB staging")?;
l[..i].rsplit(|c: char| !(c.is_ascii_digit() || c == '.')).next()?.parse().ok()
}).unwrap_or(1.1)
}
#[cfg(test)]
mod tests {
use super::*;
/// The game PC in M0 `gamemode_tune` (RTX 5070 Ti 16 GB): 2760 MiB held before the LLM there
/// included the STT model (1300 MiB), which the app starts after the LLM.
const TOTAL: i64 = 16303;
const OTHERS: i64 = 2760 - 1300;
fn probe32() -> Probe {
Probe { context: 32768, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }
}
#[test]
fn the_normal_placement_is_the_one_m0_measured_with_room_for_the_stt() {
// M0 got 1.5 GiB (the STT model counted as already held); the app counts it in the STT share.
assert_eq!(normal(TOTAL, OTHERS, &probe32(), 1024), 1.5);
// More room for other programs, fewer experts; no room at all, none.
assert!(normal(TOTAL, OTHERS, &probe32(), 2048) < 1.5);
assert_eq!(normal(TOTAL, OTHERS + 8000, &probe32(), 1024), 0.0);
}
#[test]
fn a_game_gets_its_reserve_or_what_the_smallest_placement_leaves_or_the_llm_pauses() {
// 16K saves about 0.7 GB of conversation memory (M0).
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..probe32() };
// 8 GB does not fit on 16 GB: the smallest placement, and the game gets what is left.
let g = game(TOTAL, OTHERS, &probe16, 8192);
assert_eq!((g.expert_gib, g.paused), (MIN_GAME_EXPERT_GIB, false));
let expected = TOTAL - OTHERS - probe16.model_mib - STT_MIB - MARGIN_MIB - ((2.0 - 1.1) * 1024.0) as i64;
assert_eq!(g.game_room_mib, expected);
assert!(g.game_room_mib > 0 && g.game_room_mib < 8192);
// A bigger card keeps the whole reserve, with more experts than the minimum.
let big = game(24 * 1024, OTHERS, &probe16, 8192);
assert_eq!(big.game_room_mib, 8192);
assert!(big.expert_gib > MIN_GAME_EXPERT_GIB && !big.paused);
// No room beside 2 GiB but some beside none: the 0 GiB placement, before pausing.
let tight = game(TOTAL, 2960, &Probe { model_mib: 9124, ..probe16.clone() }, 8192);
assert_eq!((tight.expert_gib, tight.paused), (0.0, false));
assert_eq!(tight.game_room_mib, TOTAL - 2960 - 9124 - STT_MIB - MARGIN_MIB);
// Other programs already hold the card: no room for a game, the LLM pauses in one.
let full = game(TOTAL, OTHERS + 4000, &probe16, 8192);
assert!(full.paused);
assert_eq!(full.game_room_mib, 0);
}
#[test]
fn the_staging_comes_from_the_engine_log() {
assert_eq!(staging_gib("strata-q36: hybrid experts: 5 of 256 per layer in VRAM (0.37 GiB), bigger prompt chunks streamed (1.10 GiB staging); 7.1 s\n"), 1.1);
assert_eq!(staging_gib("x\nstrata-q36: budget 2.00 GiB (1.25 GiB staging)"), 1.25);
assert_eq!(staging_gib("no such line"), 1.1);
}
}