fix(app): the LLM placement measures only what it can trust and keeps a game the 0 GiB step

- a measurement whose short answer fails, or whose second card reading fails or shows nothing
  more held, is not kept: the model starts with expert-vram=0 and the next start measures again
- measurements are kept per context and requests at once (<context>/<parallel>) and per card
  (its NVIDIA UUID), not by the card's size
- a server on this app's port left from a run that ended badly is stopped before the card is
  read, so its memory is not counted as other programs'
- the game placement falls back to 0 GiB of experts before pausing when 2 GiB leaves a game
  nothing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-14 10:01:53 +09:00
parent af6f2c855d
commit 1400a3ff97
4 changed files with 230 additions and 59 deletions

View File

@@ -19,12 +19,14 @@ pub const MIN_GAME_EXPERT_GIB: f64 = 2.0;
pub const GAME_CONTEXT: u32 = 16384;
/// What one start without experts at `context` showed on this PC (the first-run self-check).
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Probe {
pub context: u32,
/// The card's size when it was measured (another card means measuring again).
pub total_mib: i64,
/// Requests answered at once, which changes the engine's buffers.
pub parallel: u32,
/// The card measured (its NVIDIA UUID): another card means measuring again.
pub gpu: String,
/// What the LLM held without experts.
pub model_mib: i64,
/// The long-prompt staging the expert budget holds first (engine log, "x.xx GiB staging").
@@ -58,16 +60,22 @@ pub fn normal(total_mib: i64, used_mib: i64, p: &Probe, other_room_mib: i64) ->
}
/// The game placement: experts get what is left after the game's reserve, but never less than the
/// smallest placement; then the game gets what that leaves (shown on the dashboard), and with no
/// room at all the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
/// smallest placement (2 GiB); then the game gets what that leaves (shown on the dashboard). When
/// 2 GiB leaves a game nothing, no experts on the card (0 GiB) is the last step; with no room even
/// then the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
pub fn game(total_mib: i64, used_mib: i64, p: &Probe, reserve_mib: i64) -> GamePlacement {
let expert = expert_budget_gib(total_mib, used_mib, p, reserve_mib);
if expert >= MIN_GAME_EXPERT_GIB {
return GamePlacement { expert_gib: expert, game_room_mib: reserve_mib, paused: false };
}
// Beside the model without experts (which already holds the staging): what 2 GiB adds.
let free = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB;
let extra = ((MIN_GAME_EXPERT_GIB - p.staging_gib) * 1024.0).max(0.0) as i64;
let room = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB - extra;
GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: room.max(0), paused: room <= 0 }
if free - extra > 0 {
return GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: free - extra, paused: false };
}
// The last step before pausing: no experts on the card (slower, but the game gets room).
GamePlacement { expert_gib: 0.0, game_room_mib: free.max(0), paused: free <= 0 }
}
/// The staging the engine log reports ("(1.10 GiB staging)"), else the usual 1.1 GiB.
@@ -86,21 +94,23 @@ mod tests {
/// included the STT model (1300 MiB), which the app starts after the LLM.
const TOTAL: i64 = 16303;
const OTHERS: i64 = 2760 - 1300;
const PROBE32: Probe = Probe { context: 32768, total_mib: TOTAL, model_mib: 9874, staging_gib: 1.1 };
fn probe32() -> Probe {
Probe { context: 32768, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }
}
#[test]
fn the_normal_placement_is_the_one_m0_measured_with_room_for_the_stt() {
// M0 got 1.5 GiB (the STT model counted as already held); the app counts it in the STT share.
assert_eq!(normal(TOTAL, OTHERS, &PROBE32, 1024), 1.5);
assert_eq!(normal(TOTAL, OTHERS, &probe32(), 1024), 1.5);
// More room for other programs, fewer experts; no room at all, none.
assert!(normal(TOTAL, OTHERS, &PROBE32, 2048) < 1.5);
assert_eq!(normal(TOTAL, OTHERS + 8000, &PROBE32, 1024), 0.0);
assert!(normal(TOTAL, OTHERS, &probe32(), 2048) < 1.5);
assert_eq!(normal(TOTAL, OTHERS + 8000, &probe32(), 1024), 0.0);
}
#[test]
fn a_game_gets_its_reserve_or_what_the_smallest_placement_leaves_or_the_llm_pauses() {
// 16K saves about 0.7 GB of conversation memory (M0).
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..PROBE32 };
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..probe32() };
// 8 GB does not fit on 16 GB: the smallest placement, and the game gets what is left.
let g = game(TOTAL, OTHERS, &probe16, 8192);
assert_eq!((g.expert_gib, g.paused), (MIN_GAME_EXPERT_GIB, false));
@@ -108,11 +118,15 @@ mod tests {
assert_eq!(g.game_room_mib, expected);
assert!(g.game_room_mib > 0 && g.game_room_mib < 8192);
// A bigger card keeps the whole reserve, with more experts than the minimum.
let big = game(24 * 1024, OTHERS, &Probe { total_mib: 24 * 1024, ..probe16 }, 8192);
let big = game(24 * 1024, OTHERS, &probe16, 8192);
assert_eq!(big.game_room_mib, 8192);
assert!(big.expert_gib > MIN_GAME_EXPERT_GIB && !big.paused);
// No room beside 2 GiB but some beside none: the 0 GiB placement, before pausing.
let tight = game(TOTAL, 2960, &Probe { model_mib: 9124, ..probe16.clone() }, 8192);
assert_eq!((tight.expert_gib, tight.paused), (0.0, false));
assert_eq!(tight.game_room_mib, TOTAL - 2960 - 9124 - STT_MIB - MARGIN_MIB);
// Other programs already hold the card: no room for a game, the LLM pauses in one.
let full = game(TOTAL, OTHERS + 3000, &probe16, 8192);
let full = game(TOTAL, OTHERS + 4000, &probe16, 8192);
assert!(full.paused);
assert_eq!(full.game_room_mib, 0);
}