fix(app): the LLM placement measures only what it can trust and keeps a game the 0 GiB step
- a measurement whose short answer fails, or whose second card reading fails or shows nothing more held, is not kept: the model starts with expert-vram=0 and the next start measures again - measurements are kept per context and requests at once (<context>/<parallel>) and per card (its NVIDIA UUID), not by the card's size - a server on this app's port left from a run that ended badly is stopped before the card is read, so its memory is not counted as other programs' - the game placement falls back to 0 GiB of experts before pausing when 2 GiB leaves a game nothing Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -19,12 +19,14 @@ pub const MIN_GAME_EXPERT_GIB: f64 = 2.0;
|
||||
pub const GAME_CONTEXT: u32 = 16384;
|
||||
|
||||
/// What one start without experts at `context` showed on this PC (the first-run self-check).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Probe {
|
||||
pub context: u32,
|
||||
/// The card's size when it was measured (another card means measuring again).
|
||||
pub total_mib: i64,
|
||||
/// Requests answered at once, which changes the engine's buffers.
|
||||
pub parallel: u32,
|
||||
/// The card measured (its NVIDIA UUID): another card means measuring again.
|
||||
pub gpu: String,
|
||||
/// What the LLM held without experts.
|
||||
pub model_mib: i64,
|
||||
/// The long-prompt staging the expert budget holds first (engine log, "x.xx GiB staging").
|
||||
@@ -58,16 +60,22 @@ pub fn normal(total_mib: i64, used_mib: i64, p: &Probe, other_room_mib: i64) ->
|
||||
}
|
||||
|
||||
/// The game placement: experts get what is left after the game's reserve, but never less than the
|
||||
/// smallest placement; then the game gets what that leaves (shown on the dashboard), and with no
|
||||
/// room at all the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
|
||||
/// smallest placement (2 GiB); then the game gets what that leaves (shown on the dashboard). When
|
||||
/// 2 GiB leaves a game nothing, no experts on the card (0 GiB) is the last step; with no room even
|
||||
/// then the LLM pauses. `p` is the probe at `GAME_CONTEXT`.
|
||||
pub fn game(total_mib: i64, used_mib: i64, p: &Probe, reserve_mib: i64) -> GamePlacement {
|
||||
let expert = expert_budget_gib(total_mib, used_mib, p, reserve_mib);
|
||||
if expert >= MIN_GAME_EXPERT_GIB {
|
||||
return GamePlacement { expert_gib: expert, game_room_mib: reserve_mib, paused: false };
|
||||
}
|
||||
// Beside the model without experts (which already holds the staging): what 2 GiB adds.
|
||||
let free = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB;
|
||||
let extra = ((MIN_GAME_EXPERT_GIB - p.staging_gib) * 1024.0).max(0.0) as i64;
|
||||
let room = total_mib - used_mib - p.model_mib - STT_MIB - MARGIN_MIB - extra;
|
||||
GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: room.max(0), paused: room <= 0 }
|
||||
if free - extra > 0 {
|
||||
return GamePlacement { expert_gib: MIN_GAME_EXPERT_GIB, game_room_mib: free - extra, paused: false };
|
||||
}
|
||||
// The last step before pausing: no experts on the card (slower, but the game gets room).
|
||||
GamePlacement { expert_gib: 0.0, game_room_mib: free.max(0), paused: free <= 0 }
|
||||
}
|
||||
|
||||
/// The staging the engine log reports ("(1.10 GiB staging)"), else the usual 1.1 GiB.
|
||||
@@ -86,21 +94,23 @@ mod tests {
|
||||
/// included the STT model (1300 MiB), which the app starts after the LLM.
|
||||
const TOTAL: i64 = 16303;
|
||||
const OTHERS: i64 = 2760 - 1300;
|
||||
const PROBE32: Probe = Probe { context: 32768, total_mib: TOTAL, model_mib: 9874, staging_gib: 1.1 };
|
||||
fn probe32() -> Probe {
|
||||
Probe { context: 32768, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_normal_placement_is_the_one_m0_measured_with_room_for_the_stt() {
|
||||
// M0 got 1.5 GiB (the STT model counted as already held); the app counts it in the STT share.
|
||||
assert_eq!(normal(TOTAL, OTHERS, &PROBE32, 1024), 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS, &probe32(), 1024), 1.5);
|
||||
// More room for other programs, fewer experts; no room at all, none.
|
||||
assert!(normal(TOTAL, OTHERS, &PROBE32, 2048) < 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS + 8000, &PROBE32, 1024), 0.0);
|
||||
assert!(normal(TOTAL, OTHERS, &probe32(), 2048) < 1.5);
|
||||
assert_eq!(normal(TOTAL, OTHERS + 8000, &probe32(), 1024), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_game_gets_its_reserve_or_what_the_smallest_placement_leaves_or_the_llm_pauses() {
|
||||
// 16K saves about 0.7 GB of conversation memory (M0).
|
||||
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..PROBE32 };
|
||||
let probe16 = Probe { context: GAME_CONTEXT, model_mib: 9874 - 750, ..probe32() };
|
||||
// 8 GB does not fit on 16 GB: the smallest placement, and the game gets what is left.
|
||||
let g = game(TOTAL, OTHERS, &probe16, 8192);
|
||||
assert_eq!((g.expert_gib, g.paused), (MIN_GAME_EXPERT_GIB, false));
|
||||
@@ -108,11 +118,15 @@ mod tests {
|
||||
assert_eq!(g.game_room_mib, expected);
|
||||
assert!(g.game_room_mib > 0 && g.game_room_mib < 8192);
|
||||
// A bigger card keeps the whole reserve, with more experts than the minimum.
|
||||
let big = game(24 * 1024, OTHERS, &Probe { total_mib: 24 * 1024, ..probe16 }, 8192);
|
||||
let big = game(24 * 1024, OTHERS, &probe16, 8192);
|
||||
assert_eq!(big.game_room_mib, 8192);
|
||||
assert!(big.expert_gib > MIN_GAME_EXPERT_GIB && !big.paused);
|
||||
// No room beside 2 GiB but some beside none: the 0 GiB placement, before pausing.
|
||||
let tight = game(TOTAL, 2960, &Probe { model_mib: 9124, ..probe16.clone() }, 8192);
|
||||
assert_eq!((tight.expert_gib, tight.paused), (0.0, false));
|
||||
assert_eq!(tight.game_room_mib, TOTAL - 2960 - 9124 - STT_MIB - MARGIN_MIB);
|
||||
// Other programs already hold the card: no room for a game, the LLM pauses in one.
|
||||
let full = game(TOTAL, OTHERS + 3000, &probe16, 8192);
|
||||
let full = game(TOTAL, OTHERS + 4000, &probe16, 8192);
|
||||
assert!(full.paused);
|
||||
assert_eq!(full.game_room_mib, 0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user