Compare commits

..

2 Commits

Author SHA1 Message Date
javis-bot
1c06b16bd4 chore(app): version 0.1.25
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 15:48:01 +09:00
javis-bot
44387685af fix(app): log the card's real free memory, and short answers get their line too
Free memory is nvidia-smi's memory.free in the app and the voice worker, so the
driver's reserved share no longer counts as free. A card reading still running
when an answer ends finishes on its own and then logs, so an answer shorter
than one reading is logged too, and the answer still never waits for it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 15:48:01 +09:00
10 changed files with 94 additions and 39 deletions

2
app/Cargo.lock generated
View File

@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.24"
version = "0.1.25"
dependencies = [
"async-trait",
"base64 0.22.1",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-app"
version = "0.1.24"
version = "0.1.25"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"

4
app/shell/Cargo.lock generated
View File

@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
name = "jarvis-app"
version = "0.1.24"
version = "0.1.25"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2666,7 +2666,7 @@ dependencies = [
[[package]]
name = "jarvis-shell"
version = "0.1.24"
version = "0.1.25"
dependencies = [
"jarvis-app",
"serde_json",

View File

@@ -1,6 +1,6 @@
[package]
name = "jarvis-shell"
version = "0.1.24"
version = "0.1.25"
edition = "2021"
rust-version = "1.85"
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"

View File

@@ -709,7 +709,7 @@ start it first, so nothing answers before the brain can, and stop it last.
counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and
the model starts with `expert-vram=0`. Without a card nothing is set. The game
placement (used by the switching that comes next) takes the reserve when it fits beside more
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`card after the start: N MiB free of T MiB`).
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`memory.free`, without what the driver reserves: `card after the start: N MiB free of T MiB`).
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
settings file only) the app starts nothing; that server must list a model within 30 seconds.
- **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port`

View File

@@ -514,7 +514,7 @@ mod perf_tests {
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
Some(crate::strata::Gpu { id: "GPU-test".into(), used_mib: o2.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: 24576 })
Some(crate::strata::tests::gpu("GPU-test", o2.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, 24576))
})));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();

View File

@@ -116,6 +116,8 @@ pub struct Gpu {
pub id: String,
pub used_mib: i64,
pub total_mib: i64,
/// What is left for programs (`memory.free`): the total less what is used and what the driver reserves.
pub free_mib: i64,
}
/// Reads the card (`nvidia-smi`); None without one.
@@ -123,7 +125,7 @@ pub type GpuReader = Arc<dyn Fn() -> Option<Gpu> + Send + Sync>;
fn nvidia_memory() -> Option<Gpu> {
let mut cmd = std::process::Command::new("nvidia-smi");
cmd.args(["--query-gpu=uuid,memory.used,memory.total", "--format=csv,noheader,nounits"]);
cmd.args(["--query-gpu=uuid,memory.used,memory.total,memory.free", "--format=csv,noheader,nounits"]);
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
@@ -132,7 +134,8 @@ fn nvidia_memory() -> Option<Gpu> {
let out = cmd.output().ok()?;
let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned();
let mut f = line.split(',').map(str::trim);
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()? })
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()?,
free_mib: f.next()?.parse().ok()? })
}
/// The newest engine log under Strata's folder, its end only (the placement lines of the last start).
@@ -509,7 +512,7 @@ impl Llm {
// What the card has left for everything else once the model is in (the voice
// worker's models, a game): a slow answer later can be matched to it.
if let Some(g) = self.read_gpu().await {
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.total_mib - g.used_mib, g.total_mib);
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.free_mib, g.total_mib);
}
self.set(LlmState::Ready, None);
}
@@ -656,6 +659,11 @@ pub(crate) mod tests {
/// Other programs on the fake cards: the game PC's 1460 MiB in M0 less the TTS's share, which
/// the app keeps beside and M0 did not run, so M0's placements come out.
/// A card reading as `nvidia-smi` gives it, with nothing reserved by the driver.
pub(crate) fn gpu(id: &str, used_mib: i64, total_mib: i64) -> Gpu {
Gpu { id: id.into(), used_mib, total_mib, free_mib: total_mib - used_mib }
}
pub(crate) const OTHERS: i64 = 1460 - crate::placement::TTS_MIB;
pub(crate) fn layout(tag: &str) -> Layout {
@@ -918,10 +926,14 @@ pub(crate) mod tests {
crate::logs::install(crate::logs::LogHub::new(None));
let l = layout("llm-free");
let port = server_after_launch(&l, "list").await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
// The driver keeps some of the card for itself (`memory.reserved`): it is neither used nor free.
let inner = card(&l, "GPU-test", 0);
let reserved = 444;
let reader: GpuReader = Arc::new(move || inner().map(|g| Gpu { free_mib: g.free_mib - reserved, ..g }));
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(reader);
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let free = 16303 - OTHERS - 9874;
let free = 16303 - OTHERS - 9874 - reserved;
let lines = crate::logs::installed().unwrap()
.query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000);
assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))),
@@ -940,7 +952,7 @@ pub(crate) mod tests {
let stopped = c.contains("launch --stop");
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
let used = OTHERS + if running { 9874 } else if stopped { 0 } else { leftover };
Some(Gpu { id: id.into(), used_mib: used, total_mib: 16303 })
Some(gpu(id, used, 16303))
})
}
@@ -974,7 +986,7 @@ pub(crate) mod tests {
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
let r2 = reads.clone();
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: OTHERS, total_mib: 16303 })
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| gpu("GPU-test", OTHERS, 16303))
}));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
@@ -1080,7 +1092,7 @@ pub(crate) mod tests {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
let used = others.load(Ordering::SeqCst) + worker + if running { 9874 } else { 0 };
Some(Gpu { id: "GPU-test".into(), used_mib: used, total_mib: 16303 })
Some(gpu("GPU-test", used, 16303))
})
}
@@ -1138,7 +1150,7 @@ pub(crate) mod tests {
Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
Some(Gpu { id: "GPU-test".into(), used_mib: others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: total })
Some(gpu("GPU-test", others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total))
})
}
@@ -1191,7 +1203,7 @@ pub(crate) mod tests {
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
let r2 = reads.clone();
let llm = llm.with_gpu(Arc::new(move || {
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: OTHERS, total_mib: 16303 })
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| gpu("GPU-test", OTHERS, 16303))
}));
assert_eq!(llm.switch(&s, Stage::Gaming, 0, true).await.unwrap(), Stage::Gaming);
assert!(launches(&l).last().unwrap().contains("--set expert-vram=0 "), "{:?}", launches(&l));
@@ -1208,7 +1220,7 @@ pub(crate) mod tests {
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
Some(Gpu { id: "GPU-small".into(), used_mib: OTHERS + if running { 9874 } else { 0 }, total_mib: 12288 })
Some(gpu("GPU-small", OTHERS + if running { 9874 } else { 0 }, 12288))
}));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();

View File

@@ -45,14 +45,15 @@ LLM_WAIT_S = 180.0
def nvidia_card() -> tuple[int, int] | None:
"""The first NVIDIA card's used and total memory in MiB (``nvidia-smi``), None when unreadable."""
"""The first NVIDIA card's free and total memory in MiB (``nvidia-smi``; free leaves out what the
driver reserves), None when unreadable."""
import subprocess
try:
out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"],
out = subprocess.run(["nvidia-smi", "--query-gpu=memory.free,memory.total", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=5,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout
used, total = (int(v) for v in out.splitlines()[0].split(","))
return used, total
free, total = (int(v) for v in out.splitlines()[0].split(","))
return free, total
except Exception:
return None
@@ -120,7 +121,8 @@ class VoiceWorker:
self.history = history # jarvis.memory.history.HistoryStore or None
self.recorder = recorder # recordings.Recorder (voice.record on) or None
self.brain = brain
self.card = card # (used MiB, total MiB) of the graphics card, or None without one
self.card = card # (free MiB, total MiB) of the graphics card, or None without one
self.card_watches: set[asyncio.Task] = set() # readings still running after their answer
# One reply at a time per lane: voice answers never wait behind another person's chat answer
# (conversation.md: with an "answer everything" chat channel the LLM takes two or more
# requests at once). One person's replies stay one at a time across lanes, so their dialogue
@@ -771,30 +773,42 @@ class Session:
@contextlib.asynccontextmanager
async def _card_watch(self, label: str):
"""While an answer is written: the card's free memory, read every `CARD_EVERY_S`; the least
seen is logged once it ends (a slow answer can then be matched to a full card)."""
card = self.worker.card
seen is logged once it ends (a slow answer can then be matched to a full card). The answer
never waits for a reading: one still running when it ends finishes on its own, then logs."""
w = self.worker
card = w.card
if card is None:
yield
return
seen: list[tuple[int, int]] = []
ended = asyncio.Event()
t0, took = time.monotonic(), [0]
async def read():
async def watch():
while True:
got = await asyncio.to_thread(card)
if got:
seen.append(got)
await asyncio.sleep(CARD_EVERY_S)
if ended.is_set():
break
try:
await asyncio.wait_for(ended.wait(), CARD_EVERY_S)
break
except asyncio.TimeoutError:
pass
if seen:
total = seen[-1][1]
notice(f"{label} card: least {min(f for f, _ in seen)} MiB free of {total} MiB "
f"({len(seen)} readings, {took[0]} ms)", "llm")
t0 = time.monotonic()
task = asyncio.get_running_loop().create_task(read())
task = asyncio.get_running_loop().create_task(watch())
w.card_watches.add(task)
task.add_done_callback(w.card_watches.discard)
try:
yield
finally:
task.cancel()
if seen:
total = seen[-1][1]
notice(f"{label} card: least {min(total - u for u, _ in seen)} MiB free of {total} MiB "
f"({len(seen)} readings, {int((time.monotonic() - t0) * 1000)} ms)", "llm")
took[0] = int((time.monotonic() - t0) * 1000)
ended.set()
def _on_prepare(self, msg: dict) -> None:
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;

View File

@@ -535,6 +535,8 @@ in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches
Without debug logging too, each answer and each scheduled task logs one `llm` line once the LLM is
done with it: the least free graphics memory read while it was written, the card's total, how many
readings and how long it took (`answer <id> card: least N MiB free of T MiB (k readings, ms)`). The
card is read with `nvidia-smi` once a second while the answer is written, in a thread, so the answer
never waits for it; without `nvidia-smi` nothing is read or logged. The line carries no text, so a
card is read with `nvidia-smi` (`memory.free`, which leaves out what the driver reserves) once a
second while the answer is written, in a thread, so the answer never waits for it: a reading still
running when the answer ends finishes on its own and the line follows it, so a short answer gets its
line too. Without `nvidia-smi` nothing is read or logged. The line carries no text, so a
slow answer can be matched to a full card (the engine log has the same answer's timings).

View File

@@ -1578,8 +1578,9 @@ def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(st
import time as _time
from voice_worker import server as worker_server
monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02)
readings = iter([9000, 15000, 12000])
last = [12000]
# Free memory as nvidia-smi reports it (the driver's reserved share is not free).
readings = iter([7000, 859, 3000])
last = [3000]
def card():
last[0] = next(readings, last[0])
@@ -1602,7 +1603,7 @@ def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(st
reply = [x for x in lines if "answer r1" in x]
task = [x for x in lines if "task t1" in x]
assert len(reply) == 1 and len(task) == 1, lines
assert f"least {16303 - 15000} MiB free of 16303 MiB" in reply[0], reply
assert "least 859 MiB free of 16303 MiB" in reply[0], reply
assert "private" not in "".join(lines) and "secret" not in "".join(lines)
@@ -1616,3 +1617,29 @@ def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd):
asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None))
assert "MiB free" not in capfd.readouterr().err
@pytest.mark.unit
def test_an_answer_done_before_the_first_card_reading_is_logged_once_the_reading_ends(store, capfd):
"""A short answer is not held for the card, and still gets its line once the reading ends."""
import threading
reading = threading.Event()
def card():
reading.wait(5)
return 2048, 16303
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
before = capfd.readouterr().err
reading.set()
await asyncio.sleep(0.3)
return before
before = asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=card))
after = capfd.readouterr().err
assert "answer r1 card" not in before, "the answer did not wait for the card"
assert "answer r1 card: least 2048 MiB free of 16303 MiB (1 readings" in after, after[-1500:]