From f1600f09c25927827ba21cb17296869faf8a989e Mon Sep 17 00:00:00 2001 From: javis-bot Date: Tue, 17 Nov 2026 15:40:20 +0900 Subject: [PATCH] feat(app): log the graphics card's free memory after the LLM starts and while each answer is written The app logs how much of the card is free once the model answers, and the voice worker reads the card once a second while an answer or a scheduled task is written and logs the least free memory seen (no text, llm tab), so a slow answer can be matched to a full card. Co-Authored-By: Claude Opus 5.5 --- app/src/admin.spec.md | 2 +- app/src/strata.rs | 21 +++++++++++ src/voice_worker/__main__.py | 6 ++-- src/voice_worker/server.py | 52 ++++++++++++++++++++++++--- src/voice_worker/voice_worker.spec.md | 7 ++++ tests/test_voice_worker_brain.py | 48 +++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 7 deletions(-) diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 1fdb6c1..18e9943 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -709,7 +709,7 @@ start it first, so nothing answers before the brain can, and stop it last. counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and the model starts with `expert-vram=0`. Without a card nothing is set. The game placement (used by the switching that comes next) takes the reserve when it fits beside more - than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from. + than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`card after the start: N MiB free of T MiB`). - **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the settings file only) the app starts nothing; that server must list a model within 30 seconds. - **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port` diff --git a/app/src/strata.rs b/app/src/strata.rs index 1095d02..5767100 100644 --- a/app/src/strata.rs +++ b/app/src/strata.rs @@ -506,6 +506,11 @@ impl Llm { Ok(None) => {} Ok(Some((base, model))) => { crate::info_log!("llm", "ready: {base} model={model}"); + // What the card has left for everything else once the model is in (the voice + // worker's models, a game): a slow answer later can be matched to it. + if let Some(g) = self.read_gpu().await { + crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.total_mib - g.used_mib, g.total_mib); + } self.set(LlmState::Ready, None); } Err(e) if e == "cancelled" => self.set(LlmState::Off, None), @@ -908,6 +913,22 @@ pub(crate) mod tests { assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]); } + #[tokio::test] + async fn the_llm_log_tells_how_much_of_the_card_is_free_once_the_model_answers() { + crate::logs::install(crate::logs::LogHub::new(None)); + let l = layout("llm-free"); + let port = server_after_launch(&l, "list").await; + let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)); + let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); + tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap(); + let free = 16303 - OTHERS - 9874; + let lines = crate::logs::installed().unwrap() + .query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000); + assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))), + "{:?}", lines.iter().map(|x| &x.text).collect::>()); + llm.stop().await.unwrap(); + } + /// A fake card for `fake_placed`: 16303 MiB, `OTHERS` held by other programs (and by any /// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a /// Strata server runs (the calls file shows the last launch and stop). diff --git a/src/voice_worker/__main__.py b/src/voice_worker/__main__.py index 696d73f..adbadd8 100644 --- a/src/voice_worker/__main__.py +++ b/src/voice_worker/__main__.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import asyncio import os +import shutil import sys from .log import debug_log @@ -42,7 +43,7 @@ def main() -> int: from .brain import take_mcp_env mcp_env = take_mcp_env() - from .server import VoiceWorker, is_loopback + from .server import VoiceWorker, is_loopback, nvidia_card from .stt_whisper import WhisperTranscriber from .vad import SileroVad @@ -81,7 +82,8 @@ def main() -> int: recorder = Recorder(os.environ[RECORDINGS_ENV]) async def run() -> None: - worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder) + worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder, + card=nvidia_card if shutil.which("nvidia-smi") else None) server = await worker.serve(args.host, args.port) debug_log(f"voice worker listening on {args.host}:{args.port} tts={'on' if tts else 'off'} " f"history={'on' if history else 'off'} brain={'on' if brain else 'off'} " diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index 4fc095a..c24bfd5 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -15,7 +15,7 @@ from typing import Callable import numpy as np -from .log import debug_log +from .log import debug_log, notice from .audio import InputConverter, loudness_equalise from .brain import Brain, MemorySettings @@ -44,6 +44,19 @@ def is_loopback(host: str) -> bool: LLM_WAIT_S = 180.0 +def nvidia_card() -> tuple[int, int] | None: + """The first NVIDIA card's used and total memory in MiB (``nvidia-smi``), None when unreadable.""" + import subprocess + try: + out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout + used, total = (int(v) for v in out.splitlines()[0].split(",")) + return used, total + except Exception: + return None + + class _LlmPaused(Exception): """The app paused the LLM (a game runs): nothing is answered until it is ready again.""" @@ -53,6 +66,7 @@ MAX_NOTES = 20 MAX_NOTE_CHARS = 300 # How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts. TRACE_KEEP_DAYS = 30 +CARD_EVERY_S = 1.0 # how often the card's free memory is read while an answer is written class _LiveTrace(list): @@ -93,7 +107,8 @@ class VoiceWorker: def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream], tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05, history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0, - delivery_timeout_s: float = 30.0, recorder=None): + delivery_timeout_s: float = 30.0, recorder=None, + card: Callable[[], tuple[int, int] | None] | None = None): if not token: raise ValueError("a connection token is required") self.token = token @@ -105,6 +120,7 @@ class VoiceWorker: self.history = history # jarvis.memory.history.HistoryStore or None self.recorder = recorder # recordings.Recorder (voice.record on) or None self.brain = brain + self.card = card # (used MiB, total MiB) of the graphics card, or None without one # One reply at a time per lane: voice answers never wait behind another person's chat answer # (conversation.md: with an "answer everything" chat channel the LLM takes two or more # requests at once). One person's replies stay one at a time across lanes, so their dialogue @@ -687,7 +703,7 @@ class Session: history = HistoryContext() try: trace.append(self._records_step(history, notes)) - async with self._llm(): + async with self._llm(), self._card_watch(f"task {rid}"): answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True, notes=notes or [], trace=trace) except Exception as e: # the brain must never take the session down @@ -752,6 +768,34 @@ class Session: w.llm_active -= 1 self._llm_check_idle() + @contextlib.asynccontextmanager + async def _card_watch(self, label: str): + """While an answer is written: the card's free memory, read every `CARD_EVERY_S`; the least + seen is logged once it ends (a slow answer can then be matched to a full card).""" + card = self.worker.card + if card is None: + yield + return + seen: list[tuple[int, int]] = [] + + async def read(): + while True: + got = await asyncio.to_thread(card) + if got: + seen.append(got) + await asyncio.sleep(CARD_EVERY_S) + + t0 = time.monotonic() + task = asyncio.get_running_loop().create_task(read()) + try: + yield + finally: + task.cancel() + if seen: + total = seen[-1][1] + notice(f"{label} card: least {min(total - u for u, _ in seen)} MiB free of {total} MiB " + f"({len(seen)} readings, {int((time.monotonic() - t0) * 1000)} ms)", "llm") + def _on_prepare(self, msg: dict) -> None: """A call still being said: the brain gets ready for it. One prepare per user runs at a time; while it runs only the newest text waits, and one about the same as the last one prepared @@ -909,7 +953,7 @@ class Session: screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin) trace.append(self._records_step(history, notes)) try: - async with self._llm(rid if speak else None): + async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"): w.answering += 1 try: answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat, diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index f617f6f..e7d02ee 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -531,3 +531,10 @@ cancel, and engine failures. The category names the service, so the app's log sc in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches, the Whisper model), `tts` (sentences, cancels, MeloTTS), `llm` (the brain's replies, prepares and judgements), `voice_worker` for the rest (connections, sessions, channels). + +Without debug logging too, each answer and each scheduled task logs one `llm` line once the LLM is +done with it: the least free graphics memory read while it was written, the card's total, how many +readings and how long it took (`answer card: least N MiB free of T MiB (k readings, ms)`). The +card is read with `nvidia-smi` once a second while the answer is written, in a thread, so the answer +never waits for it; without `nvidia-smi` nothing is read or logged. The line carries no text, so a +slow answer can be matched to a full card (the engine log has the same answer's timings). diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index 4393140..0f5d59f 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -1568,3 +1568,51 @@ def test_the_judgement_is_told_who_is_in_the_channel(monkeypatch): assert "Several people" in unknown and "anything unclear" in unknown, "an app that does not say: as before" assert "one other person" in pair and "unclear, answer yes" in pair assert "5 people" in group and "Only 1 of the 5" in group and "anything unclear" in group + + +@pytest.mark.unit +def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(store, monkeypatch, capfd): + """voice_worker.spec.md, "Debug logging": an answer (and a scheduled task) logs the least free + graphics memory read while the LLM wrote it, without debug logging and without its text, so a + slow answer can be matched to a full card.""" + import time as _time + from voice_worker import server as worker_server + monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02) + readings = iter([9000, 15000, 12000]) + last = [12000] + + def card(): + last[0] = next(readings, last[0]) + return last[0], 16303 + + def slow(user, text): + _time.sleep(0.3) + return f"secret answer for {text}" + + async def body(url): + async with connect(url) as ws: + await hello(ws) + await send(ws, type="reply", id="r1", user="100", text="private question", via="chat", speak=False) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") + await send(ws, type="reply", id="t1", user="100", text="private task", via="task") + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text" and m["id"] == "t1") + + asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), card=card)) + lines = [x for x in capfd.readouterr().err.splitlines() if x.startswith(f"[{'llm':^10}]") and "free" in x] + reply = [x for x in lines if "answer r1" in x] + task = [x for x in lines if "task t1" in x] + assert len(reply) == 1 and len(task) == 1, lines + assert f"least {16303 - 15000} MiB free of 16303 MiB" in reply[0], reply + assert "private" not in "".join(lines) and "secret" not in "".join(lines) + + +@pytest.mark.unit +def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd): + async def body(url): + async with connect(url) as ws: + await hello(ws) + await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") + + asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None)) + assert "MiB free" not in capfd.readouterr().err