feat(app): log the graphics card's free memory after the LLM starts and while each answer is written

The app logs how much of the card is free once the model answers, and the
voice worker reads the card once a second while an answer or a scheduled task
is written and logs the least free memory seen (no text, llm tab), so a slow
answer can be matched to a full card.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-17 15:40:20 +09:00
parent ce03424ba4
commit f1600f09c2
6 changed files with 129 additions and 7 deletions

View File

@@ -709,7 +709,7 @@ start it first, so nothing answers before the brain can, and stop it last.
counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and
the model starts with `expert-vram=0`. Without a card nothing is set. The game
placement (used by the switching that comes next) takes the reserve when it fits beside more
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from.
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`card after the start: N MiB free of T MiB`).
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
settings file only) the app starts nothing; that server must list a model within 30 seconds.
- **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port`

View File

@@ -506,6 +506,11 @@ impl Llm {
Ok(None) => {}
Ok(Some((base, model))) => {
crate::info_log!("llm", "ready: {base} model={model}");
// What the card has left for everything else once the model is in (the voice
// worker's models, a game): a slow answer later can be matched to it.
if let Some(g) = self.read_gpu().await {
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.total_mib - g.used_mib, g.total_mib);
}
self.set(LlmState::Ready, None);
}
Err(e) if e == "cancelled" => self.set(LlmState::Off, None),
@@ -908,6 +913,22 @@ pub(crate) mod tests {
assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]);
}
#[tokio::test]
async fn the_llm_log_tells_how_much_of_the_card_is_free_once_the_model_answers() {
crate::logs::install(crate::logs::LogHub::new(None));
let l = layout("llm-free");
let port = server_after_launch(&l, "list").await;
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
let free = 16303 - OTHERS - 9874;
let lines = crate::logs::installed().unwrap()
.query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000);
assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))),
"{:?}", lines.iter().map(|x| &x.text).collect::<Vec<_>>());
llm.stop().await.unwrap();
}
/// A fake card for `fake_placed`: 16303 MiB, `OTHERS` held by other programs (and by any
/// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a
/// Strata server runs (the calls file shows the last launch and stop).

View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import asyncio
import os
import shutil
import sys
from .log import debug_log
@@ -42,7 +43,7 @@ def main() -> int:
from .brain import take_mcp_env
mcp_env = take_mcp_env()
from .server import VoiceWorker, is_loopback
from .server import VoiceWorker, is_loopback, nvidia_card
from .stt_whisper import WhisperTranscriber
from .vad import SileroVad
@@ -81,7 +82,8 @@ def main() -> int:
recorder = Recorder(os.environ[RECORDINGS_ENV])
async def run() -> None:
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder)
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder,
card=nvidia_card if shutil.which("nvidia-smi") else None)
server = await worker.serve(args.host, args.port)
debug_log(f"voice worker listening on {args.host}:{args.port} tts={'on' if tts else 'off'} "
f"history={'on' if history else 'off'} brain={'on' if brain else 'off'} "

View File

@@ -15,7 +15,7 @@ from typing import Callable
import numpy as np
from .log import debug_log
from .log import debug_log, notice
from .audio import InputConverter, loudness_equalise
from .brain import Brain, MemorySettings
@@ -44,6 +44,19 @@ def is_loopback(host: str) -> bool:
LLM_WAIT_S = 180.0
def nvidia_card() -> tuple[int, int] | None:
"""The first NVIDIA card's used and total memory in MiB (``nvidia-smi``), None when unreadable."""
import subprocess
try:
out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=5,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout
used, total = (int(v) for v in out.splitlines()[0].split(","))
return used, total
except Exception:
return None
class _LlmPaused(Exception):
"""The app paused the LLM (a game runs): nothing is answered until it is ready again."""
@@ -53,6 +66,7 @@ MAX_NOTES = 20
MAX_NOTE_CHARS = 300
# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts.
TRACE_KEEP_DAYS = 30
CARD_EVERY_S = 1.0 # how often the card's free memory is read while an answer is written
class _LiveTrace(list):
@@ -93,7 +107,8 @@ class VoiceWorker:
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0,
delivery_timeout_s: float = 30.0, recorder=None):
delivery_timeout_s: float = 30.0, recorder=None,
card: Callable[[], tuple[int, int] | None] | None = None):
if not token:
raise ValueError("a connection token is required")
self.token = token
@@ -105,6 +120,7 @@ class VoiceWorker:
self.history = history # jarvis.memory.history.HistoryStore or None
self.recorder = recorder # recordings.Recorder (voice.record on) or None
self.brain = brain
self.card = card # (used MiB, total MiB) of the graphics card, or None without one
# One reply at a time per lane: voice answers never wait behind another person's chat answer
# (conversation.md: with an "answer everything" chat channel the LLM takes two or more
# requests at once). One person's replies stay one at a time across lanes, so their dialogue
@@ -687,7 +703,7 @@ class Session:
history = HistoryContext()
try:
trace.append(self._records_step(history, notes))
async with self._llm():
async with self._llm(), self._card_watch(f"task {rid}"):
answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True,
notes=notes or [], trace=trace)
except Exception as e: # the brain must never take the session down
@@ -752,6 +768,34 @@ class Session:
w.llm_active -= 1
self._llm_check_idle()
@contextlib.asynccontextmanager
async def _card_watch(self, label: str):
"""While an answer is written: the card's free memory, read every `CARD_EVERY_S`; the least
seen is logged once it ends (a slow answer can then be matched to a full card)."""
card = self.worker.card
if card is None:
yield
return
seen: list[tuple[int, int]] = []
async def read():
while True:
got = await asyncio.to_thread(card)
if got:
seen.append(got)
await asyncio.sleep(CARD_EVERY_S)
t0 = time.monotonic()
task = asyncio.get_running_loop().create_task(read())
try:
yield
finally:
task.cancel()
if seen:
total = seen[-1][1]
notice(f"{label} card: least {min(total - u for u, _ in seen)} MiB free of {total} MiB "
f"({len(seen)} readings, {int((time.monotonic() - t0) * 1000)} ms)", "llm")
def _on_prepare(self, msg: dict) -> None:
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;
while it runs only the newest text waits, and one about the same as the last one prepared
@@ -909,7 +953,7 @@ class Session:
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
trace.append(self._records_step(history, notes))
try:
async with self._llm(rid if speak else None):
async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"):
w.answering += 1
try:
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,

View File

@@ -531,3 +531,10 @@ cancel, and engine failures. The category names the service, so the app's log sc
in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches, the Whisper model),
`tts` (sentences, cancels, MeloTTS), `llm` (the brain's replies, prepares and judgements),
`voice_worker` for the rest (connections, sessions, channels).
Without debug logging too, each answer and each scheduled task logs one `llm` line once the LLM is
done with it: the least free graphics memory read while it was written, the card's total, how many
readings and how long it took (`answer <id> card: least N MiB free of T MiB (k readings, ms)`). The
card is read with `nvidia-smi` once a second while the answer is written, in a thread, so the answer
never waits for it; without `nvidia-smi` nothing is read or logged. The line carries no text, so a
slow answer can be matched to a full card (the engine log has the same answer's timings).

View File

@@ -1568,3 +1568,51 @@ def test_the_judgement_is_told_who_is_in_the_channel(monkeypatch):
assert "Several people" in unknown and "anything unclear" in unknown, "an app that does not say: as before"
assert "one other person" in pair and "unclear, answer yes" in pair
assert "5 people" in group and "Only 1 of the 5" in group and "anything unclear" in group
@pytest.mark.unit
def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(store, monkeypatch, capfd):
"""voice_worker.spec.md, "Debug logging": an answer (and a scheduled task) logs the least free
graphics memory read while the LLM wrote it, without debug logging and without its text, so a
slow answer can be matched to a full card."""
import time as _time
from voice_worker import server as worker_server
monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02)
readings = iter([9000, 15000, 12000])
last = [12000]
def card():
last[0] = next(readings, last[0])
return last[0], 16303
def slow(user, text):
_time.sleep(0.3)
return f"secret answer for {text}"
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="private question", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
await send(ws, type="reply", id="t1", user="100", text="private task", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text" and m["id"] == "t1")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), card=card))
lines = [x for x in capfd.readouterr().err.splitlines() if x.startswith(f"[{'llm':^10}]") and "free" in x]
reply = [x for x in lines if "answer r1" in x]
task = [x for x in lines if "task t1" in x]
assert len(reply) == 1 and len(task) == 1, lines
assert f"least {16303 - 15000} MiB free of 16303 MiB" in reply[0], reply
assert "private" not in "".join(lines) and "secret" not in "".join(lines)
@pytest.mark.unit
def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd):
async def body(url):
async with connect(url) as ws:
await hello(ws)
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None))
assert "MiB free" not in capfd.readouterr().err