feat(app): log the graphics card's free memory after the LLM starts and while each answer is written
The app logs how much of the card is free once the model answers, and the voice worker reads the card once a second while an answer or a scheduled task is written and logs the least free memory seen (no text, llm tab), so a slow answer can be matched to a full card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -709,7 +709,7 @@ start it first, so nothing answers before the brain can, and stop it last.
|
||||
counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and
|
||||
the model starts with `expert-vram=0`. Without a card nothing is set. The game
|
||||
placement (used by the switching that comes next) takes the reserve when it fits beside more
|
||||
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from.
|
||||
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`card after the start: N MiB free of T MiB`).
|
||||
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
|
||||
settings file only) the app starts nothing; that server must list a model within 30 seconds.
|
||||
- **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port`
|
||||
|
||||
@@ -506,6 +506,11 @@ impl Llm {
|
||||
Ok(None) => {}
|
||||
Ok(Some((base, model))) => {
|
||||
crate::info_log!("llm", "ready: {base} model={model}");
|
||||
// What the card has left for everything else once the model is in (the voice
|
||||
// worker's models, a game): a slow answer later can be matched to it.
|
||||
if let Some(g) = self.read_gpu().await {
|
||||
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.total_mib - g.used_mib, g.total_mib);
|
||||
}
|
||||
self.set(LlmState::Ready, None);
|
||||
}
|
||||
Err(e) if e == "cancelled" => self.set(LlmState::Off, None),
|
||||
@@ -908,6 +913,22 @@ pub(crate) mod tests {
|
||||
assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_llm_log_tells_how_much_of_the_card_is_free_once_the_model_answers() {
|
||||
crate::logs::install(crate::logs::LogHub::new(None));
|
||||
let l = layout("llm-free");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let free = 16303 - OTHERS - 9874;
|
||||
let lines = crate::logs::installed().unwrap()
|
||||
.query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000);
|
||||
assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))),
|
||||
"{:?}", lines.iter().map(|x| &x.text).collect::<Vec<_>>());
|
||||
llm.stop().await.unwrap();
|
||||
}
|
||||
|
||||
/// A fake card for `fake_placed`: 16303 MiB, `OTHERS` held by other programs (and by any
|
||||
/// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a
|
||||
/// Strata server runs (the calls file shows the last launch and stop).
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
from .log import debug_log
|
||||
@@ -42,7 +43,7 @@ def main() -> int:
|
||||
from .brain import take_mcp_env
|
||||
mcp_env = take_mcp_env()
|
||||
|
||||
from .server import VoiceWorker, is_loopback
|
||||
from .server import VoiceWorker, is_loopback, nvidia_card
|
||||
from .stt_whisper import WhisperTranscriber
|
||||
from .vad import SileroVad
|
||||
|
||||
@@ -81,7 +82,8 @@ def main() -> int:
|
||||
recorder = Recorder(os.environ[RECORDINGS_ENV])
|
||||
|
||||
async def run() -> None:
|
||||
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder)
|
||||
worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder,
|
||||
card=nvidia_card if shutil.which("nvidia-smi") else None)
|
||||
server = await worker.serve(args.host, args.port)
|
||||
debug_log(f"voice worker listening on {args.host}:{args.port} tts={'on' if tts else 'off'} "
|
||||
f"history={'on' if history else 'off'} brain={'on' if brain else 'off'} "
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Callable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .log import debug_log
|
||||
from .log import debug_log, notice
|
||||
|
||||
from .audio import InputConverter, loudness_equalise
|
||||
from .brain import Brain, MemorySettings
|
||||
@@ -44,6 +44,19 @@ def is_loopback(host: str) -> bool:
|
||||
LLM_WAIT_S = 180.0
|
||||
|
||||
|
||||
def nvidia_card() -> tuple[int, int] | None:
|
||||
"""The first NVIDIA card's used and total memory in MiB (``nvidia-smi``), None when unreadable."""
|
||||
import subprocess
|
||||
try:
|
||||
out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout
|
||||
used, total = (int(v) for v in out.splitlines()[0].split(","))
|
||||
return used, total
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class _LlmPaused(Exception):
|
||||
"""The app paused the LLM (a game runs): nothing is answered until it is ready again."""
|
||||
|
||||
@@ -53,6 +66,7 @@ MAX_NOTES = 20
|
||||
MAX_NOTE_CHARS = 300
|
||||
# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts.
|
||||
TRACE_KEEP_DAYS = 30
|
||||
CARD_EVERY_S = 1.0 # how often the card's free memory is read while an answer is written
|
||||
|
||||
|
||||
class _LiveTrace(list):
|
||||
@@ -93,7 +107,8 @@ class VoiceWorker:
|
||||
def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream],
|
||||
tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05,
|
||||
history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0,
|
||||
delivery_timeout_s: float = 30.0, recorder=None):
|
||||
delivery_timeout_s: float = 30.0, recorder=None,
|
||||
card: Callable[[], tuple[int, int] | None] | None = None):
|
||||
if not token:
|
||||
raise ValueError("a connection token is required")
|
||||
self.token = token
|
||||
@@ -105,6 +120,7 @@ class VoiceWorker:
|
||||
self.history = history # jarvis.memory.history.HistoryStore or None
|
||||
self.recorder = recorder # recordings.Recorder (voice.record on) or None
|
||||
self.brain = brain
|
||||
self.card = card # (used MiB, total MiB) of the graphics card, or None without one
|
||||
# One reply at a time per lane: voice answers never wait behind another person's chat answer
|
||||
# (conversation.md: with an "answer everything" chat channel the LLM takes two or more
|
||||
# requests at once). One person's replies stay one at a time across lanes, so their dialogue
|
||||
@@ -687,7 +703,7 @@ class Session:
|
||||
history = HistoryContext()
|
||||
try:
|
||||
trace.append(self._records_step(history, notes))
|
||||
async with self._llm():
|
||||
async with self._llm(), self._card_watch(f"task {rid}"):
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True,
|
||||
notes=notes or [], trace=trace)
|
||||
except Exception as e: # the brain must never take the session down
|
||||
@@ -752,6 +768,34 @@ class Session:
|
||||
w.llm_active -= 1
|
||||
self._llm_check_idle()
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _card_watch(self, label: str):
|
||||
"""While an answer is written: the card's free memory, read every `CARD_EVERY_S`; the least
|
||||
seen is logged once it ends (a slow answer can then be matched to a full card)."""
|
||||
card = self.worker.card
|
||||
if card is None:
|
||||
yield
|
||||
return
|
||||
seen: list[tuple[int, int]] = []
|
||||
|
||||
async def read():
|
||||
while True:
|
||||
got = await asyncio.to_thread(card)
|
||||
if got:
|
||||
seen.append(got)
|
||||
await asyncio.sleep(CARD_EVERY_S)
|
||||
|
||||
t0 = time.monotonic()
|
||||
task = asyncio.get_running_loop().create_task(read())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
if seen:
|
||||
total = seen[-1][1]
|
||||
notice(f"{label} card: least {min(total - u for u, _ in seen)} MiB free of {total} MiB "
|
||||
f"({len(seen)} readings, {int((time.monotonic() - t0) * 1000)} ms)", "llm")
|
||||
|
||||
def _on_prepare(self, msg: dict) -> None:
|
||||
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;
|
||||
while it runs only the newest text waits, and one about the same as the last one prepared
|
||||
@@ -909,7 +953,7 @@ class Session:
|
||||
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
|
||||
trace.append(self._records_step(history, notes))
|
||||
try:
|
||||
async with self._llm(rid if speak else None):
|
||||
async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
||||
|
||||
@@ -531,3 +531,10 @@ cancel, and engine failures. The category names the service, so the app's log sc
|
||||
in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches, the Whisper model),
|
||||
`tts` (sentences, cancels, MeloTTS), `llm` (the brain's replies, prepares and judgements),
|
||||
`voice_worker` for the rest (connections, sessions, channels).
|
||||
|
||||
Without debug logging too, each answer and each scheduled task logs one `llm` line once the LLM is
|
||||
done with it: the least free graphics memory read while it was written, the card's total, how many
|
||||
readings and how long it took (`answer <id> card: least N MiB free of T MiB (k readings, ms)`). The
|
||||
card is read with `nvidia-smi` once a second while the answer is written, in a thread, so the answer
|
||||
never waits for it; without `nvidia-smi` nothing is read or logged. The line carries no text, so a
|
||||
slow answer can be matched to a full card (the engine log has the same answer's timings).
|
||||
|
||||
@@ -1568,3 +1568,51 @@ def test_the_judgement_is_told_who_is_in_the_channel(monkeypatch):
|
||||
assert "Several people" in unknown and "anything unclear" in unknown, "an app that does not say: as before"
|
||||
assert "one other person" in pair and "unclear, answer yes" in pair
|
||||
assert "5 people" in group and "Only 1 of the 5" in group and "anything unclear" in group
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(store, monkeypatch, capfd):
|
||||
"""voice_worker.spec.md, "Debug logging": an answer (and a scheduled task) logs the least free
|
||||
graphics memory read while the LLM wrote it, without debug logging and without its text, so a
|
||||
slow answer can be matched to a full card."""
|
||||
import time as _time
|
||||
from voice_worker import server as worker_server
|
||||
monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02)
|
||||
readings = iter([9000, 15000, 12000])
|
||||
last = [12000]
|
||||
|
||||
def card():
|
||||
last[0] = next(readings, last[0])
|
||||
return last[0], 16303
|
||||
|
||||
def slow(user, text):
|
||||
_time.sleep(0.3)
|
||||
return f"secret answer for {text}"
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="reply", id="r1", user="100", text="private question", via="chat", speak=False)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
await send(ws, type="reply", id="t1", user="100", text="private task", via="task")
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text" and m["id"] == "t1")
|
||||
|
||||
asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), card=card))
|
||||
lines = [x for x in capfd.readouterr().err.splitlines() if x.startswith(f"[{'llm':^10}]") and "free" in x]
|
||||
reply = [x for x in lines if "answer r1" in x]
|
||||
task = [x for x in lines if "task t1" in x]
|
||||
assert len(reply) == 1 and len(task) == 1, lines
|
||||
assert f"least {16303 - 15000} MiB free of 16303 MiB" in reply[0], reply
|
||||
assert "private" not in "".join(lines) and "secret" not in "".join(lines)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd):
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
|
||||
asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None))
|
||||
assert "MiB free" not in capfd.readouterr().err
|
||||
|
||||
Reference in New Issue
Block a user