diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 1fdb6c1..18e9943 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -709,7 +709,7 @@ start it first, so nothing answers before the brain can, and stop it last. counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and the model starts with `expert-vram=0`. Without a card nothing is set. The game placement (used by the switching that comes next) takes the reserve when it fits beside more - than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from. + than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`card after the start: N MiB free of T MiB`). - **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the settings file only) the app starts nothing; that server must list a model within 30 seconds. - **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port` diff --git a/app/src/strata.rs b/app/src/strata.rs index 1095d02..5767100 100644 --- a/app/src/strata.rs +++ b/app/src/strata.rs @@ -506,6 +506,11 @@ impl Llm { Ok(None) => {} Ok(Some((base, model))) => { crate::info_log!("llm", "ready: {base} model={model}"); + // What the card has left for everything else once the model is in (the voice + // worker's models, a game): a slow answer later can be matched to it. + if let Some(g) = self.read_gpu().await { + crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.total_mib - g.used_mib, g.total_mib); + } self.set(LlmState::Ready, None); } Err(e) if e == "cancelled" => self.set(LlmState::Off, None), @@ -908,6 +913,22 @@ pub(crate) mod tests { assert!(launches[0].ends_with("--set expert-vram=1.5 --set prefill=2048"), "{}", launches[0]); } + #[tokio::test] + async fn the_llm_log_tells_how_much_of_the_card_is_free_once_the_model_answers() { + crate::logs::install(crate::logs::LogHub::new(None)); + let l = layout("llm-free"); + let port = server_after_launch(&l, "list").await; + let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)); + let s = parse_settings(&json!({"llm": {"port": port}})).unwrap(); + tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap(); + let free = 16303 - OTHERS - 9874; + let lines = crate::logs::installed().unwrap() + .query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000); + assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))), + "{:?}", lines.iter().map(|x| &x.text).collect::>()); + llm.stop().await.unwrap(); + } + /// A fake card for `fake_placed`: 16303 MiB, `OTHERS` held by other programs (and by any /// server still left over, `leftover` MiB, until a `launch --stop`), and 9874 MiB more while a /// Strata server runs (the calls file shows the last launch and stop). diff --git a/src/voice_worker/__main__.py b/src/voice_worker/__main__.py index 696d73f..adbadd8 100644 --- a/src/voice_worker/__main__.py +++ b/src/voice_worker/__main__.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import asyncio import os +import shutil import sys from .log import debug_log @@ -42,7 +43,7 @@ def main() -> int: from .brain import take_mcp_env mcp_env = take_mcp_env() - from .server import VoiceWorker, is_loopback + from .server import VoiceWorker, is_loopback, nvidia_card from .stt_whisper import WhisperTranscriber from .vad import SileroVad @@ -81,7 +82,8 @@ def main() -> int: recorder = Recorder(os.environ[RECORDINGS_ENV]) async def run() -> None: - worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder) + worker = VoiceWorker(token, transcriber, vad.stream, tts, history=history, brain=brain, recorder=recorder, + card=nvidia_card if shutil.which("nvidia-smi") else None) server = await worker.serve(args.host, args.port) debug_log(f"voice worker listening on {args.host}:{args.port} tts={'on' if tts else 'off'} " f"history={'on' if history else 'off'} brain={'on' if brain else 'off'} " diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index 4fc095a..c24bfd5 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -15,7 +15,7 @@ from typing import Callable import numpy as np -from .log import debug_log +from .log import debug_log, notice from .audio import InputConverter, loudness_equalise from .brain import Brain, MemorySettings @@ -44,6 +44,19 @@ def is_loopback(host: str) -> bool: LLM_WAIT_S = 180.0 +def nvidia_card() -> tuple[int, int] | None: + """The first NVIDIA card's used and total memory in MiB (``nvidia-smi``), None when unreadable.""" + import subprocess + try: + out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout + used, total = (int(v) for v in out.splitlines()[0].split(",")) + return used, total + except Exception: + return None + + class _LlmPaused(Exception): """The app paused the LLM (a game runs): nothing is answered until it is ready again.""" @@ -53,6 +66,7 @@ MAX_NOTES = 20 MAX_NOTE_CHARS = 300 # How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts. TRACE_KEEP_DAYS = 30 +CARD_EVERY_S = 1.0 # how often the card's free memory is read while an answer is written class _LiveTrace(list): @@ -93,7 +107,8 @@ class VoiceWorker: def __init__(self, token: str, transcriber: Transcriber | None, vad_factory: Callable[[], VadStream], tts_engine: TtsEngine | None = None, hello_timeout_s: float = 5.0, tick_s: float = 0.05, history=None, brain: Brain | None = None, chat_timeout_s: float = 15.0, - delivery_timeout_s: float = 30.0, recorder=None): + delivery_timeout_s: float = 30.0, recorder=None, + card: Callable[[], tuple[int, int] | None] | None = None): if not token: raise ValueError("a connection token is required") self.token = token @@ -105,6 +120,7 @@ class VoiceWorker: self.history = history # jarvis.memory.history.HistoryStore or None self.recorder = recorder # recordings.Recorder (voice.record on) or None self.brain = brain + self.card = card # (used MiB, total MiB) of the graphics card, or None without one # One reply at a time per lane: voice answers never wait behind another person's chat answer # (conversation.md: with an "answer everything" chat channel the LLM takes two or more # requests at once). One person's replies stay one at a time across lanes, so their dialogue @@ -687,7 +703,7 @@ class Session: history = HistoryContext() try: trace.append(self._records_step(history, notes)) - async with self._llm(): + async with self._llm(), self._card_watch(f"task {rid}"): answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True, notes=notes or [], trace=trace) except Exception as e: # the brain must never take the session down @@ -752,6 +768,34 @@ class Session: w.llm_active -= 1 self._llm_check_idle() + @contextlib.asynccontextmanager + async def _card_watch(self, label: str): + """While an answer is written: the card's free memory, read every `CARD_EVERY_S`; the least + seen is logged once it ends (a slow answer can then be matched to a full card).""" + card = self.worker.card + if card is None: + yield + return + seen: list[tuple[int, int]] = [] + + async def read(): + while True: + got = await asyncio.to_thread(card) + if got: + seen.append(got) + await asyncio.sleep(CARD_EVERY_S) + + t0 = time.monotonic() + task = asyncio.get_running_loop().create_task(read()) + try: + yield + finally: + task.cancel() + if seen: + total = seen[-1][1] + notice(f"{label} card: least {min(total - u for u, _ in seen)} MiB free of {total} MiB " + f"({len(seen)} readings, {int((time.monotonic() - t0) * 1000)} ms)", "llm") + def _on_prepare(self, msg: dict) -> None: """A call still being said: the brain gets ready for it. One prepare per user runs at a time; while it runs only the newest text waits, and one about the same as the last one prepared @@ -909,7 +953,7 @@ class Session: screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin) trace.append(self._records_step(history, notes)) try: - async with self._llm(rid if speak else None): + async with self._llm(rid if speak else None), self._card_watch(f"answer {rid}"): w.answering += 1 try: answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat, diff --git a/src/voice_worker/voice_worker.spec.md b/src/voice_worker/voice_worker.spec.md index f617f6f..e7d02ee 100644 --- a/src/voice_worker/voice_worker.spec.md +++ b/src/voice_worker/voice_worker.spec.md @@ -531,3 +531,10 @@ cancel, and engine failures. The category names the service, so the app's log sc in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches, the Whisper model), `tts` (sentences, cancels, MeloTTS), `llm` (the brain's replies, prepares and judgements), `voice_worker` for the rest (connections, sessions, channels). + +Without debug logging too, each answer and each scheduled task logs one `llm` line once the LLM is +done with it: the least free graphics memory read while it was written, the card's total, how many +readings and how long it took (`answer card: least N MiB free of T MiB (k readings, ms)`). The +card is read with `nvidia-smi` once a second while the answer is written, in a thread, so the answer +never waits for it; without `nvidia-smi` nothing is read or logged. The line carries no text, so a +slow answer can be matched to a full card (the engine log has the same answer's timings). diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index 4393140..0f5d59f 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -1568,3 +1568,51 @@ def test_the_judgement_is_told_who_is_in_the_channel(monkeypatch): assert "Several people" in unknown and "anything unclear" in unknown, "an app that does not say: as before" assert "one other person" in pair and "unclear, answer yes" in pair assert "5 people" in group and "Only 1 of the 5" in group and "anything unclear" in group + + +@pytest.mark.unit +def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(store, monkeypatch, capfd): + """voice_worker.spec.md, "Debug logging": an answer (and a scheduled task) logs the least free + graphics memory read while the LLM wrote it, without debug logging and without its text, so a + slow answer can be matched to a full card.""" + import time as _time + from voice_worker import server as worker_server + monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02) + readings = iter([9000, 15000, 12000]) + last = [12000] + + def card(): + last[0] = next(readings, last[0]) + return last[0], 16303 + + def slow(user, text): + _time.sleep(0.3) + return f"secret answer for {text}" + + async def body(url): + async with connect(url) as ws: + await hello(ws) + await send(ws, type="reply", id="r1", user="100", text="private question", via="chat", speak=False) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") + await send(ws, type="reply", id="t1", user="100", text="private task", via="task") + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text" and m["id"] == "t1") + + asyncio.run(with_worker(body, store=store, brain=FakeBrain(slow), card=card)) + lines = [x for x in capfd.readouterr().err.splitlines() if x.startswith(f"[{'llm':^10}]") and "free" in x] + reply = [x for x in lines if "answer r1" in x] + task = [x for x in lines if "task t1" in x] + assert len(reply) == 1 and len(task) == 1, lines + assert f"least {16303 - 15000} MiB free of 16303 MiB" in reply[0], reply + assert "private" not in "".join(lines) and "secret" not in "".join(lines) + + +@pytest.mark.unit +def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd): + async def body(url): + async with connect(url) as ws: + await hello(ws) + await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text") + + asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None)) + assert "MiB free" not in capfd.readouterr().err