Compare commits
2 Commits
app-v0.1.2
...
app-v0.1.2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1c06b16bd4 | ||
|
|
44387685af |
2
app/Cargo.lock
generated
2
app/Cargo.lock
generated
@@ -1520,7 +1520,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 Windows app core: Discord voice, voice worker supervisor, chat channels"
|
||||
|
||||
4
app/shell/Cargo.lock
generated
4
app/shell/Cargo.lock
generated
@@ -2637,7 +2637,7 @@ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-app"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -2666,7 +2666,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
dependencies = [
|
||||
"jarvis-app",
|
||||
"serde_json",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "jarvis-shell"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
description = "Jarvis v2 management app window (Tauri 2 over the app core's AdminApi)"
|
||||
|
||||
@@ -709,7 +709,7 @@ start it first, so nothing answers before the brain can, and stop it last.
|
||||
counted as other programs' memory; when that stop is not confirmed, nothing is read or kept and
|
||||
the model starts with `expert-vram=0`. Without a card nothing is set. The game
|
||||
placement (used by the switching that comes next) takes the reserve when it fits beside more
|
||||
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`card after the start: N MiB free of T MiB`).
|
||||
than 2 GiB of experts, else 2 GiB and the room that leaves, else 0 GiB and its room, else pause. The `llm` log tab gets the placement and what it came from, and once the model answers how much of the card is left free (`memory.free`, without what the driver reserves: `card after the start: N MiB free of T MiB`).
|
||||
- **A server the app does not start**: with `llm.url` (an OpenAI-compatible `/v1` address, in the
|
||||
settings file only) the app starts nothing; that server must list a model within 30 seconds.
|
||||
- **Settings**: `llm.context` (default 32768 tokens per request, 4096-262144) and `llm.port`
|
||||
|
||||
@@ -514,7 +514,7 @@ mod perf_tests {
|
||||
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(Arc::new(move || {
|
||||
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
Some(crate::strata::Gpu { id: "GPU-test".into(), used_mib: o2.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: 24576 })
|
||||
Some(crate::strata::tests::gpu("GPU-test", o2.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, 24576))
|
||||
})));
|
||||
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
|
||||
@@ -116,6 +116,8 @@ pub struct Gpu {
|
||||
pub id: String,
|
||||
pub used_mib: i64,
|
||||
pub total_mib: i64,
|
||||
/// What is left for programs (`memory.free`): the total less what is used and what the driver reserves.
|
||||
pub free_mib: i64,
|
||||
}
|
||||
|
||||
/// Reads the card (`nvidia-smi`); None without one.
|
||||
@@ -123,7 +125,7 @@ pub type GpuReader = Arc<dyn Fn() -> Option<Gpu> + Send + Sync>;
|
||||
|
||||
fn nvidia_memory() -> Option<Gpu> {
|
||||
let mut cmd = std::process::Command::new("nvidia-smi");
|
||||
cmd.args(["--query-gpu=uuid,memory.used,memory.total", "--format=csv,noheader,nounits"]);
|
||||
cmd.args(["--query-gpu=uuid,memory.used,memory.total,memory.free", "--format=csv,noheader,nounits"]);
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::process::CommandExt;
|
||||
@@ -132,7 +134,8 @@ fn nvidia_memory() -> Option<Gpu> {
|
||||
let out = cmd.output().ok()?;
|
||||
let line = String::from_utf8_lossy(&out.stdout).lines().next()?.to_owned();
|
||||
let mut f = line.split(',').map(str::trim);
|
||||
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()? })
|
||||
Some(Gpu { id: f.next()?.to_owned(), used_mib: f.next()?.parse().ok()?, total_mib: f.next()?.parse().ok()?,
|
||||
free_mib: f.next()?.parse().ok()? })
|
||||
}
|
||||
|
||||
/// The newest engine log under Strata's folder, its end only (the placement lines of the last start).
|
||||
@@ -509,7 +512,7 @@ impl Llm {
|
||||
// What the card has left for everything else once the model is in (the voice
|
||||
// worker's models, a game): a slow answer later can be matched to it.
|
||||
if let Some(g) = self.read_gpu().await {
|
||||
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.total_mib - g.used_mib, g.total_mib);
|
||||
crate::info_log!("llm", "card after the start: {} MiB free of {} MiB", g.free_mib, g.total_mib);
|
||||
}
|
||||
self.set(LlmState::Ready, None);
|
||||
}
|
||||
@@ -656,6 +659,11 @@ pub(crate) mod tests {
|
||||
|
||||
/// Other programs on the fake cards: the game PC's 1460 MiB in M0 less the TTS's share, which
|
||||
/// the app keeps beside and M0 did not run, so M0's placements come out.
|
||||
/// A card reading as `nvidia-smi` gives it, with nothing reserved by the driver.
|
||||
pub(crate) fn gpu(id: &str, used_mib: i64, total_mib: i64) -> Gpu {
|
||||
Gpu { id: id.into(), used_mib, total_mib, free_mib: total_mib - used_mib }
|
||||
}
|
||||
|
||||
pub(crate) const OTHERS: i64 = 1460 - crate::placement::TTS_MIB;
|
||||
|
||||
pub(crate) fn layout(tag: &str) -> Layout {
|
||||
@@ -918,10 +926,14 @@ pub(crate) mod tests {
|
||||
crate::logs::install(crate::logs::LogHub::new(None));
|
||||
let l = layout("llm-free");
|
||||
let port = server_after_launch(&l, "list").await;
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0));
|
||||
// The driver keeps some of the card for itself (`memory.reserved`): it is neither used nor free.
|
||||
let inner = card(&l, "GPU-test", 0);
|
||||
let reserved = 444;
|
||||
let reader: GpuReader = Arc::new(move || inner().map(|g| Gpu { free_mib: g.free_mib - reserved, ..g }));
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(reader);
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
let free = 16303 - OTHERS - 9874;
|
||||
let free = 16303 - OTHERS - 9874 - reserved;
|
||||
let lines = crate::logs::installed().unwrap()
|
||||
.query(Some("llm"), crate::logs::Level::Info, Some("free"), None, 1000);
|
||||
assert!(lines.iter().any(|x| x.text.contains(&format!("{free} MiB free of 16303 MiB"))),
|
||||
@@ -940,7 +952,7 @@ pub(crate) mod tests {
|
||||
let stopped = c.contains("launch --stop");
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
let used = OTHERS + if running { 9874 } else if stopped { 0 } else { leftover };
|
||||
Some(Gpu { id: id.into(), used_mib: used, total_mib: 16303 })
|
||||
Some(gpu(id, used, 16303))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -974,7 +986,7 @@ pub(crate) mod tests {
|
||||
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
||||
let r2 = reads.clone();
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
|
||||
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: OTHERS, total_mib: 16303 })
|
||||
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| gpu("GPU-test", OTHERS, 16303))
|
||||
}));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(20), llm.start(&s)).await.expect("starts").unwrap();
|
||||
@@ -1080,7 +1092,7 @@ pub(crate) mod tests {
|
||||
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
let used = others.load(Ordering::SeqCst) + worker + if running { 9874 } else { 0 };
|
||||
Some(Gpu { id: "GPU-test".into(), used_mib: used, total_mib: 16303 })
|
||||
Some(gpu("GPU-test", used, 16303))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1138,7 +1150,7 @@ pub(crate) mod tests {
|
||||
Arc::new(move || {
|
||||
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
Some(Gpu { id: "GPU-test".into(), used_mib: others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total_mib: total })
|
||||
Some(gpu("GPU-test", others.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, total))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1191,7 +1203,7 @@ pub(crate) mod tests {
|
||||
let reads = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
||||
let r2 = reads.clone();
|
||||
let llm = llm.with_gpu(Arc::new(move || {
|
||||
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| Gpu { id: "GPU-test".into(), used_mib: OTHERS, total_mib: 16303 })
|
||||
(r2.fetch_add(1, Ordering::SeqCst) == 0).then(|| gpu("GPU-test", OTHERS, 16303))
|
||||
}));
|
||||
assert_eq!(llm.switch(&s, Stage::Gaming, 0, true).await.unwrap(), Stage::Gaming);
|
||||
assert!(launches(&l).last().unwrap().contains("--set expert-vram=0 "), "{:?}", launches(&l));
|
||||
@@ -1208,7 +1220,7 @@ pub(crate) mod tests {
|
||||
let llm = Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(Arc::new(move || {
|
||||
let c = std::fs::read_to_string(&calls).unwrap_or_default();
|
||||
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
|
||||
Some(Gpu { id: "GPU-small".into(), used_mib: OTHERS + if running { 9874 } else { 0 }, total_mib: 12288 })
|
||||
Some(gpu("GPU-small", OTHERS + if running { 9874 } else { 0 }, 12288))
|
||||
}));
|
||||
let s = parse_settings(&json!({"llm": {"port": port}})).unwrap();
|
||||
llm.start(&s).await.unwrap();
|
||||
|
||||
@@ -45,14 +45,15 @@ LLM_WAIT_S = 180.0
|
||||
|
||||
|
||||
def nvidia_card() -> tuple[int, int] | None:
|
||||
"""The first NVIDIA card's used and total memory in MiB (``nvidia-smi``), None when unreadable."""
|
||||
"""The first NVIDIA card's free and total memory in MiB (``nvidia-smi``; free leaves out what the
|
||||
driver reserves), None when unreadable."""
|
||||
import subprocess
|
||||
try:
|
||||
out = subprocess.run(["nvidia-smi", "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"],
|
||||
out = subprocess.run(["nvidia-smi", "--query-gpu=memory.free,memory.total", "--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout
|
||||
used, total = (int(v) for v in out.splitlines()[0].split(","))
|
||||
return used, total
|
||||
free, total = (int(v) for v in out.splitlines()[0].split(","))
|
||||
return free, total
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -120,7 +121,8 @@ class VoiceWorker:
|
||||
self.history = history # jarvis.memory.history.HistoryStore or None
|
||||
self.recorder = recorder # recordings.Recorder (voice.record on) or None
|
||||
self.brain = brain
|
||||
self.card = card # (used MiB, total MiB) of the graphics card, or None without one
|
||||
self.card = card # (free MiB, total MiB) of the graphics card, or None without one
|
||||
self.card_watches: set[asyncio.Task] = set() # readings still running after their answer
|
||||
# One reply at a time per lane: voice answers never wait behind another person's chat answer
|
||||
# (conversation.md: with an "answer everything" chat channel the LLM takes two or more
|
||||
# requests at once). One person's replies stay one at a time across lanes, so their dialogue
|
||||
@@ -771,30 +773,42 @@ class Session:
|
||||
@contextlib.asynccontextmanager
|
||||
async def _card_watch(self, label: str):
|
||||
"""While an answer is written: the card's free memory, read every `CARD_EVERY_S`; the least
|
||||
seen is logged once it ends (a slow answer can then be matched to a full card)."""
|
||||
card = self.worker.card
|
||||
seen is logged once it ends (a slow answer can then be matched to a full card). The answer
|
||||
never waits for a reading: one still running when it ends finishes on its own, then logs."""
|
||||
w = self.worker
|
||||
card = w.card
|
||||
if card is None:
|
||||
yield
|
||||
return
|
||||
seen: list[tuple[int, int]] = []
|
||||
ended = asyncio.Event()
|
||||
t0, took = time.monotonic(), [0]
|
||||
|
||||
async def read():
|
||||
async def watch():
|
||||
while True:
|
||||
got = await asyncio.to_thread(card)
|
||||
if got:
|
||||
seen.append(got)
|
||||
await asyncio.sleep(CARD_EVERY_S)
|
||||
if ended.is_set():
|
||||
break
|
||||
try:
|
||||
await asyncio.wait_for(ended.wait(), CARD_EVERY_S)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
if seen:
|
||||
total = seen[-1][1]
|
||||
notice(f"{label} card: least {min(f for f, _ in seen)} MiB free of {total} MiB "
|
||||
f"({len(seen)} readings, {took[0]} ms)", "llm")
|
||||
|
||||
t0 = time.monotonic()
|
||||
task = asyncio.get_running_loop().create_task(read())
|
||||
task = asyncio.get_running_loop().create_task(watch())
|
||||
w.card_watches.add(task)
|
||||
task.add_done_callback(w.card_watches.discard)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
if seen:
|
||||
total = seen[-1][1]
|
||||
notice(f"{label} card: least {min(total - u for u, _ in seen)} MiB free of {total} MiB "
|
||||
f"({len(seen)} readings, {int((time.monotonic() - t0) * 1000)} ms)", "llm")
|
||||
took[0] = int((time.monotonic() - t0) * 1000)
|
||||
ended.set()
|
||||
|
||||
def _on_prepare(self, msg: dict) -> None:
|
||||
"""A call still being said: the brain gets ready for it. One prepare per user runs at a time;
|
||||
|
||||
@@ -535,6 +535,8 @@ in its tab (`app/src/admin.spec.md`, "Logs"): `stt` (utterance ends, STT batches
|
||||
Without debug logging too, each answer and each scheduled task logs one `llm` line once the LLM is
|
||||
done with it: the least free graphics memory read while it was written, the card's total, how many
|
||||
readings and how long it took (`answer <id> card: least N MiB free of T MiB (k readings, ms)`). The
|
||||
card is read with `nvidia-smi` once a second while the answer is written, in a thread, so the answer
|
||||
never waits for it; without `nvidia-smi` nothing is read or logged. The line carries no text, so a
|
||||
card is read with `nvidia-smi` (`memory.free`, which leaves out what the driver reserves) once a
|
||||
second while the answer is written, in a thread, so the answer never waits for it: a reading still
|
||||
running when the answer ends finishes on its own and the line follows it, so a short answer gets its
|
||||
line too. Without `nvidia-smi` nothing is read or logged. The line carries no text, so a
|
||||
slow answer can be matched to a full card (the engine log has the same answer's timings).
|
||||
|
||||
@@ -1578,8 +1578,9 @@ def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(st
|
||||
import time as _time
|
||||
from voice_worker import server as worker_server
|
||||
monkeypatch.setattr(worker_server, "CARD_EVERY_S", 0.02)
|
||||
readings = iter([9000, 15000, 12000])
|
||||
last = [12000]
|
||||
# Free memory as nvidia-smi reports it (the driver's reserved share is not free).
|
||||
readings = iter([7000, 859, 3000])
|
||||
last = [3000]
|
||||
|
||||
def card():
|
||||
last[0] = next(readings, last[0])
|
||||
@@ -1602,7 +1603,7 @@ def test_each_answer_logs_the_least_graphics_memory_free_while_it_was_written(st
|
||||
reply = [x for x in lines if "answer r1" in x]
|
||||
task = [x for x in lines if "task t1" in x]
|
||||
assert len(reply) == 1 and len(task) == 1, lines
|
||||
assert f"least {16303 - 15000} MiB free of 16303 MiB" in reply[0], reply
|
||||
assert "least 859 MiB free of 16303 MiB" in reply[0], reply
|
||||
assert "private" not in "".join(lines) and "secret" not in "".join(lines)
|
||||
|
||||
|
||||
@@ -1616,3 +1617,29 @@ def test_without_a_card_reading_an_answer_logs_no_card_line(store, capfd):
|
||||
|
||||
asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=lambda: None))
|
||||
assert "MiB free" not in capfd.readouterr().err
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_answer_done_before_the_first_card_reading_is_logged_once_the_reading_ends(store, capfd):
|
||||
"""A short answer is not held for the card, and still gets its line once the reading ends."""
|
||||
import threading
|
||||
reading = threading.Event()
|
||||
|
||||
def card():
|
||||
reading.wait(5)
|
||||
return 2048, 16303
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
await send(ws, type="reply", id="r1", user="100", text="q", via="chat", speak=False)
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
before = capfd.readouterr().err
|
||||
reading.set()
|
||||
await asyncio.sleep(0.3)
|
||||
return before
|
||||
|
||||
before = asyncio.run(with_worker(body, store=store, brain=FakeBrain(), card=card))
|
||||
after = capfd.readouterr().err
|
||||
assert "answer r1 card" not in before, "the answer did not wait for the card"
|
||||
assert "answer r1 card: least 2048 MiB free of 16303 MiB (1 readings" in after, after[-1500:]
|
||||
|
||||
Reference in New Issue
Block a user