diff --git a/scripts/m0/README.md b/scripts/m0/README.md index 6e2ceae..22cc384 100644 --- a/scripts/m0/README.md +++ b/scripts/m0/README.md @@ -68,7 +68,7 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 `run_m0_gamemode.bat`을 더블클릭하면 게임 중 자동 조절 단계 두 개(`gamemode`, `gamemode_stt`)만 돕니다. 먼저 LLM만 배치 5개(평소, 게임 중 후보 3개, 다시 평소)로 재고, 이어서 음성 인식을 함께 켠 배치 6개를 잽니다. 모델은 이미 받아 둔 것을 쓰므로 새로 받지 않고(음성 인식 모델을 아직 안 받았으면 약 1.6GB를 받습니다), 20~30분 걸립니다. 그동안 게임이나 무거운 프로그램은 꺼 두시고 PC에 손대지 않으시면 됩니다. -`run_m0_gamemode_tune.bat`을 더블클릭하면 `gamemode_tune` 단계만 돕니다. 앞 측정에서 정한 두 가지를 확인합니다. (1) 평소 배치에 음성 인식(STT) 자리를 남겼을 때의 속도와 첫 토큰 시간입니다. LLM을 전문가 층 없이 한 번 켜서 기본 VRAM을 재고, STT가 10명 동시 전사 때 더 쓰는 VRAM과 여유분을 뺀 나머지를 전문가 층에 줍니다. (2) 게임 중 배치(전문가 2GB)에서 LLM이 답하는 동안 부분 전사를 2초마다 하거나 멈추고 최종 전사만 바로 할 때, LLM 속도와 최종 전사가 기다린 시간입니다. 게임이 없을 때는 부분 전사를 줄이지 않으므로 평소 배치는 이 비교를 하지 않습니다. 게임은 켜지 않고 잽니다. 이미 받은 모델을 쓰고 10~15분 걸립니다. 그동안 게임이나 무거운 프로그램은 꺼 두시고 PC에 손대지 않으시면 됩니다. +`run_m0_gamemode_tune.bat`을 더블클릭하면 `gamemode_tune` 단계만 돕니다. 앞 측정에서 정한 세 가지를 확인합니다. (1) 평소 배치에 음성 인식(STT) 자리와 다른 프로그램용 여유 VRAM 1GB를 남겼을 때의 속도와 첫 토큰 시간입니다. LLM을 전문가 층 없이 한 번 켜서 기본 VRAM을 재고, STT가 10명 동시 전사 때 더 쓰는 VRAM, 여유분, 다른 프로그램용 1GB를 뺀 나머지를 전문가 층에 줍니다. (2) 게임 중 배치(전문가 2GB)에서 LLM이 답하는 동안 부분 전사를 2초마다 하거나 멈추고 최종 전사만 바로 할 때, LLM 속도와 최종 전사가 기다린 시간입니다. 게임이 없을 때는 부분 전사를 줄이지 않으므로 평소 배치는 이 비교를 하지 않습니다. (3) GPU 사용률 95% 제한입니다. LLM 엔진과 STT를 200ms마다 함께 10ms씩 멈추고, 제한이 없을 때와 비교해 속도와 실제 GPU 사용률을 잽니다. 이때 두 프로그램이 잠깐씩 멈추는 것은 정상입니다. 게임은 켜지 않고 잽니다. 이미 받은 모델을 쓰고 20~30분 걸립니다. 그동안 게임이나 무거운 프로그램은 꺼 두시고 PC에 손대지 않으시면 됩니다. `run_m0_game_live.bat`을 더블클릭하면 실제 게임을 켠 상태로 잽니다(`gamemode_live`). 게임을 창 모드로 켜고 플레이 화면까지 들어간 뒤 Enter를 누르고 15초 안에 게임으로 돌아가면 됩니다. 게임 중 배치 4개(전문가 2GB·0GB, 음성 인식 GPU·CPU)를 차례로 재고 10~15분 걸리며, 끝나면 알림음이 울립니다. 게임 화면이나 게임 이름은 저장하지 않고 VRAM 사용량과 속도만 적습니다. 그래픽 메모리를 많이 쓰는 게임이면 배치가 켜지지 않을 수 있는데, 그것도 결과로 적습니다. diff --git a/scripts/m0/m0.py b/scripts/m0/m0.py index b5bcd65..7c944af 100644 --- a/scripts/m0/m0.py +++ b/scripts/m0/m0.py @@ -49,7 +49,7 @@ STAGES = [ ["model=fast", "only=together"], 0), # The fixes for what gamemode_stt showed: room for the STT in the normal placement, and fewer # STT partial passes while the LLM answers (normal and the 2 GB game placement). - ("gamemode_tune", "게임 중 자동 조절: STT 자리 남기기와 부분 전사 간격", "base", "llm.py", + ("gamemode_tune", "자동 조절: STT 자리와 VRAM 여유 1GB, 부분 전사 간격, GPU 95% 제한", "base", "llm.py", ["model=fast", "only=tune"], 0), # With the user's game running. Needs someone at the PC, so only when asked for (--only). ("gamemode_live", "게임을 켠 상태로 재기 (직접 게임을 켜 주셔야 합니다)", "base", "llm.py", diff --git a/scripts/m0/m0lib/gpucap.py b/scripts/m0/m0lib/gpucap.py new file mode 100644 index 0000000..4ef53a3 --- /dev/null +++ b/scripts/m0/m0lib/gpucap.py @@ -0,0 +1,201 @@ +"""The GPU share cap the app would put on Jarvis (user decision, 2026-11-10: 95 %). + +Jarvis's GPU work runs in two processes: the Strata engine and the voice worker (here the STT +load worker). A GPU cannot be told to give one process only part of its time, so the cap pauses +both processes together for a short slice of every period (Windows NtSuspendProcess / +NtResumeProcess): 10 ms of every 200 ms is 95 %. Work already queued on the GPU still finishes +during the pause, so what the GPU really gets back is measured (``UtilSampler``), not assumed. +""" + +from __future__ import annotations + +import subprocess +import threading +import time +from pathlib import Path +from typing import Callable, Iterable + +from m0lib.common import IS_WINDOWS, nvidia_smi, stats + +PERIOD_S = 0.2 + + +def gpu_util_pct() -> int | None: + """nvidia-smi's utilization.gpu of GPU 0: the share of the last sample period in which any + kernel ran (what the app would read through NVML), or None.""" + exe = nvidia_smi() + if not exe: + return None + try: + out = subprocess.run([exe, "--query-gpu=utilization.gpu", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=10).stdout.strip().splitlines()[0] + return int(out.strip()) + except Exception: + return None + + +class UtilSampler: + """Samples GPU utilisation in the background while a measurement runs.""" + + def __init__(self, interval: float = 0.5, read: Callable[[], int | None] | None = None): + self.interval = interval + self.read = read or gpu_util_pct + self.samples: list[int] = [] + self._stop = threading.Event() + self._t: threading.Thread | None = None + + def __enter__(self) -> "UtilSampler": + self._t = threading.Thread(target=self._loop, daemon=True) + self._t.start() + return self + + def _loop(self) -> None: + while not self._stop.is_set(): + v = self.read() + if v is not None: + self.samples.append(v) + self._stop.wait(self.interval) + + def __exit__(self, *exc) -> None: + self._stop.set() + if self._t: + self._t.join(timeout=15) + + def summary(self, cap: float | None = None) -> dict: + out = {"pct": stats(self.samples, 1)} + if cap is not None and self.samples: + out["over_cap_share"] = round(sum(1 for v in self.samples if v > cap * 100) / len(self.samples), 3) + return out + + +def process_table() -> list[dict]: + """Every process with its id, parent, executable and command line (Windows only).""" + if not IS_WINDOWS: + return [] + ps = ("Get-CimInstance Win32_Process | ForEach-Object { \"$($_.ProcessId)`t$($_.ParentProcessId)`t" + "$($_.ExecutablePath)`t$($_.CommandLine)\" }") + try: + out = subprocess.run(["powershell", "-NoProfile", "-Command", ps], capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=60).stdout + except Exception: + return [] + return parse_process_table(out) + + +def parse_process_table(text: str) -> list[dict]: + rows = [] + for line in text.splitlines(): + parts = line.rstrip("\r").split("\t", 3) + if len(parts) < 4 or not parts[0].strip().isdigit(): + continue + rows.append({"pid": int(parts[0]), "ppid": int(parts[1]) if parts[1].strip().isdigit() else None, + "exe": parts[2], "cmd": parts[3]}) + return rows + + +def with_children(procs: list[dict], roots: Iterable[int]) -> list[int]: + """The roots and everything they started (a venv python.exe on Windows is a launcher that + runs the real interpreter as its child).""" + out, todo = [], [r for r in roots if r is not None] + while todo: + pid = todo.pop() + if pid in out: + continue + out.append(pid) + todo += [p["pid"] for p in procs if p["ppid"] == pid] + return out + + +def engine_roots(procs: list[dict], home: Path) -> list[int]: + """The Strata engine serving the model: its strata.exe --serve under our components folder + (not the strata terminal that started it).""" + where = str(home / "components").lower() + return [p["pid"] for p in procs + if p["exe"].lower().startswith(where) and p["exe"].lower().endswith("strata.exe") + and "--serve" in p["cmd"]] + + +_API: dict = {} + + +def _nt(name: str, pid: int) -> bool: + import ctypes + from ctypes import wintypes + if not _API: + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + ntdll = ctypes.WinDLL("ntdll") + k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + k32.OpenProcess.restype = ctypes.c_void_p + k32.CloseHandle.argtypes = [ctypes.c_void_p] + for fn in (ntdll.NtSuspendProcess, ntdll.NtResumeProcess): + fn.argtypes = [ctypes.c_void_p] + fn.restype = ctypes.c_long + _API.update(k32=k32, ntdll=ntdll) + k32 = _API["k32"] + h = k32.OpenProcess(0x0800, False, pid) # PROCESS_SUSPEND_RESUME + if not h: + return False + try: + return getattr(_API["ntdll"], name)(h) == 0 + finally: + k32.CloseHandle(h) + + +def suspend_pid(pid: int) -> bool: + return _nt("NtSuspendProcess", pid) + + +def resume_pid(pid: int) -> bool: + return _nt("NtResumeProcess", pid) + + +class DutyCap: + """Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs; every + paused process is resumed before the block ends, also on an error.""" + + def __init__(self, pids: list[int], cap: float, period: float | None = None, + suspend: Callable[[int], bool] | None = None, resume: Callable[[int], bool] | None = None): + self.pids, self.cap, self.period = list(pids), cap, period or PERIOD_S + self.suspend, self.resume = suspend or suspend_pid, resume or resume_pid + self.pause = self.period * (1 - cap) + self.paused_s = 0.0 + self.cycles = 0 + self.failed: set[int] = set() + self._stop = threading.Event() + self._t: threading.Thread | None = None + self._t0 = 0.0 + self._t1 = 0.0 + + def __enter__(self) -> "DutyCap": + self._t0 = time.perf_counter() + if self.pids and self.pause > 0: + self._t = threading.Thread(target=self._loop, daemon=True) + self._t.start() + return self + + def _loop(self) -> None: + while not self._stop.wait(self.period - self.pause): + held = [p for p in self.pids if self.suspend(p)] + self.failed.update(p for p in self.pids if p not in held) + s0 = time.perf_counter() + try: + time.sleep(self.pause) + finally: + for p in held: + if not self.resume(p): + self.failed.add(p) + self.paused_s += time.perf_counter() - s0 + self.cycles += 1 + + def __exit__(self, *exc) -> None: + self._stop.set() + if self._t: + self._t.join(timeout=10) + self._t1 = time.perf_counter() + + def summary(self) -> dict: + span = (self._t1 or time.perf_counter()) - self._t0 + return {"cap": self.cap, "period_s": self.period, "pause_s": round(self.pause, 4), + "processes": len(self.pids), "cycles": self.cycles, + "paused_share": round(self.paused_s / span, 3) if span > 0 else None, + "failed_processes": len(self.failed)} diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 0ad5c30..6a5e4d5 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -10,8 +10,9 @@ Options (--opt key=value): only=together the same with the STT loaded next to it and transcribing ten speakers at once, on the GPU and on the CPU (needs stt_python=) only=live game placements and the STT while the user runs a real game - only=tune the normal placement with room left for the STT, and in the game - placement the STT doing fewer partial passes while the LLM answers + only=tune the normal placement with room left for the STT and 1 GB for other + programs, the game placement with the STT doing fewer partial passes + while the LLM answers, and both with Jarvis capped at 95 % of the GPU stt_python=PATH the Python of the stt venv (m0.py passes it) """ @@ -34,6 +35,7 @@ from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, ask, gpu_memory_m from m0lib.llm_bench import (bench_single, quick_bench, test_decode, test_images, test_parallel, # noqa: E402 test_tools, test_vram_shrink, warmup) from m0lib.tool_fix import test_tools_fixed, tools_passed # noqa: E402 +from m0lib import gpucap # noqa: E402 from m0lib.llm_client import ChatClient, http_json # noqa: E402 PRESETS = { @@ -111,18 +113,30 @@ LIVE = [lv for lv in TOGETHER if lv["key"].startswith("game_")] # whole run. Without a game the STT keeps every partial (user decision), so only the game # placement is paced; its back-to-back row is the baseline measured in the same run. # The 2 GB expert placement is the game minimum on a 16 GB card (faster than 0 GB in a real game). +# Then (user decision, 2026-11-10, for watching a video next to Jarvis): +# - the normal placement also leaves OTHER_ROOM_MIB free for programs opened later ("other_room_mib"); +# - Jarvis's GPU work (the engine and the STT worker together) is paused for 5 % of every 200 ms +# ("gpu_cap", gpucap.DutyCap). The rows with and without it measure what it costs and how much +# GPU time it really gives back (queued work still runs during the pause). STT_ROOM_MARGIN_MIB = 768 # what Strata's qwen36 engine itself keeps free when it places experts STT_GROWTH_FALLBACK_MIB = 1440 # M0 stt on the game PC: ten speakers at once over the loaded model +OTHER_ROOM_MIB = 1024 # the app's default room for other programs in the normal placement +GPU_CAP = 0.95 _PARTIAL_2S = {"partial_every": 2.0, "final_every": 0.5} _FINALS_ONLY = {"partial_every": None, "final_every": 0.5} -_ROOM = {"parallel": 1, "vision": "gpu", "sets": ["prefill=2048"], "stt": "cuda", "stt_room": True} +_ROOM = {"parallel": 1, "vision": "gpu", "sets": ["prefill=2048"], "stt_room": True, + "other_room_mib": OTHER_ROOM_MIB} +_GAME2 = {**_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"} TUNED = [ - {"key": "normal_stt_room_stt_gpu", **_ROOM}, - {"key": "game_experts_2g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"}, - {"key": "game_experts_2g_16k_partial_2s", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda", - "stt_pace": _PARTIAL_2S}, - {"key": "game_experts_2g_16k_finals_only", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda", - "stt_pace": _FINALS_ONLY}, + # The room is worked out in the first row, with the STT loaded; the rows after it reuse it. + {"key": "normal_room_stt_gpu", **_ROOM, "stt": "cuda"}, + {"key": "normal_room_stt_gpu_cap95", **_ROOM, "stt": "cuda", "gpu_cap": GPU_CAP}, + {"key": "normal_room_llm_only", **_ROOM}, + {"key": "normal_room_llm_only_cap95", **_ROOM, "gpu_cap": GPU_CAP}, + {"key": "game_experts_2g_16k_stt_gpu", **_GAME2}, + {"key": "game_experts_2g_16k_partial_2s", **_GAME2, "stt_pace": _PARTIAL_2S}, + {"key": "game_experts_2g_16k_partial_2s_cap95", **_GAME2, "stt_pace": _PARTIAL_2S, "gpu_cap": GPU_CAP}, + {"key": "game_experts_2g_16k_finals_only", **_GAME2, "stt_pace": _FINALS_ONLY}, ] LEVEL_SETS = {"levels": LEVELS, "together": TOGETHER, "live": LIVE, "tune": TUNED} INSTALL_URL = "https://github.com/Niko1221/Strata/releases/latest/download/install.ps1" @@ -298,6 +312,10 @@ class SttLoad: self.cmd += ["--opt", f"{key}={'none' if v is None else v}"] self.proc: subprocess.Popen | None = None + @property + def pid(self) -> int | None: + return self.proc.pid if self.proc else None + def _read(self) -> dict: try: return read_json(self.out) @@ -463,8 +481,12 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels continue sets = list(lv["sets"]) if lv.get("stt_room"): - where = (row["context"], lv["parallel"], lv["vision"]) + where = (row["context"], lv["parallel"], lv["vision"], lv.get("other_room_mib", 0)) if where not in rooms: + if not stt: + # The probe needs the STT model in VRAM, as the app has it. + r.error(f"level {lv['key']}", "no STT room was measured with the STT loaded") + continue rooms[where] = stt_room(r, s, preset, row, lv, base) row["stt_room"] = rooms[where] sets.append(f"expert-vram={row['stt_room']['expert_vram_gib']}") @@ -483,7 +505,18 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time if stt: stt.go() - row.update(quick_bench(c)) + cap = lv.get("gpu_cap") + pids = cap_pids(s, stt) if cap else [] + if cap and not pids: + r.error(f"level {lv['key']} gpu cap", "found no engine process to pause") + with gpucap.UtilSampler() as util, gpucap.DutyCap(pids, cap or 1.0) as duty: + row.update(quick_bench(c)) + row["gpu_util"] = util.summary(cap) + if cap: + row["gpu_cap"] = duty.summary() + if row["gpu_cap"]["failed_processes"]: + r.error(f"level {lv['key']} gpu cap", + f"{row['gpu_cap']['failed_processes']} process(es) could not be paused") for err in row["errors"]: r.error(f"level {lv['key']}", err) except Exception as e: @@ -540,10 +573,11 @@ def staging_gib(lines: list[str], default: float = 1.10) -> float: def expert_budget_gib(total_mib: int, used_mib: int, probe_model_mib: int, staging: float, growth_mib: int, - margin_mib: int = STT_ROOM_MARGIN_MIB) -> float: - """--expert-vram (GiB, rounded down to 0.1) that leaves the STT's growth and the margin free. - The budget first holds the staging, which the start without experts already has.""" - spare = total_mib - used_mib - probe_model_mib - growth_mib - margin_mib + margin_mib: int = STT_ROOM_MARGIN_MIB, other_mib: int = 0) -> float: + """--expert-vram (GiB, rounded down to 0.1) that leaves the STT's growth, the margin and the room + for other programs free. The budget first holds the staging, which the start without experts + already has.""" + spare = total_mib - used_mib - probe_model_mib - growth_mib - margin_mib - other_mib if spare <= 0: return 0.0 return math.floor((staging * 1024 + spare) / 1024 * 10) / 10 @@ -553,7 +587,7 @@ def stt_room(r: StageResult, s: Strata, preset: dict, row: dict, lv: dict, base: """Start the placement once without experts (the STT model already loaded), see what the LLM holds, stop it, and work out the expert budget that leaves the STT's ten-speaker growth free.""" room: dict = {"stt_growth_mib": stt_growth_mib(s.log.parent.parent), "margin_mib": STT_ROOM_MARGIN_MIB, - "expert_vram_gib": 0.0} + "other_room_mib": lv.get("other_room_mib", 0), "expert_vram_gib": 0.0} say(" STT 자리 계산: 전문가 층 없이 한 번 켜서 LLM 기본 VRAM을 잽니다") before = gpu_memory_mib() launch = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"], @@ -571,7 +605,8 @@ def stt_room(r: StageResult, s: Strata, preset: dict, row: dict, lv: dict, base: room.update(probe_model_mib=after[0] - before[0], used_before_mib=before[0], total_mib=before[1], staging_gib=staging_gib(lines)) room["expert_vram_gib"] = expert_budget_gib(before[1], before[0], room["probe_model_mib"], - room["staging_gib"], room["stt_growth_mib"]) + room["staging_gib"], room["stt_growth_mib"], + other_mib=room["other_room_mib"]) if room["expert_vram_gib"] == 0: r.note(f"{lv['key']}: no room for experts next to the STT; started with --expert-vram 0") else: @@ -580,6 +615,16 @@ def stt_room(r: StageResult, s: Strata, preset: dict, row: dict, lv: dict, base: return room +def cap_pids(s: Strata, stt: SttLoad | None) -> list[int]: + """The processes the GPU cap pauses together: the engine and the STT worker, with their children. + None when the engine is not found (pausing only the STT would not be the cap).""" + procs = gpucap.process_table() + roots = gpucap.engine_roots(procs, s.home) + if not roots: + return [] + return gpucap.with_children(procs, roots + ([stt.pid] if stt and stt.pid else [])) + + def ask_for_a_game() -> bool: """The live stage needs someone to start a game first. False when nobody is there or they say no.""" if not interactive(): diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index f989953..d11dd4f 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -16,7 +16,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0")) -from m0lib import common, llm, llm_bench # noqa: E402 +from m0lib import common, gpucap, llm, llm_bench # noqa: E402 from m0lib.llm_client import ChatClient # noqa: E402 pytestmark = pytest.mark.unit @@ -455,7 +455,7 @@ def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path): def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True, only="levels", - extra_opts=(), stt=None): + extra_opts=(), stt=None, procs=None, paused=None): """Run the stage with Strata replaced; ``gpu(running)`` is nvidia-smi's (used, total), where ``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None. ``stt`` is a _FakeStt whose loaded model adds to the used VRAM.""" @@ -492,6 +492,16 @@ def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets monkeypatch.setattr(llm, "port_in_use", lambda p: running["now"] is not None) monkeypatch.setattr(llm, "gpu_memory_mib", lambda: gpu(running["now"])) monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["now"])) + monkeypatch.setattr(gpucap, "gpu_util_pct", lambda: 99) + # The kit's own engine and STT worker; the GPU cap "pauses" them by recording it. + procs = _procs(tmp_path / "strata") if procs is None else procs + monkeypatch.setattr(gpucap, "process_table", lambda: list(procs)) + held = set() + paused = set() if paused is None else paused + monkeypatch.setattr(gpucap, "suspend_pid", lambda pid: held.add(pid) or paused.add(pid) or True) + monkeypatch.setattr(gpucap, "resume_pid", lambda pid: held.discard(pid) or True) + monkeypatch.setattr(gpucap, "PERIOD_S", 0.002) + running["held"] = held out = tmp_path / "results" / "gamemode.json" out.parent.mkdir(parents=True, exist_ok=True) opts = ["--opt", "model=fast", "--opt", f"only={only}"] @@ -659,6 +669,7 @@ def test_normal_placement_answers_one_request_at_a_time(tmp_path, monkeypatch, f STT_MODEL_MIB = 1300 # M0 stt: the loaded model STT_GROWTH_MIB = 1440 # M0 stt: what ten speakers at once add over the loaded model STT_GPU_MIB = STT_MODEL_MIB + STT_GROWTH_MIB +STT_LAUNCHER_PID = 4100 # the venv python.exe; the real interpreter is its child class _FakeStt: @@ -687,6 +698,8 @@ class _FakeStt: fake.paces.append(pace) class _One: + pid = STT_LAUNCHER_PID + def start(self, timeout=900): if fake.broken_at_load: return {"status": "failed", "errors": ["load: RuntimeError: CUDA failed"]} @@ -977,12 +990,17 @@ def test_tune_stage_leaves_the_stt_room_in_the_normal_placement(tmp_path, monkey rows = {x["key"]: x for x in r["levels"]} budgets = {_expert_vram(x) for x in started[1:] if x["vision"] == normal[0]["vision"]} assert len(budgets) == 1 and budgets.pop() > 0 - # With the STT at its ten-speaker peak the card keeps the margin free, and not much more. - continuous = [rows[lv["key"]] for lv in normal if lv.get("stt_pace") is None] + # With the STT at its ten-speaker peak the card keeps the margin and the room for other + # programs free, and not much more. + continuous = [rows[lv["key"]] for lv in normal if lv.get("stt") and lv.get("stt_pace") is None] assert continuous + keep = llm.STT_ROOM_MARGIN_MIB + llm.OTHER_ROOM_MIB for row in continuous: - assert llm.STT_ROOM_MARGIN_MIB <= row["vram_free_mib"] < llm.STT_ROOM_MARGIN_MIB + 2 * 103 + assert keep <= row["vram_free_mib"] < keep + 2 * 103 assert row["stt_room"]["stt_growth_mib"] == llm.STT_GROWTH_FALLBACK_MIB + assert row["stt_room"]["other_room_mib"] == llm.OTHER_ROOM_MIB + # The same placement is also measured without the STT, so the cap's cost on the LLM alone shows. + assert any(not lv.get("stt") for lv in normal) and any(lv.get("stt") for lv in normal) # Without a game the STT keeps every partial while the LLM answers (user decision); only the # 2 GB game placement is measured back to back, with fewer partials and with finals only. assert all(lv.get("stt_pace") is None for lv in normal) @@ -990,7 +1008,13 @@ def test_tune_stage_leaves_the_stt_room_in_the_normal_placement(tmp_path, monkey assert game and all("expert-vram=2" in lv["sets"] for lv in game) paces = {(lv.get("stt_pace") or {}).get("partial_every", "continuous") for lv in game} assert paces == {"continuous", 2.0, None} - assert stt.paces == [lv.get("stt_pace") for lv in llm.TUNED] + assert stt.paces == [lv.get("stt_pace") for lv in llm.TUNED if lv.get("stt")] + # Each capped row has its uncapped twin in the same run. + capped = [lv for lv in llm.TUNED if lv.get("gpu_cap")] + assert capped and all(lv["gpu_cap"] == llm.GPU_CAP for lv in capped) + for lv in capped: + twin = {k: v for k, v in lv.items() if k not in ("key", "gpu_cap")} + assert any({k: v for k, v in x.items() if k != "key"} == twin for x in llm.TUNED) assert stt.loaded is None and running["now"] is None @@ -1022,3 +1046,108 @@ def test_tune_stage_runs_with_the_stt_python_and_has_a_one_click_runner(tmp_path assert "gamemode_tune" in m0.NEEDS_STT and "gamemode_tune" not in m0.ON_DEMAND bat = (Path(m0.KIT) / "run_m0_gamemode_tune.bat").read_text(encoding="utf-8") assert "-Only gamemode_tune" in bat and "-Redo" in bat + + +# --- the GPU cap: Jarvis's engine and STT worker paused together for 5 % of the time -------- + + +def _procs(home): + eng = str(home / "components" / "engine" / "0.1.59-windows-x86_64-cuda-cuda13" / "strata.exe") + term = str(home / "terminal" / "current" / "bin" / "strata.exe") + return [ + {"pid": 10, "ppid": 1, "exe": term, "cmd": f"{term} launch --yes"}, + {"pid": 20, "ppid": 10, "exe": eng, "cmd": f"{eng} --serve --pack x"}, + {"pid": 21, "ppid": 20, "exe": "C:\\x\\helper.exe", "cmd": "helper"}, + {"pid": 30, "ppid": 1, "exe": eng.replace(str(home), str(home.parent / "other")), "cmd": "strata.exe --serve"}, + {"pid": STT_LAUNCHER_PID, "ppid": 1, "exe": "C:\\venv\\python.exe", "cmd": "python stt.py"}, + {"pid": STT_LAUNCHER_PID + 1, "ppid": STT_LAUNCHER_PID, "exe": "C:\\py\\python.exe", "cmd": "python stt.py"}, + {"pid": 50, "ppid": 1, "exe": "C:\\chrome.exe", "cmd": "chrome"}, + ] + + +def test_process_table_lines_are_parsed_with_tabs_in_command_lines(): + rows = gpucap.parse_process_table("20\t10\tC:\\e\\strata.exe\tstrata.exe --serve\ta\r\n\nbad line\n4\t\t\t\n") + assert rows[0] == {"pid": 20, "ppid": 10, "exe": "C:\\e\\strata.exe", "cmd": "strata.exe --serve\ta"} + assert rows[1]["pid"] == 4 and rows[1]["ppid"] is None and len(rows) == 2 + + +def test_cap_pauses_the_serving_engine_and_the_stt_worker_with_their_children(tmp_path): + procs = _procs(tmp_path / "strata") + roots = gpucap.engine_roots(procs, tmp_path / "strata") + assert roots == [20] # not the terminal that started it, nor another kit's engine + assert sorted(gpucap.with_children(procs, roots + [STT_LAUNCHER_PID])) == \ + [20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1] + + +def test_duty_cap_pauses_for_the_capped_share_and_leaves_nothing_paused(): + import time as _time + held, log = set(), [] + + def suspend(pid): + held.add(pid) + log.append(("s", pid)) + return True + + def resume(pid): + held.discard(pid) + log.append(("r", pid)) + return True + + with gpucap.DutyCap([1, 2], 0.9, period=0.05, suspend=suspend, resume=resume) as d: + _time.sleep(0.6) + s = d.summary() + assert not held + assert s["cycles"] >= 5 and s["failed_processes"] == 0 + assert 0.05 <= s["paused_share"] <= 0.2 # about 10 % of the time + # Both processes are paused together: each pause holds both before either is resumed. + for i in range(0, len(log), 4): + assert sorted(log[i:i + 4]) == [("r", 1), ("r", 2), ("s", 1), ("s", 2)] + assert {k for k, _ in log[i:i + 2]} == {"s"} + + +def test_no_cap_pauses_nothing(): + calls = [] + with gpucap.DutyCap([1], 1.0, period=0.01, suspend=calls.append, resume=calls.append) as d: + import time as _time + _time.sleep(0.05) + assert calls == [] and d.summary()["cycles"] == 0 + + +def test_util_sampler_counts_samples_over_the_cap(): + vals = iter([99, 90, 97, 80]) + with gpucap.UtilSampler(interval=0.01, read=lambda: next(vals, None)) as u: + import time as _time + _time.sleep(0.1) + s = u.summary(0.95) + assert s["pct"]["n"] == 4 and s["over_cap_share"] == 0.5 + + +def _tune_with_cap(tmp_path, monkeypatch, fake_server, procs): + paused = set() + r, _, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune", + extra_opts=["stt_python=py"], stt=_FakeStt(), procs=procs, paused=paused) + return r, running["held"], paused + + +def test_capped_rows_pause_jarvis_and_record_what_the_gpu_got(tmp_path, monkeypatch, fake_server): + home = tmp_path / "strata" + r, held, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, _procs(home)) + assert r["status"] == "ok" + assert not held # nothing is left paused + assert paused == {20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1} + rows = {x["key"]: x for x in r["levels"]} + for lv in llm.TUNED: + row = rows[lv["key"]] + assert row["gpu_util"]["pct"]["n"] > 0 # every row records the GPU's use + if lv.get("gpu_cap"): + assert row["gpu_cap"]["cap"] == llm.GPU_CAP and row["gpu_cap"]["cycles"] > 0 + assert row["gpu_util"]["over_cap_share"] == 1.0 # the fake GPU stays at 99 % + else: + assert "gpu_cap" not in row + + +def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp_path, monkeypatch, + fake_server): + r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, []) + assert r["status"] == "partial" and not paused + assert any("no engine process" in e["error"] for e in r["errors"])