User decision (2026-11-10, room to watch a video next to Jarvis): the normal placement leaves 1 GB of VRAM for programs opened later, and Jarvis's GPU work is capped at 95 %. - gamemode_tune's normal rows take OTHER_ROOM_MIB off the expert budget. - gpucap.DutyCap pauses the Strata engine (strata.exe --serve under the kit) and the STT worker with their children together for 10 ms of every 200 ms (NtSuspendProcess), and always resumes them. - Every row samples nvidia-smi utilization.gpu; capped rows have an uncapped twin (normal with and without the STT, game 2 GB with 2 s partials) so the run shows the cost and what the GPU really gets back. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
793 lines
40 KiB
Python
793 lines
40 KiB
Python
"""Stage: LLM through Strata (install, start, measure, restart timings).
|
|
|
|
Options (--opt key=value):
|
|
model=fast|recommended which preset to measure (default fast)
|
|
url=http://.../v1 measure an already running server instead (no Strata management)
|
|
context=32768 context size to start Strata with
|
|
only=tools only re-measure tool calling, before and after the Jarvis fix
|
|
only=levels the fast model in each placement the app's automatic
|
|
tuning switches between (normal, and less VRAM for a game)
|
|
only=together the same with the STT loaded next to it and transcribing ten
|
|
speakers at once, on the GPU and on the CPU (needs stt_python=)
|
|
only=live game placements and the STT while the user runs a real game
|
|
only=tune the normal placement with room left for the STT and 1 GB for other
|
|
programs, the game placement with the STT doing fewer partial passes
|
|
while the LLM answers, and both with Jarvis capped at 95 % of the GPU
|
|
stt_python=PATH the Python of the stt venv (m0.py passes it)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, ask, gpu_memory_mib, interactive, # noqa: E402
|
|
read_json, run_logged, say, scrub, stage_args, wait_enter)
|
|
from m0lib.llm_bench import (bench_single, quick_bench, test_decode, test_images, test_parallel, # noqa: E402
|
|
test_tools, test_vram_shrink, warmup)
|
|
from m0lib.tool_fix import test_tools_fixed, tools_passed # noqa: E402
|
|
from m0lib import gpucap # noqa: E402
|
|
from m0lib.llm_client import ChatClient, http_json # noqa: E402
|
|
|
|
PRESETS = {
|
|
"fast": {"family": "qwen36", "model": "ud-q4_k_xl", "label": "Qwen3.6-35B-A3B UD-Q4_K_XL"},
|
|
# Strata 0.1.59 gives Q2_0 the DFlash2 drafter alone, which needs a native pack; the canonical
|
|
# Q2_0 pack is not one, so the engine stops at start. The MTP drafter (or none) starts.
|
|
"recommended": {"family": "qwen", "model": "q2_0", "label": "Qwen3.8-Flash-Next Q2_0 (ISTA-DASLab)",
|
|
"drafting": "mtp"},
|
|
}
|
|
# Tried in order when a preset still gets a tool call wrong with the Jarvis fix
|
|
# (the user's rule: if the fix does not work, change the model). Same family,
|
|
# less compressed weights, all within 128 GB RAM. need_gb: free disk to download.
|
|
CANDIDATES = {
|
|
"fast": [
|
|
{"family": "qwen36", "model": "ud-q6_k_xl", "label": "Qwen3.6-35B-A3B UD-Q6_K_XL", "need_gb": 40},
|
|
{"family": "qwen36", "model": "q8_0", "label": "Qwen3.6-35B-A3B Q8_0", "need_gb": 45},
|
|
],
|
|
"recommended": [
|
|
{"family": "qwen", "model": "iq3_xxs", "label": "Qwen3.8-Flash-Next IQ3_XXS (ISTA-DASLab)",
|
|
"drafting": "mtp", "need_gb": 80},
|
|
{"family": "qwen", "model": "iq3_s", "label": "Qwen3.8-Flash-Next IQ3_S (ISTA-DASLab)",
|
|
"drafting": "mtp", "need_gb": 85},
|
|
],
|
|
}
|
|
PORT = 8080
|
|
|
|
# The placements the app's automatic tuning moves the one model between.
|
|
# Strata's qwen36 engine cannot give VRAM back while it runs (POST /v1/vram
|
|
# answers 400), so each placement is a restart with --expert-vram (GiB for the
|
|
# routed experts; the rest stay in RAM) and a matching prompt chunk. A game
|
|
# placement also reads pictures on the CPU, answers one request at a time and
|
|
# leaves the game half the CPU (8 engine workers cost nothing on the 9950X3D).
|
|
# The game PC runs (2026-11-10) showed that the expert budget is not the floor:
|
|
# 4.4 GiB of weights never leave the card, the conversation memory scales with
|
|
# the context, and Strata 0.1.59's qwen36 engine takes no --kv and always loads
|
|
# the MTP block, so the smaller placements shrink the context instead.
|
|
# The normal placement answers one request at a time (faster on a 16 GB card);
|
|
# two only when a chat channel answers every message.
|
|
# The last row starts the normal placement again: what returning after a game costs.
|
|
GAME_RESERVE_MIB = 8000
|
|
_GAME = {"parallel": 1, "vision": "cpu"}
|
|
_W8 = "pool-workers=8"
|
|
_EXP2 = ["expert-vram=2", "prefill=2048", _W8]
|
|
_EXP0 = ["expert-vram=0", "prefill=2048", _W8]
|
|
LEVELS = [
|
|
{"key": "normal", "parallel": 1, "vision": "gpu", "sets": []},
|
|
{"key": "game_experts_2g", **_GAME, "sets": _EXP2},
|
|
{"key": "game_experts_2g_16k", **_GAME, "context": 16384, "sets": _EXP2},
|
|
{"key": "game_experts_0g_16k", **_GAME, "context": 16384, "sets": _EXP0},
|
|
{"key": "normal_again", "parallel": 1, "vision": "gpu", "sets": []},
|
|
]
|
|
# The same with the STT worker loaded first (as the app starts it) and transcribing
|
|
# ten speakers at once while the LLM is measured: the VRAM left is then the real
|
|
# game share. "stt" is where the STT runs; on the CPU it takes no VRAM but shares
|
|
# the cores with the LLM's CPU experts.
|
|
TOGETHER = [
|
|
{"key": "normal_stt_gpu", "parallel": 1, "vision": "gpu", "sets": [], "stt": "cuda"},
|
|
{"key": "normal_parallel2_stt_gpu", "parallel": 2, "vision": "gpu", "sets": [], "stt": "cuda"},
|
|
{"key": "game_experts_2g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"},
|
|
{"key": "game_experts_2g_16k_stt_cpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cpu"},
|
|
{"key": "game_experts_0g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP0, "stt": "cuda"},
|
|
{"key": "game_experts_0g_16k_stt_cpu", **_GAME, "context": 16384, "sets": _EXP0, "stt": "cpu"},
|
|
]
|
|
# With a real game running: only game placements (the normal one would take the game's VRAM).
|
|
LIVE = [lv for lv in TOGETHER if lv["key"].startswith("game_")]
|
|
# The fixes for what gamemode_stt showed (user decision, 2026-11-10):
|
|
# - Strata's automatic placement filled the card with experts and left the STT no room (15.9 of
|
|
# 16.3 GB with the STT busy; the first token after a long context went from 1.6 to 7.5 s). The
|
|
# qwen36 engine ignores --vram-reserve-mib, so "stt_room" sets --expert-vram from what this PC
|
|
# measures: one start without experts (with the STT loaded) gives what the LLM needs before any
|
|
# expert; the experts get what is left after the STT's ten-speaker growth and a margin.
|
|
# - The STT's partial passes took the GPU from the LLM (output 1/2.5 to 1/3). "stt_pace" is what
|
|
# the app would do in a game while the LLM answers: a ten-speaker partial pass every 2 s, or
|
|
# none, and the finals at once. The LLM is measured the whole time, so the pace holds for the
|
|
# whole run. Without a game the STT keeps every partial (user decision), so only the game
|
|
# placement is paced; its back-to-back row is the baseline measured in the same run.
|
|
# The 2 GB expert placement is the game minimum on a 16 GB card (faster than 0 GB in a real game).
|
|
# Then (user decision, 2026-11-10, for watching a video next to Jarvis):
|
|
# - the normal placement also leaves OTHER_ROOM_MIB free for programs opened later ("other_room_mib");
|
|
# - Jarvis's GPU work (the engine and the STT worker together) is paused for 5 % of every 200 ms
|
|
# ("gpu_cap", gpucap.DutyCap). The rows with and without it measure what it costs and how much
|
|
# GPU time it really gives back (queued work still runs during the pause).
|
|
STT_ROOM_MARGIN_MIB = 768 # what Strata's qwen36 engine itself keeps free when it places experts
|
|
STT_GROWTH_FALLBACK_MIB = 1440 # M0 stt on the game PC: ten speakers at once over the loaded model
|
|
OTHER_ROOM_MIB = 1024 # the app's default room for other programs in the normal placement
|
|
GPU_CAP = 0.95
|
|
_PARTIAL_2S = {"partial_every": 2.0, "final_every": 0.5}
|
|
_FINALS_ONLY = {"partial_every": None, "final_every": 0.5}
|
|
_ROOM = {"parallel": 1, "vision": "gpu", "sets": ["prefill=2048"], "stt_room": True,
|
|
"other_room_mib": OTHER_ROOM_MIB}
|
|
_GAME2 = {**_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"}
|
|
TUNED = [
|
|
# The room is worked out in the first row, with the STT loaded; the rows after it reuse it.
|
|
{"key": "normal_room_stt_gpu", **_ROOM, "stt": "cuda"},
|
|
{"key": "normal_room_stt_gpu_cap95", **_ROOM, "stt": "cuda", "gpu_cap": GPU_CAP},
|
|
{"key": "normal_room_llm_only", **_ROOM},
|
|
{"key": "normal_room_llm_only_cap95", **_ROOM, "gpu_cap": GPU_CAP},
|
|
{"key": "game_experts_2g_16k_stt_gpu", **_GAME2},
|
|
{"key": "game_experts_2g_16k_partial_2s", **_GAME2, "stt_pace": _PARTIAL_2S},
|
|
{"key": "game_experts_2g_16k_partial_2s_cap95", **_GAME2, "stt_pace": _PARTIAL_2S, "gpu_cap": GPU_CAP},
|
|
{"key": "game_experts_2g_16k_finals_only", **_GAME2, "stt_pace": _FINALS_ONLY},
|
|
]
|
|
LEVEL_SETS = {"levels": LEVELS, "together": TOGETHER, "live": LIVE, "tune": TUNED}
|
|
INSTALL_URL = "https://github.com/Niko1221/Strata/releases/latest/download/install.ps1"
|
|
|
|
|
|
class Strata:
|
|
def __init__(self, work: Path, log: Path):
|
|
self.home = work / "strata"
|
|
self.models = work / "models" / "llm"
|
|
self.log = log
|
|
# Keep Strata's engine, state and models inside the work folder: the
|
|
# installer honours STRATA_DATA_DIR, the engine components
|
|
# STRATA_COMPONENTS_DIR, and the per-user state follows LOCALAPPDATA.
|
|
self.env = dict(os.environ, STRATA_DATA_DIR=str(self.home), STRATA_NO_PATH="1",
|
|
STRATA_HOME=str(self.home / "home"), STRATA_COMPONENTS_DIR=str(self.home / "components"),
|
|
PYTHONUTF8="1")
|
|
if IS_WINDOWS:
|
|
self.env["LOCALAPPDATA"] = str(self.home / "localappdata")
|
|
(self.home / "localappdata").mkdir(parents=True, exist_ok=True)
|
|
self._flags: set[str] | None = None
|
|
|
|
@property
|
|
def exe(self) -> Path:
|
|
return self.home / "terminal" / "current" / "bin" / ("strata.cmd" if IS_WINDOWS else "strata")
|
|
|
|
def install(self) -> bool:
|
|
if self.exe.exists():
|
|
return True
|
|
self.home.mkdir(parents=True, exist_ok=True)
|
|
say(" Strata 터미널 설치 중 (작업 폴더 안에만 설치하고 PATH는 바꾸지 않습니다)")
|
|
if IS_WINDOWS:
|
|
cmd = ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", f"irm {INSTALL_URL} | iex"]
|
|
else:
|
|
cmd = ["sh", "-c", f"curl -fsSL {INSTALL_URL[:-3]}sh -o install.sh && sh install.sh"]
|
|
rc = run_logged(cmd, self.log, timeout=1800, env=self.env, cwd=str(self.home), echo=True)
|
|
return rc == 0 and self.exe.exists()
|
|
|
|
def cmd(self, *args: str) -> list[str]:
|
|
return [str(self.exe), *args]
|
|
|
|
def supported_flags(self) -> set[str]:
|
|
if self._flags is None:
|
|
p = subprocess.run(self.cmd("launch", "--help"), capture_output=True, text=True, env=self.env,
|
|
encoding="utf-8", errors="replace", timeout=120)
|
|
with open(self.log, "a", encoding="utf-8") as f:
|
|
f.write(scrub(p.stdout + p.stderr))
|
|
self._flags = set(re.findall(r"--[a-z0-9][a-z0-9-]*", p.stdout + p.stderr))
|
|
return self._flags
|
|
|
|
def launch_args(self, preset: dict, context: int, parallel: int, vision: str, extra_sets: list[str],
|
|
restart: bool, kv: str | None = None, mtp: str | None = None) -> tuple[list[str], list[str]]:
|
|
want = [("--family", preset["family"]), ("--model", preset["model"]), ("--data-dir", str(self.models)),
|
|
("--context", str(context)), ("--vision", vision), ("--parallel", str(parallel)),
|
|
("--access", "local"), ("--port", str(PORT)), ("--no-api-key", None), ("--yes", None),
|
|
("--no-tui", None)]
|
|
if preset.get("drafting"):
|
|
want.append(("--drafting", preset["drafting"]))
|
|
if kv:
|
|
want.append(("--kv", kv))
|
|
if mtp:
|
|
want.append(("--mtp", mtp))
|
|
if restart:
|
|
want.append(("--restart", None))
|
|
for s in extra_sets:
|
|
want.append(("--set", s))
|
|
flags = self.supported_flags()
|
|
args, dropped = [], []
|
|
for k, v in want:
|
|
if flags and k not in flags:
|
|
dropped.append(k)
|
|
continue
|
|
args += [k] + ([v] if v is not None else [])
|
|
return args, dropped
|
|
|
|
def plan(self, preset: dict, context: int) -> object:
|
|
args, _ = self.launch_args(preset, context, 1, "on", [], False)
|
|
args = [a for a in args if a not in ("--yes", "--no-tui")] + ["--plan", "--output", "json"]
|
|
p = subprocess.run(self.cmd("launch", *args), capture_output=True, text=True, env=self.env,
|
|
encoding="utf-8", errors="replace", timeout=600)
|
|
with open(self.log, "a", encoding="utf-8") as f:
|
|
f.write(scrub(p.stdout + p.stderr))
|
|
try:
|
|
return json.loads(p.stdout)
|
|
except ValueError:
|
|
return {"exit": p.returncode, "text": scrub(p.stdout[-3000:] + p.stderr[-2000:])}
|
|
|
|
def launch(self, preset: dict, context: int, parallel: int, vision: str = "on",
|
|
extra_sets: list[str] | None = None, restart: bool = False, timeout: float = 4 * 3600,
|
|
kv: str | None = None, mtp: str | None = None) -> dict:
|
|
args, dropped = self.launch_args(preset, context, parallel, vision, extra_sets or [], restart, kv, mtp)
|
|
last = [0.0]
|
|
|
|
def progress(line: str) -> bool:
|
|
# Downloads print many progress lines; show one every 15 s.
|
|
if "%" in line:
|
|
if time.monotonic() - last[0] < 15:
|
|
return False
|
|
last[0] = time.monotonic()
|
|
return True
|
|
|
|
self.mark_engine_logs()
|
|
t0 = time.perf_counter()
|
|
rc = run_logged(self.cmd("launch", *args), self.log, timeout=timeout, env=self.env, echo=True,
|
|
echo_filter=progress)
|
|
t_cmd = time.perf_counter() - t0
|
|
# Only a successful launch counts: after a failed one, whatever still
|
|
# answers on the port is not the model that was asked for.
|
|
ready = wait_ready(f"http://127.0.0.1:{PORT}", timeout=1800) if rc == 0 else None
|
|
if ready is None:
|
|
self.keep_engine_log()
|
|
return {"args": scrub(" ".join(args)), "dropped_flags": dropped, "exit": rc,
|
|
"command_seconds": round(t_cmd, 1), "ready": ready is not None,
|
|
"ready_seconds": round(time.perf_counter() - t0, 1) if ready is not None else None}
|
|
|
|
def keep_engine_log(self) -> None:
|
|
"""Copy the newest engine log (scrubbed, its end) next to our log so the results zip carries it:
|
|
Strata's one-line reason can name a side effect, not the cause."""
|
|
found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime)
|
|
if not found:
|
|
return
|
|
src = found[-1]
|
|
text = src.read_text(encoding="utf-8", errors="replace")[-200_000:]
|
|
dest = self.log.with_name(f"{self.log.stem}_engine_{src.name.removesuffix('.engine.log')}.log")
|
|
dest.write_text(scrub(text), encoding="utf-8")
|
|
|
|
def mark_engine_logs(self) -> None:
|
|
"""Remember how long each engine log is, so engine_lines() reads only what the next start writes."""
|
|
self._log_marks = {p: p.stat().st_size for p in self.home.rglob("*.engine.log")}
|
|
|
|
def engine_lines(self, prefix: str = "strata-q36:", limit: int = 20) -> list[str]:
|
|
"""The placement lines (how much went to VRAM and RAM) the newest engine log got since the
|
|
last start, scrubbed. The log is appended across starts; one written again is read whole."""
|
|
found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime)
|
|
if not found:
|
|
return []
|
|
src = found[-1]
|
|
start = getattr(self, "_log_marks", {}).get(src, 0)
|
|
if src.stat().st_size < start:
|
|
start = 0
|
|
with open(src, "rb") as f:
|
|
f.seek(start)
|
|
text = f.read()[-400_000:].decode("utf-8", errors="replace")
|
|
return [scrub(x.strip())[:400] for x in text.splitlines() if x.lstrip().startswith(prefix)][:limit]
|
|
|
|
def stop(self) -> None:
|
|
run_logged(self.cmd("launch", "--stop"), self.log, timeout=300, env=self.env)
|
|
time.sleep(3)
|
|
|
|
def status(self) -> object:
|
|
p = subprocess.run(self.cmd("launch", "--status", "--output", "json"), capture_output=True, text=True,
|
|
env=self.env, encoding="utf-8", errors="replace", timeout=120)
|
|
try:
|
|
return json.loads(p.stdout)
|
|
except ValueError:
|
|
return scrub(p.stdout[-2000:])
|
|
|
|
|
|
class SttLoad:
|
|
"""The STT worker (stt.py mode=load in the stt venv) as a separate process next to the LLM."""
|
|
|
|
def __init__(self, python: str, work: Path, device: str, log: Path, pace: dict | None = None):
|
|
d = work / "cache" / f"stt_load_{device}"
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
d.mkdir(parents=True)
|
|
self.out, self.go_file, self.stop_file = d / "result.json", d / "go", d / "stop"
|
|
self.log = log
|
|
self.cmd = [python, str(Path(__file__).resolve().parent / "stt.py"), "--out", str(self.out),
|
|
"--work", str(work), "--opt", "mode=load", "--opt", f"device={device}",
|
|
"--opt", f"go={self.go_file}", "--opt", f"stop={self.stop_file}"]
|
|
if pace is not None:
|
|
for key in ("partial_every", "final_every"):
|
|
v = pace.get(key)
|
|
self.cmd += ["--opt", f"{key}={'none' if v is None else v}"]
|
|
self.proc: subprocess.Popen | None = None
|
|
|
|
@property
|
|
def pid(self) -> int | None:
|
|
return self.proc.pid if self.proc else None
|
|
|
|
def _read(self) -> dict:
|
|
try:
|
|
return read_json(self.out)
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
def start(self, timeout: float = 900) -> dict:
|
|
"""Load the model; returns the worker's result ("loaded", or "failed" with the reason)."""
|
|
self._logf = open(self.log, "a", encoding="utf-8")
|
|
self.proc = subprocess.Popen(self.cmd, stdout=self._logf, stderr=subprocess.STDOUT)
|
|
t0 = time.monotonic()
|
|
while time.monotonic() - t0 < timeout:
|
|
got = self._read()
|
|
if got.get("status") in ("loaded", "failed"):
|
|
return got
|
|
if self.proc.poll() is not None:
|
|
return got or {"status": "failed", "errors": [f"stt worker exited with {self.proc.returncode}"]}
|
|
time.sleep(0.5)
|
|
return {"status": "failed", "errors": ["stt model did not load in time"]}
|
|
|
|
def go(self) -> None:
|
|
self.go_file.write_text("", encoding="utf-8")
|
|
|
|
def finish(self, timeout: float = 300) -> dict:
|
|
"""Stop the loop; the CPU can be in the middle of a long ten-speaker batch."""
|
|
self.stop_file.write_text("", encoding="utf-8")
|
|
try:
|
|
self.proc.wait(timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
# Whatever the file holds is the "loaded" note, not timings; close() kills the worker.
|
|
return {"status": "timeout", "errors": [f"stt worker did not stop within {timeout:.0f} s"]}
|
|
return self._read()
|
|
|
|
def close(self) -> None:
|
|
if self.proc and self.proc.poll() is None:
|
|
self.proc.kill()
|
|
self.proc.wait(timeout=30)
|
|
if getattr(self, "_logf", None):
|
|
self._logf.close()
|
|
|
|
|
|
def stt_gaps(busy: dict, pace: dict | None = None) -> list[str]:
|
|
"""Why an STT load next to the LLM did not measure what it should: its errors, a worker that
|
|
did not finish (timed out, died, never started), or no ten-speaker or single-clip timings
|
|
(no ten-speaker ones are expected when the pace turns the partials off)."""
|
|
gaps = list(busy.get("errors") or [])
|
|
if busy.get("status") != "done":
|
|
if not gaps:
|
|
gaps.append(f"stt worker ended as {busy.get('status') or 'nothing written'}")
|
|
return gaps
|
|
partials = pace is None or pace.get("partial_every") is not None
|
|
for key in (("batch10_s", "single_s") if partials else ("single_s",)):
|
|
if not (busy.get(key) or {}).get("n"):
|
|
gaps.append(f"stt worker measured no {key}")
|
|
return gaps
|
|
|
|
|
|
def port_in_use(port: int) -> bool:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.settimeout(2)
|
|
return sock.connect_ex(("127.0.0.1", port)) == 0
|
|
|
|
|
|
def wait_ready(root: str, timeout: float) -> float | None:
|
|
t0 = time.perf_counter()
|
|
while time.perf_counter() - t0 < timeout:
|
|
try:
|
|
code, body = http_json("GET", root + "/v1/models", timeout=5)
|
|
if code == 200 and isinstance(body, dict) and body.get("data"):
|
|
return time.perf_counter() - t0
|
|
except Exception:
|
|
pass
|
|
time.sleep(2)
|
|
return None
|
|
|
|
|
|
def server_info(base: str) -> dict:
|
|
info = {}
|
|
for path in ("/models", "/status"):
|
|
try:
|
|
code, body = http_json("GET", base + path, timeout=10)
|
|
info[path] = body if code == 200 else {"http": code}
|
|
except Exception as e:
|
|
info[path] = {"error": str(e)}
|
|
return info
|
|
|
|
|
|
def tools_only(r: StageResult, c: ChatClient) -> dict | None:
|
|
"""Tool calling as first measured (baseline) and with the Jarvis fix.
|
|
|
|
Returns the fixed run's summary (None when it could not run)."""
|
|
summary = None
|
|
for name, fn in (("warmup", lambda: warmup(r, c)), ("tools", lambda: test_tools(r, c)),
|
|
("fixed", lambda: test_tools_fixed(r, c))):
|
|
try:
|
|
out = fn()
|
|
if name == "fixed":
|
|
summary = out
|
|
except Exception as e:
|
|
r.error("tools", e)
|
|
r.save()
|
|
return summary
|
|
|
|
|
|
def try_candidates(r: StageResult, which: str, default_summary: dict | None, measure, free_gb) -> None:
|
|
"""If the default preset fails with the fix, measure the next candidates.
|
|
|
|
``measure(preset)`` starts the preset and returns its fixed-run summary;
|
|
``free_gb()`` is the free disk in GB. Stops at the first passing model and
|
|
records which one Jarvis should switch to (None if none passed).
|
|
"""
|
|
verdict: dict = {"default": PRESETS[which]["label"], "default_passed": tools_passed(default_summary),
|
|
"candidates": [], "chosen": PRESETS[which]["label"] if tools_passed(default_summary) else None}
|
|
r["tools_verdict"] = verdict
|
|
if verdict["default_passed"]:
|
|
return
|
|
for cand in CANDIDATES[which]:
|
|
row: dict = {"label": cand["label"]}
|
|
verdict["candidates"].append(row)
|
|
if free_gb() < cand["need_gb"]:
|
|
row["skipped"] = "disk"
|
|
say(f" {cand['label']}: 디스크 공간이 부족해 건너뜁니다 (필요 약 {cand['need_gb']} GB)")
|
|
continue
|
|
say(f" 수정 후에도 틀린 도구 호출이 있어 다른 모델을 잽니다: {cand['label']} (처음이면 내려받습니다)")
|
|
try:
|
|
row["summary"] = measure(cand)
|
|
except Exception as e:
|
|
row["error"] = str(e)[:300]
|
|
r.save()
|
|
if tools_passed(row.get("summary")):
|
|
verdict["chosen"] = cand["label"]
|
|
return
|
|
|
|
|
|
def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels: list[dict] = LEVELS,
|
|
stt_python: str = "", choose: bool = True) -> None:
|
|
"""Start the model in each placement and measure what the switch and the placement cost.
|
|
A level with "stt" first loads the STT worker there and keeps it transcribing during the
|
|
measurement, so the VRAM left is what a game really gets."""
|
|
base = f"http://127.0.0.1:{PORT}/v1"
|
|
rows: list[dict] = []
|
|
r["levels"] = rows
|
|
rooms: dict = {} # one probe per placement that leaves the STT room
|
|
for lv in levels:
|
|
row: dict = {"key": lv["key"], "parallel": lv["parallel"], "vision": lv["vision"], "sets": lv["sets"],
|
|
"context": lv.get("context", context), "kv": lv.get("kv"), "mtp": lv.get("mtp")}
|
|
rows.append(row)
|
|
say(f" - 배치 {lv['key']}: 다시 시작해서 재는 중")
|
|
t0 = time.perf_counter()
|
|
s.stop()
|
|
idle = gpu_memory_mib()
|
|
row["vram_idle_mib"] = idle[0] if idle else None
|
|
stt = None
|
|
with GpuSampler(interval=1) as g:
|
|
try:
|
|
if lv.get("stt"):
|
|
stt = SttLoad(stt_python, s.log.parent.parent, lv["stt"], s.log.with_name("stt_load.log"),
|
|
pace=lv.get("stt_pace"))
|
|
row["stt"] = {"device": lv["stt"], "pace": lv.get("stt_pace"), "load": stt.start()}
|
|
if row["stt"]["load"].get("status") != "loaded":
|
|
r.error(f"level {lv['key']}", "; ".join(row["stt"]["load"].get("errors") or [])
|
|
or "the STT did not load")
|
|
continue
|
|
sets = list(lv["sets"])
|
|
if lv.get("stt_room"):
|
|
where = (row["context"], lv["parallel"], lv["vision"], lv.get("other_room_mib", 0))
|
|
if where not in rooms:
|
|
if not stt:
|
|
# The probe needs the STT model in VRAM, as the app has it.
|
|
r.error(f"level {lv['key']}", "no STT room was measured with the STT loaded")
|
|
continue
|
|
rooms[where] = stt_room(r, s, preset, row, lv, base)
|
|
row["stt_room"] = rooms[where]
|
|
sets.append(f"expert-vram={row['stt_room']['expert_vram_gib']}")
|
|
row["sets"] = sets
|
|
t0 = time.perf_counter() # the probe is not part of the switch
|
|
row["launch"] = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"],
|
|
extra_sets=sets, kv=lv.get("kv"), mtp=lv.get("mtp"), timeout=3600)
|
|
ready = row["launch"].get("ready")
|
|
row["switch_s"] = round(time.perf_counter() - t0, 1) if ready else None
|
|
row["engine"] = s.engine_lines()
|
|
if ready:
|
|
try:
|
|
c = ChatClient(base)
|
|
if "server" not in r.data:
|
|
r["server"] = {"model": c.model, "info": server_info(base)}
|
|
c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time
|
|
if stt:
|
|
stt.go()
|
|
cap = lv.get("gpu_cap")
|
|
pids = cap_pids(s, stt) if cap else []
|
|
if cap and not pids:
|
|
r.error(f"level {lv['key']} gpu cap", "found no engine process to pause")
|
|
with gpucap.UtilSampler() as util, gpucap.DutyCap(pids, cap or 1.0) as duty:
|
|
row.update(quick_bench(c))
|
|
row["gpu_util"] = util.summary(cap)
|
|
if cap:
|
|
row["gpu_cap"] = duty.summary()
|
|
if row["gpu_cap"]["failed_processes"]:
|
|
r.error(f"level {lv['key']} gpu cap",
|
|
f"{row['gpu_cap']['failed_processes']} process(es) could not be paused")
|
|
for err in row["errors"]:
|
|
r.error(f"level {lv['key']}", err)
|
|
except Exception as e:
|
|
r.error(f"level {lv['key']}", e)
|
|
if stt:
|
|
busy = stt.finish()
|
|
row["stt"].update({k: busy.get(k) for k in ("status", "batch10_s", "single_s", "errors",
|
|
"seconds", "busy_share", "final_wait_s")
|
|
if k in busy})
|
|
for err in stt_gaps(busy, lv.get("stt_pace")):
|
|
r.error(f"level {lv['key']} stt", err)
|
|
else:
|
|
r.error(f"level {lv['key']}", "the placement did not start")
|
|
now = gpu_memory_mib() if ready else None # the sampler can miss a short run
|
|
finally:
|
|
if stt:
|
|
stt.close()
|
|
peak = max(x for x in (g.peak, now[0] if now else None) if x is not None) \
|
|
if g.peak is not None or now else None
|
|
if now:
|
|
row["vram_after_mib"] = now[0] # settled; the peak (during the start) is what a running game must survive
|
|
if peak is not None and idle:
|
|
row["vram_peak_mib"] = peak
|
|
row["vram_free_mib"] = idle[1] - peak
|
|
say(f" 전환 {row.get('switch_s')}s, 출력 {(row.get('decode_tok_s') or {}).get('median')} tok/s, "
|
|
f"남는 VRAM {row.get('vram_free_mib')} MiB")
|
|
r.save()
|
|
s.stop()
|
|
if not choose:
|
|
return
|
|
r["game_choice"] = pick_game_level(rows)
|
|
measured = [x for x in rows if x["key"].startswith("game_") and x.get("vram_free_mib") is not None]
|
|
if r["game_choice"] is None and measured:
|
|
most = max(measured, key=lambda x: x["vram_free_mib"])
|
|
r.note(f"no game placement leaves {GAME_RESERVE_MIB} MiB free; the most free is "
|
|
f"{most['key']}: {most['vram_free_mib']} MiB")
|
|
|
|
|
|
def stt_growth_mib(work: Path) -> int:
|
|
"""What ten speakers at once add over the loaded STT model, from this PC's own stt stage."""
|
|
try:
|
|
return int(read_json(work / "results" / "stt.json")["run_int8_float16"]["batch_speakers"]["vram_added_mib"])
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
return STT_GROWTH_FALLBACK_MIB
|
|
|
|
|
|
def staging_gib(lines: list[str], default: float = 1.10) -> float:
|
|
"""The long-prompt staging the expert budget holds first (engine log: "(1.10 GiB staging)")."""
|
|
for line in lines:
|
|
m = re.search(r"([0-9.]+) GiB staging", line)
|
|
if m:
|
|
return float(m.group(1))
|
|
return default
|
|
|
|
|
|
def expert_budget_gib(total_mib: int, used_mib: int, probe_model_mib: int, staging: float, growth_mib: int,
|
|
margin_mib: int = STT_ROOM_MARGIN_MIB, other_mib: int = 0) -> float:
|
|
"""--expert-vram (GiB, rounded down to 0.1) that leaves the STT's growth, the margin and the room
|
|
for other programs free. The budget first holds the staging, which the start without experts
|
|
already has."""
|
|
spare = total_mib - used_mib - probe_model_mib - growth_mib - margin_mib - other_mib
|
|
if spare <= 0:
|
|
return 0.0
|
|
return math.floor((staging * 1024 + spare) / 1024 * 10) / 10
|
|
|
|
|
|
def stt_room(r: StageResult, s: Strata, preset: dict, row: dict, lv: dict, base: str) -> dict:
|
|
"""Start the placement once without experts (the STT model already loaded), see what the LLM
|
|
holds, stop it, and work out the expert budget that leaves the STT's ten-speaker growth free."""
|
|
room: dict = {"stt_growth_mib": stt_growth_mib(s.log.parent.parent), "margin_mib": STT_ROOM_MARGIN_MIB,
|
|
"other_room_mib": lv.get("other_room_mib", 0), "expert_vram_gib": 0.0}
|
|
say(" STT 자리 계산: 전문가 층 없이 한 번 켜서 LLM 기본 VRAM을 잽니다")
|
|
before = gpu_memory_mib()
|
|
launch = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"],
|
|
extra_sets=["expert-vram=0"] + [x for x in lv["sets"] if not x.startswith("expert-vram=")],
|
|
timeout=3600)
|
|
room["probe_ready"] = bool(launch.get("ready"))
|
|
if launch.get("ready"):
|
|
try:
|
|
ChatClient(base).chat([{"role": "user", "content": "안녕"}], max_tokens=8)
|
|
except Exception as e:
|
|
r.error(f"level {lv['key']} stt room", e)
|
|
after = gpu_memory_mib()
|
|
lines = s.engine_lines()
|
|
if before and after:
|
|
room.update(probe_model_mib=after[0] - before[0], used_before_mib=before[0], total_mib=before[1],
|
|
staging_gib=staging_gib(lines))
|
|
room["expert_vram_gib"] = expert_budget_gib(before[1], before[0], room["probe_model_mib"],
|
|
room["staging_gib"], room["stt_growth_mib"],
|
|
other_mib=room["other_room_mib"])
|
|
if room["expert_vram_gib"] == 0:
|
|
r.note(f"{lv['key']}: no room for experts next to the STT; started with --expert-vram 0")
|
|
else:
|
|
r.error(f"level {lv['key']} stt room", "the start without experts did not come up; --expert-vram 0")
|
|
s.stop()
|
|
return room
|
|
|
|
|
|
def cap_pids(s: Strata, stt: SttLoad | None) -> list[int]:
|
|
"""The processes the GPU cap pauses together: the engine and the STT worker, with their children.
|
|
None when the engine is not found (pausing only the STT would not be the cap)."""
|
|
procs = gpucap.process_table()
|
|
roots = gpucap.engine_roots(procs, s.home)
|
|
if not roots:
|
|
return []
|
|
return gpucap.with_children(procs, roots + ([stt.pid] if stt and stt.pid else []))
|
|
|
|
|
|
def ask_for_a_game() -> bool:
|
|
"""The live stage needs someone to start a game first. False when nobody is there or they say no."""
|
|
if not interactive():
|
|
return False
|
|
say(" 게임을 켠 상태로 LLM과 음성 인식을 잽니다. 배치 4개를 재고 10~15분 걸립니다.")
|
|
if ask(" 지금 게임을 켜고 재시겠습니까?") != "y":
|
|
return False
|
|
wait_enter(" 게임을 창 모드(테두리 없는 창이면 가장 좋음)로 켜고 플레이 화면까지 들어간 뒤 Enter를 누르고, "
|
|
"15초 안에 게임으로 돌아가 주세요. 끝나면 알림음이 울립니다:")
|
|
time.sleep(15)
|
|
return True
|
|
|
|
|
|
def pick_game_level(rows: list[dict], reserve_mib: int = GAME_RESERVE_MIB) -> dict | None:
|
|
"""The game placement the app would pick on this PC: the fastest one that
|
|
leaves the game its reserve of VRAM. None when no game placement does."""
|
|
fits = [x for x in rows if x["key"].startswith("game_") and (x.get("vram_free_mib") or 0) >= reserve_mib
|
|
and (x.get("decode_tok_s") or {}).get("median")]
|
|
if not fits:
|
|
return None
|
|
best = max(fits, key=lambda x: x["decode_tok_s"]["median"])
|
|
return {"key": best["key"], "reserve_mib": reserve_mib, "vram_free_mib": best["vram_free_mib"],
|
|
"decode_tok_s": best["decode_tok_s"]["median"], "switch_s": best.get("switch_s"),
|
|
"voice_first_sentence": (best.get("voice_first_sentence") or {}).get("median")}
|
|
|
|
|
|
def measure_external(r: StageResult, url: str, context: int, only: str = "") -> None:
|
|
c = ChatClient(url)
|
|
r["server"] = {"url": url, "model": c.model, "info": server_info(c.base)}
|
|
if only == "tools":
|
|
tools_only(r, c)
|
|
return
|
|
if only in LEVEL_SETS:
|
|
r.note("placements need Strata; not measured against an external server")
|
|
return
|
|
with GpuSampler() as g:
|
|
bench_single(r, c, context)
|
|
test_parallel(r, c)
|
|
r["gpu"] = g.summary()
|
|
|
|
|
|
def measure_strata(r: StageResult, work: Path, which: str, context: int, only: str = "",
|
|
stt_python: str = "") -> None:
|
|
preset = PRESETS[which]
|
|
r["preset"] = dict(preset, key=which, context=context)
|
|
s = Strata(work, work / "logs" / f"strata_{which}.log")
|
|
if not s.install():
|
|
r.error("install", "Strata 설치 실패 (logs/strata_*.log 참고)")
|
|
return
|
|
r["plan"] = s.plan(preset, context)
|
|
r.save()
|
|
if only in LEVEL_SETS:
|
|
s.stop()
|
|
if port_in_use(PORT):
|
|
r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.")
|
|
return
|
|
if only != "levels" and not stt_python:
|
|
r.error("stt", "STT 환경을 찾지 못했습니다")
|
|
return
|
|
if only == "live":
|
|
if not ask_for_a_game():
|
|
r["skipped"] = "no game started"
|
|
return
|
|
r.note("measured while the user's game was running")
|
|
measure_levels(r, s, preset, context, LEVEL_SETS[only], stt_python=stt_python,
|
|
choose=only in ("levels", "together"))
|
|
if only == "live":
|
|
say("\a 게임 중 측정이 끝났습니다. 게임을 꺼도 됩니다.")
|
|
try:
|
|
import winsound
|
|
winsound.MessageBeep()
|
|
except Exception:
|
|
pass
|
|
return
|
|
|
|
# 1) Start (the first start downloads the model and engine).
|
|
say(f" {preset['label']} 시작 중. 처음에는 모델을 내려받으므로 오래 걸립니다.")
|
|
s.stop()
|
|
if port_in_use(PORT):
|
|
# Another program owns the port; measuring it would report the wrong model.
|
|
r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.")
|
|
return
|
|
with GpuSampler(interval=2) as g:
|
|
r["launch_cold"] = s.launch(preset, context, parallel=1)
|
|
r["launch_cold"].update(g.summary())
|
|
r.save()
|
|
if not r["launch_cold"]["ready"]:
|
|
r.error("launch", "서버가 준비되지 않았습니다")
|
|
r["status_after_fail"] = s.status()
|
|
return
|
|
base = f"http://127.0.0.1:{PORT}/v1"
|
|
c = ChatClient(base)
|
|
r["server"] = {"model": c.model, "info": server_info(base), "status": s.status()}
|
|
if only == "tools":
|
|
default_summary = tools_only(r, c)
|
|
|
|
def measure(cand: dict) -> dict | None:
|
|
key = f"tools_fixed_{cand['model']}"
|
|
res = s.launch(cand, context, parallel=1, restart=True)
|
|
r[f"launch_{cand['model']}"] = res
|
|
if not res["ready"]:
|
|
return None
|
|
return test_tools_fixed(r, ChatClient(base), key=key)
|
|
|
|
try_candidates(r, which, default_summary, measure,
|
|
free_gb=lambda: shutil.disk_usage(work).free / 1e9)
|
|
s.stop()
|
|
return
|
|
with GpuSampler() as g:
|
|
bench_single(r, c, context)
|
|
r["gpu_single"] = g.summary()
|
|
r.save()
|
|
|
|
# 2) Restart with the files already in RAM/page cache: what a game-mode switch costs.
|
|
# Also: two slots, the elastic VRAM cache and the GPU picture reader.
|
|
say(" 다시 시작 시간 측정 (동시 2개, VRAM 조절 켜기, 이미지 GPU 처리)")
|
|
r["launch_restart"] = s.launch(preset, context, parallel=2, vision="gpu", extra_sets=["vram-elastic"],
|
|
restart=True, timeout=3600)
|
|
r.save()
|
|
if r["launch_restart"]["ready"]:
|
|
c = ChatClient(base)
|
|
r["server_restart"] = {"status": s.status()}
|
|
for fn in (lambda: warmup(r, c), lambda: test_decode(r, c, key="decode_parallel2"), lambda: test_parallel(r, c),
|
|
lambda: test_images(r, c, key="images_gpu_reader"), lambda: test_vram_shrink(r, c)):
|
|
try:
|
|
fn()
|
|
except Exception as e:
|
|
r.error("restart tests", e)
|
|
r.save()
|
|
|
|
# 3) Game mode (b): switching from the recommended model to the fast one.
|
|
if which == "recommended" and (work / "results" / "llm_fast.json").exists():
|
|
say(" 게임 모드 (b): 권장 모델 -> 빠름 모델 전환 시간")
|
|
r["switch_to_fast"] = s.launch(PRESETS["fast"], context, parallel=1, restart=True, timeout=3600)
|
|
s.stop()
|
|
|
|
|
|
def main() -> int:
|
|
out, work, opts = stage_args()
|
|
which = opts.get("model", "fast")
|
|
context = int(opts.get("context", "32768"))
|
|
only = opts.get("only", "")
|
|
name = {"tools": f"tools_{which}", "levels": "gamemode", "together": "gamemode_stt",
|
|
"live": "gamemode_live", "tune": "gamemode_tune"}.get(only, f"llm_{which}")
|
|
r = StageResult(name, out)
|
|
try:
|
|
if opts.get("url"):
|
|
measure_external(r, opts["url"], context, only)
|
|
else:
|
|
measure_strata(r, work, which, context, only, opts.get("stt_python", ""))
|
|
except Exception as e:
|
|
r.error("llm", e)
|
|
if r.get("skipped"):
|
|
r.save("skipped")
|
|
return 0
|
|
# No server reached means nothing was measured: "failed" so the next run retries it.
|
|
if "server" not in r.data:
|
|
r.save("failed")
|
|
else:
|
|
r.save("ok" if not r["errors"] else "partial")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|