A resume that failed was only counted, and the process stayed suspended after the cap ended. DutyCap now keeps what it still owes per process, does not stack another suspend on it, retries the resume every period and at the end, treats a raising call as a failure, and reports any process it could not resume as left_paused (an error for the row). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
797 lines
40 KiB
Python
797 lines
40 KiB
Python
"""Stage: LLM through Strata (install, start, measure, restart timings).
|
|
|
|
Options (--opt key=value):
|
|
model=fast|recommended which preset to measure (default fast)
|
|
url=http://.../v1 measure an already running server instead (no Strata management)
|
|
context=32768 context size to start Strata with
|
|
only=tools only re-measure tool calling, before and after the Jarvis fix
|
|
only=levels the fast model in each placement the app's automatic
|
|
tuning switches between (normal, and less VRAM for a game)
|
|
only=together the same with the STT loaded next to it and transcribing ten
|
|
speakers at once, on the GPU and on the CPU (needs stt_python=)
|
|
only=live game placements and the STT while the user runs a real game
|
|
only=tune the normal placement with room left for the STT and 1 GB for other
|
|
programs, the game placement with the STT doing fewer partial passes
|
|
while the LLM answers, and both with Jarvis capped at 95 % of the GPU
|
|
stt_python=PATH the Python of the stt venv (m0.py passes it)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, ask, gpu_memory_mib, interactive, # noqa: E402
|
|
read_json, run_logged, say, scrub, stage_args, wait_enter)
|
|
from m0lib.llm_bench import (bench_single, quick_bench, test_decode, test_images, test_parallel, # noqa: E402
|
|
test_tools, test_vram_shrink, warmup)
|
|
from m0lib.tool_fix import test_tools_fixed, tools_passed # noqa: E402
|
|
from m0lib import gpucap # noqa: E402
|
|
from m0lib.llm_client import ChatClient, http_json # noqa: E402
|
|
|
|
PRESETS = {
|
|
"fast": {"family": "qwen36", "model": "ud-q4_k_xl", "label": "Qwen3.6-35B-A3B UD-Q4_K_XL"},
|
|
# Strata 0.1.59 gives Q2_0 the DFlash2 drafter alone, which needs a native pack; the canonical
|
|
# Q2_0 pack is not one, so the engine stops at start. The MTP drafter (or none) starts.
|
|
"recommended": {"family": "qwen", "model": "q2_0", "label": "Qwen3.8-Flash-Next Q2_0 (ISTA-DASLab)",
|
|
"drafting": "mtp"},
|
|
}
|
|
# Tried in order when a preset still gets a tool call wrong with the Jarvis fix
|
|
# (the user's rule: if the fix does not work, change the model). Same family,
|
|
# less compressed weights, all within 128 GB RAM. need_gb: free disk to download.
|
|
CANDIDATES = {
|
|
"fast": [
|
|
{"family": "qwen36", "model": "ud-q6_k_xl", "label": "Qwen3.6-35B-A3B UD-Q6_K_XL", "need_gb": 40},
|
|
{"family": "qwen36", "model": "q8_0", "label": "Qwen3.6-35B-A3B Q8_0", "need_gb": 45},
|
|
],
|
|
"recommended": [
|
|
{"family": "qwen", "model": "iq3_xxs", "label": "Qwen3.8-Flash-Next IQ3_XXS (ISTA-DASLab)",
|
|
"drafting": "mtp", "need_gb": 80},
|
|
{"family": "qwen", "model": "iq3_s", "label": "Qwen3.8-Flash-Next IQ3_S (ISTA-DASLab)",
|
|
"drafting": "mtp", "need_gb": 85},
|
|
],
|
|
}
|
|
PORT = 8080
|
|
|
|
# The placements the app's automatic tuning moves the one model between.
|
|
# Strata's qwen36 engine cannot give VRAM back while it runs (POST /v1/vram
|
|
# answers 400), so each placement is a restart with --expert-vram (GiB for the
|
|
# routed experts; the rest stay in RAM) and a matching prompt chunk. A game
|
|
# placement also reads pictures on the CPU, answers one request at a time and
|
|
# leaves the game half the CPU (8 engine workers cost nothing on the 9950X3D).
|
|
# The game PC runs (2026-11-10) showed that the expert budget is not the floor:
|
|
# 4.4 GiB of weights never leave the card, the conversation memory scales with
|
|
# the context, and Strata 0.1.59's qwen36 engine takes no --kv and always loads
|
|
# the MTP block, so the smaller placements shrink the context instead.
|
|
# The normal placement answers one request at a time (faster on a 16 GB card);
|
|
# two only when a chat channel answers every message.
|
|
# The last row starts the normal placement again: what returning after a game costs.
|
|
GAME_RESERVE_MIB = 8000
|
|
_GAME = {"parallel": 1, "vision": "cpu"}
|
|
_W8 = "pool-workers=8"
|
|
_EXP2 = ["expert-vram=2", "prefill=2048", _W8]
|
|
_EXP0 = ["expert-vram=0", "prefill=2048", _W8]
|
|
LEVELS = [
|
|
{"key": "normal", "parallel": 1, "vision": "gpu", "sets": []},
|
|
{"key": "game_experts_2g", **_GAME, "sets": _EXP2},
|
|
{"key": "game_experts_2g_16k", **_GAME, "context": 16384, "sets": _EXP2},
|
|
{"key": "game_experts_0g_16k", **_GAME, "context": 16384, "sets": _EXP0},
|
|
{"key": "normal_again", "parallel": 1, "vision": "gpu", "sets": []},
|
|
]
|
|
# The same with the STT worker loaded first (as the app starts it) and transcribing
|
|
# ten speakers at once while the LLM is measured: the VRAM left is then the real
|
|
# game share. "stt" is where the STT runs; on the CPU it takes no VRAM but shares
|
|
# the cores with the LLM's CPU experts.
|
|
TOGETHER = [
|
|
{"key": "normal_stt_gpu", "parallel": 1, "vision": "gpu", "sets": [], "stt": "cuda"},
|
|
{"key": "normal_parallel2_stt_gpu", "parallel": 2, "vision": "gpu", "sets": [], "stt": "cuda"},
|
|
{"key": "game_experts_2g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"},
|
|
{"key": "game_experts_2g_16k_stt_cpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cpu"},
|
|
{"key": "game_experts_0g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP0, "stt": "cuda"},
|
|
{"key": "game_experts_0g_16k_stt_cpu", **_GAME, "context": 16384, "sets": _EXP0, "stt": "cpu"},
|
|
]
|
|
# With a real game running: only game placements (the normal one would take the game's VRAM).
|
|
LIVE = [lv for lv in TOGETHER if lv["key"].startswith("game_")]
|
|
# The fixes for what gamemode_stt showed (user decision, 2026-11-10):
|
|
# - Strata's automatic placement filled the card with experts and left the STT no room (15.9 of
|
|
# 16.3 GB with the STT busy; the first token after a long context went from 1.6 to 7.5 s). The
|
|
# qwen36 engine ignores --vram-reserve-mib, so "stt_room" sets --expert-vram from what this PC
|
|
# measures: one start without experts (with the STT loaded) gives what the LLM needs before any
|
|
# expert; the experts get what is left after the STT's ten-speaker growth and a margin.
|
|
# - The STT's partial passes took the GPU from the LLM (output 1/2.5 to 1/3). "stt_pace" is what
|
|
# the app would do in a game while the LLM answers: a ten-speaker partial pass every 2 s, or
|
|
# none, and the finals at once. The LLM is measured the whole time, so the pace holds for the
|
|
# whole run. Without a game the STT keeps every partial (user decision), so only the game
|
|
# placement is paced; its back-to-back row is the baseline measured in the same run.
|
|
# The 2 GB expert placement is the game minimum on a 16 GB card (faster than 0 GB in a real game).
|
|
# Then (user decision, 2026-11-10, for watching a video next to Jarvis):
|
|
# - the normal placement also leaves OTHER_ROOM_MIB free for programs opened later ("other_room_mib");
|
|
# - Jarvis's GPU work (the engine and the STT worker together) is paused for 5 % of every 200 ms
|
|
# ("gpu_cap", gpucap.DutyCap). The rows with and without it measure what it costs and how much
|
|
# GPU time it really gives back (queued work still runs during the pause).
|
|
STT_ROOM_MARGIN_MIB = 768 # what Strata's qwen36 engine itself keeps free when it places experts
|
|
STT_GROWTH_FALLBACK_MIB = 1440 # M0 stt on the game PC: ten speakers at once over the loaded model
|
|
OTHER_ROOM_MIB = 1024 # the app's default room for other programs in the normal placement
|
|
GPU_CAP = 0.95
|
|
_PARTIAL_2S = {"partial_every": 2.0, "final_every": 0.5}
|
|
_FINALS_ONLY = {"partial_every": None, "final_every": 0.5}
|
|
_ROOM = {"parallel": 1, "vision": "gpu", "sets": ["prefill=2048"], "stt_room": True,
|
|
"other_room_mib": OTHER_ROOM_MIB}
|
|
_GAME2 = {**_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"}
|
|
TUNED = [
|
|
# The room is worked out in the first row, with the STT loaded; the rows after it reuse it.
|
|
{"key": "normal_room_stt_gpu", **_ROOM, "stt": "cuda"},
|
|
{"key": "normal_room_stt_gpu_cap95", **_ROOM, "stt": "cuda", "gpu_cap": GPU_CAP},
|
|
{"key": "normal_room_llm_only", **_ROOM},
|
|
{"key": "normal_room_llm_only_cap95", **_ROOM, "gpu_cap": GPU_CAP},
|
|
{"key": "game_experts_2g_16k_stt_gpu", **_GAME2},
|
|
{"key": "game_experts_2g_16k_partial_2s", **_GAME2, "stt_pace": _PARTIAL_2S},
|
|
{"key": "game_experts_2g_16k_partial_2s_cap95", **_GAME2, "stt_pace": _PARTIAL_2S, "gpu_cap": GPU_CAP},
|
|
{"key": "game_experts_2g_16k_finals_only", **_GAME2, "stt_pace": _FINALS_ONLY},
|
|
]
|
|
LEVEL_SETS = {"levels": LEVELS, "together": TOGETHER, "live": LIVE, "tune": TUNED}
|
|
INSTALL_URL = "https://github.com/Niko1221/Strata/releases/latest/download/install.ps1"
|
|
|
|
|
|
class Strata:
|
|
def __init__(self, work: Path, log: Path):
|
|
self.home = work / "strata"
|
|
self.models = work / "models" / "llm"
|
|
self.log = log
|
|
# Keep Strata's engine, state and models inside the work folder: the
|
|
# installer honours STRATA_DATA_DIR, the engine components
|
|
# STRATA_COMPONENTS_DIR, and the per-user state follows LOCALAPPDATA.
|
|
self.env = dict(os.environ, STRATA_DATA_DIR=str(self.home), STRATA_NO_PATH="1",
|
|
STRATA_HOME=str(self.home / "home"), STRATA_COMPONENTS_DIR=str(self.home / "components"),
|
|
PYTHONUTF8="1")
|
|
if IS_WINDOWS:
|
|
self.env["LOCALAPPDATA"] = str(self.home / "localappdata")
|
|
(self.home / "localappdata").mkdir(parents=True, exist_ok=True)
|
|
self._flags: set[str] | None = None
|
|
|
|
@property
|
|
def exe(self) -> Path:
|
|
return self.home / "terminal" / "current" / "bin" / ("strata.cmd" if IS_WINDOWS else "strata")
|
|
|
|
def install(self) -> bool:
|
|
if self.exe.exists():
|
|
return True
|
|
self.home.mkdir(parents=True, exist_ok=True)
|
|
say(" Strata 터미널 설치 중 (작업 폴더 안에만 설치하고 PATH는 바꾸지 않습니다)")
|
|
if IS_WINDOWS:
|
|
cmd = ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", f"irm {INSTALL_URL} | iex"]
|
|
else:
|
|
cmd = ["sh", "-c", f"curl -fsSL {INSTALL_URL[:-3]}sh -o install.sh && sh install.sh"]
|
|
rc = run_logged(cmd, self.log, timeout=1800, env=self.env, cwd=str(self.home), echo=True)
|
|
return rc == 0 and self.exe.exists()
|
|
|
|
def cmd(self, *args: str) -> list[str]:
|
|
return [str(self.exe), *args]
|
|
|
|
def supported_flags(self) -> set[str]:
|
|
if self._flags is None:
|
|
p = subprocess.run(self.cmd("launch", "--help"), capture_output=True, text=True, env=self.env,
|
|
encoding="utf-8", errors="replace", timeout=120)
|
|
with open(self.log, "a", encoding="utf-8") as f:
|
|
f.write(scrub(p.stdout + p.stderr))
|
|
self._flags = set(re.findall(r"--[a-z0-9][a-z0-9-]*", p.stdout + p.stderr))
|
|
return self._flags
|
|
|
|
def launch_args(self, preset: dict, context: int, parallel: int, vision: str, extra_sets: list[str],
|
|
restart: bool, kv: str | None = None, mtp: str | None = None) -> tuple[list[str], list[str]]:
|
|
want = [("--family", preset["family"]), ("--model", preset["model"]), ("--data-dir", str(self.models)),
|
|
("--context", str(context)), ("--vision", vision), ("--parallel", str(parallel)),
|
|
("--access", "local"), ("--port", str(PORT)), ("--no-api-key", None), ("--yes", None),
|
|
("--no-tui", None)]
|
|
if preset.get("drafting"):
|
|
want.append(("--drafting", preset["drafting"]))
|
|
if kv:
|
|
want.append(("--kv", kv))
|
|
if mtp:
|
|
want.append(("--mtp", mtp))
|
|
if restart:
|
|
want.append(("--restart", None))
|
|
for s in extra_sets:
|
|
want.append(("--set", s))
|
|
flags = self.supported_flags()
|
|
args, dropped = [], []
|
|
for k, v in want:
|
|
if flags and k not in flags:
|
|
dropped.append(k)
|
|
continue
|
|
args += [k] + ([v] if v is not None else [])
|
|
return args, dropped
|
|
|
|
def plan(self, preset: dict, context: int) -> object:
|
|
args, _ = self.launch_args(preset, context, 1, "on", [], False)
|
|
args = [a for a in args if a not in ("--yes", "--no-tui")] + ["--plan", "--output", "json"]
|
|
p = subprocess.run(self.cmd("launch", *args), capture_output=True, text=True, env=self.env,
|
|
encoding="utf-8", errors="replace", timeout=600)
|
|
with open(self.log, "a", encoding="utf-8") as f:
|
|
f.write(scrub(p.stdout + p.stderr))
|
|
try:
|
|
return json.loads(p.stdout)
|
|
except ValueError:
|
|
return {"exit": p.returncode, "text": scrub(p.stdout[-3000:] + p.stderr[-2000:])}
|
|
|
|
def launch(self, preset: dict, context: int, parallel: int, vision: str = "on",
|
|
extra_sets: list[str] | None = None, restart: bool = False, timeout: float = 4 * 3600,
|
|
kv: str | None = None, mtp: str | None = None) -> dict:
|
|
args, dropped = self.launch_args(preset, context, parallel, vision, extra_sets or [], restart, kv, mtp)
|
|
last = [0.0]
|
|
|
|
def progress(line: str) -> bool:
|
|
# Downloads print many progress lines; show one every 15 s.
|
|
if "%" in line:
|
|
if time.monotonic() - last[0] < 15:
|
|
return False
|
|
last[0] = time.monotonic()
|
|
return True
|
|
|
|
self.mark_engine_logs()
|
|
t0 = time.perf_counter()
|
|
rc = run_logged(self.cmd("launch", *args), self.log, timeout=timeout, env=self.env, echo=True,
|
|
echo_filter=progress)
|
|
t_cmd = time.perf_counter() - t0
|
|
# Only a successful launch counts: after a failed one, whatever still
|
|
# answers on the port is not the model that was asked for.
|
|
ready = wait_ready(f"http://127.0.0.1:{PORT}", timeout=1800) if rc == 0 else None
|
|
if ready is None:
|
|
self.keep_engine_log()
|
|
return {"args": scrub(" ".join(args)), "dropped_flags": dropped, "exit": rc,
|
|
"command_seconds": round(t_cmd, 1), "ready": ready is not None,
|
|
"ready_seconds": round(time.perf_counter() - t0, 1) if ready is not None else None}
|
|
|
|
def keep_engine_log(self) -> None:
|
|
"""Copy the newest engine log (scrubbed, its end) next to our log so the results zip carries it:
|
|
Strata's one-line reason can name a side effect, not the cause."""
|
|
found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime)
|
|
if not found:
|
|
return
|
|
src = found[-1]
|
|
text = src.read_text(encoding="utf-8", errors="replace")[-200_000:]
|
|
dest = self.log.with_name(f"{self.log.stem}_engine_{src.name.removesuffix('.engine.log')}.log")
|
|
dest.write_text(scrub(text), encoding="utf-8")
|
|
|
|
def mark_engine_logs(self) -> None:
|
|
"""Remember how long each engine log is, so engine_lines() reads only what the next start writes."""
|
|
self._log_marks = {p: p.stat().st_size for p in self.home.rglob("*.engine.log")}
|
|
|
|
def engine_lines(self, prefix: str = "strata-q36:", limit: int = 20) -> list[str]:
|
|
"""The placement lines (how much went to VRAM and RAM) the newest engine log got since the
|
|
last start, scrubbed. The log is appended across starts; one written again is read whole."""
|
|
found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime)
|
|
if not found:
|
|
return []
|
|
src = found[-1]
|
|
start = getattr(self, "_log_marks", {}).get(src, 0)
|
|
if src.stat().st_size < start:
|
|
start = 0
|
|
with open(src, "rb") as f:
|
|
f.seek(start)
|
|
text = f.read()[-400_000:].decode("utf-8", errors="replace")
|
|
return [scrub(x.strip())[:400] for x in text.splitlines() if x.lstrip().startswith(prefix)][:limit]
|
|
|
|
def stop(self) -> None:
|
|
run_logged(self.cmd("launch", "--stop"), self.log, timeout=300, env=self.env)
|
|
time.sleep(3)
|
|
|
|
def status(self) -> object:
|
|
p = subprocess.run(self.cmd("launch", "--status", "--output", "json"), capture_output=True, text=True,
|
|
env=self.env, encoding="utf-8", errors="replace", timeout=120)
|
|
try:
|
|
return json.loads(p.stdout)
|
|
except ValueError:
|
|
return scrub(p.stdout[-2000:])
|
|
|
|
|
|
class SttLoad:
|
|
"""The STT worker (stt.py mode=load in the stt venv) as a separate process next to the LLM."""
|
|
|
|
def __init__(self, python: str, work: Path, device: str, log: Path, pace: dict | None = None):
|
|
d = work / "cache" / f"stt_load_{device}"
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
d.mkdir(parents=True)
|
|
self.out, self.go_file, self.stop_file = d / "result.json", d / "go", d / "stop"
|
|
self.log = log
|
|
self.cmd = [python, str(Path(__file__).resolve().parent / "stt.py"), "--out", str(self.out),
|
|
"--work", str(work), "--opt", "mode=load", "--opt", f"device={device}",
|
|
"--opt", f"go={self.go_file}", "--opt", f"stop={self.stop_file}"]
|
|
if pace is not None:
|
|
for key in ("partial_every", "final_every"):
|
|
v = pace.get(key)
|
|
self.cmd += ["--opt", f"{key}={'none' if v is None else v}"]
|
|
self.proc: subprocess.Popen | None = None
|
|
|
|
@property
|
|
def pid(self) -> int | None:
|
|
return self.proc.pid if self.proc else None
|
|
|
|
def _read(self) -> dict:
|
|
try:
|
|
return read_json(self.out)
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
def start(self, timeout: float = 900) -> dict:
|
|
"""Load the model; returns the worker's result ("loaded", or "failed" with the reason)."""
|
|
self._logf = open(self.log, "a", encoding="utf-8")
|
|
self.proc = subprocess.Popen(self.cmd, stdout=self._logf, stderr=subprocess.STDOUT)
|
|
t0 = time.monotonic()
|
|
while time.monotonic() - t0 < timeout:
|
|
got = self._read()
|
|
if got.get("status") in ("loaded", "failed"):
|
|
return got
|
|
if self.proc.poll() is not None:
|
|
return got or {"status": "failed", "errors": [f"stt worker exited with {self.proc.returncode}"]}
|
|
time.sleep(0.5)
|
|
return {"status": "failed", "errors": ["stt model did not load in time"]}
|
|
|
|
def go(self) -> None:
|
|
self.go_file.write_text("", encoding="utf-8")
|
|
|
|
def finish(self, timeout: float = 300) -> dict:
|
|
"""Stop the loop; the CPU can be in the middle of a long ten-speaker batch."""
|
|
self.stop_file.write_text("", encoding="utf-8")
|
|
try:
|
|
self.proc.wait(timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
# Whatever the file holds is the "loaded" note, not timings; close() kills the worker.
|
|
return {"status": "timeout", "errors": [f"stt worker did not stop within {timeout:.0f} s"]}
|
|
return self._read()
|
|
|
|
def close(self) -> None:
|
|
if self.proc and self.proc.poll() is None:
|
|
self.proc.kill()
|
|
self.proc.wait(timeout=30)
|
|
if getattr(self, "_logf", None):
|
|
self._logf.close()
|
|
|
|
|
|
def stt_gaps(busy: dict, pace: dict | None = None) -> list[str]:
|
|
"""Why an STT load next to the LLM did not measure what it should: its errors, a worker that
|
|
did not finish (timed out, died, never started), or no ten-speaker or single-clip timings
|
|
(no ten-speaker ones are expected when the pace turns the partials off)."""
|
|
gaps = list(busy.get("errors") or [])
|
|
if busy.get("status") != "done":
|
|
if not gaps:
|
|
gaps.append(f"stt worker ended as {busy.get('status') or 'nothing written'}")
|
|
return gaps
|
|
partials = pace is None or pace.get("partial_every") is not None
|
|
for key in (("batch10_s", "single_s") if partials else ("single_s",)):
|
|
if not (busy.get(key) or {}).get("n"):
|
|
gaps.append(f"stt worker measured no {key}")
|
|
return gaps
|
|
|
|
|
|
def port_in_use(port: int) -> bool:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.settimeout(2)
|
|
return sock.connect_ex(("127.0.0.1", port)) == 0
|
|
|
|
|
|
def wait_ready(root: str, timeout: float) -> float | None:
|
|
t0 = time.perf_counter()
|
|
while time.perf_counter() - t0 < timeout:
|
|
try:
|
|
code, body = http_json("GET", root + "/v1/models", timeout=5)
|
|
if code == 200 and isinstance(body, dict) and body.get("data"):
|
|
return time.perf_counter() - t0
|
|
except Exception:
|
|
pass
|
|
time.sleep(2)
|
|
return None
|
|
|
|
|
|
def server_info(base: str) -> dict:
|
|
info = {}
|
|
for path in ("/models", "/status"):
|
|
try:
|
|
code, body = http_json("GET", base + path, timeout=10)
|
|
info[path] = body if code == 200 else {"http": code}
|
|
except Exception as e:
|
|
info[path] = {"error": str(e)}
|
|
return info
|
|
|
|
|
|
def tools_only(r: StageResult, c: ChatClient) -> dict | None:
|
|
"""Tool calling as first measured (baseline) and with the Jarvis fix.
|
|
|
|
Returns the fixed run's summary (None when it could not run)."""
|
|
summary = None
|
|
for name, fn in (("warmup", lambda: warmup(r, c)), ("tools", lambda: test_tools(r, c)),
|
|
("fixed", lambda: test_tools_fixed(r, c))):
|
|
try:
|
|
out = fn()
|
|
if name == "fixed":
|
|
summary = out
|
|
except Exception as e:
|
|
r.error("tools", e)
|
|
r.save()
|
|
return summary
|
|
|
|
|
|
def try_candidates(r: StageResult, which: str, default_summary: dict | None, measure, free_gb) -> None:
|
|
"""If the default preset fails with the fix, measure the next candidates.
|
|
|
|
``measure(preset)`` starts the preset and returns its fixed-run summary;
|
|
``free_gb()`` is the free disk in GB. Stops at the first passing model and
|
|
records which one Jarvis should switch to (None if none passed).
|
|
"""
|
|
verdict: dict = {"default": PRESETS[which]["label"], "default_passed": tools_passed(default_summary),
|
|
"candidates": [], "chosen": PRESETS[which]["label"] if tools_passed(default_summary) else None}
|
|
r["tools_verdict"] = verdict
|
|
if verdict["default_passed"]:
|
|
return
|
|
for cand in CANDIDATES[which]:
|
|
row: dict = {"label": cand["label"]}
|
|
verdict["candidates"].append(row)
|
|
if free_gb() < cand["need_gb"]:
|
|
row["skipped"] = "disk"
|
|
say(f" {cand['label']}: 디스크 공간이 부족해 건너뜁니다 (필요 약 {cand['need_gb']} GB)")
|
|
continue
|
|
say(f" 수정 후에도 틀린 도구 호출이 있어 다른 모델을 잽니다: {cand['label']} (처음이면 내려받습니다)")
|
|
try:
|
|
row["summary"] = measure(cand)
|
|
except Exception as e:
|
|
row["error"] = str(e)[:300]
|
|
r.save()
|
|
if tools_passed(row.get("summary")):
|
|
verdict["chosen"] = cand["label"]
|
|
return
|
|
|
|
|
|
def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels: list[dict] = LEVELS,
|
|
stt_python: str = "", choose: bool = True) -> None:
|
|
"""Start the model in each placement and measure what the switch and the placement cost.
|
|
A level with "stt" first loads the STT worker there and keeps it transcribing during the
|
|
measurement, so the VRAM left is what a game really gets."""
|
|
base = f"http://127.0.0.1:{PORT}/v1"
|
|
rows: list[dict] = []
|
|
r["levels"] = rows
|
|
rooms: dict = {} # one probe per placement that leaves the STT room
|
|
for lv in levels:
|
|
row: dict = {"key": lv["key"], "parallel": lv["parallel"], "vision": lv["vision"], "sets": lv["sets"],
|
|
"context": lv.get("context", context), "kv": lv.get("kv"), "mtp": lv.get("mtp")}
|
|
rows.append(row)
|
|
say(f" - 배치 {lv['key']}: 다시 시작해서 재는 중")
|
|
t0 = time.perf_counter()
|
|
s.stop()
|
|
idle = gpu_memory_mib()
|
|
row["vram_idle_mib"] = idle[0] if idle else None
|
|
stt = None
|
|
with GpuSampler(interval=1) as g:
|
|
try:
|
|
if lv.get("stt"):
|
|
stt = SttLoad(stt_python, s.log.parent.parent, lv["stt"], s.log.with_name("stt_load.log"),
|
|
pace=lv.get("stt_pace"))
|
|
row["stt"] = {"device": lv["stt"], "pace": lv.get("stt_pace"), "load": stt.start()}
|
|
if row["stt"]["load"].get("status") != "loaded":
|
|
r.error(f"level {lv['key']}", "; ".join(row["stt"]["load"].get("errors") or [])
|
|
or "the STT did not load")
|
|
continue
|
|
sets = list(lv["sets"])
|
|
if lv.get("stt_room"):
|
|
where = (row["context"], lv["parallel"], lv["vision"], lv.get("other_room_mib", 0))
|
|
if where not in rooms:
|
|
if not stt:
|
|
# The probe needs the STT model in VRAM, as the app has it.
|
|
r.error(f"level {lv['key']}", "no STT room was measured with the STT loaded")
|
|
continue
|
|
rooms[where] = stt_room(r, s, preset, row, lv, base)
|
|
row["stt_room"] = rooms[where]
|
|
sets.append(f"expert-vram={row['stt_room']['expert_vram_gib']}")
|
|
row["sets"] = sets
|
|
t0 = time.perf_counter() # the probe is not part of the switch
|
|
row["launch"] = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"],
|
|
extra_sets=sets, kv=lv.get("kv"), mtp=lv.get("mtp"), timeout=3600)
|
|
ready = row["launch"].get("ready")
|
|
row["switch_s"] = round(time.perf_counter() - t0, 1) if ready else None
|
|
row["engine"] = s.engine_lines()
|
|
if ready:
|
|
try:
|
|
c = ChatClient(base)
|
|
if "server" not in r.data:
|
|
r["server"] = {"model": c.model, "info": server_info(base)}
|
|
c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time
|
|
if stt:
|
|
stt.go()
|
|
cap = lv.get("gpu_cap")
|
|
pids = cap_pids(s, stt) if cap else []
|
|
if cap and not pids:
|
|
r.error(f"level {lv['key']} gpu cap", "found no engine process to pause")
|
|
with gpucap.UtilSampler() as util, gpucap.DutyCap(pids, cap or 1.0) as duty:
|
|
row.update(quick_bench(c))
|
|
row["gpu_util"] = util.summary(cap)
|
|
if cap:
|
|
row["gpu_cap"] = duty.summary()
|
|
if row["gpu_cap"]["left_paused"]:
|
|
r.error(f"level {lv['key']} gpu cap",
|
|
f"{row['gpu_cap']['left_paused']} process(es) are still paused")
|
|
elif row["gpu_cap"]["failed_processes"]:
|
|
r.error(f"level {lv['key']} gpu cap",
|
|
f"{row['gpu_cap']['failed_processes']} process(es) could not be "
|
|
"paused or resumed at once")
|
|
for err in row["errors"]:
|
|
r.error(f"level {lv['key']}", err)
|
|
except Exception as e:
|
|
r.error(f"level {lv['key']}", e)
|
|
if stt:
|
|
busy = stt.finish()
|
|
row["stt"].update({k: busy.get(k) for k in ("status", "batch10_s", "single_s", "errors",
|
|
"seconds", "busy_share", "final_wait_s")
|
|
if k in busy})
|
|
for err in stt_gaps(busy, lv.get("stt_pace")):
|
|
r.error(f"level {lv['key']} stt", err)
|
|
else:
|
|
r.error(f"level {lv['key']}", "the placement did not start")
|
|
now = gpu_memory_mib() if ready else None # the sampler can miss a short run
|
|
finally:
|
|
if stt:
|
|
stt.close()
|
|
peak = max(x for x in (g.peak, now[0] if now else None) if x is not None) \
|
|
if g.peak is not None or now else None
|
|
if now:
|
|
row["vram_after_mib"] = now[0] # settled; the peak (during the start) is what a running game must survive
|
|
if peak is not None and idle:
|
|
row["vram_peak_mib"] = peak
|
|
row["vram_free_mib"] = idle[1] - peak
|
|
say(f" 전환 {row.get('switch_s')}s, 출력 {(row.get('decode_tok_s') or {}).get('median')} tok/s, "
|
|
f"남는 VRAM {row.get('vram_free_mib')} MiB")
|
|
r.save()
|
|
s.stop()
|
|
if not choose:
|
|
return
|
|
r["game_choice"] = pick_game_level(rows)
|
|
measured = [x for x in rows if x["key"].startswith("game_") and x.get("vram_free_mib") is not None]
|
|
if r["game_choice"] is None and measured:
|
|
most = max(measured, key=lambda x: x["vram_free_mib"])
|
|
r.note(f"no game placement leaves {GAME_RESERVE_MIB} MiB free; the most free is "
|
|
f"{most['key']}: {most['vram_free_mib']} MiB")
|
|
|
|
|
|
def stt_growth_mib(work: Path) -> int:
|
|
"""What ten speakers at once add over the loaded STT model, from this PC's own stt stage."""
|
|
try:
|
|
return int(read_json(work / "results" / "stt.json")["run_int8_float16"]["batch_speakers"]["vram_added_mib"])
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
return STT_GROWTH_FALLBACK_MIB
|
|
|
|
|
|
def staging_gib(lines: list[str], default: float = 1.10) -> float:
|
|
"""The long-prompt staging the expert budget holds first (engine log: "(1.10 GiB staging)")."""
|
|
for line in lines:
|
|
m = re.search(r"([0-9.]+) GiB staging", line)
|
|
if m:
|
|
return float(m.group(1))
|
|
return default
|
|
|
|
|
|
def expert_budget_gib(total_mib: int, used_mib: int, probe_model_mib: int, staging: float, growth_mib: int,
|
|
margin_mib: int = STT_ROOM_MARGIN_MIB, other_mib: int = 0) -> float:
|
|
"""--expert-vram (GiB, rounded down to 0.1) that leaves the STT's growth, the margin and the room
|
|
for other programs free. The budget first holds the staging, which the start without experts
|
|
already has."""
|
|
spare = total_mib - used_mib - probe_model_mib - growth_mib - margin_mib - other_mib
|
|
if spare <= 0:
|
|
return 0.0
|
|
return math.floor((staging * 1024 + spare) / 1024 * 10) / 10
|
|
|
|
|
|
def stt_room(r: StageResult, s: Strata, preset: dict, row: dict, lv: dict, base: str) -> dict:
|
|
"""Start the placement once without experts (the STT model already loaded), see what the LLM
|
|
holds, stop it, and work out the expert budget that leaves the STT's ten-speaker growth free."""
|
|
room: dict = {"stt_growth_mib": stt_growth_mib(s.log.parent.parent), "margin_mib": STT_ROOM_MARGIN_MIB,
|
|
"other_room_mib": lv.get("other_room_mib", 0), "expert_vram_gib": 0.0}
|
|
say(" STT 자리 계산: 전문가 층 없이 한 번 켜서 LLM 기본 VRAM을 잽니다")
|
|
before = gpu_memory_mib()
|
|
launch = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"],
|
|
extra_sets=["expert-vram=0"] + [x for x in lv["sets"] if not x.startswith("expert-vram=")],
|
|
timeout=3600)
|
|
room["probe_ready"] = bool(launch.get("ready"))
|
|
if launch.get("ready"):
|
|
try:
|
|
ChatClient(base).chat([{"role": "user", "content": "안녕"}], max_tokens=8)
|
|
except Exception as e:
|
|
r.error(f"level {lv['key']} stt room", e)
|
|
after = gpu_memory_mib()
|
|
lines = s.engine_lines()
|
|
if before and after:
|
|
room.update(probe_model_mib=after[0] - before[0], used_before_mib=before[0], total_mib=before[1],
|
|
staging_gib=staging_gib(lines))
|
|
room["expert_vram_gib"] = expert_budget_gib(before[1], before[0], room["probe_model_mib"],
|
|
room["staging_gib"], room["stt_growth_mib"],
|
|
other_mib=room["other_room_mib"])
|
|
if room["expert_vram_gib"] == 0:
|
|
r.note(f"{lv['key']}: no room for experts next to the STT; started with --expert-vram 0")
|
|
else:
|
|
r.error(f"level {lv['key']} stt room", "the start without experts did not come up; --expert-vram 0")
|
|
s.stop()
|
|
return room
|
|
|
|
|
|
def cap_pids(s: Strata, stt: SttLoad | None) -> list[int]:
|
|
"""The processes the GPU cap pauses together: the engine and the STT worker, with their children.
|
|
None when the engine is not found (pausing only the STT would not be the cap)."""
|
|
procs = gpucap.process_table()
|
|
roots = gpucap.engine_roots(procs, s.home)
|
|
if not roots:
|
|
return []
|
|
return gpucap.with_children(procs, roots + ([stt.pid] if stt and stt.pid else []))
|
|
|
|
|
|
def ask_for_a_game() -> bool:
|
|
"""The live stage needs someone to start a game first. False when nobody is there or they say no."""
|
|
if not interactive():
|
|
return False
|
|
say(" 게임을 켠 상태로 LLM과 음성 인식을 잽니다. 배치 4개를 재고 10~15분 걸립니다.")
|
|
if ask(" 지금 게임을 켜고 재시겠습니까?") != "y":
|
|
return False
|
|
wait_enter(" 게임을 창 모드(테두리 없는 창이면 가장 좋음)로 켜고 플레이 화면까지 들어간 뒤 Enter를 누르고, "
|
|
"15초 안에 게임으로 돌아가 주세요. 끝나면 알림음이 울립니다:")
|
|
time.sleep(15)
|
|
return True
|
|
|
|
|
|
def pick_game_level(rows: list[dict], reserve_mib: int = GAME_RESERVE_MIB) -> dict | None:
|
|
"""The game placement the app would pick on this PC: the fastest one that
|
|
leaves the game its reserve of VRAM. None when no game placement does."""
|
|
fits = [x for x in rows if x["key"].startswith("game_") and (x.get("vram_free_mib") or 0) >= reserve_mib
|
|
and (x.get("decode_tok_s") or {}).get("median")]
|
|
if not fits:
|
|
return None
|
|
best = max(fits, key=lambda x: x["decode_tok_s"]["median"])
|
|
return {"key": best["key"], "reserve_mib": reserve_mib, "vram_free_mib": best["vram_free_mib"],
|
|
"decode_tok_s": best["decode_tok_s"]["median"], "switch_s": best.get("switch_s"),
|
|
"voice_first_sentence": (best.get("voice_first_sentence") or {}).get("median")}
|
|
|
|
|
|
def measure_external(r: StageResult, url: str, context: int, only: str = "") -> None:
|
|
c = ChatClient(url)
|
|
r["server"] = {"url": url, "model": c.model, "info": server_info(c.base)}
|
|
if only == "tools":
|
|
tools_only(r, c)
|
|
return
|
|
if only in LEVEL_SETS:
|
|
r.note("placements need Strata; not measured against an external server")
|
|
return
|
|
with GpuSampler() as g:
|
|
bench_single(r, c, context)
|
|
test_parallel(r, c)
|
|
r["gpu"] = g.summary()
|
|
|
|
|
|
def measure_strata(r: StageResult, work: Path, which: str, context: int, only: str = "",
|
|
stt_python: str = "") -> None:
|
|
preset = PRESETS[which]
|
|
r["preset"] = dict(preset, key=which, context=context)
|
|
s = Strata(work, work / "logs" / f"strata_{which}.log")
|
|
if not s.install():
|
|
r.error("install", "Strata 설치 실패 (logs/strata_*.log 참고)")
|
|
return
|
|
r["plan"] = s.plan(preset, context)
|
|
r.save()
|
|
if only in LEVEL_SETS:
|
|
s.stop()
|
|
if port_in_use(PORT):
|
|
r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.")
|
|
return
|
|
if only != "levels" and not stt_python:
|
|
r.error("stt", "STT 환경을 찾지 못했습니다")
|
|
return
|
|
if only == "live":
|
|
if not ask_for_a_game():
|
|
r["skipped"] = "no game started"
|
|
return
|
|
r.note("measured while the user's game was running")
|
|
measure_levels(r, s, preset, context, LEVEL_SETS[only], stt_python=stt_python,
|
|
choose=only in ("levels", "together"))
|
|
if only == "live":
|
|
say("\a 게임 중 측정이 끝났습니다. 게임을 꺼도 됩니다.")
|
|
try:
|
|
import winsound
|
|
winsound.MessageBeep()
|
|
except Exception:
|
|
pass
|
|
return
|
|
|
|
# 1) Start (the first start downloads the model and engine).
|
|
say(f" {preset['label']} 시작 중. 처음에는 모델을 내려받으므로 오래 걸립니다.")
|
|
s.stop()
|
|
if port_in_use(PORT):
|
|
# Another program owns the port; measuring it would report the wrong model.
|
|
r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.")
|
|
return
|
|
with GpuSampler(interval=2) as g:
|
|
r["launch_cold"] = s.launch(preset, context, parallel=1)
|
|
r["launch_cold"].update(g.summary())
|
|
r.save()
|
|
if not r["launch_cold"]["ready"]:
|
|
r.error("launch", "서버가 준비되지 않았습니다")
|
|
r["status_after_fail"] = s.status()
|
|
return
|
|
base = f"http://127.0.0.1:{PORT}/v1"
|
|
c = ChatClient(base)
|
|
r["server"] = {"model": c.model, "info": server_info(base), "status": s.status()}
|
|
if only == "tools":
|
|
default_summary = tools_only(r, c)
|
|
|
|
def measure(cand: dict) -> dict | None:
|
|
key = f"tools_fixed_{cand['model']}"
|
|
res = s.launch(cand, context, parallel=1, restart=True)
|
|
r[f"launch_{cand['model']}"] = res
|
|
if not res["ready"]:
|
|
return None
|
|
return test_tools_fixed(r, ChatClient(base), key=key)
|
|
|
|
try_candidates(r, which, default_summary, measure,
|
|
free_gb=lambda: shutil.disk_usage(work).free / 1e9)
|
|
s.stop()
|
|
return
|
|
with GpuSampler() as g:
|
|
bench_single(r, c, context)
|
|
r["gpu_single"] = g.summary()
|
|
r.save()
|
|
|
|
# 2) Restart with the files already in RAM/page cache: what a game-mode switch costs.
|
|
# Also: two slots, the elastic VRAM cache and the GPU picture reader.
|
|
say(" 다시 시작 시간 측정 (동시 2개, VRAM 조절 켜기, 이미지 GPU 처리)")
|
|
r["launch_restart"] = s.launch(preset, context, parallel=2, vision="gpu", extra_sets=["vram-elastic"],
|
|
restart=True, timeout=3600)
|
|
r.save()
|
|
if r["launch_restart"]["ready"]:
|
|
c = ChatClient(base)
|
|
r["server_restart"] = {"status": s.status()}
|
|
for fn in (lambda: warmup(r, c), lambda: test_decode(r, c, key="decode_parallel2"), lambda: test_parallel(r, c),
|
|
lambda: test_images(r, c, key="images_gpu_reader"), lambda: test_vram_shrink(r, c)):
|
|
try:
|
|
fn()
|
|
except Exception as e:
|
|
r.error("restart tests", e)
|
|
r.save()
|
|
|
|
# 3) Game mode (b): switching from the recommended model to the fast one.
|
|
if which == "recommended" and (work / "results" / "llm_fast.json").exists():
|
|
say(" 게임 모드 (b): 권장 모델 -> 빠름 모델 전환 시간")
|
|
r["switch_to_fast"] = s.launch(PRESETS["fast"], context, parallel=1, restart=True, timeout=3600)
|
|
s.stop()
|
|
|
|
|
|
def main() -> int:
|
|
out, work, opts = stage_args()
|
|
which = opts.get("model", "fast")
|
|
context = int(opts.get("context", "32768"))
|
|
only = opts.get("only", "")
|
|
name = {"tools": f"tools_{which}", "levels": "gamemode", "together": "gamemode_stt",
|
|
"live": "gamemode_live", "tune": "gamemode_tune"}.get(only, f"llm_{which}")
|
|
r = StageResult(name, out)
|
|
try:
|
|
if opts.get("url"):
|
|
measure_external(r, opts["url"], context, only)
|
|
else:
|
|
measure_strata(r, work, which, context, only, opts.get("stt_python", ""))
|
|
except Exception as e:
|
|
r.error("llm", e)
|
|
if r.get("skipped"):
|
|
r.save("skipped")
|
|
return 0
|
|
# No server reached means nothing was measured: "failed" so the next run retries it.
|
|
if "server" not in r.data:
|
|
r.save("failed")
|
|
else:
|
|
r.save("ok" if not r["errors"] else "partial")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|