Files
javis_bot/scripts/m0/m0lib/llm.py
javis-bot 165142be9a fix(m0): measure smaller game placements and read only the latest engine start
The first game PC run showed that a 2 GB expert budget still left a 16 GB
card only 4.3 GB free: the weights, the 32K int8 conversation memory, the
MTP drafter and buffers take about 8.7 GB on their own. The game placements
now also shrink the memory format (q4_0), the context (16K, 8K) and the
drafter, with half the CPU workers (no speed cost measured). The engine log
is appended across starts, so each placement now reads only its own lines;
when no placement leaves the reserve the result names the most free one.
The gamemode runner skips the desktop stage, which is already measured.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 18:23:28 +09:00

500 lines
23 KiB
Python

"""Stage: LLM through Strata (install, start, measure, restart timings).
Options (--opt key=value):
model=fast|recommended which preset to measure (default fast)
url=http://.../v1 measure an already running server instead (no Strata management)
context=32768 context size to start Strata with
only=tools only re-measure tool calling, before and after the Jarvis fix
only=levels the fast model in each placement the app's automatic
tuning switches between (normal, and less VRAM for a game)
"""
from __future__ import annotations
import json
import os
import re
import shutil
import socket
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, gpu_memory_mib, run_logged, say, # noqa: E402
scrub, stage_args)
from m0lib.llm_bench import (bench_single, quick_bench, test_decode, test_images, test_parallel, # noqa: E402
test_tools, test_vram_shrink, warmup)
from m0lib.tool_fix import test_tools_fixed, tools_passed # noqa: E402
from m0lib.llm_client import ChatClient, http_json # noqa: E402
PRESETS = {
"fast": {"family": "qwen36", "model": "ud-q4_k_xl", "label": "Qwen3.6-35B-A3B UD-Q4_K_XL"},
# Strata 0.1.59 gives Q2_0 the DFlash2 drafter alone, which needs a native pack; the canonical
# Q2_0 pack is not one, so the engine stops at start. The MTP drafter (or none) starts.
"recommended": {"family": "qwen", "model": "q2_0", "label": "Qwen3.8-Flash-Next Q2_0 (ISTA-DASLab)",
"drafting": "mtp"},
}
# Tried in order when a preset still gets a tool call wrong with the Jarvis fix
# (the user's rule: if the fix does not work, change the model). Same family,
# less compressed weights, all within 128 GB RAM. need_gb: free disk to download.
CANDIDATES = {
"fast": [
{"family": "qwen36", "model": "ud-q6_k_xl", "label": "Qwen3.6-35B-A3B UD-Q6_K_XL", "need_gb": 40},
{"family": "qwen36", "model": "q8_0", "label": "Qwen3.6-35B-A3B Q8_0", "need_gb": 45},
],
"recommended": [
{"family": "qwen", "model": "iq3_xxs", "label": "Qwen3.8-Flash-Next IQ3_XXS (ISTA-DASLab)",
"drafting": "mtp", "need_gb": 80},
{"family": "qwen", "model": "iq3_s", "label": "Qwen3.8-Flash-Next IQ3_S (ISTA-DASLab)",
"drafting": "mtp", "need_gb": 85},
],
}
PORT = 8080
# The placements the app's automatic tuning moves the one model between.
# Strata's qwen36 engine cannot give VRAM back while it runs (POST /v1/vram
# answers 400), so each placement is a restart with --expert-vram (GiB for the
# routed experts; the rest stay in RAM) and a matching prompt chunk. A game
# placement also reads pictures on the CPU, answers one request at a time and
# leaves the game half the CPU (8 engine workers cost nothing on the 9950X3D).
# The first game PC run (2026-11-10) showed that the expert budget is not the
# floor: at 2 GB the model still held 10.8 GB and a 16 GB card kept only
# 4.3 GB free. The rest is 4.4 GiB of weights that never leave the card, the
# conversation memory (1.43 GiB per 32K in int8; q4_0 is half), the MTP
# drafter (about 1 GB) and buffers, so the smaller placements also shrink those.
# The last row starts the normal placement again: what returning after a game costs.
GAME_RESERVE_MIB = 8000
_GAME = {"parallel": 1, "vision": "cpu"}
_W8 = "pool-workers=8"
LEVELS = [
{"key": "normal", "parallel": 2, "vision": "gpu", "sets": []},
{"key": "game_experts_2g", **_GAME, "sets": ["expert-vram=2", "prefill=2048", _W8]},
{"key": "game_experts_1g_kv_q4_16k", **_GAME, "context": 16384, "kv": "q4_0",
"sets": ["expert-vram=1", "prefill=2048", _W8]},
{"key": "game_experts_1g_kv_q4_16k_no_mtp", **_GAME, "context": 16384, "kv": "q4_0", "mtp": "off",
"sets": ["expert-vram=1", "prefill=2048", _W8]},
{"key": "game_experts_0g_kv_q4_16k_no_mtp", **_GAME, "context": 16384, "kv": "q4_0", "mtp": "off",
"sets": ["expert-vram=0", "prefill=2048", _W8]},
{"key": "game_experts_0g_kv_q4_8k_no_mtp", **_GAME, "context": 8192, "kv": "q4_0", "mtp": "off",
"sets": ["expert-vram=0", "prefill=2048", _W8]},
{"key": "normal_again", "parallel": 2, "vision": "gpu", "sets": []},
]
INSTALL_URL = "https://github.com/Niko1221/Strata/releases/latest/download/install.ps1"
class Strata:
def __init__(self, work: Path, log: Path):
self.home = work / "strata"
self.models = work / "models" / "llm"
self.log = log
# Keep Strata's engine, state and models inside the work folder: the
# installer honours STRATA_DATA_DIR, the engine components
# STRATA_COMPONENTS_DIR, and the per-user state follows LOCALAPPDATA.
self.env = dict(os.environ, STRATA_DATA_DIR=str(self.home), STRATA_NO_PATH="1",
STRATA_HOME=str(self.home / "home"), STRATA_COMPONENTS_DIR=str(self.home / "components"),
PYTHONUTF8="1")
if IS_WINDOWS:
self.env["LOCALAPPDATA"] = str(self.home / "localappdata")
(self.home / "localappdata").mkdir(parents=True, exist_ok=True)
self._flags: set[str] | None = None
@property
def exe(self) -> Path:
return self.home / "terminal" / "current" / "bin" / ("strata.cmd" if IS_WINDOWS else "strata")
def install(self) -> bool:
if self.exe.exists():
return True
self.home.mkdir(parents=True, exist_ok=True)
say(" Strata 터미널 설치 중 (작업 폴더 안에만 설치하고 PATH는 바꾸지 않습니다)")
if IS_WINDOWS:
cmd = ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", f"irm {INSTALL_URL} | iex"]
else:
cmd = ["sh", "-c", f"curl -fsSL {INSTALL_URL[:-3]}sh -o install.sh && sh install.sh"]
rc = run_logged(cmd, self.log, timeout=1800, env=self.env, cwd=str(self.home), echo=True)
return rc == 0 and self.exe.exists()
def cmd(self, *args: str) -> list[str]:
return [str(self.exe), *args]
def supported_flags(self) -> set[str]:
if self._flags is None:
p = subprocess.run(self.cmd("launch", "--help"), capture_output=True, text=True, env=self.env,
encoding="utf-8", errors="replace", timeout=120)
with open(self.log, "a", encoding="utf-8") as f:
f.write(scrub(p.stdout + p.stderr))
self._flags = set(re.findall(r"--[a-z0-9][a-z0-9-]*", p.stdout + p.stderr))
return self._flags
def launch_args(self, preset: dict, context: int, parallel: int, vision: str, extra_sets: list[str],
restart: bool, kv: str | None = None, mtp: str | None = None) -> tuple[list[str], list[str]]:
want = [("--family", preset["family"]), ("--model", preset["model"]), ("--data-dir", str(self.models)),
("--context", str(context)), ("--vision", vision), ("--parallel", str(parallel)),
("--access", "local"), ("--port", str(PORT)), ("--no-api-key", None), ("--yes", None),
("--no-tui", None)]
if preset.get("drafting"):
want.append(("--drafting", preset["drafting"]))
if kv:
want.append(("--kv", kv))
if mtp:
want.append(("--mtp", mtp))
if restart:
want.append(("--restart", None))
for s in extra_sets:
want.append(("--set", s))
flags = self.supported_flags()
args, dropped = [], []
for k, v in want:
if flags and k not in flags:
dropped.append(k)
continue
args += [k] + ([v] if v is not None else [])
return args, dropped
def plan(self, preset: dict, context: int) -> object:
args, _ = self.launch_args(preset, context, 1, "on", [], False)
args = [a for a in args if a not in ("--yes", "--no-tui")] + ["--plan", "--output", "json"]
p = subprocess.run(self.cmd("launch", *args), capture_output=True, text=True, env=self.env,
encoding="utf-8", errors="replace", timeout=600)
with open(self.log, "a", encoding="utf-8") as f:
f.write(scrub(p.stdout + p.stderr))
try:
return json.loads(p.stdout)
except ValueError:
return {"exit": p.returncode, "text": scrub(p.stdout[-3000:] + p.stderr[-2000:])}
def launch(self, preset: dict, context: int, parallel: int, vision: str = "on",
extra_sets: list[str] | None = None, restart: bool = False, timeout: float = 4 * 3600,
kv: str | None = None, mtp: str | None = None) -> dict:
args, dropped = self.launch_args(preset, context, parallel, vision, extra_sets or [], restart, kv, mtp)
last = [0.0]
def progress(line: str) -> bool:
# Downloads print many progress lines; show one every 15 s.
if "%" in line:
if time.monotonic() - last[0] < 15:
return False
last[0] = time.monotonic()
return True
self.mark_engine_logs()
t0 = time.perf_counter()
rc = run_logged(self.cmd("launch", *args), self.log, timeout=timeout, env=self.env, echo=True,
echo_filter=progress)
t_cmd = time.perf_counter() - t0
# Only a successful launch counts: after a failed one, whatever still
# answers on the port is not the model that was asked for.
ready = wait_ready(f"http://127.0.0.1:{PORT}", timeout=1800) if rc == 0 else None
if ready is None:
self.keep_engine_log()
return {"args": scrub(" ".join(args)), "dropped_flags": dropped, "exit": rc,
"command_seconds": round(t_cmd, 1), "ready": ready is not None,
"ready_seconds": round(time.perf_counter() - t0, 1) if ready is not None else None}
def keep_engine_log(self) -> None:
"""Copy the newest engine log (scrubbed, its end) next to our log so the results zip carries it:
Strata's one-line reason can name a side effect, not the cause."""
found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime)
if not found:
return
src = found[-1]
text = src.read_text(encoding="utf-8", errors="replace")[-200_000:]
dest = self.log.with_name(f"{self.log.stem}_engine_{src.name.removesuffix('.engine.log')}.log")
dest.write_text(scrub(text), encoding="utf-8")
def mark_engine_logs(self) -> None:
"""Remember how long each engine log is, so engine_lines() reads only what the next start writes."""
self._log_marks = {p: p.stat().st_size for p in self.home.rglob("*.engine.log")}
def engine_lines(self, prefix: str = "strata-q36:", limit: int = 20) -> list[str]:
"""The placement lines (how much went to VRAM and RAM) the newest engine log got since the
last start, scrubbed. The log is appended across starts; one written again is read whole."""
found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime)
if not found:
return []
src = found[-1]
start = getattr(self, "_log_marks", {}).get(src, 0)
if src.stat().st_size < start:
start = 0
with open(src, "rb") as f:
f.seek(start)
text = f.read()[-400_000:].decode("utf-8", errors="replace")
return [scrub(x.strip())[:400] for x in text.splitlines() if x.lstrip().startswith(prefix)][:limit]
def stop(self) -> None:
run_logged(self.cmd("launch", "--stop"), self.log, timeout=300, env=self.env)
time.sleep(3)
def status(self) -> object:
p = subprocess.run(self.cmd("launch", "--status", "--output", "json"), capture_output=True, text=True,
env=self.env, encoding="utf-8", errors="replace", timeout=120)
try:
return json.loads(p.stdout)
except ValueError:
return scrub(p.stdout[-2000:])
def port_in_use(port: int) -> bool:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(2)
return sock.connect_ex(("127.0.0.1", port)) == 0
def wait_ready(root: str, timeout: float) -> float | None:
t0 = time.perf_counter()
while time.perf_counter() - t0 < timeout:
try:
code, body = http_json("GET", root + "/v1/models", timeout=5)
if code == 200 and isinstance(body, dict) and body.get("data"):
return time.perf_counter() - t0
except Exception:
pass
time.sleep(2)
return None
def server_info(base: str) -> dict:
info = {}
for path in ("/models", "/status"):
try:
code, body = http_json("GET", base + path, timeout=10)
info[path] = body if code == 200 else {"http": code}
except Exception as e:
info[path] = {"error": str(e)}
return info
def tools_only(r: StageResult, c: ChatClient) -> dict | None:
"""Tool calling as first measured (baseline) and with the Jarvis fix.
Returns the fixed run's summary (None when it could not run)."""
summary = None
for name, fn in (("warmup", lambda: warmup(r, c)), ("tools", lambda: test_tools(r, c)),
("fixed", lambda: test_tools_fixed(r, c))):
try:
out = fn()
if name == "fixed":
summary = out
except Exception as e:
r.error("tools", e)
r.save()
return summary
def try_candidates(r: StageResult, which: str, default_summary: dict | None, measure, free_gb) -> None:
"""If the default preset fails with the fix, measure the next candidates.
``measure(preset)`` starts the preset and returns its fixed-run summary;
``free_gb()`` is the free disk in GB. Stops at the first passing model and
records which one Jarvis should switch to (None if none passed).
"""
verdict: dict = {"default": PRESETS[which]["label"], "default_passed": tools_passed(default_summary),
"candidates": [], "chosen": PRESETS[which]["label"] if tools_passed(default_summary) else None}
r["tools_verdict"] = verdict
if verdict["default_passed"]:
return
for cand in CANDIDATES[which]:
row: dict = {"label": cand["label"]}
verdict["candidates"].append(row)
if free_gb() < cand["need_gb"]:
row["skipped"] = "disk"
say(f" {cand['label']}: 디스크 공간이 부족해 건너뜁니다 (필요 약 {cand['need_gb']} GB)")
continue
say(f" 수정 후에도 틀린 도구 호출이 있어 다른 모델을 잽니다: {cand['label']} (처음이면 내려받습니다)")
try:
row["summary"] = measure(cand)
except Exception as e:
row["error"] = str(e)[:300]
r.save()
if tools_passed(row.get("summary")):
verdict["chosen"] = cand["label"]
return
def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels: list[dict] = LEVELS) -> None:
"""Start the model in each placement and measure what the switch and the placement cost."""
base = f"http://127.0.0.1:{PORT}/v1"
rows: list[dict] = []
r["levels"] = rows
for lv in levels:
row: dict = {"key": lv["key"], "parallel": lv["parallel"], "vision": lv["vision"], "sets": lv["sets"],
"context": lv.get("context", context), "kv": lv.get("kv"), "mtp": lv.get("mtp")}
rows.append(row)
say(f" - 배치 {lv['key']}: 다시 시작해서 재는 중")
t0 = time.perf_counter()
s.stop()
idle = gpu_memory_mib()
row["vram_idle_mib"] = idle[0] if idle else None
with GpuSampler(interval=1) as g:
row["launch"] = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"],
extra_sets=lv["sets"], kv=lv.get("kv"), mtp=lv.get("mtp"), timeout=3600)
ready = row["launch"].get("ready")
row["switch_s"] = round(time.perf_counter() - t0, 1) if ready else None
row["engine"] = s.engine_lines()
if ready:
try:
c = ChatClient(base)
if "server" not in r.data:
r["server"] = {"model": c.model, "info": server_info(base)}
c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time
row.update(quick_bench(c))
for err in row["errors"]:
r.error(f"level {lv['key']}", err)
except Exception as e:
r.error(f"level {lv['key']}", e)
else:
r.error(f"level {lv['key']}", "the placement did not start")
now = gpu_memory_mib() if ready else None # the sampler can miss a short run
peak = max(x for x in (g.peak, now[0] if now else None) if x is not None) \
if g.peak is not None or now else None
if now:
row["vram_after_mib"] = now[0] # settled; the peak (during the start) is what a running game must survive
if peak is not None and idle:
row["vram_peak_mib"] = peak
row["vram_free_mib"] = idle[1] - peak
say(f" 전환 {row['switch_s']}s, 출력 {(row.get('decode_tok_s') or {}).get('median')} tok/s, "
f"남는 VRAM {row.get('vram_free_mib')} MiB")
r.save()
s.stop()
r["game_choice"] = pick_game_level(rows)
measured = [x for x in rows if x["key"].startswith("game_") and x.get("vram_free_mib") is not None]
if r["game_choice"] is None and measured:
most = max(measured, key=lambda x: x["vram_free_mib"])
r.note(f"no game placement leaves {GAME_RESERVE_MIB} MiB free; the most free is "
f"{most['key']}: {most['vram_free_mib']} MiB")
def pick_game_level(rows: list[dict], reserve_mib: int = GAME_RESERVE_MIB) -> dict | None:
"""The game placement the app would pick on this PC: the fastest one that
leaves the game its reserve of VRAM. None when no game placement does."""
fits = [x for x in rows if x["key"].startswith("game_") and (x.get("vram_free_mib") or 0) >= reserve_mib
and (x.get("decode_tok_s") or {}).get("median")]
if not fits:
return None
best = max(fits, key=lambda x: x["decode_tok_s"]["median"])
return {"key": best["key"], "reserve_mib": reserve_mib, "vram_free_mib": best["vram_free_mib"],
"decode_tok_s": best["decode_tok_s"]["median"], "switch_s": best.get("switch_s"),
"voice_first_sentence": (best.get("voice_first_sentence") or {}).get("median")}
def measure_external(r: StageResult, url: str, context: int, only: str = "") -> None:
c = ChatClient(url)
r["server"] = {"url": url, "model": c.model, "info": server_info(c.base)}
if only == "tools":
tools_only(r, c)
return
if only == "levels":
r.note("placements need Strata; not measured against an external server")
return
with GpuSampler() as g:
bench_single(r, c, context)
test_parallel(r, c)
r["gpu"] = g.summary()
def measure_strata(r: StageResult, work: Path, which: str, context: int, only: str = "") -> None:
preset = PRESETS[which]
r["preset"] = dict(preset, key=which, context=context)
s = Strata(work, work / "logs" / f"strata_{which}.log")
if not s.install():
r.error("install", "Strata 설치 실패 (logs/strata_*.log 참고)")
return
r["plan"] = s.plan(preset, context)
r.save()
if only == "levels":
s.stop()
if port_in_use(PORT):
r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.")
return
measure_levels(r, s, preset, context)
return
# 1) Start (the first start downloads the model and engine).
say(f" {preset['label']} 시작 중. 처음에는 모델을 내려받으므로 오래 걸립니다.")
s.stop()
if port_in_use(PORT):
# Another program owns the port; measuring it would report the wrong model.
r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.")
return
with GpuSampler(interval=2) as g:
r["launch_cold"] = s.launch(preset, context, parallel=1)
r["launch_cold"].update(g.summary())
r.save()
if not r["launch_cold"]["ready"]:
r.error("launch", "서버가 준비되지 않았습니다")
r["status_after_fail"] = s.status()
return
base = f"http://127.0.0.1:{PORT}/v1"
c = ChatClient(base)
r["server"] = {"model": c.model, "info": server_info(base), "status": s.status()}
if only == "tools":
default_summary = tools_only(r, c)
def measure(cand: dict) -> dict | None:
key = f"tools_fixed_{cand['model']}"
res = s.launch(cand, context, parallel=1, restart=True)
r[f"launch_{cand['model']}"] = res
if not res["ready"]:
return None
return test_tools_fixed(r, ChatClient(base), key=key)
try_candidates(r, which, default_summary, measure,
free_gb=lambda: shutil.disk_usage(work).free / 1e9)
s.stop()
return
with GpuSampler() as g:
bench_single(r, c, context)
r["gpu_single"] = g.summary()
r.save()
# 2) Restart with the files already in RAM/page cache: what a game-mode switch costs.
# Also: two slots, the elastic VRAM cache and the GPU picture reader.
say(" 다시 시작 시간 측정 (동시 2개, VRAM 조절 켜기, 이미지 GPU 처리)")
r["launch_restart"] = s.launch(preset, context, parallel=2, vision="gpu", extra_sets=["vram-elastic"],
restart=True, timeout=3600)
r.save()
if r["launch_restart"]["ready"]:
c = ChatClient(base)
r["server_restart"] = {"status": s.status()}
for fn in (lambda: warmup(r, c), lambda: test_decode(r, c, key="decode_parallel2"), lambda: test_parallel(r, c),
lambda: test_images(r, c, key="images_gpu_reader"), lambda: test_vram_shrink(r, c)):
try:
fn()
except Exception as e:
r.error("restart tests", e)
r.save()
# 3) Game mode (b): switching from the recommended model to the fast one.
if which == "recommended" and (work / "results" / "llm_fast.json").exists():
say(" 게임 모드 (b): 권장 모델 -> 빠름 모델 전환 시간")
r["switch_to_fast"] = s.launch(PRESETS["fast"], context, parallel=1, restart=True, timeout=3600)
s.stop()
def main() -> int:
out, work, opts = stage_args()
which = opts.get("model", "fast")
context = int(opts.get("context", "32768"))
only = opts.get("only", "")
name = {"tools": f"tools_{which}", "levels": "gamemode"}.get(only, f"llm_{which}")
r = StageResult(name, out)
try:
if opts.get("url"):
measure_external(r, opts["url"], context, only)
else:
measure_strata(r, work, which, context, only)
except Exception as e:
r.error("llm", e)
# No server reached means nothing was measured: "failed" so the next run retries it.
if "server" not in r.data:
r.save("failed")
else:
r.save("ok" if not r["errors"] else "partial")
return 0
if __name__ == "__main__":
sys.exit(main())