One double-click kit (scripts/m0/run_m0.bat) that measures the gaming PC and packs the results into a single zip: - PC info (GPU, driver, RAM, disks, monitors) - Windows checks with the user: bot browser window hidden from Alt+Tab but listed in Discord's share picker and still drawing while covered; Discord pop-out stream window identification and occlusion; installed games, anti-cheat services and the borderless game detector - MeloTTS, CosyVoice2 and Fun-CosyVoice3 Korean synthesis speed - faster-whisper large-v3-turbo: final/partial latency, CER, wake-word hint, batched decoding for several speakers - Strata LLM (fast and recommended presets): TTFT by context length, decode speed, tool calls, picture reading per quality tier, Korean answers, parallel requests, VRAM give-back and restart/switch timings uv and Python, per-stage venvs, models and Strata all live inside the work folder. Results are scrubbed of user/computer names and home paths. Verified on Linux GPU (LLM client against Ollama, STT, MeloTTS, CosyVoice2/3) and on the Windows 11 CI VM (window checks, game scan, MeloTTS/CosyVoice2/STT installs, Strata install and plan). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
322 lines
11 KiB
Python
322 lines
11 KiB
Python
"""Shared helpers for the M0 measurement kit (standard library only).
|
|
|
|
Every stage script imports this module, both from the orchestrator's Python
|
|
and from the per-stage virtual environments, so it must not import anything
|
|
outside the standard library.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as _dt
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterable
|
|
|
|
IS_WINDOWS = os.name == "nt"
|
|
|
|
|
|
def now_iso() -> str:
|
|
return _dt.datetime.now().astimezone().isoformat(timespec="seconds")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Privacy: results leave the PC, so personal paths and names are masked.
|
|
|
|
def _personal_tokens() -> list[str]:
|
|
tokens = []
|
|
for key in ("USERNAME", "USER", "COMPUTERNAME", "USERDOMAIN"):
|
|
v = os.environ.get(key, "").strip()
|
|
if len(v) >= 2 and v.lower() not in ("root", "admin", "user", "workgroup"):
|
|
tokens.append(v)
|
|
home = str(Path.home())
|
|
if home:
|
|
tokens.append(home)
|
|
# Longest first so a home path is masked before the bare user name.
|
|
return sorted(set(tokens), key=len, reverse=True)
|
|
|
|
|
|
_TOKENS = _personal_tokens()
|
|
|
|
|
|
def scrub(text: str) -> str:
|
|
"""Mask the home folder, user name and computer name in ``text``."""
|
|
if not text:
|
|
return text
|
|
out = text
|
|
for t in _TOKENS:
|
|
if "\\" in t or "/" in t:
|
|
out = out.replace(t, "<HOME>")
|
|
out = out.replace(t.replace("\\", "/"), "<HOME>")
|
|
out = out.replace(t.replace("\\", "\\\\"), "<HOME>")
|
|
else:
|
|
# Whole words only: a short account name such as "ci" must not
|
|
# turn "insufficient" into "insuffi<USER>ent".
|
|
out = re.sub(rf"(?<![A-Za-z0-9]){re.escape(t)}(?![A-Za-z0-9])", "<USER>", out, flags=re.IGNORECASE)
|
|
return out
|
|
|
|
|
|
def scrub_obj(obj: Any) -> Any:
|
|
if isinstance(obj, str):
|
|
return scrub(obj)
|
|
if isinstance(obj, list):
|
|
return [scrub_obj(x) for x in obj]
|
|
if isinstance(obj, tuple):
|
|
return [scrub_obj(x) for x in obj]
|
|
if isinstance(obj, dict):
|
|
return {scrub(str(k)): scrub_obj(v) for k, v in obj.items()}
|
|
return obj
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Console output. Messages for the user are Korean; keep them short.
|
|
|
|
_print_lock = threading.Lock()
|
|
|
|
|
|
def say(msg: str) -> None:
|
|
with _print_lock:
|
|
print(msg, flush=True)
|
|
|
|
|
|
def step(msg: str) -> None:
|
|
say(f"\n=== {msg} ===")
|
|
|
|
|
|
def interactive() -> bool:
|
|
"""False when stdin is not a console or M0_AUTO=1 (unattended test runs)."""
|
|
return bool(sys.stdin) and sys.stdin.isatty() and os.environ.get("M0_AUTO") != "1"
|
|
|
|
|
|
def ask(question: str, choices: str = "yn", default: str | None = None) -> str:
|
|
"""Ask a one-letter question. Returns the chosen letter (lower case).
|
|
|
|
When not interactive the default is returned (or the first choice).
|
|
"""
|
|
fallback = default or choices[0]
|
|
if not interactive():
|
|
say(f"{question} -> (자동 응답) {fallback}")
|
|
return fallback
|
|
hint = "/".join(choices)
|
|
while True:
|
|
try:
|
|
ans = input(f"{question} [{hint}] ").strip().lower()
|
|
except EOFError:
|
|
return fallback
|
|
if not ans and default:
|
|
return default
|
|
if ans and ans[0] in choices:
|
|
return ans[0]
|
|
say(f" {hint} 중 하나를 입력해 주세요.")
|
|
|
|
|
|
def wait_enter(msg: str) -> str:
|
|
"""Wait for Enter. Returns 's' when the user typed s (skip)."""
|
|
if not interactive():
|
|
say(f"{msg} -> (자동 진행)")
|
|
return ""
|
|
try:
|
|
return input(f"{msg} ").strip().lower()[:1]
|
|
except EOFError:
|
|
return ""
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Results and logs
|
|
|
|
def write_json(path: Path, data: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
tmp.write_text(json.dumps(scrub_obj(data), ensure_ascii=False, indent=2), encoding="utf-8")
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def read_json(path: Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
class StageResult:
|
|
"""Collects one stage's measurements and always writes them, even on error."""
|
|
|
|
def __init__(self, stage: str, out_path: Path):
|
|
self.stage = stage
|
|
self.out_path = out_path
|
|
self.data: dict[str, Any] = {"stage": stage, "started": now_iso(), "status": "running",
|
|
"errors": [], "notes": []}
|
|
self._t0 = time.perf_counter()
|
|
|
|
def __setitem__(self, key: str, value: Any) -> None:
|
|
self.data[key] = value
|
|
|
|
def __getitem__(self, key: str) -> Any:
|
|
return self.data[key]
|
|
|
|
def get(self, key: str, default: Any = None) -> Any:
|
|
return self.data.get(key, default)
|
|
|
|
def error(self, where: str, exc: BaseException | str) -> None:
|
|
text = exc if isinstance(exc, str) else f"{type(exc).__name__}: {exc}"
|
|
self.data["errors"].append({"where": where, "error": text[:2000]})
|
|
say(f" [오류] {where}: {text[:300]}")
|
|
|
|
def note(self, text: str) -> None:
|
|
self.data["notes"].append(text)
|
|
|
|
def save(self, status: str | None = None) -> None:
|
|
if status:
|
|
self.data["status"] = status
|
|
self.data["seconds"] = round(time.perf_counter() - self._t0, 1)
|
|
self.data["finished"] = now_iso()
|
|
write_json(self.out_path, self.data)
|
|
|
|
|
|
def run_logged(cmd: list[str], log_path: Path, *, timeout: float | None = None,
|
|
env: dict[str, str] | None = None, cwd: str | None = None,
|
|
echo: bool = False, echo_filter: Callable[[str], bool] | None = None) -> int:
|
|
"""Run ``cmd`` and append its combined output to ``log_path``.
|
|
|
|
Returns the exit code, or -9 on timeout. With ``echo`` the lines are also
|
|
printed (optionally filtered) so long downloads show progress.
|
|
"""
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(log_path, "a", encoding="utf-8", errors="replace") as log:
|
|
log.write(f"\n$ {scrub(' '.join(cmd))}\n")
|
|
log.flush()
|
|
proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, cwd=cwd,
|
|
text=True, encoding="utf-8", errors="replace", bufsize=1)
|
|
deadline = time.monotonic() + timeout if timeout else None
|
|
timed_out = threading.Event()
|
|
|
|
def _watch() -> None:
|
|
while proc.poll() is None:
|
|
if deadline and time.monotonic() > deadline:
|
|
timed_out.set()
|
|
proc.kill()
|
|
return
|
|
time.sleep(1)
|
|
|
|
threading.Thread(target=_watch, daemon=True).start()
|
|
assert proc.stdout is not None
|
|
for line in proc.stdout:
|
|
log.write(scrub(line))
|
|
if echo and (echo_filter is None or echo_filter(line)):
|
|
say(" " + line.rstrip()[:200])
|
|
proc.wait()
|
|
if timed_out.is_set():
|
|
log.write(f"[timeout after {timeout}s]\n")
|
|
return -9
|
|
log.write(f"[exit {proc.returncode}]\n")
|
|
return proc.returncode
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# GPU sampling via nvidia-smi (present with every NVIDIA driver)
|
|
|
|
def nvidia_smi() -> str | None:
|
|
exe = shutil.which("nvidia-smi")
|
|
if exe:
|
|
return exe
|
|
if IS_WINDOWS:
|
|
for p in (r"C:\Windows\System32\nvidia-smi.exe",
|
|
r"C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe"):
|
|
if os.path.exists(p):
|
|
return p
|
|
return None
|
|
|
|
|
|
def gpu_memory_mib() -> tuple[int, int] | None:
|
|
"""(used, total) MiB of GPU 0, or None."""
|
|
exe = nvidia_smi()
|
|
if not exe:
|
|
return None
|
|
try:
|
|
out = subprocess.run([exe, "--query-gpu=memory.used,memory.total", "--format=csv,noheader,nounits"],
|
|
capture_output=True, text=True, timeout=10).stdout.strip().splitlines()[0]
|
|
used, total = (int(x.strip()) for x in out.split(","))
|
|
return used, total
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
class GpuSampler:
|
|
"""Samples GPU memory in the background; ``peak`` is the max used MiB."""
|
|
|
|
def __init__(self, interval: float = 0.5):
|
|
self.interval = interval
|
|
self.peak: int | None = None
|
|
self.start_used: int | None = None
|
|
self._stop = threading.Event()
|
|
self._t: threading.Thread | None = None
|
|
|
|
def __enter__(self) -> "GpuSampler":
|
|
m = gpu_memory_mib()
|
|
self.start_used = m[0] if m else None
|
|
self.peak = self.start_used
|
|
self._t = threading.Thread(target=self._loop, daemon=True)
|
|
self._t.start()
|
|
return self
|
|
|
|
def _loop(self) -> None:
|
|
while not self._stop.is_set():
|
|
m = gpu_memory_mib()
|
|
if m and (self.peak is None or m[0] > self.peak):
|
|
self.peak = m[0]
|
|
self._stop.wait(self.interval)
|
|
|
|
def __exit__(self, *exc: Any) -> None:
|
|
self._stop.set()
|
|
if self._t:
|
|
self._t.join(timeout=5)
|
|
|
|
def summary(self) -> dict[str, Any]:
|
|
out = {"vram_used_before_mib": self.start_used, "vram_peak_mib": self.peak}
|
|
if self.start_used is not None and self.peak is not None:
|
|
out["vram_added_mib"] = self.peak - self.start_used
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Small numeric helpers
|
|
|
|
def stats(values: Iterable[float], digits: int = 3) -> dict[str, Any]:
|
|
v = [float(x) for x in values if x is not None]
|
|
if not v:
|
|
return {"n": 0}
|
|
return {"n": len(v), "median": round(statistics.median(v), digits), "min": round(min(v), digits),
|
|
"max": round(max(v), digits)}
|
|
|
|
|
|
def cer(ref: str, hyp: str) -> float:
|
|
"""Character error rate after removing spaces and punctuation."""
|
|
def norm(s: str) -> str:
|
|
return re.sub(r"[\W_]+", "", s, flags=re.UNICODE).lower()
|
|
r, h = norm(ref), norm(hyp)
|
|
if not r:
|
|
return 0.0 if not h else 1.0
|
|
prev = list(range(len(h) + 1))
|
|
for i, rc in enumerate(r, 1):
|
|
cur = [i] + [0] * len(h)
|
|
for j, hc in enumerate(h, 1):
|
|
cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (rc != hc))
|
|
prev = cur
|
|
return round(prev[-1] / len(r), 4)
|
|
|
|
|
|
def stage_args(argv: list[str] | None = None) -> tuple[Path, Path, dict[str, str]]:
|
|
"""Common CLI for stage scripts: --out FILE --work DIR [--opt k=v ...]."""
|
|
import argparse
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--out", required=True)
|
|
p.add_argument("--work", required=True)
|
|
p.add_argument("--opt", action="append", default=[])
|
|
a = p.parse_args(argv)
|
|
opts = dict(o.split("=", 1) for o in a.opt if "=" in o)
|
|
return Path(a.out), Path(a.work), opts
|