- Frame rate, 1 % low and hitches come from msBetweenDisplayChange of the frames that reached the screen; frames never shown are counted apart and the presented rate is kept as presented_fps. A game presenting 60 fps while the screen changes every 167 ms now shows as the stutter it is. - A window whose capture failed, a busy window where the LLM finished no answer, and every busy window when the STT load failed are not counted. - A yielding window counts only when every Jarvis process got both its CPU and its GPU priority lowered. - The "game hardly drawing" floor applies only with Jarvis off or idle, so a game Jarvis slows that far still counts as harm. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
515 lines
23 KiB
Python
515 lines
23 KiB
Python
"""Stage part: does Jarvis make the user's game drop frames, and does yielding to the game help?
|
|
|
|
The GPU cap is gone (2026-11-11): pausing Jarvis for 5 % of every 200 ms did not lower nvidia-smi's
|
|
utilisation, cost 6 to 10 % of the LLM's output, and a YouTube video next to a busy Jarvis did not
|
|
stutter with or without it. What is left unmeasured is a game. So this stage measures what the user
|
|
would notice, the game's own frame times, instead of a GPU-wide number:
|
|
|
|
- PresentMon (Intel's open-source tool, pinned by hash) records when the game presents each frame
|
|
through Windows' event tracing. It sees no picture; the kit keeps only the frame-time numbers and
|
|
deletes its CSV (which names the game) right after reading it. Event tracing needs administrator
|
|
rights, so this stage runs elevated (run_m0_game_frames.bat asks once, before the game is started).
|
|
- Windows: the game alone, Jarvis loaded and idle, Jarvis busy in the game placement the app uses
|
|
(the LLM answering, the STT doing only finals, as decided for a game), and busy while Jarvis yields:
|
|
its processes at below-normal CPU priority and below-normal GPU scheduling priority. Unlike the cap
|
|
nothing is paused; Windows hands the game the cores and the GPU first only when it wants them. The
|
|
busy rows run A B B A (plain, yield, yield, plain) so a slow drift does not count for one side.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from m0lib import gpucap, llm
|
|
from m0lib.common import IS_WINDOWS, StageResult, gpu_memory_mib, interactive, ask, say, stats, wait_enter
|
|
from m0lib.llm_bench import KOREAN_PROMPTS, NO_THINK, VOICE_SYSTEM
|
|
from m0lib.llm_client import ChatClient
|
|
|
|
PRESENTMON_VERSION = "2.6.0"
|
|
PRESENTMON_URL = (f"https://github.com/GameTechDev/PresentMon/releases/download/v{PRESENTMON_VERSION}/"
|
|
f"PresentMon-{PRESENTMON_VERSION}-x64.exe")
|
|
PRESENTMON_SHA256 = "b2a706bc6ad475749e3b7e3409263aa1e6906d45bdcf993f6dbc0f660188f1af"
|
|
WINDOW_S = 60
|
|
SETTLE_S = 5 # Jarvis busy (or yielding) before a window starts
|
|
HITCH_MS = (50, 100) # a frame this long is a visible hitch
|
|
MIN_FRAMES = 300 # under this in a minute (5 fps) with Jarvis off or idle, the game was not really drawing
|
|
ROUNDS = ("busy", "busy_yield", "busy_yield_2", "busy_2")
|
|
# The foreground window belongs to one of these when the user did not get back to the game.
|
|
NOT_A_GAME = {"windowsterminal.exe", "openconsole.exe", "conhost.exe", "cmd.exe", "powershell.exe",
|
|
"pwsh.exe", "explorer.exe"}
|
|
# Windows' values: process priority classes and D3DKMT_SCHEDULINGPRIORITYCLASS.
|
|
CPU_NORMAL, CPU_BELOW_NORMAL = 0x20, 0x4000
|
|
GPU_NORMAL, GPU_BELOW_NORMAL = 2, 1
|
|
GAME_LEVEL = next(lv for lv in llm.TOGETHER if lv["key"] == "game_experts_2g_16k_stt_gpu")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Windows: rights, the game's process, priorities.
|
|
|
|
def is_admin() -> bool:
|
|
if not IS_WINDOWS:
|
|
return False
|
|
import ctypes
|
|
try:
|
|
return bool(ctypes.windll.shell32.IsUserAnAdmin())
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def foreground_process() -> tuple[int, str] | None:
|
|
"""(pid, exe file name in lower case) of the window in front, or None."""
|
|
if not IS_WINDOWS:
|
|
return None
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
user32 = ctypes.WinDLL("user32")
|
|
k32 = ctypes.WinDLL("kernel32")
|
|
user32.GetForegroundWindow.restype = ctypes.c_void_p
|
|
user32.GetWindowThreadProcessId.argtypes = [ctypes.c_void_p, ctypes.POINTER(wintypes.DWORD)]
|
|
k32.OpenProcess.restype = ctypes.c_void_p
|
|
k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
k32.QueryFullProcessImageNameW.argtypes = [ctypes.c_void_p, wintypes.DWORD, wintypes.LPWSTR,
|
|
ctypes.POINTER(wintypes.DWORD)]
|
|
k32.CloseHandle.argtypes = [ctypes.c_void_p]
|
|
hwnd = user32.GetForegroundWindow()
|
|
if not hwnd:
|
|
return None
|
|
pid = wintypes.DWORD()
|
|
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
|
|
h = k32.OpenProcess(0x1000, False, pid.value) # PROCESS_QUERY_LIMITED_INFORMATION
|
|
name = ""
|
|
if h:
|
|
try:
|
|
buf = ctypes.create_unicode_buffer(1024)
|
|
n = wintypes.DWORD(len(buf))
|
|
if k32.QueryFullProcessImageNameW(h, 0, buf, ctypes.byref(n)):
|
|
name = Path(buf.value).name.lower()
|
|
finally:
|
|
k32.CloseHandle(h)
|
|
return pid.value, name
|
|
|
|
|
|
_API: dict = {}
|
|
|
|
|
|
def _api() -> dict:
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
if not _API:
|
|
k32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
gdi = ctypes.WinDLL("gdi32")
|
|
k32.OpenProcess.restype = ctypes.c_void_p
|
|
k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
k32.CloseHandle.argtypes = [ctypes.c_void_p]
|
|
k32.GetPriorityClass.argtypes = [ctypes.c_void_p]
|
|
k32.GetPriorityClass.restype = wintypes.DWORD
|
|
k32.SetPriorityClass.argtypes = [ctypes.c_void_p, wintypes.DWORD]
|
|
gdi.D3DKMTGetProcessSchedulingPriorityClass.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)]
|
|
gdi.D3DKMTGetProcessSchedulingPriorityClass.restype = ctypes.c_long
|
|
gdi.D3DKMTSetProcessSchedulingPriorityClass.argtypes = [ctypes.c_void_p, ctypes.c_int]
|
|
gdi.D3DKMTSetProcessSchedulingPriorityClass.restype = ctypes.c_long
|
|
_API.update(k32=k32, gdi=gdi, ctypes=ctypes)
|
|
return _API
|
|
|
|
|
|
def get_priority(pid: int) -> tuple[int | None, int | None]:
|
|
"""(CPU priority class, GPU scheduling priority class) of a process; None where unreadable."""
|
|
a = _api()
|
|
h = a["k32"].OpenProcess(0x0200 | 0x1000, False, pid) # SET_INFORMATION | QUERY_LIMITED
|
|
if not h:
|
|
return None, None
|
|
try:
|
|
cpu = a["k32"].GetPriorityClass(h) or None
|
|
val = a["ctypes"].c_int()
|
|
gpu = val.value if a["gdi"].D3DKMTGetProcessSchedulingPriorityClass(h, a["ctypes"].byref(val)) == 0 else None
|
|
return cpu, gpu
|
|
finally:
|
|
a["k32"].CloseHandle(h)
|
|
|
|
|
|
def set_priority(pid: int, cpu: int | None, gpu: int | None) -> tuple[bool, bool]:
|
|
"""Set what is given; (CPU done, GPU done)."""
|
|
a = _api()
|
|
h = a["k32"].OpenProcess(0x0200 | 0x1000, False, pid)
|
|
if not h:
|
|
return False, False
|
|
try:
|
|
ok_cpu = cpu is None or bool(a["k32"].SetPriorityClass(h, cpu))
|
|
ok_gpu = gpu is None or a["gdi"].D3DKMTSetProcessSchedulingPriorityClass(h, gpu) == 0
|
|
return ok_cpu, ok_gpu
|
|
finally:
|
|
a["k32"].CloseHandle(h)
|
|
|
|
|
|
class Yield:
|
|
"""Jarvis's processes at below-normal CPU and GPU priority while the block runs, put back after.
|
|
Each lowering is read back, so a priority Windows ignored does not count as lowered."""
|
|
|
|
def __init__(self, pids: list[int]):
|
|
self.pids = list(pids)
|
|
self.before: dict[int, tuple[int | None, int | None]] = {}
|
|
self.cpu_lowered = self.gpu_lowered = self.restored = 0
|
|
self.not_restored: list[int] = []
|
|
|
|
def __enter__(self) -> "Yield":
|
|
for pid in self.pids:
|
|
self.before[pid] = get_priority(pid)
|
|
set_priority(pid, CPU_BELOW_NORMAL, GPU_BELOW_NORMAL)
|
|
cpu, gpu = get_priority(pid)
|
|
self.cpu_lowered += cpu == CPU_BELOW_NORMAL
|
|
self.gpu_lowered += gpu == GPU_BELOW_NORMAL
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
for pid, (cpu, gpu) in self.before.items():
|
|
ok = set_priority(pid, cpu or CPU_NORMAL, GPU_NORMAL if gpu is None else gpu)
|
|
if all(ok):
|
|
self.restored += 1
|
|
else:
|
|
self.not_restored.append(pid)
|
|
|
|
def summary(self) -> dict:
|
|
return {"processes": len(self.pids), "cpu_lowered": self.cpu_lowered, "gpu_lowered": self.gpu_lowered,
|
|
"restored": self.restored, "not_restored": len(self.not_restored)}
|
|
|
|
|
|
def yield_problems(summary: dict) -> list[str]:
|
|
"""Why a yielding window did not yield: no process, a CPU or GPU priority left normal on any of
|
|
Jarvis's processes (yielding with some of them is not the yield being measured), or one not put back."""
|
|
out = []
|
|
n = summary.get("processes") or 0
|
|
if not n:
|
|
out.append("yield found no Jarvis process")
|
|
for key, what in (("cpu_lowered", "CPU"), ("gpu_lowered", "GPU")):
|
|
if n and summary.get(key, 0) < n:
|
|
out.append(f"yield lowered the {what} priority of only {summary.get(key, 0)} of {n} processes")
|
|
if summary.get("not_restored"):
|
|
out.append(f"{summary['not_restored']} process(es) kept the lowered priority")
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# PresentMon and the frame times.
|
|
|
|
def ensure_presentmon(work: Path) -> Path:
|
|
exe = work / "tools" / f"PresentMon-{PRESENTMON_VERSION}-x64.exe"
|
|
if exe.exists() and hashlib.sha256(exe.read_bytes()).hexdigest() == PRESENTMON_SHA256:
|
|
return exe
|
|
exe.parent.mkdir(parents=True, exist_ok=True)
|
|
part = exe.with_suffix(".part")
|
|
with urllib.request.urlopen(PRESENTMON_URL, timeout=120) as resp:
|
|
part.write_bytes(resp.read())
|
|
got = hashlib.sha256(part.read_bytes()).hexdigest()
|
|
if got != PRESENTMON_SHA256:
|
|
part.unlink()
|
|
raise RuntimeError(f"PresentMon download has the wrong hash ({got[:12]})")
|
|
part.replace(exe)
|
|
return exe
|
|
|
|
|
|
def presentmon_cmd(exe: Path, pid: int, seconds: int, out_csv: Path) -> list[str]:
|
|
return [str(exe), "--process_id", str(pid), "--output_file", str(out_csv), "--timed", str(seconds),
|
|
"--terminate_after_timed", "--no_console_stats", "--v1_metrics", "--no_track_input",
|
|
"--session_name", "JarvisM0", "--stop_existing_session"]
|
|
|
|
|
|
def capture(exe: Path, pid: int, seconds: int, out_csv: Path, log: Path) -> tuple[list[dict], str | None]:
|
|
"""The game's frames over ``seconds`` (rows of PresentMon's CSV) and an error, if any. The CSV
|
|
names the game, so it is deleted as soon as it is read."""
|
|
out_csv.unlink(missing_ok=True)
|
|
with open(log, "a", encoding="utf-8") as f:
|
|
try:
|
|
p = subprocess.run(presentmon_cmd(exe, pid, seconds, out_csv), stdout=f, stderr=subprocess.STDOUT,
|
|
timeout=seconds + 60)
|
|
code = p.returncode
|
|
except subprocess.TimeoutExpired:
|
|
code = "timeout"
|
|
try:
|
|
text = out_csv.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return [], f"PresentMon wrote no frames (exit {code})"
|
|
finally:
|
|
out_csv.unlink(missing_ok=True)
|
|
return parse_csv(text), None if code == 0 else f"PresentMon exited with {code}"
|
|
|
|
|
|
def parse_csv(text: str) -> list[dict]:
|
|
"""PresentMon's rows with lower-case column names (1.x writes msBetweenPresents, 2.x MsBetweenPresents)."""
|
|
return [{(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
|
|
for row in csv.DictReader(io.StringIO(text))]
|
|
|
|
|
|
def _num(value: str) -> float | None:
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def frame_stats(rows: list[dict], seconds: float, min_frames: int = 0) -> dict:
|
|
"""What the screen showed of the game's main swap chain (the one with the most frames): the rate of
|
|
shown frames, the 1 % low (the 99th-percentile time between screen updates as a rate) and hitches,
|
|
all from the time each shown frame stayed before the next (msBetweenDisplayChange). A frame the
|
|
game presented but the screen never showed is counted in ``not_shown``; the rate the game
|
|
presented at is kept apart (``presented_fps``), since a game can present at 60 fps while the
|
|
screen changes far less often. ``min_frames`` (per minute) marks a window where the game was
|
|
hardly drawing; it is only given with Jarvis off or idle, so a game Jarvis slows down that much
|
|
still counts as the harm it is."""
|
|
chains: dict[str, list[dict]] = {}
|
|
for x in rows:
|
|
chains.setdefault(x.get("swapchainaddress", ""), []).append(x)
|
|
main = (max(chains.values(), key=len) if chains else [])[1:] # the first reaches back before the capture
|
|
presents = [v for v in (_num(x.get("msbetweenpresents")) for x in main) if v is not None and v > 0]
|
|
shown = [x for x in main if x.get("dropped") != "1"]
|
|
ms = [v for v in (_num(x.get("msbetweendisplaychange")) for x in shown) if v is not None and v > 0][1:]
|
|
out: dict[str, Any] = {"seconds": round(seconds, 1), "frames": len(ms), "presented": len(presents),
|
|
"not_shown": len(main) - len(shown)}
|
|
if presents:
|
|
out["presented_fps"] = round(len(presents) / (sum(presents) / 1000), 1)
|
|
if not ms:
|
|
why = "the game drew no frames" if not presents else "PresentMon saw none of the frames reach the screen"
|
|
return {**out, "valid": False, "invalid_because": [why]}
|
|
srt = sorted(ms)
|
|
p99 = srt[min(len(srt) - 1, int(len(srt) * 0.99))]
|
|
out.update(fps=round(len(ms) / (sum(ms) / 1000), 1), low1_fps=round(1000 / p99, 1),
|
|
p99_ms=round(p99, 2), max_ms=round(srt[-1], 1),
|
|
hitches_over_ms={str(h): sum(v > h for v in ms) for h in HITCH_MS})
|
|
reasons = []
|
|
if len(ms) < min_frames * seconds / 60:
|
|
reasons.append(f"the screen showed only {len(ms)} of the game's frames")
|
|
out["valid"] = not reasons
|
|
if reasons:
|
|
out["invalid_because"] = reasons
|
|
return out
|
|
|
|
|
|
def compare(rows: list[dict]) -> dict:
|
|
"""The game alone, next to busy Jarvis and next to yielding Jarvis, over the valid windows, with
|
|
the change against the game alone in per cent."""
|
|
def pool(kind: str) -> dict:
|
|
valid = [x["game"] for x in rows if x.get("kind") == kind and (x.get("game") or {}).get("valid")]
|
|
if not valid:
|
|
return {"windows": 0}
|
|
n = len(valid)
|
|
return {"windows": n, "fps": round(sum(v["fps"] for v in valid) / n, 1),
|
|
"low1_fps": round(sum(v["low1_fps"] for v in valid) / n, 1),
|
|
**{f"hitches_over_{h}ms": sum(v["hitches_over_ms"][str(h)] for v in valid) for h in HITCH_MS}}
|
|
|
|
out = {k: pool(k) for k in ("off", "busy", "busy_yield")}
|
|
off = out["off"]
|
|
for k in ("busy", "busy_yield"):
|
|
if off["windows"] and out[k]["windows"]:
|
|
out[k]["vs_off_pct"] = {m: round((out[k][m] - off[m]) / off[m] * 100, 1) for m in ("fps", "low1_fps")}
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Jarvis busy for as long as a window lasts.
|
|
|
|
class BusyLlm:
|
|
"""The LLM answering back to back (a long answer, then a voice answer) until the block ends."""
|
|
|
|
def __init__(self, c: ChatClient):
|
|
self.c = c
|
|
self.stop = threading.Event()
|
|
self.decode: list[float] = []
|
|
self.voice: list[float] = []
|
|
self.errors: list[str] = []
|
|
self.thread = threading.Thread(target=self._loop, daemon=True)
|
|
|
|
def _loop(self) -> None:
|
|
voice = [p for kind, p in KOREAN_PROMPTS if kind == "voice"]
|
|
i = 0
|
|
while not self.stop.is_set():
|
|
try:
|
|
a = self.c.chat([{"role": "user", "content": f"({i}) 한국의 사계절을 각각 세 문장씩 자세히 설명해 줘."}],
|
|
max_tokens=320, extra=NO_THINK)
|
|
if a.get("decode_tok_s"):
|
|
self.decode.append(a["decode_tok_s"])
|
|
if voice and not self.stop.is_set():
|
|
a = self.c.chat([{"role": "system", "content": VOICE_SYSTEM},
|
|
{"role": "user", "content": voice[i % len(voice)]}], max_tokens=120, extra=NO_THINK)
|
|
if a.get("first_sentence") is not None:
|
|
self.voice.append(a["first_sentence"])
|
|
except Exception as e:
|
|
self.errors.append(f"{type(e).__name__}")
|
|
if len(self.errors) > 20:
|
|
return
|
|
time.sleep(1)
|
|
i += 1
|
|
|
|
def __enter__(self) -> "BusyLlm":
|
|
self.thread.start()
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self.stop.set()
|
|
self.thread.join(timeout=120)
|
|
|
|
def summary(self) -> dict:
|
|
return {"decode_tok_s": stats(self.decode), "voice_first_sentence": stats(self.voice),
|
|
"errors": sorted(set(self.errors))}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The stage.
|
|
|
|
def ask_for_a_game() -> bool:
|
|
if not interactive():
|
|
return False
|
|
say(" 게임을 켠 채로 게임 화면의 프레임을 잽니다. 10~15분 걸립니다.")
|
|
if ask(" 지금 게임을 켜고 재시겠습니까?") != "y":
|
|
return False
|
|
wait_enter(" 게임을 켜고 플레이 화면까지 들어간 뒤 Enter를 누르고, 15초 안에 게임으로 돌아가 주세요. "
|
|
"끝날 때까지 캐릭터를 한자리에 세워 두고 마우스·키보드에 손대지 말아 주세요. 끝나면 알림음이 울립니다:")
|
|
time.sleep(15)
|
|
return True
|
|
|
|
|
|
def _spoil(x: dict, why: str) -> None:
|
|
x["game"]["valid"] = False
|
|
x["game"]["invalid_because"] = x["game"].get("invalid_because", []) + [why]
|
|
|
|
|
|
def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_python: str) -> None:
|
|
work = s.log.parent.parent
|
|
pm_log = s.log.with_name("presentmon.log")
|
|
rows: list[dict] = []
|
|
r["frame_rows"] = rows
|
|
if not is_admin():
|
|
r.error("admin", "게임 프레임은 관리자 권한이 있어야 잴 수 있습니다. run_m0_game_frames.bat으로 실행해 주세요.")
|
|
return
|
|
try:
|
|
pm = ensure_presentmon(work)
|
|
except Exception as e:
|
|
r.error("presentmon", e)
|
|
return
|
|
r["presentmon"] = PRESENTMON_VERSION
|
|
if not ask_for_a_game():
|
|
r["skipped"] = "no game started"
|
|
return
|
|
fg = foreground_process()
|
|
if fg is None or fg[1] in NOT_A_GAME:
|
|
r.error("game", "게임 창이 앞에 없었습니다. 게임으로 돌아간 뒤 다시 실행해 주세요.")
|
|
return
|
|
game = fg[0]
|
|
r.note("measured while the user's game was running (its name is not kept)")
|
|
base = f"http://127.0.0.1:{llm.PORT}/v1"
|
|
|
|
def window(key: str, kind: str) -> dict:
|
|
x: dict = {"key": key, "kind": kind}
|
|
used = gpu_memory_mib()
|
|
if used:
|
|
x["vram_free_mib"] = used[1] - used[0]
|
|
with gpucap.UtilSampler() as util:
|
|
frames, err = capture(pm, game, WINDOW_S, work / "cache" / "frames.csv", pm_log)
|
|
g = frame_stats(frames, WINDOW_S, MIN_FRAMES if kind in ("off", "idle") else 0)
|
|
g["gpu_util"] = util.summary()
|
|
now = foreground_process()
|
|
if now is not None and now[0] != game:
|
|
g["valid"] = False
|
|
g["invalid_because"] = g.get("invalid_because", []) + ["the game was not in front at the end"]
|
|
if err:
|
|
# Frames from a capture that failed or was cut off do not cover the window.
|
|
r.error(f"frames {key}", err)
|
|
g["valid"] = False
|
|
g["invalid_because"] = g.get("invalid_because", []) + [f"the capture failed ({err})"]
|
|
x["game"] = g
|
|
rows.append(x)
|
|
if not g["valid"]:
|
|
r.note(f"{key}: not counted ({'; '.join(g['invalid_because'])})")
|
|
say(f" {key}: {g.get('fps')} fps, 1% low {g.get('low1_fps')} fps, "
|
|
f"{HITCH_MS[0]}ms 넘는 프레임 {(g.get('hitches_over_ms') or {}).get(str(HITCH_MS[0]))}개")
|
|
r.save()
|
|
return x
|
|
|
|
stt = None
|
|
try:
|
|
say(" - 게임만")
|
|
window("game_only", "off")
|
|
|
|
say(" - Jarvis 켜기 (게임 중 배치: 전문가 2GB, 맥락 16K, 음성 인식 GPU)")
|
|
s.stop()
|
|
stt = llm.SttLoad(stt_python, work, "cuda", s.log.with_name("stt_load.log"), pace=llm._FINALS_ONLY)
|
|
load = stt.start()
|
|
r["stt_load"] = load
|
|
if load.get("status") != "loaded":
|
|
r.error("frames stt", "; ".join(load.get("errors") or []) or "the STT did not load")
|
|
return
|
|
lv = GAME_LEVEL
|
|
r["placement"] = {"key": lv["key"], "sets": lv["sets"], "context": lv["context"], "parallel": lv["parallel"],
|
|
"vision": lv["vision"], "stt_pace": llm._FINALS_ONLY}
|
|
launch = s.launch(preset, lv["context"], parallel=lv["parallel"], vision=lv["vision"],
|
|
extra_sets=list(lv["sets"]), timeout=3600)
|
|
if not launch.get("ready"):
|
|
r.error("frames llm", "the game placement did not start")
|
|
return
|
|
c = ChatClient(base)
|
|
r["server"] = {"model": c.model}
|
|
c.chat([{"role": "user", "content": "안녕"}], max_tokens=8)
|
|
|
|
say(" - Jarvis 켜 두고 쉬는 중")
|
|
window("jarvis_idle", "idle")
|
|
|
|
stt.go()
|
|
for key in ROUNDS:
|
|
yielding = "yield" in key
|
|
say(f" - Jarvis가 답하고 음성 인식이 최종 전사를 하는 중{' (게임에 양보)' if yielding else ''}")
|
|
pids = llm.cap_pids(s, stt) if yielding else []
|
|
y = Yield(pids)
|
|
with y if yielding else contextlib.nullcontext(), BusyLlm(c) as busy:
|
|
time.sleep(SETTLE_S)
|
|
x = window(key, "busy_yield" if yielding else "busy")
|
|
x["llm"] = busy.summary()
|
|
for err in x["llm"]["errors"]:
|
|
r.error(f"frames {key} llm", err)
|
|
if not x["llm"]["decode_tok_s"].get("n"):
|
|
# Jarvis was not busy, so the window measured the game next to an idle Jarvis.
|
|
r.error(f"frames {key} llm", "no answer finished during the window")
|
|
_spoil(x, "the LLM finished no answer during the window")
|
|
if yielding:
|
|
x["yield"] = y.summary()
|
|
problems = yield_problems(x["yield"])
|
|
for why in problems:
|
|
r.error(f"frames {key} yield", why)
|
|
for why in problems:
|
|
_spoil(x, why)
|
|
r.save()
|
|
busy_stt = stt.finish()
|
|
r["stt_busy"] = {k: busy_stt.get(k) for k in ("status", "single_s", "errors", "final_wait_s")
|
|
if k in busy_stt}
|
|
for err in llm.stt_gaps(busy_stt, llm._FINALS_ONLY):
|
|
r.error("frames stt", err)
|
|
# The STT's part of the load is not known to have run, in any busy window.
|
|
for x in rows:
|
|
if x["kind"].startswith("busy"):
|
|
_spoil(x, f"the STT load failed ({err})")
|
|
stt.close()
|
|
stt = None
|
|
s.stop()
|
|
|
|
say(" - Jarvis 끄고 게임만 다시")
|
|
window("game_only_again", "off")
|
|
r["comparison"] = compare(rows)
|
|
except Exception as e:
|
|
r.error("frames", e)
|
|
finally:
|
|
if stt:
|
|
stt.close()
|
|
s.stop()
|
|
say("\a 게임 프레임 측정이 끝났습니다. 게임을 꺼도 됩니다.")
|
|
try:
|
|
import winsound
|
|
winsound.MessageBeep()
|
|
except Exception:
|
|
pass
|