Files
javis_bot/tests/test_m0_kit.py
javis-bot dff18c5c7e feat(m0): measure the game's own frame times and try yielding by priority instead of a GPU cap
The 95 % GPU cap is gone: it did not lower utilisation, slowed the LLM and
a YouTube video did not stutter without it. What it was meant to protect
is now measured directly in a game. The new on-demand game_frames stage
records the game's frame times with PresentMon (pinned by hash, game
process only, CSV deleted after reading) with Jarvis off, idle and busy in
the game placement, and busy while Jarvis yields: its processes at
below-normal CPU and GPU scheduling priority, read back to confirm, then
restored. Busy rows run A B B A. The design docs record the candidate and
the pre-registered rule for adopting it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 09:05:55 +09:00

1844 lines
88 KiB
Python

"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
The kit runs once on the gaming PC and its results leave the PC, so the
privacy scrub and the timing client must behave before it is handed over.
"""
from __future__ import annotations
import json
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
from m0lib import common, gpucap, llm, llm_bench # noqa: E402
from m0lib.llm_client import ChatClient # noqa: E402
pytestmark = pytest.mark.unit
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
out = common.scrub(text)
assert "minsu" not in out.lower()
assert "<HOME>" in out and "<USER>" in out
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
monkeypatch.setattr(common, "_TOKENS", ["ci"])
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
target = tmp_path / "r.json"
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
data = json.loads(target.read_text(encoding="utf-8"))
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
def test_cer_ignores_spacing_and_punctuation():
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
assert common.cer("가나다라", "가나다마") == 0.25
class _FakeOpenAI(BaseHTTPRequestHandler):
"""Streams a tool call split over chunks, then text, then usage."""
def log_message(self, *a):
pass
def do_GET(self):
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
request_sizes: list[int] = []
def do_POST(self):
n = int(self.headers.get("Content-Length", 0))
self.rfile.read(n)
_FakeOpenAI.request_sizes.append(n)
chunks = [
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
]
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.end_headers()
for c in chunks:
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
self.wfile.flush()
self.wfile.write(b"data: [DONE]\n\n")
@pytest.fixture()
def fake_server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
t = threading.Thread(target=srv.serve_forever, daemon=True)
t.start()
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
srv.shutdown()
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
c = ChatClient(fake_server)
assert c.model == "fake-model"
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
assert a["finish"] == "tool_calls"
assert a["usage"]["completion_tokens"] == 9
# The first sentence ends at "요. ", before the whole answer is done.
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
# The fake server always reports 12 prompt tokens. The context test must not
# trust that blindly and grow each next prompt without limit (it used to
# reach gigabytes and exhaust RAM).
_FakeOpenAI.request_sizes.clear()
largest_target = 24576
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
assert _FakeOpenAI.request_sizes
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
# --- LLM stage: a failed start must not count as measured -------------------
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
out = tmp_path / "results" / "llm_fast.json"
out.parent.mkdir(parents=True)
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
llm.main()
return json.loads(out.read_text(encoding="utf-8"))
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
# A failed stage is retried on the next run; "partial" would be skipped.
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
monkeypatch.setattr(llm, "PORT", port)
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
launched = []
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
r = _run_llm_stage(tmp_path, monkeypatch)
assert r["status"] == "failed"
assert launched == []
assert "server" not in r
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
monkeypatch.setattr(llm, "PORT", port)
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
s = llm.Strata(tmp_path, tmp_path / "strata.log")
monkeypatch.setattr(s, "supported_flags", lambda: set())
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
# so the engine's own log must reach the results zip, which packs logs/*.log.
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
logs = tmp_path / "logs"
logs.mkdir()
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
monkeypatch.setattr(s, "supported_flags", lambda: set())
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
engine_dir.mkdir(parents=True, exist_ok=True)
home = str(Path.home())
(engine_dir / "q2_0-32k.engine.log").write_text(
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
assert len(kept) == 1
text = kept[0].read_text(encoding="utf-8")
assert "out of memory" in text
assert home not in text
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
# canonical Q2_0 pack is not a native pack, so the engine stops at start
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
ran = []
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
s = llm.Strata(tmp_path, tmp_path / "strata.log")
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
for parallel in (1, 2):
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
for cmd in ran:
assert "--drafting" in cmd
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
def test_virtual_display_structures_match_the_win32_layout():
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
import ctypes
from m0lib import winvdd
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
if ctypes.sizeof(ctypes.c_void_p) == 8:
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
def test_virtual_display_goes_where_it_only_touches_a_corner():
from m0lib import winvdd
# Two side-by-side 1440p monitors, the second one lower than the first.
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
x, y = winvdd.corner_position(physical)
assert (x, y) == (5120, 1640)
for d in physical:
left, top, right, bottom = d["rect"]
# Only a point is shared: no edge of length > 0 with any real monitor.
assert not (left <= x < right and top <= y < bottom)
def test_virtual_display_remove_sends_the_index_the_driver_expects():
from m0lib import winvdd
sent = []
vdd = object.__new__(winvdd.Vdd)
vdd.added = [1]
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
vdd.remove(1)
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
assert vdd.added == []
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
import m0
keys = [s[0] for s in m0.STAGES]
assert keys.index("vdisplay") < keys.index("tts_melo")
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
def test_virtual_display_names_are_read_from_utf16_buffers():
from m0lib import winvdd
dd = winvdd.DISPLAY_DEVICEW()
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
# ---------------------------------------------------------------------------
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
class _ScriptedClient:
"""Answers chat calls from a script; records what was sent."""
model = "scripted"
def __init__(self, answers):
self.answers = list(answers)
self.sent = []
def chat(self, messages, **kwargs):
self.sent.append((messages, kwargs))
text, calls = self.answers.pop(0)
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
def test_kit_uses_the_production_prompt_rules_and_planner():
from jarvis import system_prompt
from m0lib import tool_fix
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
# The real jarvis package stays importable next to the kit's private copy.
import jarvis.reply.planner as real_planner
assert real_planner.named_tool_steps is not planner.named_tool_steps
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
from m0lib import tool_fix
case = tool_fix.case_for("서울 날씨 어때?")
client = _ScriptedClient([
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
("서울은 24도, 맑음입니다.", []), # model skips the tool
("서울은 13도이고 비가 옵니다.", []), # answer from the result
])
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
row = tool_fix.run_case(client, "system", planner, case)
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
assert row["final_ok"] is True
final_messages = client.sent[-1][0]
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
def test_model_that_calls_the_tool_is_scored_without_the_guard():
from m0lib import tool_fix
case = tool_fix.case_for("5분 타이머 맞춰 줘.")
client = _ScriptedClient([
("set_timer minutes='5'", []),
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
("5분 타이머를 맞췄습니다.", []),
])
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
row = tool_fix.run_case(client, "system", planner, case)
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
assert row["text_with_call"] is False
# ---------------------------------------------------------------------------
# "If the fix does not work, change the model": the tools stage tries the
# next model of the same family until one passes every scored case.
def _summary(ok, total=21):
return {"ok": ok, "model_ok": ok, "total": total, "guard_ran": 0, "text_with_call": 0}
def test_passing_default_tries_no_other_model(tmp_path):
r = common.StageResult("tools_fast", tmp_path / "r.json")
tried = []
llm.try_candidates(r, "fast", _summary(21), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 999)
assert tried == [] and r["tools_verdict"]["chosen"] == llm.PRESETS["fast"]["label"]
def test_failing_default_stops_at_the_first_candidate_that_passes(tmp_path):
r = common.StageResult("tools_fast", tmp_path / "r.json")
results = iter([_summary(20), _summary(21)])
tried = []
llm.try_candidates(r, "fast", _summary(18), lambda p: tried.append(p["model"]) or next(results),
free_gb=lambda: 999)
assert tried == [c["model"] for c in llm.CANDIDATES["fast"][:2]]
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][1]["label"]
def test_candidate_without_disk_space_is_skipped_not_downloaded(tmp_path):
r = common.StageResult("tools_recommended", tmp_path / "r.json")
tried = []
llm.try_candidates(r, "recommended", _summary(18), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 1)
assert tried == []
assert r["tools_verdict"]["chosen"] is None
assert all(c["skipped"] == "disk" for c in r["tools_verdict"]["candidates"])
def _run_scripted(case_prompt, answers):
from m0lib import tool_fix
client = _ScriptedClient(answers)
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
return tool_fix.run_case(client, "system", planner, tool_fix.case_for(case_prompt)), client
def test_right_tool_with_a_made_up_final_answer_fails():
# The weather tool ran and said 13 degrees; an answer of "24도, 맑음" is still a lie.
row, _ = _run_scripted("서울 날씨 어때?", [
("get_weather city='서울'", []),
("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]),
("서울은 지금 24도, 맑음입니다.", []),
])
assert row["model_ok"] is True and row["final_ok"] is False and row["ok"] is False
def test_claiming_an_action_no_tool_can_do_fails():
claimed, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불을 껐습니다.", [])])
honest, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불은 제가 끌 수 없습니다.", [])])
assert claimed["ok"] is False and honest["ok"] is True
def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine():
row, client = _run_scripted("서울 날씨 어때?", [
("get_weather town='서울'", []), # key get_weather does not take
("서울은 24도, 맑음입니다.", []), # model skips the tool
("지금은 날씨를 확인하지 못했습니다.", []), # answer from the failed result
])
assert row["guard_ran"] is False and row["ok"] is False
assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다."
last = client.sent[-1][0][-1]
assert last["role"] == "tool" and "could not be run" in last["content"]
# Final-answer judging: the reviewer's counterexamples must fail, honest answers pass.
@pytest.mark.parametrize("prompt,answer,expected", [
("서울 날씨 어때?", "서울은 지금 13도이고 비가 오고 있습니다.", True),
("서울 날씨 어때?", "서울은 지금 24도, 맑음입니다.", False),
("서울 날씨 어때?", "서울은 13도이고 비가 오지 않습니다.", False),
("서울 날씨 어때?", "서울은 13도, 어제는 20도였고 비가 옵니다.", False),
("서울 날씨 어때?", "서울은 13.9도이고 비가 옵니다.", False),
("서울 날씨 어때?", "서울은 -13도이고 비가 옵니다.", False),
("서울 날씨 어때?", "서울은 영하 13도이고 비가 옵니다.", False),
("서울 날씨 어때?", "서울은 13.0도이고 비가 옵니다.", True),
("부산 지금 비 와?", "부산은 맑고 비가 오지 않아요, 17.5도입니다.", False),
("나 지금 서울인데 우산 챙겨야 할까?", "영하 13도에 비가 오고 있으니 우산을 챙기세요.", False),
("5분 타이머 맞춰 줘.", "5.5분 타이머를 맞췄습니다.", False),
("5분 타이머 맞춰 줘.", "-5분 타이머를 맞췄습니다.", False),
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오고 있으니 우산을 챙기세요.", True),
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오지 않으니 우산은 필요 없습니다.", False),
("부산 지금 비 와?", "부산은 지금 맑고 비가 오지 않아요, 17도입니다.", True),
("부산 지금 비 와?", "부산은 지금 비가 와요, 우산 챙기세요.", False),
("5분 타이머 맞춰 줘.", "5분 타이머를 맞췄습니다.", True),
("5분 타이머 맞춰 줘.", "15분 타이머를 맞췄습니다.", False),
("5분 타이머 맞춰 줘.", "50분 타이머를 맞췄습니다.", False),
("10분 뒤에 알려 줘.", "10분 뒤에 알려 드릴게요.", True),
("10분 뒤에 알려 줘.", "110분 뒤에 알려 드릴게요.", False),
("거실 불 꺼 줘.", "거실 불은 제가 끌 수 없습니다.", True),
("거실 불 꺼 줘.", "거실 불을 껐습니다.", False),
("거실 불 꺼 줘.", "거실 불을 껐습니다. 다시 켜는 기능은 지원하지 않습니다.", False),
("3분 타이머 맞춰 줘.", "저는 타이머를 맞출 수 없습니다.", True),
("3분 타이머 맞춰 줘.", "3분 타이머를 맞췄습니다.", False),
("3분 타이머 맞춰 줘.", "타이머는 지원하지 않지만 3분 뒤에 알려 드릴게요.", False),
# Real refusals from the gaming-PC run (2026-11-10) that name the missing tool.
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 없습니다.", True),
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 준비되어 있지 않습니다. "
"직접 스위치를 조작하거나 스마트 홈 기기를 통해 조절해 주셔야 합니다.", True),
("3분 타이머 맞춰 줘.", "죄송합니다만, 현재 타이머를 설정할 수 있는 도구는 준비되어 있지 않습니다. "
"직접 알람을 설정해 주시거나 다른 방법을 찾아보시길 바랍니다.", True),
("거실 불 꺼 줘.", "거실 불을 끌 수 있는 도구가 없지만 대신 꺼 드렸습니다.", False),
("3분 타이머 맞춰 줘.", "타이머 도구는 없지만 3분 타이머를 맞췄습니다.", False),
])
def test_final_answer_judge(prompt, answer, expected):
from m0lib import tool_fix
assert tool_fix.case_for(prompt).judge(answer) is expected
def test_no_timer_tool_case_offers_no_timer_tool():
from m0lib import tool_fix
names = [t["function"]["name"] for t in tool_fix.case_for("3분 타이머 맞춰 줘.").offered()]
assert "set_timer" not in names and names
class _LyingClient:
"""Never calls a tool and always answers with a made-up weather report."""
model = "liar"
def chat(self, messages, **kwargs):
if kwargs.get("tools") is None: # planner call
return {"text": "Reply to the user.", "tool_calls": [], "first_tool": None, "ttft": 0.1, "finish": "stop"}
return {"text": "서울은 24도, 맑음입니다. 5분 타이머를 맞췄습니다.", "tool_calls": [], "first_tool": None,
"ttft": 0.1, "finish": "stop"}
def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path):
from m0lib import tool_fix
r = common.StageResult("tools_fast", tmp_path / "r.json")
summary = tool_fix.test_tools_fixed(r, _LyingClient(), tries=1)
assert not tool_fix.tools_passed(summary)
tried = []
llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999)
assert tried == [llm.CANDIDATES["fast"][0]["model"]]
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"]
# --- gamemode stage: the one model in each placement the app switches to ----
def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True, only="levels",
extra_opts=(), stt=None, procs=None, paused=None, suspend_ok=True):
"""Run the stage with Strata replaced; ``gpu(running)`` is nvidia-smi's (used, total), where
``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None. ``stt`` is a
_FakeStt whose loaded model adds to the used VRAM."""
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
monkeypatch.setattr(llm, "PORT", port)
started = []
running = {"now": None}
if stt is not None:
stt.running = running
monkeypatch.setattr(llm, "SttLoad", stt.factory)
base_gpu = gpu
def gpu(now):
used, total = base_gpu(now)
return used + stt.used_mib(), total
def launch(self, preset, context, parallel, vision="on", extra_sets=None, restart=False, timeout=0,
kv=None, mtp=None):
spec = {"parallel": parallel, "vision": vision, "sets": list(extra_sets or []), "context": context,
"kv": kv, "mtp": mtp}
started.append(spec)
if stt is not None:
stt.events.append("llm_start")
running["now"] = spec if ready(extra_sets or []) else None
return {"ready": running["now"] is not None}
def stop(self):
running["now"] = None
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
monkeypatch.setattr(llm.Strata, "launch", launch)
monkeypatch.setattr(llm.Strata, "stop", stop)
monkeypatch.setattr(llm, "port_in_use", lambda p: running["now"] is not None)
monkeypatch.setattr(llm, "gpu_memory_mib", lambda: gpu(running["now"]))
monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["now"]))
monkeypatch.setattr(gpucap, "gpu_util_pct", lambda: 99)
# The kit's own engine and STT worker; the GPU cap "pauses" them by recording it.
procs = _procs(tmp_path / "strata") if procs is None else procs
monkeypatch.setattr(gpucap, "process_table", lambda: list(procs))
held = set()
paused = set() if paused is None else paused
monkeypatch.setattr(gpucap, "suspend_pid",
lambda pid: (held.add(pid) or paused.add(pid) or True) if suspend_ok else False)
monkeypatch.setattr(gpucap, "resume_pid", lambda pid: held.discard(pid) or True)
monkeypatch.setattr(gpucap, "PERIOD_S", 0.002)
running["held"] = held
out = tmp_path / "results" / "gamemode.json"
out.parent.mkdir(parents=True, exist_ok=True)
opts = ["--opt", "model=fast", "--opt", f"only={only}"]
for o in extra_opts:
opts += ["--opt", o]
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), *opts])
llm.main()
return json.loads(out.read_text(encoding="utf-8")), started, running
def _model_mib(spec):
"""Roughly what the 2026-11-10 game PC showed: 4.4 GiB of weights that never leave the card, the
expert budget (which first holds the 1.1 GiB long-prompt staging, so 0 and 1 cost the same), the
conversation memory (int8, 1.43 GiB per 32K; q4_0 half), about 1 GB for the MTP drafter and
1.5 GB of buffers. With no expert budget Strata fills the card."""
experts = next((float(s.split("=")[1]) for s in spec["sets"] if s.startswith("expert-vram=")), None)
if experts is None:
return 13400
kv = 1464 * spec["context"] / 32768 * (0.5 if spec["kv"] == "q4_0" else 1)
return int(4506 + max(experts, 1.1) * 1024 + kv + (0 if spec["mtp"] == "off" else 1000) + 1500)
def _vram(running):
"""16 GB card, 2 GB used by Windows."""
return (2000 if running is None else 2000 + _model_mib(running)), 16303
def _vram_20g(running):
"""A 20 GB card, where the smaller game placements do leave the reserve."""
return _vram(running)[0], 20480
def _vram_big_floor(running):
"""A card where even the smallest placement leaves the game less than its reserve."""
return (2000 if running is None else 2000 + _model_mib(running) + 3000), 16303
def test_gamemode_measures_every_placement_and_returns_to_normal(tmp_path, monkeypatch, fake_server):
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_20g)
assert r["stage"] == "gamemode" and r["status"] == "ok"
assert [x["sets"] for x in started] == [lv["sets"] for lv in llm.LEVELS]
# Each placement starts with its own context, conversation memory format and drafter.
for spec, lv in zip(started, llm.LEVELS):
assert spec["context"] == lv.get("context", 32768)
assert (spec["kv"], spec["mtp"]) == (lv.get("kv"), lv.get("mtp"))
assert started[-1] == started[0] # the last start is the normal placement again
rows = {x["key"]: x for x in r["levels"]}
for lv in llm.LEVELS:
assert rows[lv["key"]]["switch_s"] is not None
assert rows[lv["key"]]["voice_first_sentence"]["n"] > 0
# The game placements go below the 2 GB expert budget that still left a 16 GB card only 4 GB.
game = [rows[lv["key"]] for lv in llm.LEVELS if lv["key"].startswith("game_")]
assert max(x["vram_free_mib"] for x in game) > min(x["vram_free_mib"] for x in game)
assert min(x["vram_free_mib"] for x in game) > rows["normal"]["vram_free_mib"]
# On this card a smaller placement does leave the reserve, and the choice is one that does.
assert r["game_choice"]["vram_free_mib"] >= llm.GAME_RESERVE_MIB
assert running["now"] is None # the model is not left running after the stage
def test_when_no_placement_leaves_the_reserve_the_result_says_how_much_is_free(tmp_path, monkeypatch,
fake_server):
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_big_floor)
assert r["game_choice"] is None
assert r["status"] == "ok" # every placement was measured; it is the card that is too small
most = max(x["vram_free_mib"] for x in r["levels"] if x["key"].startswith("game_"))
assert any(str(most) in n for n in r["notes"])
def test_engine_lines_are_only_those_of_the_latest_start(tmp_path):
# The engine log is appended across starts: reading it from the top repeated an old start's
# placement lines for every placement.
s = llm.Strata(tmp_path, tmp_path / "strata.log")
log = tmp_path / "strata" / "localappdata" / "Strata" / "launch" / "ud-q4_k_xl-32k.engine.log"
log.parent.mkdir(parents=True)
log.write_text("strata-q36: hybrid experts: 59 of 256 per layer in VRAM (4.32 GiB)\n", encoding="utf-8")
s.mark_engine_logs()
with open(log, "a", encoding="utf-8") as f:
f.write("strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)\n")
assert s.engine_lines() == ["strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)"]
# A log written again from the start (shorter than the mark) is read whole.
log.write_text("strata-q36: context 8192 cells\n", encoding="utf-8")
assert s.engine_lines() == ["strata-q36: context 8192 cells"]
def test_placement_memory_format_and_drafter_reach_the_launch_command(tmp_path, monkeypatch):
ran = []
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
s = llm.Strata(tmp_path, tmp_path / "strata.log")
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--context", "--kv", "--mtp"})
s.launch(llm.PRESETS["fast"], 8192, parallel=1, kv="q4_0", mtp="off")
s.launch(llm.PRESETS["fast"], 32768, parallel=1)
with_opts, plain = ran
assert with_opts[with_opts.index("--kv") + 1] == "q4_0" and with_opts[with_opts.index("--mtp") + 1] == "off"
assert with_opts[with_opts.index("--context") + 1] == "8192"
assert "--kv" not in plain and "--mtp" not in plain
def test_a_placement_that_does_not_start_is_recorded_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
ready=lambda sets: "expert-vram=0" not in sets)
assert len(started) == len(llm.LEVELS)
failed = [lv["key"] for lv in llm.LEVELS if "expert-vram=0" in lv["sets"]]
row = next(x for x in r["levels"] if x["key"] == failed[0])
assert row["switch_s"] is None and "decode_tok_s" not in row
# A placement that never started is a gap in the measurement, not a pass.
assert r["status"] == "partial"
assert any(failed[0] in e["where"] for e in r["errors"])
def test_only_the_normal_placement_starting_is_not_ok(tmp_path, monkeypatch, fake_server):
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
ready=lambda sets: not any(s.startswith("expert-vram=") for s in sets))
assert r["game_choice"] is None
assert r["status"] == "partial"
game = [lv for lv in llm.LEVELS if lv["key"].startswith("game_")]
assert sum(1 for e in r["errors"] if e["where"].startswith("level game_")) == len(game)
def _row(key, free, tok_s):
return {"key": key, "vram_free_mib": free, "decode_tok_s": {"median": tok_s}, "switch_s": 50.0}
def test_game_choice_is_the_fastest_placement_that_leaves_the_reserve_free():
rows = [_row("normal", 900, 82.0), _row("game_experts_6g", 6000, 70.0),
_row("game_experts_4g", 8100, 60.0), _row("game_experts_2g", 10100, 45.0)]
choice = llm.pick_game_level(rows, reserve_mib=8000)
assert choice["key"] == "game_experts_4g" and choice["decode_tok_s"] == 60.0
def test_no_game_choice_when_no_placement_leaves_the_reserve():
rows = [_row("normal", 900, 82.0), _row("game_experts_2g", 7000, 45.0)]
assert llm.pick_game_level(rows, reserve_mib=8000) is None
def test_kit_has_a_one_click_runner_for_gamemode():
import m0
keys = [s[0] for s in m0.STAGES]
assert "gamemode" in keys
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
assert "-Only gamemode" in bat and "-Redo" in bat
def test_a_measurement_error_inside_a_placement_marks_the_stage_partial(tmp_path, monkeypatch, fake_server):
monkeypatch.setattr(llm, "quick_bench", lambda c: {"errors": ["decode: TimeoutError"], "decode_tok_s": {"n": 0}})
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
assert r["status"] == "partial"
assert any(e["error"] == "decode: TimeoutError" for e in r["errors"])
def test_gamemode_measures_the_2g_expert_placement_with_a_16k_context(tmp_path, monkeypatch, fake_server):
# The one placement between "2 GB, 32K" and "0 GB, 16K" that was never measured.
_, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
assert any("expert-vram=2" in x["sets"] and x["context"] == 16384 for x in started)
def test_normal_placement_answers_one_request_at_a_time(tmp_path, monkeypatch, fake_server):
# One request at a time is the default; two only when a chat channel answers every message.
_, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
assert started[0]["parallel"] == 1 and started[-1]["parallel"] == 1
# --- gamemode_stt / gamemode_live: the LLM next to the STT that runs with it --
STT_MODEL_MIB = 1300 # M0 stt: the loaded model
STT_GROWTH_MIB = 1440 # M0 stt: what ten speakers at once add over the loaded model
STT_GPU_MIB = STT_MODEL_MIB + STT_GROWTH_MIB
STT_LAUNCHER_PID = 4100 # the venv python.exe; the real interpreter is its child
class _FakeStt:
"""Stands in for the STT worker process: on cuda its model holds VRAM while it is loaded, and
ten speakers at once hold more once it is transcribing (with partials on)."""
def __init__(self, fail_device=None, broken_at_load=False, finish_result=None):
self.finish_result = finish_result
self.events: list[str] = []
self.devices: list[str] = []
self.paces: list = []
self.loaded: str | None = None
self.busy = False
self.running: dict = {}
self.fail_device = fail_device
self.broken_at_load = broken_at_load
def used_mib(self) -> int:
if self.loaded != "cuda":
return 0
return STT_GPU_MIB if self.busy else STT_MODEL_MIB
def factory(self, python, work, device, log, pace=None):
fake = self
fake.devices.append(device)
fake.paces.append(pace)
class _One:
pid = STT_LAUNCHER_PID
def start(self, timeout=900):
if fake.broken_at_load:
return {"status": "failed", "errors": ["load: RuntimeError: CUDA failed"]}
fake.loaded = device
fake.events.append("stt_loaded")
return {"status": "loaded", "load_s": 3.0}
def go(self):
assert fake.running.get("now") is not None, "STT load must overlap a running LLM"
fake.events.append("stt_go")
fake.busy = pace is None or pace.get("partial_every") is not None
def finish(self, timeout=120):
fake.events.append("stt_finish")
if fake.finish_result is not None:
return fake.finish_result
errs = ["RuntimeError: CUDA out of memory"] if device == fake.fail_device else []
partials = pace is None or pace.get("partial_every") is not None
return {"status": "done", "batch10_s": {"n": 4 if partials else 0, "median": 0.9},
"single_s": {"n": 4, "median": 0.2}, "errors": errs, "pace": pace}
def close(self):
fake.loaded = None
fake.busy = False
return _One()
def test_together_stage_measures_the_llm_with_the_stt_loaded_and_busy(tmp_path, monkeypatch, fake_server):
stt = _FakeStt()
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
extra_opts=["stt_python=py"], stt=stt)
assert r["stage"] == "gamemode_stt" and r["status"] == "ok"
assert len(started) == len(llm.TOGETHER)
# The STT model is in VRAM before the LLM picks its placement, and its ten-speaker loop runs
# while the LLM is measured (asserted by _FakeStt.go).
for i in range(len(llm.TOGETHER)):
assert stt.events[i * 4:(i + 1) * 4] == ["stt_loaded", "llm_start", "stt_go", "stt_finish"]
rows = {x["key"]: x for x in r["levels"]}
for lv in llm.TOGETHER:
row = rows[lv["key"]]
assert row["stt"]["device"] == lv["stt"]
assert row["stt"]["batch10_s"]["n"] > 0 and row["decode_tok_s"]["n"] > 0
# What is left for the game already has the STT taken out when it runs on the GPU.
_, total = _vram(None)
assert row["vram_free_mib"] == total - row["vram_peak_mib"]
# Both places for the STT are measured in the same game placement.
game = [lv for lv in llm.TOGETHER if lv["key"].startswith("game_")]
assert {lv["stt"] for lv in game} == {"cuda", "cpu"}
pairs = [(rows[a["key"]], rows[b["key"]]) for a in game for b in game
if a["stt"] == "cuda" and b["stt"] == "cpu" and (a["sets"], a.get("context")) == (b["sets"], b.get("context"))]
assert pairs
for on_gpu, on_cpu in pairs:
assert on_cpu["vram_free_mib"] - on_gpu["vram_free_mib"] == STT_GPU_MIB
# A chat channel that answers every message runs two requests at once: measured with the STT too.
assert any(lv["parallel"] == 2 for lv in llm.TOGETHER if lv["key"].startswith("normal"))
assert stt.loaded is None and running["now"] is None # nothing is left running
def test_stt_error_next_to_the_llm_marks_the_together_stage_partial(tmp_path, monkeypatch, fake_server):
stt = _FakeStt(fail_device="cuda")
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
extra_opts=["stt_python=py"], stt=stt)
assert r["status"] == "partial"
assert any("out of memory" in e["error"] for e in r["errors"])
def test_stt_that_does_not_load_skips_that_placement_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
stt = _FakeStt(broken_at_load=True)
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
extra_opts=["stt_python=py"], stt=stt)
assert started == [] # an LLM measured without its STT would be the wrong number
assert len(stt.devices) == len(llm.TOGETHER)
assert r["status"] == "failed"
def test_together_stage_without_the_stt_environment_measures_nothing(tmp_path, monkeypatch, fake_server):
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together")
assert started == [] and r["status"] == "failed"
def test_live_stage_is_skipped_when_nobody_is_there_to_start_a_game(tmp_path, monkeypatch, fake_server):
monkeypatch.setattr(llm, "interactive", lambda: False)
stt = _FakeStt()
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live",
extra_opts=["stt_python=py"], stt=stt)
assert r["status"] == "skipped" and started == []
def test_live_stage_measures_game_placements_while_the_game_runs(tmp_path, monkeypatch, fake_server):
monkeypatch.setattr(llm, "interactive", lambda: True)
monkeypatch.setattr(llm, "ask", lambda *a, **k: "y")
monkeypatch.setattr(llm, "wait_enter", lambda *a, **k: "")
monkeypatch.setattr(llm.time, "sleep", lambda s: None)
stt = _FakeStt()
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live",
extra_opts=["stt_python=py"], stt=stt)
assert r["stage"] == "gamemode_live" and r["status"] == "ok"
# Only game placements: starting the normal one would take the game's VRAM.
assert len(started) == len(llm.LIVE) and all(lv["key"].startswith("game_") for lv in llm.LIVE)
assert {lv["stt"] for lv in llm.LIVE} == {"cuda", "cpu"}
assert all(x["vram_idle_mib"] is not None for x in r["levels"]) # Windows and the game
assert running["now"] is None and stt.loaded is None
def _fake_kit_run(tmp_path, monkeypatch, *args):
import m0
ran = []
monkeypatch.setattr(m0, "run_stage", lambda key, *a, **k: ran.append(key))
monkeypatch.setattr(m0, "pack", lambda work: tmp_path / "x.zip")
monkeypatch.setattr(m0.subprocess, "run", lambda *a, **k: None)
monkeypatch.setattr(sys, "argv", ["m0.py", "--work", str(tmp_path / "w"), *args])
m0.main()
return ran
def test_full_kit_run_leaves_out_the_stage_that_needs_a_running_game(tmp_path, monkeypatch):
ran = _fake_kit_run(tmp_path, monkeypatch)
assert "gamemode_stt" in ran and "gamemode_live" not in ran
assert _fake_kit_run(tmp_path, monkeypatch, "--only", "gamemode_live", "--redo") == ["gamemode_live"]
def test_stages_that_run_the_stt_next_to_the_llm_get_the_stt_python(tmp_path, monkeypatch):
import m0
cmds = []
monkeypatch.setattr(m0.envs, "ensure_env", lambda name, work, log: Path(f"/venvs/{name}/python"))
monkeypatch.setattr(m0.subprocess, "run", lambda cmd, **k: cmds.append(cmd) or type("P", (), {"returncode": 0}))
for key in ("gamemode_stt", "gamemode"):
m0.run_stage(key, "base", "llm.py", ["model=fast"], tmp_path, {})
together, alone = cmds
assert f"stt_python={Path('/venvs/stt/python')}" in together
assert not any(str(x).startswith("stt_python=") for x in alone)
def test_one_click_runners_cover_the_new_gamemode_stages():
import m0
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
assert "-Only gamemode,gamemode_stt" in bat
live = (Path(m0.KIT) / "run_m0_game_live.bat").read_text(encoding="utf-8")
assert "-Only gamemode_live" in live and "-Redo" in live
def test_stt_load_loop_waits_for_go_and_runs_until_stop(tmp_path):
from m0lib import stt
go, stop = tmp_path / "go", tmp_path / "stop"
calls = []
def batch():
calls.append("b")
if not go.exists():
raise AssertionError("ran before go")
def single():
calls.append("s")
if calls.count("s") == 3:
stop.write_text("")
go.write_text("")
res = stt.load_loop(batch, single, go, stop)
assert res["status"] == "done" and res["batch10_s"]["n"] == 3 and res["single_s"]["n"] == 3
assert calls == ["b", "s"] * 3
def test_stt_load_loop_keeps_an_out_of_memory_and_stops(tmp_path):
from m0lib import stt
go, stop = tmp_path / "go", tmp_path / "stop"
go.write_text("")
def batch():
raise RuntimeError("CUDA failed with error out of memory")
res = stt.load_loop(batch, lambda: None, go, stop)
assert res["batch10_s"]["n"] == 0 and "out of memory" in res["errors"][0]
def test_stt_load_loop_does_not_run_when_stopped_before_go(tmp_path):
from m0lib import stt
stop = tmp_path / "stop"
stop.write_text("")
res = stt.load_loop(lambda: 1 / 0, lambda: 1 / 0, tmp_path / "go", stop)
assert res["status"] == "not_started" and res["errors"] == []
@pytest.mark.parametrize("busy", [
{"status": "timeout", "errors": []}, # the worker did not stop in time
{"status": "loaded"}, # it died before writing its timings
{"status": "not_started", "batch10_s": {"n": 0}, "single_s": {"n": 0}, "errors": []},
{"status": "done", "batch10_s": {"n": 0}, "single_s": {"n": 3, "median": 0.2}, "errors": []},
])
def test_stt_load_without_timings_is_a_gap_not_a_pass(tmp_path, monkeypatch, fake_server, busy):
stt = _FakeStt(finish_result=busy)
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
extra_opts=["stt_python=py"], stt=stt)
assert r["status"] == "partial"
assert sum(1 for e in r["errors"] if e["where"].endswith(" stt")) == len(llm.TOGETHER)
def test_stt_worker_that_ignores_stop_is_reported_as_timed_out_and_killed(tmp_path):
s = llm.SttLoad(sys.executable, tmp_path, "cpu", tmp_path / "stt.log")
s.cmd = [sys.executable, "-c",
"import json,sys,time; open(sys.argv[1],'w').write(json.dumps({'status':'loaded'})); time.sleep(60)",
str(s.out)]
assert s.start(timeout=30)["status"] == "loaded"
s.go()
res = s.finish(timeout=1)
assert res["status"] == "timeout" and res["errors"]
s.close()
assert s.proc.poll() is not None
# --- gamemode_tune: room for the STT in the normal placement, fewer partials while the LLM answers --
class _FakeClock:
"""time.monotonic/perf_counter/sleep for the STT loop: passes advance it, nothing waits."""
def __init__(self):
self.t = 0.0
def monotonic(self):
return self.t
perf_counter = monotonic
def sleep(self, s):
self.t += s
def _paced_run(tmp_path, monkeypatch, pace, run_s=10.0):
from m0lib import stt
clock = _FakeClock()
monkeypatch.setattr(stt, "time", clock)
go, stop = tmp_path / "go", tmp_path / "stop"
go.write_text("")
def work(seconds):
def fn():
clock.t += seconds
if clock.t >= run_s:
stop.write_text("")
return fn
return stt.load_loop(work(0.9), work(0.2), go, stop, pace=pace)
def test_paced_stt_runs_partials_at_their_interval_and_finals_in_between(tmp_path, monkeypatch):
res = _paced_run(tmp_path, monkeypatch, {"partial_every": 2.0, "final_every": 0.5})
assert res["status"] == "done" and res["pace"] == {"partial_every": 2.0, "final_every": 0.5}
assert 5 <= res["batch10_s"]["n"] <= 6 # one ten-speaker pass every 2 s over 10 s
assert res["single_s"]["n"] >= 12 # finals keep coming in between
# A final waits at most for the partial pass that is running when it is due.
assert res["final_wait_s"]["max"] <= 0.9 + 1e-6
assert res["busy_share"] < 1.0 # the GPU gets time back for the LLM
def test_stt_without_a_pace_transcribes_back_to_back(tmp_path, monkeypatch):
res = _paced_run(tmp_path, monkeypatch, None)
assert res["busy_share"] == 1.0 and res["batch10_s"]["n"] == res["single_s"]["n"]
def test_finals_only_stt_runs_no_ten_speaker_pass(tmp_path, monkeypatch):
res = _paced_run(tmp_path, monkeypatch, {"partial_every": None, "final_every": 0.5})
assert res["batch10_s"]["n"] == 0 and res["single_s"]["n"] >= 15
assert res["busy_share"] < 0.5
def test_stt_worker_command_carries_the_pace(tmp_path):
s = llm.SttLoad(sys.executable, tmp_path, "cuda", tmp_path / "stt.log",
pace={"partial_every": None, "final_every": 0.5})
assert "partial_every=none" in s.cmd and "final_every=0.5" in s.cmd
plain = llm.SttLoad(sys.executable, tmp_path, "cuda", tmp_path / "stt2.log")
assert not any(str(x).startswith(("partial_every=", "final_every=")) for x in plain.cmd)
def _expert_vram(spec):
return next((float(s.split("=")[1]) for s in spec["sets"] if s.startswith("expert-vram=")), None)
def test_tune_stage_leaves_the_stt_room_in_the_normal_placement(tmp_path, monkeypatch, fake_server):
stt = _FakeStt()
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune",
extra_opts=["stt_python=py"], stt=stt)
assert r["stage"] == "gamemode_tune" and r["status"] == "ok"
normal = [lv for lv in llm.TUNED if lv.get("stt_room")]
# One start without experts measures what the LLM needs before any expert, with the STT loaded.
probes = [x for x in started if _expert_vram(x) == 0 and x["vision"] == normal[0]["vision"]]
assert len(probes) == 1 and started[0] is probes[0]
rows = {x["key"]: x for x in r["levels"]}
budgets = {_expert_vram(x) for x in started[1:] if x["vision"] == normal[0]["vision"]}
assert len(budgets) == 1 and budgets.pop() > 0
# With the STT at its ten-speaker peak the card keeps the margin and the room for other
# programs free, and not much more.
continuous = [rows[lv["key"]] for lv in normal if lv.get("stt") and lv.get("stt_pace") is None]
assert continuous
keep = llm.STT_ROOM_MARGIN_MIB + llm.OTHER_ROOM_MIB
for row in continuous:
assert keep <= row["vram_free_mib"] < keep + 2 * 103
assert row["stt_room"]["stt_growth_mib"] == llm.STT_GROWTH_FALLBACK_MIB
assert row["stt_room"]["other_room_mib"] == llm.OTHER_ROOM_MIB
# The same placement is also measured without the STT, so the cap's cost on the LLM alone shows.
assert any(not lv.get("stt") for lv in normal) and any(lv.get("stt") for lv in normal)
# Without a game the STT keeps every partial while the LLM answers (user decision); only the
# 2 GB game placement is measured back to back, with fewer partials and with finals only.
assert all(lv.get("stt_pace") is None for lv in normal)
game = [lv for lv in llm.TUNED if lv["key"].startswith("game_")]
assert game and all("expert-vram=2" in lv["sets"] for lv in game)
paces = {(lv.get("stt_pace") or {}).get("partial_every", "continuous") for lv in game}
assert paces == {"continuous", 2.0, None}
assert stt.paces == [lv.get("stt_pace") for lv in llm.TUNED if lv.get("stt")]
# Each capped row has its uncapped twin in the same run.
capped = [lv for lv in llm.TUNED if lv.get("gpu_cap")]
assert capped and all(lv["gpu_cap"] == llm.GPU_CAP for lv in capped)
for lv in capped:
twin = {k: v for k, v in lv.items() if k not in ("key", "gpu_cap")}
assert any({k: v for k, v in x.items() if k != "key"} == twin for x in llm.TUNED)
assert stt.loaded is None and running["now"] is None
def test_stt_room_comes_from_this_pcs_own_stt_measurement(tmp_path, monkeypatch, fake_server):
(tmp_path / "results").mkdir(parents=True)
(tmp_path / "results" / "stt.json").write_text(json.dumps(
{"run_int8_float16": {"batch_speakers": {"vram_added_mib": STT_GROWTH_MIB + 512}}}), encoding="utf-8")
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune",
extra_opts=["stt_python=py"], stt=_FakeStt())
row = next(x for x in r["levels"] if x.get("stt_room"))
assert row["stt_room"]["stt_growth_mib"] == STT_GROWTH_MIB + 512
assert row["vram_free_mib"] >= llm.STT_ROOM_MARGIN_MIB + 512
def test_card_without_room_for_any_expert_starts_the_normal_placement_with_none(tmp_path, monkeypatch,
fake_server):
def small(running):
return _vram(running)[0], 12000
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, small, only="tune",
extra_opts=["stt_python=py"], stt=_FakeStt())
row = next(x for x in r["levels"] if x.get("stt_room"))
assert row["stt_room"]["expert_vram_gib"] == 0
assert any("no room" in n for n in r["notes"])
def test_tune_stage_runs_with_the_stt_python_and_has_a_one_click_runner(tmp_path, monkeypatch):
import m0
assert "gamemode_tune" in m0.NEEDS_STT and "gamemode_tune" not in m0.ON_DEMAND
bat = (Path(m0.KIT) / "run_m0_gamemode_tune.bat").read_text(encoding="utf-8")
assert "-Only gamemode_tune" in bat and "-Redo" in bat
# --- the GPU cap: Jarvis's engine and STT worker paused together for 5 % of the time --------
def _procs(home):
eng = str(home / "components" / "engine" / "0.1.59-windows-x86_64-cuda-cuda13" / "strata.exe")
term = str(home / "terminal" / "current" / "bin" / "strata.exe")
return [
{"pid": 10, "ppid": 1, "exe": term, "cmd": f"{term} launch --yes"},
{"pid": 20, "ppid": 10, "exe": eng, "cmd": f"{eng} --serve --pack x"},
{"pid": 21, "ppid": 20, "exe": "C:\\x\\helper.exe", "cmd": "helper"},
{"pid": 30, "ppid": 1, "exe": eng.replace(str(home), str(home.parent / "other")), "cmd": "strata.exe --serve"},
{"pid": STT_LAUNCHER_PID, "ppid": 1, "exe": "C:\\venv\\python.exe", "cmd": "python stt.py"},
{"pid": STT_LAUNCHER_PID + 1, "ppid": STT_LAUNCHER_PID, "exe": "C:\\py\\python.exe", "cmd": "python stt.py"},
{"pid": 50, "ppid": 1, "exe": "C:\\chrome.exe", "cmd": "chrome"},
]
def test_process_table_lines_are_parsed_with_tabs_in_command_lines():
rows = gpucap.parse_process_table("20\t10\tC:\\e\\strata.exe\tstrata.exe --serve\ta\r\n\nbad line\n4\t\t\t\n")
assert rows[0] == {"pid": 20, "ppid": 10, "exe": "C:\\e\\strata.exe", "cmd": "strata.exe --serve\ta"}
assert rows[1]["pid"] == 4 and rows[1]["ppid"] is None and len(rows) == 2
def test_cap_pauses_the_serving_engine_and_the_stt_worker_with_their_children(tmp_path):
procs = _procs(tmp_path / "strata")
roots = gpucap.engine_roots(procs, tmp_path / "strata")
assert roots == [20] # not the terminal that started it, nor another kit's engine
assert sorted(gpucap.with_children(procs, roots + [STT_LAUNCHER_PID])) == \
[20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1]
def test_duty_cap_pauses_for_the_capped_share_and_leaves_nothing_paused():
import time as _time
held, log = set(), []
def suspend(pid):
held.add(pid)
log.append(("s", pid))
return True
def resume(pid):
held.discard(pid)
log.append(("r", pid))
return True
with gpucap.DutyCap([1, 2], 0.9, period=0.05, suspend=suspend, resume=resume) as d:
_time.sleep(0.6)
s = d.summary()
assert not held
assert s["cycles"] >= 5 and s["failed_processes"] == 0
assert 0.05 <= s["paused_share"] <= 0.2 # about 10 % of the time
# Both processes are paused together: each pause holds both before either is resumed.
for i in range(0, len(log), 4):
assert sorted(log[i:i + 4]) == [("r", 1), ("r", 2), ("s", 1), ("s", 2)]
assert {k for k, _ in log[i:i + 2]} == {"s"}
def test_no_cap_pauses_nothing():
calls = []
with gpucap.DutyCap([1], 1.0, period=0.01, suspend=calls.append, resume=calls.append) as d:
import time as _time
_time.sleep(0.05)
assert calls == [] and d.summary()["cycles"] == 0
def test_util_sampler_counts_samples_over_the_cap():
vals = iter([99, 90, 97, 80])
with gpucap.UtilSampler(interval=0.01, read=lambda: next(vals, None)) as u:
import time as _time
_time.sleep(0.1)
s = u.summary(0.95)
assert s["pct"]["n"] == 4 and s["over_cap_share"] == 0.5
def _tune_with_cap(tmp_path, monkeypatch, fake_server, procs):
paused = set()
r, _, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune",
extra_opts=["stt_python=py"], stt=_FakeStt(), procs=procs, paused=paused)
return r, running["held"], paused
def test_capped_rows_pause_jarvis_and_record_what_the_gpu_got(tmp_path, monkeypatch, fake_server):
home = tmp_path / "strata"
r, held, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, _procs(home))
assert r["status"] == "ok"
assert not held # nothing is left paused
assert paused == {20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1}
rows = {x["key"]: x for x in r["levels"]}
for lv in llm.TUNED:
row = rows[lv["key"]]
assert row["gpu_util"]["pct"]["n"] > 0 # every row records the GPU's use
if lv.get("gpu_cap"):
assert row["gpu_cap"]["cap"] == llm.GPU_CAP and row["gpu_cap"]["cycles"] > 0
assert row["gpu_util"]["over_cap_share"] == 1.0 # the fake GPU stays at 99 %
else:
assert "gpu_cap" not in row
def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp_path, monkeypatch,
fake_server):
r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, [])
assert r["status"] == "partial" and not paused
assert any("no engine process" in e["error"] for e in r["errors"])
class _CountingProcs:
"""NtSuspendProcess/NtResumeProcess on fake processes: each suspend adds one to the process's
suspend count and each successful resume takes one off; ``fail`` says when a resume fails."""
def __init__(self, fail):
self.count: dict[int, int] = {}
self.fail = fail
self.resumes = 0
def suspend(self, pid):
self.count[pid] = self.count.get(pid, 0) + 1
return True
def resume(self, pid):
self.resumes += 1
if self.fail(pid, self.resumes):
return False
self.count[pid] -= 1
return True
def test_a_resume_that_fails_once_is_retried_and_nothing_is_left_paused():
import time as _time
procs = _CountingProcs(fail=lambda pid, n: pid == 1 and n == 1)
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
_time.sleep(0.2)
s = d.summary()
assert all(v == 0 for v in procs.count.values())
assert s["failed_processes"] == 1 and s["left_paused"] == 0
def test_a_process_that_cannot_be_resumed_is_reported_as_left_paused():
import time as _time
procs = _CountingProcs(fail=lambda pid, n: pid == 1)
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
_time.sleep(0.2)
assert procs.count[1] == 1 and procs.count[2] == 0 # paused once, never stacked
assert d.summary()["left_paused"] == 1
def test_a_suspend_or_resume_that_raises_does_not_end_the_cap_with_processes_paused():
import time as _time
procs = _CountingProcs(fail=lambda pid, n: False)
def resume(pid):
if procs.resumes == 0:
procs.resumes += 1
raise OSError("access denied")
return procs.resume(pid)
with gpucap.DutyCap([1], 0.9, period=0.02, suspend=procs.suspend, resume=resume) as d:
_time.sleep(0.2)
assert procs.count[1] == 0 and d.summary()["left_paused"] == 0
# --- video_drops: a YouTube video's dropped frames next to Jarvis, with and without the cap --
def _server_frame(payload: bytes, opcode: int = 1, fin: bool = True) -> bytes:
"""A frame as Chrome sends it (servers do not mask)."""
import struct
n = len(payload)
head = bytes([(0x80 if fin else 0) | opcode])
if n < 126:
head += bytes([n])
elif n < 1 << 16:
head += bytes([126]) + struct.pack(">H", n)
else:
head += bytes([127]) + struct.pack(">Q", n)
return head + payload
def _read_client_frame(sock) -> tuple[int, bytes]:
"""Unmask one frame the client sent (clients must mask)."""
import struct
b0, b1 = sock.recv(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", sock.recv(2))[0]
elif n == 127:
n = struct.unpack(">Q", sock.recv(8))[0]
assert b1 & 0x80, "a client frame must be masked"
mask = sock.recv(4)
data = b""
while len(data) < n:
data += sock.recv(n - len(data))
return b0 & 0x0F, bytes(b ^ mask[i % 4] for i, b in enumerate(data))
def test_devtools_call_returns_its_own_answer_across_events_pings_and_fragments():
import socket
from m0lib import video
ours, chrome = socket.socketpair()
cdp = video.Cdp("", sock=ours)
big = json.dumps({"id": 1, "result": {"result": {"value": "x" * 70000}}}).encode()
chrome.sendall(_server_frame(json.dumps({"method": "Page.frameNavigated"}).encode()) # an event
+ _server_frame(b"are you there", opcode=9) # a ping
+ _server_frame(big[:300], fin=False) + _server_frame(big[300:], opcode=0))
got = {}
t = threading.Thread(target=lambda: got.update(v=cdp.evaluate("1")))
t.start()
op, sent = _read_client_frame(chrome)
assert op == 1 and json.loads(sent)["method"] == "Runtime.evaluate"
op, pong = _read_client_frame(chrome)
assert (op, pong) == (10, b"are you there")
t.join(timeout=10)
assert got["v"] == "x" * 70000
cdp.close()
chrome.close()
def test_devtools_errors_and_a_closed_socket_are_raised_not_hidden():
import socket
from m0lib import video
ours, chrome = socket.socketpair()
cdp = video.Cdp("", sock=ours)
chrome.sendall(_server_frame(json.dumps({"id": 1, "error": {"message": "no such method"}}).encode()))
with pytest.raises(RuntimeError, match="no such method"):
cdp.call("Nope.nothing")
chrome.sendall(_server_frame(json.dumps({"id": 2, "result": {
"exceptionDetails": {"text": "Uncaught", "exception": {"description": "TypeError: m is undefined"}}}}).encode()))
with pytest.raises(RuntimeError, match="m is undefined"):
cdp.evaluate("m.v")
chrome.sendall(_server_frame(b"", opcode=8))
with pytest.raises(ConnectionError):
cdp.call("Runtime.evaluate")
cdp.close()
chrome.close()
def _snap(total, dropped, **kw):
return {"total": total, "dropped": dropped, "shown": kw.pop("shown", total), "gaps": kw.pop("gaps", [0, 0]),
"max_gap_ms": kw.pop("max_gap_ms", 17.0), "waits": 0, "pauses": 0, "seeks": 0,
"quality": kw.pop("quality", "hd1080"), "height": kw.pop("height", 1080), **kw}
def test_a_window_counts_the_frames_the_video_dropped_meanwhile():
from m0lib import video
w = video.window_stats(_snap(1000, 5), _snap(4600, 23, gaps=[4, 1], max_gap_ms=140.0), 60.0)
assert w["valid"] and w["frames"] == 3600 and w["dropped"] == 18
assert w["fps"] == 60.0 and w["dropped_per_min"] == 18.0 and w["dropped_share"] == 0.005
assert w["gaps_over_ms"] == {str(video.GAP_MS[0]): 4, str(video.GAP_MS[1]): 1} and w["max_gap_ms"] == 140.0
@pytest.mark.parametrize("after", [_snap(4600, 9, ad=True), _snap(4600, 9, paused=True), _snap(4600, 9, ended=True),
_snap(300, 0), _snap(1000, 5),
# paused, jumped or buffered for a while and playing again at the end
_snap(4600, 9, pauses=1), _snap(4600, 9, seeks=1), _snap(4600, 9, waits=2)])
def test_a_window_with_an_ad_a_pause_the_end_a_new_video_or_no_frames_is_kept_but_not_counted(after):
from m0lib import video
w = video.window_stats(_snap(1000, 5), after, 60.0)
assert not w["valid"] and w["invalid_because"]
def test_the_comparison_adds_up_only_valid_busy_windows_on_each_side_of_the_cap():
from m0lib import video
def row(jarvis, cap, frames, dropped, valid=True):
return {"jarvis": jarvis, "gpu_cap": {"cap": 0.95} if cap else None,
"video": {"valid": valid, "frames": frames, "dropped": dropped, "seconds": 60.0,
"gaps_over_ms": {str(g): dropped for g in video.GAP_MS}}}
rows = [row("off", False, 3600, 0), row("busy", False, 3600, 36), row("busy", True, 3600, 6),
row("busy", True, 3600, 1000, valid=False), row("busy", False, 3600, 18)]
c = video.compare(rows)
assert c["no_cap"]["windows"] == 2 and c["no_cap"]["dropped"] == 54 and c["no_cap"]["dropped_per_min"] == 27.0
assert c["cap95"]["windows"] == 1 and c["cap95"]["dropped"] == 6 and c["cap95"]["dropped_share"] == round(6 / 3600, 4)
class _FakePage:
"""The watch page: 60 frames shown per snapshot step, a few dropped; records what the stage did.
``breaks_at`` lists the snapshots (counted from 1) at which the page was loaded again meanwhile;
the player then stays broken until ``keep_room`` loads it again."""
def __init__(self, starts=True, quality="hd1080", height=1080, breaks_at=()):
self.starts, self.quality, self.height = starts, quality, height
self.total = self.dropped = 0
self.events: list[str] = []
self.closed = False
self.breaks_at, self.snaps, self.broken, self.reloads = set(breaks_at), 0, False, 0
def start(self, quality="hd1080", timeout=180):
self.events.append("video_start")
return {"started": self.starts, "quality": self.quality if self.starts else None,
"height": self.height if self.starts else None, "ad_seen": False}
def keep_room(self):
if not self.broken:
return []
self.broken, self.total, self.dropped = False, 0, 0
self.reloads += 1
return ["loaded the page again"]
def reset(self):
pass
def snapshot(self):
self.snaps += 1
if self.snaps in self.breaks_at:
self.broken = True
if self.broken:
return {"lost": True}
self.total += 3600
self.dropped += 2
return _snap(self.total, self.dropped, quality=self.quality, height=self.height)
def close(self):
self.closed = True
def _run_video_stage(tmp_path, monkeypatch, fake_server, page, stt=None, **kw):
from m0lib import video
monkeypatch.setattr(video, "QUIET_S", 0.01)
monkeypatch.setattr(video, "open_video_page", lambda work, *a, **k: page)
stt = _FakeStt() if stt is None else stt
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="video",
extra_opts=["stt_python=py"], stt=stt, **kw)
return r, started, running, stt
def test_video_stage_compares_the_video_alone_jarvis_idle_and_busy_with_and_without_the_cap(
tmp_path, monkeypatch, fake_server):
from m0lib import video
page = _FakePage()
r, started, running, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
assert r["stage"] == "video_drops" and r["status"] == "ok"
rows = r["video_rows"]
# Alone first and last; idle before busy; the busy rows alternate A B B A around the cap.
assert [x["jarvis"] for x in rows] == ["off", "idle"] + ["busy"] * len(video.ROUNDS) + ["off"]
busy = [x for x in rows if x["jarvis"] == "busy"]
capped = [bool(x["gpu_cap"]) for x in busy]
assert capped == [False, True, True, False]
assert all(x["video"]["valid"] and x["video"]["frames"] > 0 for x in rows)
for x in busy:
assert x["llm"]["decode_tok_s"]["n"] > 0 # the LLM answered during the window
if x["gpu_cap"]:
assert x["gpu_cap"]["cap"] == llm.GPU_CAP and x["gpu_cap"]["cycles"] > 0
assert x["gpu_cap"]["left_paused"] == 0
# Jarvis is the normal placement: the STT room probe, then one start with the experts it leaves.
assert len(started) == 2 and _expert_vram(started[0]) == 0 and _expert_vram(started[1]) > 0
assert r["placement"]["sets"] == started[1]["sets"]
# The STT is loaded before the LLM and transcribes only once Jarvis is busy.
assert stt.events[:4] == ["stt_loaded", "llm_start", "llm_start", "stt_go"]
assert set(r["comparison"]) == {"no_cap", "cap95"} and r["comparison"]["cap95"]["windows"] == 2
# Nothing is left behind: the browser closed, the model stopped, the STT gone, nothing paused.
assert page.closed and running["now"] is None and stt.loaded is None and not running["held"]
def test_the_cap_in_the_video_stage_pauses_jarvis_and_never_the_browser(tmp_path, monkeypatch, fake_server):
paused = set()
from m0lib import video
page = _FakePage()
monkeypatch.setattr(video, "QUIET_S", 0.01)
monkeypatch.setattr(video, "open_video_page", lambda work, *a, **k: page)
_run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="video", extra_opts=["stt_python=py"],
stt=_FakeStt(), paused=paused)
browser = next(p["pid"] for p in _procs(tmp_path / "strata") if "chrome" in p["exe"])
assert paused == {20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1} and browser not in paused
def test_video_stage_without_a_browser_or_a_playing_video_starts_no_jarvis(tmp_path, monkeypatch, fake_server):
r, started, _, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, None)
assert r["status"] == "failed" and not started and not stt.events
page = _FakePage(starts=False)
r, started, _, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
assert r["status"] == "failed" and not started and not stt.events and page.closed
def test_video_stage_is_on_demand_with_the_stt_python_and_a_one_click_runner():
import m0
assert "video_drops" in m0.ON_DEMAND and "video_drops" in m0.NEEDS_STT
bat = (Path(m0.KIT) / "run_m0_video.bat").read_text(encoding="utf-8")
assert "-Only video_drops" in bat and "-Redo" in bat
@pytest.mark.parametrize("before,after,why", [
(_snap(1000, 5, quality="medium", height=360), _snap(4600, 9, quality="medium", height=360), "quality"),
(_snap(1000, 5), _snap(4600, 9, min_height=720, max_height=1080), "lines high"), # dipped to 720p meanwhile
(_snap(1000, 5), _snap(2800, 9), "fps"), # a 30 fps format
])
def test_a_window_that_played_another_quality_or_frame_rate_is_not_counted(before, after, why):
from m0lib import video
w = video.window_stats(before, after, 60.0, video.WANT)
assert not w["valid"] and any(why in x for x in w["invalid_because"])
assert video.window_stats(_snap(1000, 5), _snap(4600, 9, min_height=1080, max_height=1080), 60.0,
video.WANT)["valid"]
def test_a_video_stuck_below_the_asked_quality_counts_no_window(tmp_path, monkeypatch, fake_server):
r, _, _, _ = _run_video_stage(tmp_path, monkeypatch, fake_server, _FakePage(quality="medium", height=360))
assert r["status"] == "partial" and any("medium" in e["error"] for e in r["errors"])
assert not any(x["video"]["valid"] for x in r["video_rows"])
# A lower quality may come from the GPU itself, so such a window is not measured again.
assert not any(x["video"].get("earlier_attempts") for x in r["video_rows"])
assert r["comparison"]["no_cap"]["windows"] == 0 and r["comparison"]["cap95"]["windows"] == 0
def test_a_cap_that_could_not_pause_jarvis_does_not_count_as_a_capped_window(tmp_path, monkeypatch, fake_server):
r, _, running, _ = _run_video_stage(tmp_path, monkeypatch, fake_server, _FakePage(), suspend_ok=False)
capped = [x for x in r["video_rows"] if x["gpu_cap"]]
assert capped and all(not x["video"]["valid"] for x in capped)
assert r["comparison"]["cap95"]["windows"] == 0 and r["comparison"]["no_cap"]["windows"] == 2
assert r["status"] == "partial" and any("could not be paused" in e["error"] for e in r["errors"])
assert not running["held"]
@pytest.mark.parametrize("before,after,retry", [
(_snap(1000, 5), {"lost": True}, True), # the page was loaded again
(_snap(1000, 5), _snap(4600, 9, error=True, error_text="x"), True), # the player's error screen
(_snap(1000, 5), _snap(640, 0, paused=True, height=0), True), # what the gaming PC reported
(_snap(1000, 5), _snap(4600, 9, ad=True), True),
(_snap(1000, 5), _snap(4600, 9, waits=2), True),
(_snap(1000, 5, quality="medium", height=360), _snap(4600, 9, quality="medium", height=360), False),
(_snap(1000, 5), _snap(2800, 9), False), # the frame rate fell
])
def test_only_a_window_the_player_spoilt_is_worth_measuring_again(before, after, retry):
from m0lib import video
w = video.window_stats(before, after, 60.0, video.WANT)
assert not w["valid"] and w["retry"] is retry
def test_a_reloaded_page_is_measured_again_and_the_stage_goes_on(tmp_path, monkeypatch, fake_server):
from m0lib import video
# Two snapshots a window, one more pair for the window measured again: the video alone, then the first busy one.
page = _FakePage(breaks_at={2, 8})
r, _, running, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
assert r["status"] == "ok" and page.reloads == 2
rows = {x["key"]: x for x in r["video_rows"]}
assert all(x["video"]["valid"] for x in rows.values())
for key in ("video_only", "busy"):
assert len(rows[key]["video"]["earlier_attempts"]) == 1
assert rows[key]["video"]["player_fixes"] == ["loaded the page again"]
assert any(key in n and "measured again" in n for n in r["notes"])
assert rows["busy"]["llm"]["decode_tok_s"]["n"] > 0
assert r["comparison"]["no_cap"]["windows"] == 2 and r["comparison"]["cap95"]["windows"] == 2
assert page.closed and running["now"] is None and stt.loaded is None and not running["held"]
def test_a_player_that_keeps_breaking_is_tried_a_few_times_per_window_and_never_stops_the_stage(
tmp_path, monkeypatch, fake_server):
from m0lib import video
page = _FakePage(breaks_at=set(range(2, 400, 2))) # every window's end
r, _, running, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
rows = r["video_rows"]
assert [x["key"] for x in rows][-1] == "video_only_again"
assert all(not x["video"]["valid"] and len(x["video"]["earlier_attempts"]) == video.ATTEMPTS - 1 for x in rows)
assert r["comparison"]["no_cap"]["windows"] == 0 and r["comparison"]["cap95"]["windows"] == 0
assert page.closed and running["now"] is None and stt.loaded is None and not running["held"]
class _ScriptedCdp:
"""The DevTools side of a watch page whose player is stopped or broken until it is loaded again."""
def __init__(self, first: dict, plays_again: bool):
from m0lib import video
self.v = video
self.first, self.plays_again = first, plays_again
self.navigated: list[str] = []
self.pressed = 0
def _ok(self) -> bool:
return bool(self.navigated) or (self.plays_again and self.pressed > 0)
def evaluate(self, expr):
if expr == self.v._SNAPSHOT:
if not self._ok():
return dict(self.first)
return {**_snap(1000, 0), "paused": False, "ad_now": False, "time": 30.0, "duration": 634.0}
if expr == self.v._STEP:
self.pressed += 1
return {"state": "playing" if self._ok() else "starting", "levels": ["hd1080"]}
return True
def call(self, method, params=None):
if method == "Page.navigate":
self.navigated.append(params["url"])
return {}
@pytest.mark.parametrize("first,plays_again,reloaded", [
({"lost": True}, False, True), # loaded again by hand: counters gone
({**_snap(640, 0), "error": True, "error_text": "문제가 발생했습니다. (재생 ID: Ab12Cd)"}, False, True),
({**_snap(640, 0), "paused": True, "height": 0}, True, False), # play is enough
({**_snap(640, 0), "paused": True, "height": 0}, False, True), # play is not
])
def test_before_a_window_a_stopped_or_broken_player_is_started_again(monkeypatch, first, plays_again, reloaded):
from m0lib import video
monkeypatch.setattr(video, "SETTLE_S", 0)
monkeypatch.setattr(video, "REPLAY_S", 0.01)
monkeypatch.setattr(video.time, "sleep", lambda s: None)
cdp = _ScriptedCdp(first, plays_again)
page = video.VideoPage(cdp, url="https://example.invalid/watch")
did = page.keep_room()
assert cdp.navigated == (["https://example.invalid/watch"] if reloaded else [])
assert ("loaded the page again" in did) is reloaded and did
assert not any("Ab12Cd" in d for d in did) # YouTube's playback ID is not kept
assert not page.snapshot().get("lost")
def test_the_video_browser_does_not_tell_pages_a_program_drives_it(tmp_path, monkeypatch):
"""With DevTools on, Chrome sets navigator.webdriver, and YouTube then stops the video about a
minute in with its error screen (seen on the gaming PC and reproduced on Linux)."""
from m0lib import video
if not video.find_browser():
pytest.skip("no Chrome or Edge on this machine")
class Page(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-Type", "text/html")
self.end_headers()
self.wfile.write(b"<title>watch</title>")
def log_message(self, *a):
pass
srv = ThreadingHTTPServer(("127.0.0.1", 0), Page)
threading.Thread(target=srv.serve_forever, daemon=True).start()
# The page is found by "youtube" in its address; headless keeps the test off the screen.
url = f"http://127.0.0.1:{srv.server_port}/youtube"
page = None
try:
page = video.open_video_page(tmp_path, url=url, extra_args=("--headless=new",))
assert page.cdp.evaluate("navigator.webdriver") is False
page.close()
page = None
# A browser that still says so (one that ignores the switch) is refused at once and closed,
# before Jarvis is started for nothing.
class Says(video.Cdp):
def evaluate(self, expression):
return True if expression == "navigator.webdriver" else super().evaluate(expression)
monkeypatch.setattr(video, "Cdp", Says)
with pytest.raises(RuntimeError, match="webdriver"):
video.open_video_page(tmp_path, url=url, extra_args=("--headless=new",))
assert not (tmp_path / "cache" / "video" / "browser-profile").exists()
finally:
if page:
page.close()
srv.shutdown()
def test_a_page_script_that_fails_while_the_page_loads_reads_as_a_lost_page():
from m0lib import video
class Broken:
def evaluate(self, expr):
raise RuntimeError("page script failed: Execution context was destroyed.")
assert video.VideoPage(Broken()).snapshot()["lost"] is True
class _DyingCdp:
"""A watch page that plays, then loses its player for good (loading it again does not help);
like Chrome, a page script that reaches for the missing player throws."""
def __init__(self, die_at: int):
from m0lib import video
self.v, self.die_at, self.snaps, self.total, self.navigated = video, die_at, 0, 0, 0
def evaluate(self, expr):
dead = self.snaps >= self.die_at
if expr == self.v._SNAPSHOT:
self.snaps += 1
if dead:
return {"lost": True, "error": False, "error_text": None}
self.total += 3600
return {**_snap(self.total, 0), "paused": False, "ad_now": False, "time": 30.0, "duration": 634.0}
if expr == self.v._STEP:
return {"state": "loading"} if dead else {"state": "playing", "levels": ["hd1080"]}
if expr in (self.v._INSTALL, self.v._RESET):
return not dead
if dead and "movie_player" in expr:
raise RuntimeError("page script failed: TypeError: Cannot read properties of null (reading 'seekTo')")
return True
def call(self, method, params=None):
self.navigated += method == "Page.navigate"
return {}
def test_a_player_that_never_comes_back_is_tried_again_per_window_and_the_stage_goes_on(
tmp_path, monkeypatch, fake_server):
from m0lib import video
for name, value in (("SETTLE_S", 0), ("REPLAY_S", 0.01), ("RELOAD_S", 0.01)):
monkeypatch.setattr(video, name, value)
cdp = _DyingCdp(die_at=4) # two snapshots while starting, then it dies during the first window
page = video.VideoPage(cdp)
closed = []
monkeypatch.setattr(page, "close", lambda: closed.append(True))
r, _, running, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
rows = r["video_rows"]
assert [x["key"] for x in rows] == ["video_only", "jarvis_idle", *video.ROUNDS, "video_only_again"]
assert all(not x["video"]["valid"] for x in rows)
# Every window was tried the full number of times, each time loading the page again.
assert all(len(x["video"]["earlier_attempts"]) == video.ATTEMPTS - 1 for x in rows)
assert all("the video did not play again" in x["video"]["player_fixes"] for x in rows)
assert cdp.navigated >= len(rows) * (video.ATTEMPTS - 1)
assert not any("seekTo" in e["error"] for e in r["errors"])
assert closed and running["now"] is None and stt.loaded is None and not running["held"]
# --- game_frames: the user's game's frame times next to Jarvis, plain and yielding by priority --
_PM_HEAD = ("Application,ProcessID,SwapChainAddress,Runtime,SyncInterval,PresentFlags,Dropped,TimeInSeconds,"
"msInPresentAPI,msBetweenPresents\n")
def _pm_csv(frames_ms, chain="0x1", pid=7000, dropped=(), head=_PM_HEAD):
t, lines = 0.0, []
for i, ms in enumerate(frames_ms):
t += ms / 1000
lines.append(f"game.exe,{pid},{chain},DXGI,0,0,{1 if i in dropped else 0},{t:.4f},0.1,{ms}")
return head + "\n".join(lines) + "\n"
def test_frame_stats_read_the_main_swap_chain_with_rate_one_percent_low_and_hitches():
from m0lib import frames
main = [10.0] * 200 + [60.0, 120.0]
text = _pm_csv(main, dropped={5}) + _pm_csv([500.0] * 5, chain="0x2", head="")
g = frames.frame_stats(frames.parse_csv(text), 3.0)
counted = main[1:] # the first row's interval reaches back before the capture
assert g["valid"] and g["frames"] == len(counted)
assert g["fps"] == round(len(counted) / (sum(counted) / 1000), 1)
p99 = sorted(counted)[int(len(counted) * 0.99)]
assert g["low1_fps"] == round(1000 / p99, 1) and g["max_ms"] == 120.0
assert g["hitches_over_ms"] == {"50": 2, "100": 1} and g["not_shown"] == 1
def test_frame_stats_read_presentmon_2_column_names_too():
from m0lib import frames
text = _pm_csv([16.7] * 100).replace("msBetweenPresents", "MsBetweenPresents")
assert frames.frame_stats(frames.parse_csv(text), 1.5)["fps"] == round(1000 / 16.7, 1)
def test_a_window_where_the_game_drew_nothing_or_almost_nothing_is_not_counted():
from m0lib import frames
assert not frames.frame_stats([], 60)["valid"]
few = frames.frame_stats(frames.parse_csv(_pm_csv([1000.0] * 30)), 60)
assert not few["valid"] and "frames" in few["invalid_because"][0]
def test_compare_pools_the_valid_windows_against_the_game_alone():
from m0lib import frames
def row(kind, fps, low, valid=True):
return {"kind": kind, "game": {"valid": valid, "fps": fps, "low1_fps": low,
"hitches_over_ms": {str(h): 1 for h in frames.HITCH_MS}}}
c = frames.compare([row("off", 100, 80), row("idle", 99, 79), row("busy", 90, 60), row("busy_yield", 98, 76),
row("busy_yield", 10, 5, valid=False), row("busy", 90, 60), row("off", 100, 80)])
assert c["off"]["windows"] == 2 and c["busy_yield"]["windows"] == 1
assert c["busy"]["vs_off_pct"] == {"fps": -10.0, "low1_fps": -25.0}
assert c["busy_yield"]["vs_off_pct"] == {"fps": -2.0, "low1_fps": -5.0}
class _FakePriorities:
"""Process priorities as Windows keeps them; ``ignored`` pids accept nothing."""
def __init__(self, ignored=()):
self.now: dict = {}
self.ignored = set(ignored)
self.touched: set = set()
def get(self, pid):
return self.now.get(pid, (0x20, 2))
def set(self, pid, cpu, gpu):
self.touched.add(pid)
if pid in self.ignored:
return False, False
self.now[pid] = (cpu, gpu)
return True, True
def test_yield_lowers_cpu_and_gpu_priority_and_puts_back_what_each_process_had(monkeypatch):
from m0lib import frames
pr = _FakePriorities(ignored={3})
pr.now[2] = (0x8000, 2) # already above normal: it gets that back, not plain normal
monkeypatch.setattr(frames, "get_priority", pr.get)
monkeypatch.setattr(frames, "set_priority", pr.set)
with frames.Yield([1, 2, 3]) as y:
assert pr.now[1] == pr.now[2] == (frames.CPU_BELOW_NORMAL, frames.GPU_BELOW_NORMAL)
assert pr.now[1] == (0x20, 2) and pr.now[2] == (0x8000, 2)
s = y.summary()
assert s == {"processes": 3, "cpu_lowered": 2, "gpu_lowered": 2, "restored": 2, "not_restored": 1}
assert frames.yield_problems(s) == ["1 process(es) kept the lowered priority"]
assert frames.yield_problems(frames.Yield([]).summary()) == ["yield found no Jarvis process"]
def test_presentmon_records_only_the_game_for_a_fixed_time_and_a_bad_download_is_refused(tmp_path, monkeypatch):
from m0lib import frames
cmd = frames.presentmon_cmd(Path("pm.exe"), 7000, 60, Path("f.csv"))
assert cmd[cmd.index("--process_id") + 1] == "7000" and cmd[cmd.index("--timed") + 1] == "60"
assert "--terminate_after_timed" in cmd and "--output_file" in cmd
class _Resp:
def __enter__(self):
return self
def __exit__(self, *a):
return None
def read(self):
return b"not presentmon"
monkeypatch.setattr(frames.urllib.request, "urlopen", lambda url, timeout=0: _Resp())
with pytest.raises(RuntimeError, match="hash"):
frames.ensure_presentmon(tmp_path)
assert not list((tmp_path / "tools").iterdir())
GAME_PID = 7000
def _run_frames_stage(tmp_path, monkeypatch, fake_server, admin=True, front=(GAME_PID, "game.exe"), pr=None):
from m0lib import frames
pr = _FakePriorities() if pr is None else pr
captured = []
monkeypatch.setattr(frames, "is_admin", lambda: admin)
monkeypatch.setattr(frames, "ensure_presentmon", lambda work: Path("pm.exe"))
monkeypatch.setattr(frames, "ask_for_a_game", lambda: True)
monkeypatch.setattr(frames, "foreground_process", lambda: front)
monkeypatch.setattr(frames, "get_priority", pr.get)
monkeypatch.setattr(frames, "set_priority", pr.set)
monkeypatch.setattr(frames, "SETTLE_S", 0)
def capture(exe, pid, seconds, out_csv, log):
captured.append((pid, dict(pr.now)))
return frames.parse_csv(_pm_csv([16.0] * 4000, pid=pid)), None
monkeypatch.setattr(frames, "capture", capture)
stt = _FakeStt()
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="frames",
extra_opts=["stt_python=py"], stt=stt)
return r, started, running, stt, pr, captured
def test_frames_stage_measures_the_game_alone_next_to_jarvis_idle_busy_and_yielding(
tmp_path, monkeypatch, fake_server):
from m0lib import frames
r, started, running, stt, pr, captured = _run_frames_stage(tmp_path, monkeypatch, fake_server)
assert r["stage"] == "game_frames" and r["status"] == "ok"
rows = r["frame_rows"]
# Alone first and last; idle before busy; plain and yielding rows alternate A B B A.
assert [x["kind"] for x in rows] == ["off", "idle", "busy", "busy_yield", "busy_yield", "busy", "off"]
assert all(x["game"]["valid"] for x in rows) and {pid for pid, _ in captured} == {GAME_PID}
for x in rows:
if x["kind"].startswith("busy"):
assert x["llm"]["decode_tok_s"]["n"] > 0 # the LLM answered during the window
# Jarvis runs the app's game placement, the STT doing finals only (no partials while it answers).
assert len(started) == 1 and started[0]["sets"] == frames.GAME_LEVEL["sets"]
assert stt.paces == [llm._FINALS_ONLY] and stt.events[:3] == ["stt_loaded", "llm_start", "stt_go"]
# Yielding lowers Jarvis's processes during those windows only, never the game, and puts them back.
jarvis = {20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1}
low = (frames.CPU_BELOW_NORMAL, frames.GPU_BELOW_NORMAL)
for (pid, prio), x in zip(captured, rows):
lowered = {p for p, v in prio.items() if v == low}
assert lowered == (jarvis if x["kind"] == "busy_yield" else set())
assert GAME_PID not in pr.touched and all(v == (0x20, 2) for v in pr.now.values())
assert set(r["comparison"]) == {"off", "busy", "busy_yield"} and r["comparison"]["busy_yield"]["windows"] == 2
assert running["now"] is None and stt.loaded is None
def test_frames_stage_needs_administrator_rights_and_the_game_in_front_before_starting_jarvis(
tmp_path, monkeypatch, fake_server):
r, started, _, stt, _, captured = _run_frames_stage(tmp_path, monkeypatch, fake_server, admin=False)
assert r["status"] == "failed" and not started and not stt.events and not captured
assert any("관리자" in e["error"] for e in r["errors"])
r, started, _, stt, _, captured = _run_frames_stage(tmp_path, monkeypatch, fake_server,
front=(1234, "windowsterminal.exe"))
assert r["status"] == "failed" and not started and not stt.events and not captured
def test_a_yield_that_lowered_nothing_does_not_count_as_a_yielding_window(tmp_path, monkeypatch, fake_server):
pr = _FakePriorities(ignored={20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1})
r, *_ = _run_frames_stage(tmp_path, monkeypatch, fake_server, pr=pr)
assert r["comparison"]["busy_yield"]["windows"] == 0 and r["comparison"]["busy"]["windows"] == 2
assert r["status"] == "partial" and any("lowered no priority" in e["error"] for e in r["errors"])
def test_frames_stage_is_on_demand_with_the_stt_python_and_an_elevating_one_click_runner():
import m0
assert "game_frames" in m0.ON_DEMAND and "game_frames" in m0.NEEDS_STT
bat = (Path(m0.KIT) / "run_m0_game_frames.bat").read_text(encoding="utf-8")
assert "-Only game_frames" in bat and "-Verb RunAs" in bat