User decision (2026-11-11): the 95 % GPU cap is off by default because gamemode_tune showed it did not lower nvidia-smi's utilisation and cost 6 to 10 % of the LLM's output. The worry behind it was a video stuttering next to Jarvis, so the new on-demand stage video_drops measures that: - A public Blender upload plays at 1080p60 in a throwaway Chrome/Edge profile (removed afterwards), driven over DevTools with a small standard-library WebSocket client, so no package is added to the already built venvs. - Each window counts the video's dropped frames (the player's own counter) and the gaps over 50/100 ms between shown frames (requestVideoFrameCallback); windows with an ad, a pause, the end or a new video are kept but not counted. - Rows: video alone, Jarvis idle (normal placement with the STT room and 1 GB for other programs), Jarvis busy (LLM answering, STT transcribing ten speakers back to back) without and with the cap in A B B A order, then the video alone again. The cap pauses the engine and the STT worker only, never the browser. - run_m0_video.bat runs just this stage. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1414 lines
66 KiB
Python
1414 lines
66 KiB
Python
"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
|
|
|
|
The kit runs once on the gaming PC and its results leave the PC, so the
|
|
privacy scrub and the timing client must behave before it is handed over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import common, gpucap, llm, llm_bench # noqa: E402
|
|
from m0lib.llm_client import ChatClient # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
|
|
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
|
|
out = common.scrub(text)
|
|
assert "minsu" not in out.lower()
|
|
assert "<HOME>" in out and "<USER>" in out
|
|
|
|
|
|
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["ci"])
|
|
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
|
|
|
|
|
|
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
|
|
target = tmp_path / "r.json"
|
|
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
|
|
|
|
|
|
def test_cer_ignores_spacing_and_punctuation():
|
|
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
|
|
assert common.cer("가나다라", "가나다마") == 0.25
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
"""Streams a tool call split over chunks, then text, then usage."""
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
request_sizes: list[int] = []
|
|
|
|
def do_POST(self):
|
|
n = int(self.headers.get("Content-Length", 0))
|
|
self.rfile.read(n)
|
|
_FakeOpenAI.request_sizes.append(n)
|
|
chunks = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
|
|
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
|
|
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
|
|
]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.end_headers()
|
|
for c in chunks:
|
|
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
|
|
self.wfile.flush()
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
|
c = ChatClient(fake_server)
|
|
assert c.model == "fake-model"
|
|
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
|
|
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
|
|
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
|
|
assert a["finish"] == "tool_calls"
|
|
assert a["usage"]["completion_tokens"] == 9
|
|
# The first sentence ends at "요. ", before the whole answer is done.
|
|
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|
|
|
|
|
|
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
|
|
# The fake server always reports 12 prompt tokens. The context test must not
|
|
# trust that blindly and grow each next prompt without limit (it used to
|
|
# reach gigabytes and exhaust RAM).
|
|
_FakeOpenAI.request_sizes.clear()
|
|
largest_target = 24576
|
|
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
|
|
assert _FakeOpenAI.request_sizes
|
|
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
|
|
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
|
|
|
|
|
|
# --- LLM stage: a failed start must not count as measured -------------------
|
|
|
|
|
|
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
|
|
out = tmp_path / "results" / "llm_fast.json"
|
|
out.parent.mkdir(parents=True)
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
|
|
# A failed stage is retried on the next run; "partial" would be skipped.
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
|
|
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
|
|
|
|
|
|
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
|
|
launched = []
|
|
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
|
|
r = _run_llm_stage(tmp_path, monkeypatch)
|
|
assert r["status"] == "failed"
|
|
assert launched == []
|
|
assert "server" not in r
|
|
|
|
|
|
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
|
|
|
|
|
|
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
|
|
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
|
|
# so the engine's own log must reach the results zip, which packs logs/*.log.
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
|
|
logs = tmp_path / "logs"
|
|
logs.mkdir()
|
|
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
|
|
engine_dir.mkdir(parents=True, exist_ok=True)
|
|
home = str(Path.home())
|
|
(engine_dir / "q2_0-32k.engine.log").write_text(
|
|
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
|
|
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
|
|
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
|
|
assert len(kept) == 1
|
|
text = kept[0].read_text(encoding="utf-8")
|
|
assert "out of memory" in text
|
|
assert home not in text
|
|
|
|
|
|
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
|
|
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
|
|
# canonical Q2_0 pack is not a native pack, so the engine stops at start
|
|
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
|
|
for parallel in (1, 2):
|
|
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
|
|
for cmd in ran:
|
|
assert "--drafting" in cmd
|
|
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
|
|
|
|
|
|
def test_virtual_display_structures_match_the_win32_layout():
|
|
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
|
|
import ctypes
|
|
|
|
from m0lib import winvdd
|
|
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
|
|
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
|
|
if ctypes.sizeof(ctypes.c_void_p) == 8:
|
|
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
|
|
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
|
|
|
|
|
|
def test_virtual_display_goes_where_it_only_touches_a_corner():
|
|
from m0lib import winvdd
|
|
# Two side-by-side 1440p monitors, the second one lower than the first.
|
|
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
|
|
x, y = winvdd.corner_position(physical)
|
|
assert (x, y) == (5120, 1640)
|
|
for d in physical:
|
|
left, top, right, bottom = d["rect"]
|
|
# Only a point is shared: no edge of length > 0 with any real monitor.
|
|
assert not (left <= x < right and top <= y < bottom)
|
|
|
|
|
|
def test_virtual_display_remove_sends_the_index_the_driver_expects():
|
|
from m0lib import winvdd
|
|
sent = []
|
|
vdd = object.__new__(winvdd.Vdd)
|
|
vdd.added = [1]
|
|
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
|
|
vdd.remove(1)
|
|
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
|
|
assert vdd.added == []
|
|
|
|
|
|
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert keys.index("vdisplay") < keys.index("tts_melo")
|
|
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
|
|
|
|
|
|
def test_virtual_display_names_are_read_from_utf16_buffers():
|
|
from m0lib import winvdd
|
|
dd = winvdd.DISPLAY_DEVICEW()
|
|
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
|
|
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
|
|
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
|
|
|
|
class _ScriptedClient:
|
|
"""Answers chat calls from a script; records what was sent."""
|
|
|
|
model = "scripted"
|
|
|
|
def __init__(self, answers):
|
|
self.answers = list(answers)
|
|
self.sent = []
|
|
|
|
def chat(self, messages, **kwargs):
|
|
self.sent.append((messages, kwargs))
|
|
text, calls = self.answers.pop(0)
|
|
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_kit_uses_the_production_prompt_rules_and_planner():
|
|
from jarvis import system_prompt
|
|
from m0lib import tool_fix
|
|
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
|
|
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
|
|
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
|
|
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
|
|
# The real jarvis package stays importable next to the kit's private copy.
|
|
import jarvis.reply.planner as real_planner
|
|
assert real_planner.named_tool_steps is not planner.named_tool_steps
|
|
|
|
|
|
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("서울 날씨 어때?")
|
|
client = _ScriptedClient([
|
|
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("서울은 13도이고 비가 옵니다.", []), # answer from the result
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
|
|
assert row["final_ok"] is True
|
|
final_messages = client.sent[-1][0]
|
|
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
|
|
|
|
|
|
def test_model_that_calls_the_tool_is_scored_without_the_guard():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("5분 타이머 맞춰 줘.")
|
|
client = _ScriptedClient([
|
|
("set_timer minutes='5'", []),
|
|
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
|
|
("5분 타이머를 맞췄습니다.", []),
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
|
|
assert row["text_with_call"] is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# "If the fix does not work, change the model": the tools stage tries the
|
|
# next model of the same family until one passes every scored case.
|
|
|
|
def _summary(ok, total=21):
|
|
return {"ok": ok, "model_ok": ok, "total": total, "guard_ran": 0, "text_with_call": 0}
|
|
|
|
|
|
def test_passing_default_tries_no_other_model(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(21), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 999)
|
|
assert tried == [] and r["tools_verdict"]["chosen"] == llm.PRESETS["fast"]["label"]
|
|
|
|
|
|
def test_failing_default_stops_at_the_first_candidate_that_passes(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
results = iter([_summary(20), _summary(21)])
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(18), lambda p: tried.append(p["model"]) or next(results),
|
|
free_gb=lambda: 999)
|
|
assert tried == [c["model"] for c in llm.CANDIDATES["fast"][:2]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][1]["label"]
|
|
|
|
|
|
def test_candidate_without_disk_space_is_skipped_not_downloaded(tmp_path):
|
|
r = common.StageResult("tools_recommended", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "recommended", _summary(18), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 1)
|
|
assert tried == []
|
|
assert r["tools_verdict"]["chosen"] is None
|
|
assert all(c["skipped"] == "disk" for c in r["tools_verdict"]["candidates"])
|
|
|
|
|
|
|
|
def _run_scripted(case_prompt, answers):
|
|
from m0lib import tool_fix
|
|
client = _ScriptedClient(answers)
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
return tool_fix.run_case(client, "system", planner, tool_fix.case_for(case_prompt)), client
|
|
|
|
|
|
def test_right_tool_with_a_made_up_final_answer_fails():
|
|
# The weather tool ran and said 13 degrees; an answer of "24도, 맑음" is still a lie.
|
|
row, _ = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather city='서울'", []),
|
|
("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]),
|
|
("서울은 지금 24도, 맑음입니다.", []),
|
|
])
|
|
assert row["model_ok"] is True and row["final_ok"] is False and row["ok"] is False
|
|
|
|
|
|
def test_claiming_an_action_no_tool_can_do_fails():
|
|
claimed, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불을 껐습니다.", [])])
|
|
honest, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불은 제가 끌 수 없습니다.", [])])
|
|
assert claimed["ok"] is False and honest["ok"] is True
|
|
|
|
|
|
def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine():
|
|
row, client = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather town='서울'", []), # key get_weather does not take
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("지금은 날씨를 확인하지 못했습니다.", []), # answer from the failed result
|
|
])
|
|
assert row["guard_ran"] is False and row["ok"] is False
|
|
assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다."
|
|
last = client.sent[-1][0][-1]
|
|
assert last["role"] == "tool" and "could not be run" in last["content"]
|
|
|
|
|
|
|
|
# Final-answer judging: the reviewer's counterexamples must fail, honest answers pass.
|
|
@pytest.mark.parametrize("prompt,answer,expected", [
|
|
("서울 날씨 어때?", "서울은 지금 13도이고 비가 오고 있습니다.", True),
|
|
("서울 날씨 어때?", "서울은 지금 24도, 맑음입니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도이고 비가 오지 않습니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도, 어제는 20도였고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.9도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 -13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 영하 13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.0도이고 비가 옵니다.", True),
|
|
("부산 지금 비 와?", "부산은 맑고 비가 오지 않아요, 17.5도입니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "영하 13도에 비가 오고 있으니 우산을 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5.5분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "-5분 타이머를 맞췄습니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오고 있으니 우산을 챙기세요.", True),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오지 않으니 우산은 필요 없습니다.", False),
|
|
("부산 지금 비 와?", "부산은 지금 맑고 비가 오지 않아요, 17도입니다.", True),
|
|
("부산 지금 비 와?", "부산은 지금 비가 와요, 우산 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5분 타이머를 맞췄습니다.", True),
|
|
("5분 타이머 맞춰 줘.", "15분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "50분 타이머를 맞췄습니다.", False),
|
|
("10분 뒤에 알려 줘.", "10분 뒤에 알려 드릴게요.", True),
|
|
("10분 뒤에 알려 줘.", "110분 뒤에 알려 드릴게요.", False),
|
|
("거실 불 꺼 줘.", "거실 불은 제가 끌 수 없습니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다.", False),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다. 다시 켜는 기능은 지원하지 않습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "저는 타이머를 맞출 수 없습니다.", True),
|
|
("3분 타이머 맞춰 줘.", "3분 타이머를 맞췄습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머는 지원하지 않지만 3분 뒤에 알려 드릴게요.", False),
|
|
# Real refusals from the gaming-PC run (2026-11-10) that name the missing tool.
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 없습니다.", True),
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 스위치를 조작하거나 스마트 홈 기기를 통해 조절해 주셔야 합니다.", True),
|
|
("3분 타이머 맞춰 줘.", "죄송합니다만, 현재 타이머를 설정할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 알람을 설정해 주시거나 다른 방법을 찾아보시길 바랍니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 끌 수 있는 도구가 없지만 대신 꺼 드렸습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머 도구는 없지만 3분 타이머를 맞췄습니다.", False),
|
|
])
|
|
def test_final_answer_judge(prompt, answer, expected):
|
|
from m0lib import tool_fix
|
|
assert tool_fix.case_for(prompt).judge(answer) is expected
|
|
|
|
|
|
def test_no_timer_tool_case_offers_no_timer_tool():
|
|
from m0lib import tool_fix
|
|
names = [t["function"]["name"] for t in tool_fix.case_for("3분 타이머 맞춰 줘.").offered()]
|
|
assert "set_timer" not in names and names
|
|
|
|
|
|
class _LyingClient:
|
|
"""Never calls a tool and always answers with a made-up weather report."""
|
|
|
|
model = "liar"
|
|
|
|
def chat(self, messages, **kwargs):
|
|
if kwargs.get("tools") is None: # planner call
|
|
return {"text": "Reply to the user.", "tool_calls": [], "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
return {"text": "서울은 24도, 맑음입니다. 5분 타이머를 맞췄습니다.", "tool_calls": [], "first_tool": None,
|
|
"ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path):
|
|
from m0lib import tool_fix
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
summary = tool_fix.test_tools_fixed(r, _LyingClient(), tries=1)
|
|
assert not tool_fix.tools_passed(summary)
|
|
tried = []
|
|
llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999)
|
|
assert tried == [llm.CANDIDATES["fast"][0]["model"]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"]
|
|
|
|
|
|
# --- gamemode stage: the one model in each placement the app switches to ----
|
|
|
|
|
|
def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True, only="levels",
|
|
extra_opts=(), stt=None, procs=None, paused=None):
|
|
"""Run the stage with Strata replaced; ``gpu(running)`` is nvidia-smi's (used, total), where
|
|
``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None. ``stt`` is a
|
|
_FakeStt whose loaded model adds to the used VRAM."""
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
started = []
|
|
running = {"now": None}
|
|
if stt is not None:
|
|
stt.running = running
|
|
monkeypatch.setattr(llm, "SttLoad", stt.factory)
|
|
base_gpu = gpu
|
|
|
|
def gpu(now):
|
|
used, total = base_gpu(now)
|
|
return used + stt.used_mib(), total
|
|
|
|
def launch(self, preset, context, parallel, vision="on", extra_sets=None, restart=False, timeout=0,
|
|
kv=None, mtp=None):
|
|
spec = {"parallel": parallel, "vision": vision, "sets": list(extra_sets or []), "context": context,
|
|
"kv": kv, "mtp": mtp}
|
|
started.append(spec)
|
|
if stt is not None:
|
|
stt.events.append("llm_start")
|
|
running["now"] = spec if ready(extra_sets or []) else None
|
|
return {"ready": running["now"] is not None}
|
|
|
|
def stop(self):
|
|
running["now"] = None
|
|
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "launch", launch)
|
|
monkeypatch.setattr(llm.Strata, "stop", stop)
|
|
monkeypatch.setattr(llm, "port_in_use", lambda p: running["now"] is not None)
|
|
monkeypatch.setattr(llm, "gpu_memory_mib", lambda: gpu(running["now"]))
|
|
monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["now"]))
|
|
monkeypatch.setattr(gpucap, "gpu_util_pct", lambda: 99)
|
|
# The kit's own engine and STT worker; the GPU cap "pauses" them by recording it.
|
|
procs = _procs(tmp_path / "strata") if procs is None else procs
|
|
monkeypatch.setattr(gpucap, "process_table", lambda: list(procs))
|
|
held = set()
|
|
paused = set() if paused is None else paused
|
|
monkeypatch.setattr(gpucap, "suspend_pid", lambda pid: held.add(pid) or paused.add(pid) or True)
|
|
monkeypatch.setattr(gpucap, "resume_pid", lambda pid: held.discard(pid) or True)
|
|
monkeypatch.setattr(gpucap, "PERIOD_S", 0.002)
|
|
running["held"] = held
|
|
out = tmp_path / "results" / "gamemode.json"
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
opts = ["--opt", "model=fast", "--opt", f"only={only}"]
|
|
for o in extra_opts:
|
|
opts += ["--opt", o]
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), *opts])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8")), started, running
|
|
|
|
|
|
def _model_mib(spec):
|
|
"""Roughly what the 2026-11-10 game PC showed: 4.4 GiB of weights that never leave the card, the
|
|
expert budget (which first holds the 1.1 GiB long-prompt staging, so 0 and 1 cost the same), the
|
|
conversation memory (int8, 1.43 GiB per 32K; q4_0 half), about 1 GB for the MTP drafter and
|
|
1.5 GB of buffers. With no expert budget Strata fills the card."""
|
|
experts = next((float(s.split("=")[1]) for s in spec["sets"] if s.startswith("expert-vram=")), None)
|
|
if experts is None:
|
|
return 13400
|
|
kv = 1464 * spec["context"] / 32768 * (0.5 if spec["kv"] == "q4_0" else 1)
|
|
return int(4506 + max(experts, 1.1) * 1024 + kv + (0 if spec["mtp"] == "off" else 1000) + 1500)
|
|
|
|
|
|
def _vram(running):
|
|
"""16 GB card, 2 GB used by Windows."""
|
|
return (2000 if running is None else 2000 + _model_mib(running)), 16303
|
|
|
|
|
|
def _vram_20g(running):
|
|
"""A 20 GB card, where the smaller game placements do leave the reserve."""
|
|
return _vram(running)[0], 20480
|
|
|
|
|
|
def _vram_big_floor(running):
|
|
"""A card where even the smallest placement leaves the game less than its reserve."""
|
|
return (2000 if running is None else 2000 + _model_mib(running) + 3000), 16303
|
|
|
|
|
|
def test_gamemode_measures_every_placement_and_returns_to_normal(tmp_path, monkeypatch, fake_server):
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_20g)
|
|
assert r["stage"] == "gamemode" and r["status"] == "ok"
|
|
assert [x["sets"] for x in started] == [lv["sets"] for lv in llm.LEVELS]
|
|
# Each placement starts with its own context, conversation memory format and drafter.
|
|
for spec, lv in zip(started, llm.LEVELS):
|
|
assert spec["context"] == lv.get("context", 32768)
|
|
assert (spec["kv"], spec["mtp"]) == (lv.get("kv"), lv.get("mtp"))
|
|
assert started[-1] == started[0] # the last start is the normal placement again
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
for lv in llm.LEVELS:
|
|
assert rows[lv["key"]]["switch_s"] is not None
|
|
assert rows[lv["key"]]["voice_first_sentence"]["n"] > 0
|
|
# The game placements go below the 2 GB expert budget that still left a 16 GB card only 4 GB.
|
|
game = [rows[lv["key"]] for lv in llm.LEVELS if lv["key"].startswith("game_")]
|
|
assert max(x["vram_free_mib"] for x in game) > min(x["vram_free_mib"] for x in game)
|
|
assert min(x["vram_free_mib"] for x in game) > rows["normal"]["vram_free_mib"]
|
|
# On this card a smaller placement does leave the reserve, and the choice is one that does.
|
|
assert r["game_choice"]["vram_free_mib"] >= llm.GAME_RESERVE_MIB
|
|
assert running["now"] is None # the model is not left running after the stage
|
|
|
|
|
|
def test_when_no_placement_leaves_the_reserve_the_result_says_how_much_is_free(tmp_path, monkeypatch,
|
|
fake_server):
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_big_floor)
|
|
assert r["game_choice"] is None
|
|
assert r["status"] == "ok" # every placement was measured; it is the card that is too small
|
|
most = max(x["vram_free_mib"] for x in r["levels"] if x["key"].startswith("game_"))
|
|
assert any(str(most) in n for n in r["notes"])
|
|
|
|
|
|
def test_engine_lines_are_only_those_of_the_latest_start(tmp_path):
|
|
# The engine log is appended across starts: reading it from the top repeated an old start's
|
|
# placement lines for every placement.
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
log = tmp_path / "strata" / "localappdata" / "Strata" / "launch" / "ud-q4_k_xl-32k.engine.log"
|
|
log.parent.mkdir(parents=True)
|
|
log.write_text("strata-q36: hybrid experts: 59 of 256 per layer in VRAM (4.32 GiB)\n", encoding="utf-8")
|
|
s.mark_engine_logs()
|
|
with open(log, "a", encoding="utf-8") as f:
|
|
f.write("strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)\n")
|
|
assert s.engine_lines() == ["strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)"]
|
|
# A log written again from the start (shorter than the mark) is read whole.
|
|
log.write_text("strata-q36: context 8192 cells\n", encoding="utf-8")
|
|
assert s.engine_lines() == ["strata-q36: context 8192 cells"]
|
|
|
|
|
|
def test_placement_memory_format_and_drafter_reach_the_launch_command(tmp_path, monkeypatch):
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--context", "--kv", "--mtp"})
|
|
s.launch(llm.PRESETS["fast"], 8192, parallel=1, kv="q4_0", mtp="off")
|
|
s.launch(llm.PRESETS["fast"], 32768, parallel=1)
|
|
with_opts, plain = ran
|
|
assert with_opts[with_opts.index("--kv") + 1] == "q4_0" and with_opts[with_opts.index("--mtp") + 1] == "off"
|
|
assert with_opts[with_opts.index("--context") + 1] == "8192"
|
|
assert "--kv" not in plain and "--mtp" not in plain
|
|
|
|
|
|
def test_a_placement_that_does_not_start_is_recorded_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
|
|
ready=lambda sets: "expert-vram=0" not in sets)
|
|
assert len(started) == len(llm.LEVELS)
|
|
failed = [lv["key"] for lv in llm.LEVELS if "expert-vram=0" in lv["sets"]]
|
|
row = next(x for x in r["levels"] if x["key"] == failed[0])
|
|
assert row["switch_s"] is None and "decode_tok_s" not in row
|
|
# A placement that never started is a gap in the measurement, not a pass.
|
|
assert r["status"] == "partial"
|
|
assert any(failed[0] in e["where"] for e in r["errors"])
|
|
|
|
|
|
def test_only_the_normal_placement_starting_is_not_ok(tmp_path, monkeypatch, fake_server):
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
|
|
ready=lambda sets: not any(s.startswith("expert-vram=") for s in sets))
|
|
assert r["game_choice"] is None
|
|
assert r["status"] == "partial"
|
|
game = [lv for lv in llm.LEVELS if lv["key"].startswith("game_")]
|
|
assert sum(1 for e in r["errors"] if e["where"].startswith("level game_")) == len(game)
|
|
|
|
|
|
def _row(key, free, tok_s):
|
|
return {"key": key, "vram_free_mib": free, "decode_tok_s": {"median": tok_s}, "switch_s": 50.0}
|
|
|
|
|
|
def test_game_choice_is_the_fastest_placement_that_leaves_the_reserve_free():
|
|
rows = [_row("normal", 900, 82.0), _row("game_experts_6g", 6000, 70.0),
|
|
_row("game_experts_4g", 8100, 60.0), _row("game_experts_2g", 10100, 45.0)]
|
|
choice = llm.pick_game_level(rows, reserve_mib=8000)
|
|
assert choice["key"] == "game_experts_4g" and choice["decode_tok_s"] == 60.0
|
|
|
|
|
|
def test_no_game_choice_when_no_placement_leaves_the_reserve():
|
|
rows = [_row("normal", 900, 82.0), _row("game_experts_2g", 7000, 45.0)]
|
|
assert llm.pick_game_level(rows, reserve_mib=8000) is None
|
|
|
|
|
|
def test_kit_has_a_one_click_runner_for_gamemode():
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert "gamemode" in keys
|
|
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode" in bat and "-Redo" in bat
|
|
|
|
|
|
def test_a_measurement_error_inside_a_placement_marks_the_stage_partial(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "quick_bench", lambda c: {"errors": ["decode: TimeoutError"], "decode_tok_s": {"n": 0}})
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert r["status"] == "partial"
|
|
assert any(e["error"] == "decode: TimeoutError" for e in r["errors"])
|
|
|
|
|
|
def test_gamemode_measures_the_2g_expert_placement_with_a_16k_context(tmp_path, monkeypatch, fake_server):
|
|
# The one placement between "2 GB, 32K" and "0 GB, 16K" that was never measured.
|
|
_, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert any("expert-vram=2" in x["sets"] and x["context"] == 16384 for x in started)
|
|
|
|
|
|
def test_normal_placement_answers_one_request_at_a_time(tmp_path, monkeypatch, fake_server):
|
|
# One request at a time is the default; two only when a chat channel answers every message.
|
|
_, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert started[0]["parallel"] == 1 and started[-1]["parallel"] == 1
|
|
|
|
|
|
# --- gamemode_stt / gamemode_live: the LLM next to the STT that runs with it --
|
|
|
|
|
|
STT_MODEL_MIB = 1300 # M0 stt: the loaded model
|
|
STT_GROWTH_MIB = 1440 # M0 stt: what ten speakers at once add over the loaded model
|
|
STT_GPU_MIB = STT_MODEL_MIB + STT_GROWTH_MIB
|
|
STT_LAUNCHER_PID = 4100 # the venv python.exe; the real interpreter is its child
|
|
|
|
|
|
class _FakeStt:
|
|
"""Stands in for the STT worker process: on cuda its model holds VRAM while it is loaded, and
|
|
ten speakers at once hold more once it is transcribing (with partials on)."""
|
|
|
|
def __init__(self, fail_device=None, broken_at_load=False, finish_result=None):
|
|
self.finish_result = finish_result
|
|
self.events: list[str] = []
|
|
self.devices: list[str] = []
|
|
self.paces: list = []
|
|
self.loaded: str | None = None
|
|
self.busy = False
|
|
self.running: dict = {}
|
|
self.fail_device = fail_device
|
|
self.broken_at_load = broken_at_load
|
|
|
|
def used_mib(self) -> int:
|
|
if self.loaded != "cuda":
|
|
return 0
|
|
return STT_GPU_MIB if self.busy else STT_MODEL_MIB
|
|
|
|
def factory(self, python, work, device, log, pace=None):
|
|
fake = self
|
|
fake.devices.append(device)
|
|
fake.paces.append(pace)
|
|
|
|
class _One:
|
|
pid = STT_LAUNCHER_PID
|
|
|
|
def start(self, timeout=900):
|
|
if fake.broken_at_load:
|
|
return {"status": "failed", "errors": ["load: RuntimeError: CUDA failed"]}
|
|
fake.loaded = device
|
|
fake.events.append("stt_loaded")
|
|
return {"status": "loaded", "load_s": 3.0}
|
|
|
|
def go(self):
|
|
assert fake.running.get("now") is not None, "STT load must overlap a running LLM"
|
|
fake.events.append("stt_go")
|
|
fake.busy = pace is None or pace.get("partial_every") is not None
|
|
|
|
def finish(self, timeout=120):
|
|
fake.events.append("stt_finish")
|
|
if fake.finish_result is not None:
|
|
return fake.finish_result
|
|
errs = ["RuntimeError: CUDA out of memory"] if device == fake.fail_device else []
|
|
partials = pace is None or pace.get("partial_every") is not None
|
|
return {"status": "done", "batch10_s": {"n": 4 if partials else 0, "median": 0.9},
|
|
"single_s": {"n": 4, "median": 0.2}, "errors": errs, "pace": pace}
|
|
|
|
def close(self):
|
|
fake.loaded = None
|
|
fake.busy = False
|
|
|
|
return _One()
|
|
|
|
|
|
def test_together_stage_measures_the_llm_with_the_stt_loaded_and_busy(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt()
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["stage"] == "gamemode_stt" and r["status"] == "ok"
|
|
assert len(started) == len(llm.TOGETHER)
|
|
# The STT model is in VRAM before the LLM picks its placement, and its ten-speaker loop runs
|
|
# while the LLM is measured (asserted by _FakeStt.go).
|
|
for i in range(len(llm.TOGETHER)):
|
|
assert stt.events[i * 4:(i + 1) * 4] == ["stt_loaded", "llm_start", "stt_go", "stt_finish"]
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
for lv in llm.TOGETHER:
|
|
row = rows[lv["key"]]
|
|
assert row["stt"]["device"] == lv["stt"]
|
|
assert row["stt"]["batch10_s"]["n"] > 0 and row["decode_tok_s"]["n"] > 0
|
|
# What is left for the game already has the STT taken out when it runs on the GPU.
|
|
_, total = _vram(None)
|
|
assert row["vram_free_mib"] == total - row["vram_peak_mib"]
|
|
# Both places for the STT are measured in the same game placement.
|
|
game = [lv for lv in llm.TOGETHER if lv["key"].startswith("game_")]
|
|
assert {lv["stt"] for lv in game} == {"cuda", "cpu"}
|
|
pairs = [(rows[a["key"]], rows[b["key"]]) for a in game for b in game
|
|
if a["stt"] == "cuda" and b["stt"] == "cpu" and (a["sets"], a.get("context")) == (b["sets"], b.get("context"))]
|
|
assert pairs
|
|
for on_gpu, on_cpu in pairs:
|
|
assert on_cpu["vram_free_mib"] - on_gpu["vram_free_mib"] == STT_GPU_MIB
|
|
# A chat channel that answers every message runs two requests at once: measured with the STT too.
|
|
assert any(lv["parallel"] == 2 for lv in llm.TOGETHER if lv["key"].startswith("normal"))
|
|
assert stt.loaded is None and running["now"] is None # nothing is left running
|
|
|
|
|
|
def test_stt_error_next_to_the_llm_marks_the_together_stage_partial(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt(fail_device="cuda")
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["status"] == "partial"
|
|
assert any("out of memory" in e["error"] for e in r["errors"])
|
|
|
|
|
|
def test_stt_that_does_not_load_skips_that_placement_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt(broken_at_load=True)
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert started == [] # an LLM measured without its STT would be the wrong number
|
|
assert len(stt.devices) == len(llm.TOGETHER)
|
|
assert r["status"] == "failed"
|
|
|
|
|
|
def test_together_stage_without_the_stt_environment_measures_nothing(tmp_path, monkeypatch, fake_server):
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together")
|
|
assert started == [] and r["status"] == "failed"
|
|
|
|
|
|
def test_live_stage_is_skipped_when_nobody_is_there_to_start_a_game(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "interactive", lambda: False)
|
|
stt = _FakeStt()
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["status"] == "skipped" and started == []
|
|
|
|
|
|
def test_live_stage_measures_game_placements_while_the_game_runs(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "interactive", lambda: True)
|
|
monkeypatch.setattr(llm, "ask", lambda *a, **k: "y")
|
|
monkeypatch.setattr(llm, "wait_enter", lambda *a, **k: "")
|
|
monkeypatch.setattr(llm.time, "sleep", lambda s: None)
|
|
stt = _FakeStt()
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["stage"] == "gamemode_live" and r["status"] == "ok"
|
|
# Only game placements: starting the normal one would take the game's VRAM.
|
|
assert len(started) == len(llm.LIVE) and all(lv["key"].startswith("game_") for lv in llm.LIVE)
|
|
assert {lv["stt"] for lv in llm.LIVE} == {"cuda", "cpu"}
|
|
assert all(x["vram_idle_mib"] is not None for x in r["levels"]) # Windows and the game
|
|
assert running["now"] is None and stt.loaded is None
|
|
|
|
|
|
def _fake_kit_run(tmp_path, monkeypatch, *args):
|
|
import m0
|
|
ran = []
|
|
monkeypatch.setattr(m0, "run_stage", lambda key, *a, **k: ran.append(key))
|
|
monkeypatch.setattr(m0, "pack", lambda work: tmp_path / "x.zip")
|
|
monkeypatch.setattr(m0.subprocess, "run", lambda *a, **k: None)
|
|
monkeypatch.setattr(sys, "argv", ["m0.py", "--work", str(tmp_path / "w"), *args])
|
|
m0.main()
|
|
return ran
|
|
|
|
|
|
def test_full_kit_run_leaves_out_the_stage_that_needs_a_running_game(tmp_path, monkeypatch):
|
|
ran = _fake_kit_run(tmp_path, monkeypatch)
|
|
assert "gamemode_stt" in ran and "gamemode_live" not in ran
|
|
assert _fake_kit_run(tmp_path, monkeypatch, "--only", "gamemode_live", "--redo") == ["gamemode_live"]
|
|
|
|
|
|
def test_stages_that_run_the_stt_next_to_the_llm_get_the_stt_python(tmp_path, monkeypatch):
|
|
import m0
|
|
cmds = []
|
|
monkeypatch.setattr(m0.envs, "ensure_env", lambda name, work, log: Path(f"/venvs/{name}/python"))
|
|
monkeypatch.setattr(m0.subprocess, "run", lambda cmd, **k: cmds.append(cmd) or type("P", (), {"returncode": 0}))
|
|
for key in ("gamemode_stt", "gamemode"):
|
|
m0.run_stage(key, "base", "llm.py", ["model=fast"], tmp_path, {})
|
|
together, alone = cmds
|
|
assert f"stt_python={Path('/venvs/stt/python')}" in together
|
|
assert not any(str(x).startswith("stt_python=") for x in alone)
|
|
|
|
|
|
def test_one_click_runners_cover_the_new_gamemode_stages():
|
|
import m0
|
|
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode,gamemode_stt" in bat
|
|
live = (Path(m0.KIT) / "run_m0_game_live.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode_live" in live and "-Redo" in live
|
|
|
|
|
|
def test_stt_load_loop_waits_for_go_and_runs_until_stop(tmp_path):
|
|
from m0lib import stt
|
|
go, stop = tmp_path / "go", tmp_path / "stop"
|
|
calls = []
|
|
|
|
def batch():
|
|
calls.append("b")
|
|
if not go.exists():
|
|
raise AssertionError("ran before go")
|
|
|
|
def single():
|
|
calls.append("s")
|
|
if calls.count("s") == 3:
|
|
stop.write_text("")
|
|
|
|
go.write_text("")
|
|
res = stt.load_loop(batch, single, go, stop)
|
|
assert res["status"] == "done" and res["batch10_s"]["n"] == 3 and res["single_s"]["n"] == 3
|
|
assert calls == ["b", "s"] * 3
|
|
|
|
|
|
def test_stt_load_loop_keeps_an_out_of_memory_and_stops(tmp_path):
|
|
from m0lib import stt
|
|
go, stop = tmp_path / "go", tmp_path / "stop"
|
|
go.write_text("")
|
|
|
|
def batch():
|
|
raise RuntimeError("CUDA failed with error out of memory")
|
|
|
|
res = stt.load_loop(batch, lambda: None, go, stop)
|
|
assert res["batch10_s"]["n"] == 0 and "out of memory" in res["errors"][0]
|
|
|
|
|
|
def test_stt_load_loop_does_not_run_when_stopped_before_go(tmp_path):
|
|
from m0lib import stt
|
|
stop = tmp_path / "stop"
|
|
stop.write_text("")
|
|
res = stt.load_loop(lambda: 1 / 0, lambda: 1 / 0, tmp_path / "go", stop)
|
|
assert res["status"] == "not_started" and res["errors"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("busy", [
|
|
{"status": "timeout", "errors": []}, # the worker did not stop in time
|
|
{"status": "loaded"}, # it died before writing its timings
|
|
{"status": "not_started", "batch10_s": {"n": 0}, "single_s": {"n": 0}, "errors": []},
|
|
{"status": "done", "batch10_s": {"n": 0}, "single_s": {"n": 3, "median": 0.2}, "errors": []},
|
|
])
|
|
def test_stt_load_without_timings_is_a_gap_not_a_pass(tmp_path, monkeypatch, fake_server, busy):
|
|
stt = _FakeStt(finish_result=busy)
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["status"] == "partial"
|
|
assert sum(1 for e in r["errors"] if e["where"].endswith(" stt")) == len(llm.TOGETHER)
|
|
|
|
|
|
def test_stt_worker_that_ignores_stop_is_reported_as_timed_out_and_killed(tmp_path):
|
|
s = llm.SttLoad(sys.executable, tmp_path, "cpu", tmp_path / "stt.log")
|
|
s.cmd = [sys.executable, "-c",
|
|
"import json,sys,time; open(sys.argv[1],'w').write(json.dumps({'status':'loaded'})); time.sleep(60)",
|
|
str(s.out)]
|
|
assert s.start(timeout=30)["status"] == "loaded"
|
|
s.go()
|
|
res = s.finish(timeout=1)
|
|
assert res["status"] == "timeout" and res["errors"]
|
|
s.close()
|
|
assert s.proc.poll() is not None
|
|
|
|
|
|
# --- gamemode_tune: room for the STT in the normal placement, fewer partials while the LLM answers --
|
|
|
|
|
|
class _FakeClock:
|
|
"""time.monotonic/perf_counter/sleep for the STT loop: passes advance it, nothing waits."""
|
|
|
|
def __init__(self):
|
|
self.t = 0.0
|
|
|
|
def monotonic(self):
|
|
return self.t
|
|
|
|
perf_counter = monotonic
|
|
|
|
def sleep(self, s):
|
|
self.t += s
|
|
|
|
|
|
def _paced_run(tmp_path, monkeypatch, pace, run_s=10.0):
|
|
from m0lib import stt
|
|
clock = _FakeClock()
|
|
monkeypatch.setattr(stt, "time", clock)
|
|
go, stop = tmp_path / "go", tmp_path / "stop"
|
|
go.write_text("")
|
|
|
|
def work(seconds):
|
|
def fn():
|
|
clock.t += seconds
|
|
if clock.t >= run_s:
|
|
stop.write_text("")
|
|
return fn
|
|
|
|
return stt.load_loop(work(0.9), work(0.2), go, stop, pace=pace)
|
|
|
|
|
|
def test_paced_stt_runs_partials_at_their_interval_and_finals_in_between(tmp_path, monkeypatch):
|
|
res = _paced_run(tmp_path, monkeypatch, {"partial_every": 2.0, "final_every": 0.5})
|
|
assert res["status"] == "done" and res["pace"] == {"partial_every": 2.0, "final_every": 0.5}
|
|
assert 5 <= res["batch10_s"]["n"] <= 6 # one ten-speaker pass every 2 s over 10 s
|
|
assert res["single_s"]["n"] >= 12 # finals keep coming in between
|
|
# A final waits at most for the partial pass that is running when it is due.
|
|
assert res["final_wait_s"]["max"] <= 0.9 + 1e-6
|
|
assert res["busy_share"] < 1.0 # the GPU gets time back for the LLM
|
|
|
|
|
|
def test_stt_without_a_pace_transcribes_back_to_back(tmp_path, monkeypatch):
|
|
res = _paced_run(tmp_path, monkeypatch, None)
|
|
assert res["busy_share"] == 1.0 and res["batch10_s"]["n"] == res["single_s"]["n"]
|
|
|
|
|
|
def test_finals_only_stt_runs_no_ten_speaker_pass(tmp_path, monkeypatch):
|
|
res = _paced_run(tmp_path, monkeypatch, {"partial_every": None, "final_every": 0.5})
|
|
assert res["batch10_s"]["n"] == 0 and res["single_s"]["n"] >= 15
|
|
assert res["busy_share"] < 0.5
|
|
|
|
|
|
def test_stt_worker_command_carries_the_pace(tmp_path):
|
|
s = llm.SttLoad(sys.executable, tmp_path, "cuda", tmp_path / "stt.log",
|
|
pace={"partial_every": None, "final_every": 0.5})
|
|
assert "partial_every=none" in s.cmd and "final_every=0.5" in s.cmd
|
|
plain = llm.SttLoad(sys.executable, tmp_path, "cuda", tmp_path / "stt2.log")
|
|
assert not any(str(x).startswith(("partial_every=", "final_every=")) for x in plain.cmd)
|
|
|
|
|
|
def _expert_vram(spec):
|
|
return next((float(s.split("=")[1]) for s in spec["sets"] if s.startswith("expert-vram=")), None)
|
|
|
|
|
|
def test_tune_stage_leaves_the_stt_room_in_the_normal_placement(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt()
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["stage"] == "gamemode_tune" and r["status"] == "ok"
|
|
normal = [lv for lv in llm.TUNED if lv.get("stt_room")]
|
|
# One start without experts measures what the LLM needs before any expert, with the STT loaded.
|
|
probes = [x for x in started if _expert_vram(x) == 0 and x["vision"] == normal[0]["vision"]]
|
|
assert len(probes) == 1 and started[0] is probes[0]
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
budgets = {_expert_vram(x) for x in started[1:] if x["vision"] == normal[0]["vision"]}
|
|
assert len(budgets) == 1 and budgets.pop() > 0
|
|
# With the STT at its ten-speaker peak the card keeps the margin and the room for other
|
|
# programs free, and not much more.
|
|
continuous = [rows[lv["key"]] for lv in normal if lv.get("stt") and lv.get("stt_pace") is None]
|
|
assert continuous
|
|
keep = llm.STT_ROOM_MARGIN_MIB + llm.OTHER_ROOM_MIB
|
|
for row in continuous:
|
|
assert keep <= row["vram_free_mib"] < keep + 2 * 103
|
|
assert row["stt_room"]["stt_growth_mib"] == llm.STT_GROWTH_FALLBACK_MIB
|
|
assert row["stt_room"]["other_room_mib"] == llm.OTHER_ROOM_MIB
|
|
# The same placement is also measured without the STT, so the cap's cost on the LLM alone shows.
|
|
assert any(not lv.get("stt") for lv in normal) and any(lv.get("stt") for lv in normal)
|
|
# Without a game the STT keeps every partial while the LLM answers (user decision); only the
|
|
# 2 GB game placement is measured back to back, with fewer partials and with finals only.
|
|
assert all(lv.get("stt_pace") is None for lv in normal)
|
|
game = [lv for lv in llm.TUNED if lv["key"].startswith("game_")]
|
|
assert game and all("expert-vram=2" in lv["sets"] for lv in game)
|
|
paces = {(lv.get("stt_pace") or {}).get("partial_every", "continuous") for lv in game}
|
|
assert paces == {"continuous", 2.0, None}
|
|
assert stt.paces == [lv.get("stt_pace") for lv in llm.TUNED if lv.get("stt")]
|
|
# Each capped row has its uncapped twin in the same run.
|
|
capped = [lv for lv in llm.TUNED if lv.get("gpu_cap")]
|
|
assert capped and all(lv["gpu_cap"] == llm.GPU_CAP for lv in capped)
|
|
for lv in capped:
|
|
twin = {k: v for k, v in lv.items() if k not in ("key", "gpu_cap")}
|
|
assert any({k: v for k, v in x.items() if k != "key"} == twin for x in llm.TUNED)
|
|
assert stt.loaded is None and running["now"] is None
|
|
|
|
|
|
def test_stt_room_comes_from_this_pcs_own_stt_measurement(tmp_path, monkeypatch, fake_server):
|
|
(tmp_path / "results").mkdir(parents=True)
|
|
(tmp_path / "results" / "stt.json").write_text(json.dumps(
|
|
{"run_int8_float16": {"batch_speakers": {"vram_added_mib": STT_GROWTH_MIB + 512}}}), encoding="utf-8")
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune",
|
|
extra_opts=["stt_python=py"], stt=_FakeStt())
|
|
row = next(x for x in r["levels"] if x.get("stt_room"))
|
|
assert row["stt_room"]["stt_growth_mib"] == STT_GROWTH_MIB + 512
|
|
assert row["vram_free_mib"] >= llm.STT_ROOM_MARGIN_MIB + 512
|
|
|
|
|
|
def test_card_without_room_for_any_expert_starts_the_normal_placement_with_none(tmp_path, monkeypatch,
|
|
fake_server):
|
|
def small(running):
|
|
return _vram(running)[0], 12000
|
|
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, small, only="tune",
|
|
extra_opts=["stt_python=py"], stt=_FakeStt())
|
|
row = next(x for x in r["levels"] if x.get("stt_room"))
|
|
assert row["stt_room"]["expert_vram_gib"] == 0
|
|
assert any("no room" in n for n in r["notes"])
|
|
|
|
|
|
def test_tune_stage_runs_with_the_stt_python_and_has_a_one_click_runner(tmp_path, monkeypatch):
|
|
import m0
|
|
assert "gamemode_tune" in m0.NEEDS_STT and "gamemode_tune" not in m0.ON_DEMAND
|
|
bat = (Path(m0.KIT) / "run_m0_gamemode_tune.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode_tune" in bat and "-Redo" in bat
|
|
|
|
|
|
# --- the GPU cap: Jarvis's engine and STT worker paused together for 5 % of the time --------
|
|
|
|
|
|
def _procs(home):
|
|
eng = str(home / "components" / "engine" / "0.1.59-windows-x86_64-cuda-cuda13" / "strata.exe")
|
|
term = str(home / "terminal" / "current" / "bin" / "strata.exe")
|
|
return [
|
|
{"pid": 10, "ppid": 1, "exe": term, "cmd": f"{term} launch --yes"},
|
|
{"pid": 20, "ppid": 10, "exe": eng, "cmd": f"{eng} --serve --pack x"},
|
|
{"pid": 21, "ppid": 20, "exe": "C:\\x\\helper.exe", "cmd": "helper"},
|
|
{"pid": 30, "ppid": 1, "exe": eng.replace(str(home), str(home.parent / "other")), "cmd": "strata.exe --serve"},
|
|
{"pid": STT_LAUNCHER_PID, "ppid": 1, "exe": "C:\\venv\\python.exe", "cmd": "python stt.py"},
|
|
{"pid": STT_LAUNCHER_PID + 1, "ppid": STT_LAUNCHER_PID, "exe": "C:\\py\\python.exe", "cmd": "python stt.py"},
|
|
{"pid": 50, "ppid": 1, "exe": "C:\\chrome.exe", "cmd": "chrome"},
|
|
]
|
|
|
|
|
|
def test_process_table_lines_are_parsed_with_tabs_in_command_lines():
|
|
rows = gpucap.parse_process_table("20\t10\tC:\\e\\strata.exe\tstrata.exe --serve\ta\r\n\nbad line\n4\t\t\t\n")
|
|
assert rows[0] == {"pid": 20, "ppid": 10, "exe": "C:\\e\\strata.exe", "cmd": "strata.exe --serve\ta"}
|
|
assert rows[1]["pid"] == 4 and rows[1]["ppid"] is None and len(rows) == 2
|
|
|
|
|
|
def test_cap_pauses_the_serving_engine_and_the_stt_worker_with_their_children(tmp_path):
|
|
procs = _procs(tmp_path / "strata")
|
|
roots = gpucap.engine_roots(procs, tmp_path / "strata")
|
|
assert roots == [20] # not the terminal that started it, nor another kit's engine
|
|
assert sorted(gpucap.with_children(procs, roots + [STT_LAUNCHER_PID])) == \
|
|
[20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1]
|
|
|
|
|
|
def test_duty_cap_pauses_for_the_capped_share_and_leaves_nothing_paused():
|
|
import time as _time
|
|
held, log = set(), []
|
|
|
|
def suspend(pid):
|
|
held.add(pid)
|
|
log.append(("s", pid))
|
|
return True
|
|
|
|
def resume(pid):
|
|
held.discard(pid)
|
|
log.append(("r", pid))
|
|
return True
|
|
|
|
with gpucap.DutyCap([1, 2], 0.9, period=0.05, suspend=suspend, resume=resume) as d:
|
|
_time.sleep(0.6)
|
|
s = d.summary()
|
|
assert not held
|
|
assert s["cycles"] >= 5 and s["failed_processes"] == 0
|
|
assert 0.05 <= s["paused_share"] <= 0.2 # about 10 % of the time
|
|
# Both processes are paused together: each pause holds both before either is resumed.
|
|
for i in range(0, len(log), 4):
|
|
assert sorted(log[i:i + 4]) == [("r", 1), ("r", 2), ("s", 1), ("s", 2)]
|
|
assert {k for k, _ in log[i:i + 2]} == {"s"}
|
|
|
|
|
|
def test_no_cap_pauses_nothing():
|
|
calls = []
|
|
with gpucap.DutyCap([1], 1.0, period=0.01, suspend=calls.append, resume=calls.append) as d:
|
|
import time as _time
|
|
_time.sleep(0.05)
|
|
assert calls == [] and d.summary()["cycles"] == 0
|
|
|
|
|
|
def test_util_sampler_counts_samples_over_the_cap():
|
|
vals = iter([99, 90, 97, 80])
|
|
with gpucap.UtilSampler(interval=0.01, read=lambda: next(vals, None)) as u:
|
|
import time as _time
|
|
_time.sleep(0.1)
|
|
s = u.summary(0.95)
|
|
assert s["pct"]["n"] == 4 and s["over_cap_share"] == 0.5
|
|
|
|
|
|
def _tune_with_cap(tmp_path, monkeypatch, fake_server, procs):
|
|
paused = set()
|
|
r, _, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="tune",
|
|
extra_opts=["stt_python=py"], stt=_FakeStt(), procs=procs, paused=paused)
|
|
return r, running["held"], paused
|
|
|
|
|
|
def test_capped_rows_pause_jarvis_and_record_what_the_gpu_got(tmp_path, monkeypatch, fake_server):
|
|
home = tmp_path / "strata"
|
|
r, held, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, _procs(home))
|
|
assert r["status"] == "ok"
|
|
assert not held # nothing is left paused
|
|
assert paused == {20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1}
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
for lv in llm.TUNED:
|
|
row = rows[lv["key"]]
|
|
assert row["gpu_util"]["pct"]["n"] > 0 # every row records the GPU's use
|
|
if lv.get("gpu_cap"):
|
|
assert row["gpu_cap"]["cap"] == llm.GPU_CAP and row["gpu_cap"]["cycles"] > 0
|
|
assert row["gpu_util"]["over_cap_share"] == 1.0 # the fake GPU stays at 99 %
|
|
else:
|
|
assert "gpu_cap" not in row
|
|
|
|
|
|
def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp_path, monkeypatch,
|
|
fake_server):
|
|
r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, [])
|
|
assert r["status"] == "partial" and not paused
|
|
assert any("no engine process" in e["error"] for e in r["errors"])
|
|
|
|
|
|
class _CountingProcs:
|
|
"""NtSuspendProcess/NtResumeProcess on fake processes: each suspend adds one to the process's
|
|
suspend count and each successful resume takes one off; ``fail`` says when a resume fails."""
|
|
|
|
def __init__(self, fail):
|
|
self.count: dict[int, int] = {}
|
|
self.fail = fail
|
|
self.resumes = 0
|
|
|
|
def suspend(self, pid):
|
|
self.count[pid] = self.count.get(pid, 0) + 1
|
|
return True
|
|
|
|
def resume(self, pid):
|
|
self.resumes += 1
|
|
if self.fail(pid, self.resumes):
|
|
return False
|
|
self.count[pid] -= 1
|
|
return True
|
|
|
|
|
|
def test_a_resume_that_fails_once_is_retried_and_nothing_is_left_paused():
|
|
import time as _time
|
|
procs = _CountingProcs(fail=lambda pid, n: pid == 1 and n == 1)
|
|
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
|
|
_time.sleep(0.2)
|
|
s = d.summary()
|
|
assert all(v == 0 for v in procs.count.values())
|
|
assert s["failed_processes"] == 1 and s["left_paused"] == 0
|
|
|
|
|
|
def test_a_process_that_cannot_be_resumed_is_reported_as_left_paused():
|
|
import time as _time
|
|
procs = _CountingProcs(fail=lambda pid, n: pid == 1)
|
|
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
|
|
_time.sleep(0.2)
|
|
assert procs.count[1] == 1 and procs.count[2] == 0 # paused once, never stacked
|
|
assert d.summary()["left_paused"] == 1
|
|
|
|
|
|
def test_a_suspend_or_resume_that_raises_does_not_end_the_cap_with_processes_paused():
|
|
import time as _time
|
|
procs = _CountingProcs(fail=lambda pid, n: False)
|
|
|
|
def resume(pid):
|
|
if procs.resumes == 0:
|
|
procs.resumes += 1
|
|
raise OSError("access denied")
|
|
return procs.resume(pid)
|
|
|
|
with gpucap.DutyCap([1], 0.9, period=0.02, suspend=procs.suspend, resume=resume) as d:
|
|
_time.sleep(0.2)
|
|
assert procs.count[1] == 0 and d.summary()["left_paused"] == 0
|
|
|
|
|
|
# --- video_drops: a YouTube video's dropped frames next to Jarvis, with and without the cap --
|
|
|
|
|
|
def _server_frame(payload: bytes, opcode: int = 1, fin: bool = True) -> bytes:
|
|
"""A frame as Chrome sends it (servers do not mask)."""
|
|
import struct
|
|
n = len(payload)
|
|
head = bytes([(0x80 if fin else 0) | opcode])
|
|
if n < 126:
|
|
head += bytes([n])
|
|
elif n < 1 << 16:
|
|
head += bytes([126]) + struct.pack(">H", n)
|
|
else:
|
|
head += bytes([127]) + struct.pack(">Q", n)
|
|
return head + payload
|
|
|
|
|
|
def _read_client_frame(sock) -> tuple[int, bytes]:
|
|
"""Unmask one frame the client sent (clients must mask)."""
|
|
import struct
|
|
b0, b1 = sock.recv(2)
|
|
n = b1 & 0x7F
|
|
if n == 126:
|
|
n = struct.unpack(">H", sock.recv(2))[0]
|
|
elif n == 127:
|
|
n = struct.unpack(">Q", sock.recv(8))[0]
|
|
assert b1 & 0x80, "a client frame must be masked"
|
|
mask = sock.recv(4)
|
|
data = b""
|
|
while len(data) < n:
|
|
data += sock.recv(n - len(data))
|
|
return b0 & 0x0F, bytes(b ^ mask[i % 4] for i, b in enumerate(data))
|
|
|
|
|
|
def test_devtools_call_returns_its_own_answer_across_events_pings_and_fragments():
|
|
import socket
|
|
from m0lib import video
|
|
ours, chrome = socket.socketpair()
|
|
cdp = video.Cdp("", sock=ours)
|
|
big = json.dumps({"id": 1, "result": {"result": {"value": "x" * 70000}}}).encode()
|
|
chrome.sendall(_server_frame(json.dumps({"method": "Page.frameNavigated"}).encode()) # an event
|
|
+ _server_frame(b"are you there", opcode=9) # a ping
|
|
+ _server_frame(big[:300], fin=False) + _server_frame(big[300:], opcode=0))
|
|
|
|
got = {}
|
|
t = threading.Thread(target=lambda: got.update(v=cdp.evaluate("1")))
|
|
t.start()
|
|
op, sent = _read_client_frame(chrome)
|
|
assert op == 1 and json.loads(sent)["method"] == "Runtime.evaluate"
|
|
op, pong = _read_client_frame(chrome)
|
|
assert (op, pong) == (10, b"are you there")
|
|
t.join(timeout=10)
|
|
assert got["v"] == "x" * 70000
|
|
cdp.close()
|
|
chrome.close()
|
|
|
|
|
|
def test_devtools_errors_and_a_closed_socket_are_raised_not_hidden():
|
|
import socket
|
|
from m0lib import video
|
|
ours, chrome = socket.socketpair()
|
|
cdp = video.Cdp("", sock=ours)
|
|
chrome.sendall(_server_frame(json.dumps({"id": 1, "error": {"message": "no such method"}}).encode()))
|
|
with pytest.raises(RuntimeError, match="no such method"):
|
|
cdp.call("Nope.nothing")
|
|
chrome.sendall(_server_frame(json.dumps({"id": 2, "result": {
|
|
"exceptionDetails": {"text": "Uncaught", "exception": {"description": "TypeError: m is undefined"}}}}).encode()))
|
|
with pytest.raises(RuntimeError, match="m is undefined"):
|
|
cdp.evaluate("m.v")
|
|
chrome.sendall(_server_frame(b"", opcode=8))
|
|
with pytest.raises(ConnectionError):
|
|
cdp.call("Runtime.evaluate")
|
|
cdp.close()
|
|
chrome.close()
|
|
|
|
|
|
def _snap(total, dropped, **kw):
|
|
return {"total": total, "dropped": dropped, "shown": kw.pop("shown", total), "gaps": kw.pop("gaps", [0, 0]),
|
|
"max_gap_ms": kw.pop("max_gap_ms", 17.0), "waits": 0, "quality": "hd1080", "height": 1080, **kw}
|
|
|
|
|
|
def test_a_window_counts_the_frames_the_video_dropped_meanwhile():
|
|
from m0lib import video
|
|
w = video.window_stats(_snap(1000, 5), _snap(4600, 23, gaps=[4, 1], max_gap_ms=140.0), 60.0)
|
|
assert w["valid"] and w["frames"] == 3600 and w["dropped"] == 18
|
|
assert w["fps"] == 60.0 and w["dropped_per_min"] == 18.0 and w["dropped_share"] == 0.005
|
|
assert w["gaps_over_ms"] == {str(video.GAP_MS[0]): 4, str(video.GAP_MS[1]): 1} and w["max_gap_ms"] == 140.0
|
|
|
|
|
|
@pytest.mark.parametrize("after", [_snap(4600, 9, ad=True), _snap(4600, 9, paused=True), _snap(4600, 9, ended=True),
|
|
_snap(300, 0), _snap(1000, 5)])
|
|
def test_a_window_with_an_ad_a_pause_the_end_a_new_video_or_no_frames_is_kept_but_not_counted(after):
|
|
from m0lib import video
|
|
w = video.window_stats(_snap(1000, 5), after, 60.0)
|
|
assert not w["valid"] and w["invalid_because"]
|
|
|
|
|
|
def test_the_comparison_adds_up_only_valid_busy_windows_on_each_side_of_the_cap():
|
|
from m0lib import video
|
|
|
|
def row(jarvis, cap, frames, dropped, valid=True):
|
|
return {"jarvis": jarvis, "gpu_cap": {"cap": 0.95} if cap else None,
|
|
"video": {"valid": valid, "frames": frames, "dropped": dropped, "seconds": 60.0,
|
|
"gaps_over_ms": {str(g): dropped for g in video.GAP_MS}}}
|
|
|
|
rows = [row("off", False, 3600, 0), row("busy", False, 3600, 36), row("busy", True, 3600, 6),
|
|
row("busy", True, 3600, 1000, valid=False), row("busy", False, 3600, 18)]
|
|
c = video.compare(rows)
|
|
assert c["no_cap"]["windows"] == 2 and c["no_cap"]["dropped"] == 54 and c["no_cap"]["dropped_per_min"] == 27.0
|
|
assert c["cap95"]["windows"] == 1 and c["cap95"]["dropped"] == 6 and c["cap95"]["dropped_share"] == round(6 / 3600, 4)
|
|
|
|
|
|
class _FakePage:
|
|
"""The watch page: 60 frames shown per snapshot step, a few dropped; records what the stage did."""
|
|
|
|
def __init__(self, starts=True):
|
|
self.starts = starts
|
|
self.total = self.dropped = 0
|
|
self.events: list[str] = []
|
|
self.closed = False
|
|
|
|
def start(self, quality="hd1080", timeout=180):
|
|
self.events.append("video_start")
|
|
return {"started": self.starts, "quality": quality if self.starts else None, "ad_seen": False}
|
|
|
|
def keep_room(self):
|
|
pass
|
|
|
|
def reset(self):
|
|
pass
|
|
|
|
def snapshot(self):
|
|
self.total += 3600
|
|
self.dropped += 2
|
|
return _snap(self.total, self.dropped)
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
def _run_video_stage(tmp_path, monkeypatch, fake_server, page, stt=None):
|
|
from m0lib import video
|
|
monkeypatch.setattr(video, "QUIET_S", 0.01)
|
|
monkeypatch.setattr(video, "open_video_page", lambda work, *a, **k: page)
|
|
stt = _FakeStt() if stt is None else stt
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="video",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
return r, started, running, stt
|
|
|
|
|
|
def test_video_stage_compares_the_video_alone_jarvis_idle_and_busy_with_and_without_the_cap(
|
|
tmp_path, monkeypatch, fake_server):
|
|
from m0lib import video
|
|
page = _FakePage()
|
|
r, started, running, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
|
|
assert r["stage"] == "video_drops" and r["status"] == "ok"
|
|
rows = r["video_rows"]
|
|
# Alone first and last; idle before busy; the busy rows alternate A B B A around the cap.
|
|
assert [x["jarvis"] for x in rows] == ["off", "idle"] + ["busy"] * len(video.ROUNDS) + ["off"]
|
|
busy = [x for x in rows if x["jarvis"] == "busy"]
|
|
capped = [bool(x["gpu_cap"]) for x in busy]
|
|
assert capped == [False, True, True, False]
|
|
assert all(x["video"]["valid"] and x["video"]["frames"] > 0 for x in rows)
|
|
for x in busy:
|
|
assert x["llm"]["decode_tok_s"]["n"] > 0 # the LLM answered during the window
|
|
if x["gpu_cap"]:
|
|
assert x["gpu_cap"]["cap"] == llm.GPU_CAP and x["gpu_cap"]["cycles"] > 0
|
|
assert x["gpu_cap"]["left_paused"] == 0
|
|
# Jarvis is the normal placement: the STT room probe, then one start with the experts it leaves.
|
|
assert len(started) == 2 and _expert_vram(started[0]) == 0 and _expert_vram(started[1]) > 0
|
|
assert r["placement"]["sets"] == started[1]["sets"]
|
|
# The STT is loaded before the LLM and transcribes only once Jarvis is busy.
|
|
assert stt.events[:4] == ["stt_loaded", "llm_start", "llm_start", "stt_go"]
|
|
assert set(r["comparison"]) == {"no_cap", "cap95"} and r["comparison"]["cap95"]["windows"] == 2
|
|
# Nothing is left behind: the browser closed, the model stopped, the STT gone, nothing paused.
|
|
assert page.closed and running["now"] is None and stt.loaded is None and not running["held"]
|
|
|
|
|
|
def test_the_cap_in_the_video_stage_pauses_jarvis_and_never_the_browser(tmp_path, monkeypatch, fake_server):
|
|
paused = set()
|
|
from m0lib import video
|
|
page = _FakePage()
|
|
monkeypatch.setattr(video, "QUIET_S", 0.01)
|
|
monkeypatch.setattr(video, "open_video_page", lambda work, *a, **k: page)
|
|
_run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="video", extra_opts=["stt_python=py"],
|
|
stt=_FakeStt(), paused=paused)
|
|
browser = next(p["pid"] for p in _procs(tmp_path / "strata") if "chrome" in p["exe"])
|
|
assert paused == {20, 21, STT_LAUNCHER_PID, STT_LAUNCHER_PID + 1} and browser not in paused
|
|
|
|
|
|
def test_video_stage_without_a_browser_or_a_playing_video_starts_no_jarvis(tmp_path, monkeypatch, fake_server):
|
|
r, started, _, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, None)
|
|
assert r["status"] == "failed" and not started and not stt.events
|
|
page = _FakePage(starts=False)
|
|
r, started, _, stt = _run_video_stage(tmp_path, monkeypatch, fake_server, page)
|
|
assert r["status"] == "failed" and not started and not stt.events and page.closed
|
|
|
|
|
|
def test_video_stage_is_on_demand_with_the_stt_python_and_a_one_click_runner():
|
|
import m0
|
|
assert "video_drops" in m0.ON_DEMAND and "video_drops" in m0.NEEDS_STT
|
|
bat = (Path(m0.KIT) / "run_m0_video.bat").read_text(encoding="utf-8")
|
|
assert "-Only video_drops" in bat and "-Redo" in bat
|