A worker that did not stop in time returned its earlier "loaded" note, was killed, and the placement still counted as ok with no transcription times. finish() now reports a timeout, and a worker that timed out, died, never started or measured no ten-speaker or single-clip timing is an error, so the stage is partial and the next run retries it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
886 lines
42 KiB
Python
886 lines
42 KiB
Python
"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
|
|
|
|
The kit runs once on the gaming PC and its results leave the PC, so the
|
|
privacy scrub and the timing client must behave before it is handed over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import common, llm, llm_bench # noqa: E402
|
|
from m0lib.llm_client import ChatClient # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
|
|
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
|
|
out = common.scrub(text)
|
|
assert "minsu" not in out.lower()
|
|
assert "<HOME>" in out and "<USER>" in out
|
|
|
|
|
|
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["ci"])
|
|
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
|
|
|
|
|
|
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
|
|
target = tmp_path / "r.json"
|
|
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
|
|
|
|
|
|
def test_cer_ignores_spacing_and_punctuation():
|
|
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
|
|
assert common.cer("가나다라", "가나다마") == 0.25
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
"""Streams a tool call split over chunks, then text, then usage."""
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
request_sizes: list[int] = []
|
|
|
|
def do_POST(self):
|
|
n = int(self.headers.get("Content-Length", 0))
|
|
self.rfile.read(n)
|
|
_FakeOpenAI.request_sizes.append(n)
|
|
chunks = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
|
|
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
|
|
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
|
|
]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.end_headers()
|
|
for c in chunks:
|
|
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
|
|
self.wfile.flush()
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
|
c = ChatClient(fake_server)
|
|
assert c.model == "fake-model"
|
|
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
|
|
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
|
|
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
|
|
assert a["finish"] == "tool_calls"
|
|
assert a["usage"]["completion_tokens"] == 9
|
|
# The first sentence ends at "요. ", before the whole answer is done.
|
|
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|
|
|
|
|
|
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
|
|
# The fake server always reports 12 prompt tokens. The context test must not
|
|
# trust that blindly and grow each next prompt without limit (it used to
|
|
# reach gigabytes and exhaust RAM).
|
|
_FakeOpenAI.request_sizes.clear()
|
|
largest_target = 24576
|
|
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
|
|
assert _FakeOpenAI.request_sizes
|
|
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
|
|
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
|
|
|
|
|
|
# --- LLM stage: a failed start must not count as measured -------------------
|
|
|
|
|
|
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
|
|
out = tmp_path / "results" / "llm_fast.json"
|
|
out.parent.mkdir(parents=True)
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
|
|
# A failed stage is retried on the next run; "partial" would be skipped.
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
|
|
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
|
|
|
|
|
|
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
|
|
launched = []
|
|
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
|
|
r = _run_llm_stage(tmp_path, monkeypatch)
|
|
assert r["status"] == "failed"
|
|
assert launched == []
|
|
assert "server" not in r
|
|
|
|
|
|
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
|
|
|
|
|
|
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
|
|
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
|
|
# so the engine's own log must reach the results zip, which packs logs/*.log.
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
|
|
logs = tmp_path / "logs"
|
|
logs.mkdir()
|
|
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
|
|
engine_dir.mkdir(parents=True, exist_ok=True)
|
|
home = str(Path.home())
|
|
(engine_dir / "q2_0-32k.engine.log").write_text(
|
|
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
|
|
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
|
|
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
|
|
assert len(kept) == 1
|
|
text = kept[0].read_text(encoding="utf-8")
|
|
assert "out of memory" in text
|
|
assert home not in text
|
|
|
|
|
|
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
|
|
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
|
|
# canonical Q2_0 pack is not a native pack, so the engine stops at start
|
|
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
|
|
for parallel in (1, 2):
|
|
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
|
|
for cmd in ran:
|
|
assert "--drafting" in cmd
|
|
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
|
|
|
|
|
|
def test_virtual_display_structures_match_the_win32_layout():
|
|
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
|
|
import ctypes
|
|
|
|
from m0lib import winvdd
|
|
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
|
|
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
|
|
if ctypes.sizeof(ctypes.c_void_p) == 8:
|
|
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
|
|
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
|
|
|
|
|
|
def test_virtual_display_goes_where_it_only_touches_a_corner():
|
|
from m0lib import winvdd
|
|
# Two side-by-side 1440p monitors, the second one lower than the first.
|
|
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
|
|
x, y = winvdd.corner_position(physical)
|
|
assert (x, y) == (5120, 1640)
|
|
for d in physical:
|
|
left, top, right, bottom = d["rect"]
|
|
# Only a point is shared: no edge of length > 0 with any real monitor.
|
|
assert not (left <= x < right and top <= y < bottom)
|
|
|
|
|
|
def test_virtual_display_remove_sends_the_index_the_driver_expects():
|
|
from m0lib import winvdd
|
|
sent = []
|
|
vdd = object.__new__(winvdd.Vdd)
|
|
vdd.added = [1]
|
|
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
|
|
vdd.remove(1)
|
|
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
|
|
assert vdd.added == []
|
|
|
|
|
|
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert keys.index("vdisplay") < keys.index("tts_melo")
|
|
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
|
|
|
|
|
|
def test_virtual_display_names_are_read_from_utf16_buffers():
|
|
from m0lib import winvdd
|
|
dd = winvdd.DISPLAY_DEVICEW()
|
|
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
|
|
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
|
|
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
|
|
|
|
class _ScriptedClient:
|
|
"""Answers chat calls from a script; records what was sent."""
|
|
|
|
model = "scripted"
|
|
|
|
def __init__(self, answers):
|
|
self.answers = list(answers)
|
|
self.sent = []
|
|
|
|
def chat(self, messages, **kwargs):
|
|
self.sent.append((messages, kwargs))
|
|
text, calls = self.answers.pop(0)
|
|
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_kit_uses_the_production_prompt_rules_and_planner():
|
|
from jarvis import system_prompt
|
|
from m0lib import tool_fix
|
|
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
|
|
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
|
|
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
|
|
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
|
|
# The real jarvis package stays importable next to the kit's private copy.
|
|
import jarvis.reply.planner as real_planner
|
|
assert real_planner.named_tool_steps is not planner.named_tool_steps
|
|
|
|
|
|
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("서울 날씨 어때?")
|
|
client = _ScriptedClient([
|
|
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("서울은 13도이고 비가 옵니다.", []), # answer from the result
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
|
|
assert row["final_ok"] is True
|
|
final_messages = client.sent[-1][0]
|
|
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
|
|
|
|
|
|
def test_model_that_calls_the_tool_is_scored_without_the_guard():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("5분 타이머 맞춰 줘.")
|
|
client = _ScriptedClient([
|
|
("set_timer minutes='5'", []),
|
|
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
|
|
("5분 타이머를 맞췄습니다.", []),
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
|
|
assert row["text_with_call"] is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# "If the fix does not work, change the model": the tools stage tries the
|
|
# next model of the same family until one passes every scored case.
|
|
|
|
def _summary(ok, total=21):
|
|
return {"ok": ok, "model_ok": ok, "total": total, "guard_ran": 0, "text_with_call": 0}
|
|
|
|
|
|
def test_passing_default_tries_no_other_model(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(21), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 999)
|
|
assert tried == [] and r["tools_verdict"]["chosen"] == llm.PRESETS["fast"]["label"]
|
|
|
|
|
|
def test_failing_default_stops_at_the_first_candidate_that_passes(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
results = iter([_summary(20), _summary(21)])
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(18), lambda p: tried.append(p["model"]) or next(results),
|
|
free_gb=lambda: 999)
|
|
assert tried == [c["model"] for c in llm.CANDIDATES["fast"][:2]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][1]["label"]
|
|
|
|
|
|
def test_candidate_without_disk_space_is_skipped_not_downloaded(tmp_path):
|
|
r = common.StageResult("tools_recommended", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "recommended", _summary(18), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 1)
|
|
assert tried == []
|
|
assert r["tools_verdict"]["chosen"] is None
|
|
assert all(c["skipped"] == "disk" for c in r["tools_verdict"]["candidates"])
|
|
|
|
|
|
|
|
def _run_scripted(case_prompt, answers):
|
|
from m0lib import tool_fix
|
|
client = _ScriptedClient(answers)
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
return tool_fix.run_case(client, "system", planner, tool_fix.case_for(case_prompt)), client
|
|
|
|
|
|
def test_right_tool_with_a_made_up_final_answer_fails():
|
|
# The weather tool ran and said 13 degrees; an answer of "24도, 맑음" is still a lie.
|
|
row, _ = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather city='서울'", []),
|
|
("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]),
|
|
("서울은 지금 24도, 맑음입니다.", []),
|
|
])
|
|
assert row["model_ok"] is True and row["final_ok"] is False and row["ok"] is False
|
|
|
|
|
|
def test_claiming_an_action_no_tool_can_do_fails():
|
|
claimed, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불을 껐습니다.", [])])
|
|
honest, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불은 제가 끌 수 없습니다.", [])])
|
|
assert claimed["ok"] is False and honest["ok"] is True
|
|
|
|
|
|
def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine():
|
|
row, client = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather town='서울'", []), # key get_weather does not take
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("지금은 날씨를 확인하지 못했습니다.", []), # answer from the failed result
|
|
])
|
|
assert row["guard_ran"] is False and row["ok"] is False
|
|
assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다."
|
|
last = client.sent[-1][0][-1]
|
|
assert last["role"] == "tool" and "could not be run" in last["content"]
|
|
|
|
|
|
|
|
# Final-answer judging: the reviewer's counterexamples must fail, honest answers pass.
|
|
@pytest.mark.parametrize("prompt,answer,expected", [
|
|
("서울 날씨 어때?", "서울은 지금 13도이고 비가 오고 있습니다.", True),
|
|
("서울 날씨 어때?", "서울은 지금 24도, 맑음입니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도이고 비가 오지 않습니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도, 어제는 20도였고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.9도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 -13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 영하 13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.0도이고 비가 옵니다.", True),
|
|
("부산 지금 비 와?", "부산은 맑고 비가 오지 않아요, 17.5도입니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "영하 13도에 비가 오고 있으니 우산을 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5.5분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "-5분 타이머를 맞췄습니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오고 있으니 우산을 챙기세요.", True),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오지 않으니 우산은 필요 없습니다.", False),
|
|
("부산 지금 비 와?", "부산은 지금 맑고 비가 오지 않아요, 17도입니다.", True),
|
|
("부산 지금 비 와?", "부산은 지금 비가 와요, 우산 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5분 타이머를 맞췄습니다.", True),
|
|
("5분 타이머 맞춰 줘.", "15분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "50분 타이머를 맞췄습니다.", False),
|
|
("10분 뒤에 알려 줘.", "10분 뒤에 알려 드릴게요.", True),
|
|
("10분 뒤에 알려 줘.", "110분 뒤에 알려 드릴게요.", False),
|
|
("거실 불 꺼 줘.", "거실 불은 제가 끌 수 없습니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다.", False),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다. 다시 켜는 기능은 지원하지 않습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "저는 타이머를 맞출 수 없습니다.", True),
|
|
("3분 타이머 맞춰 줘.", "3분 타이머를 맞췄습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머는 지원하지 않지만 3분 뒤에 알려 드릴게요.", False),
|
|
# Real refusals from the gaming-PC run (2026-11-10) that name the missing tool.
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 없습니다.", True),
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 스위치를 조작하거나 스마트 홈 기기를 통해 조절해 주셔야 합니다.", True),
|
|
("3분 타이머 맞춰 줘.", "죄송합니다만, 현재 타이머를 설정할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 알람을 설정해 주시거나 다른 방법을 찾아보시길 바랍니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 끌 수 있는 도구가 없지만 대신 꺼 드렸습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머 도구는 없지만 3분 타이머를 맞췄습니다.", False),
|
|
])
|
|
def test_final_answer_judge(prompt, answer, expected):
|
|
from m0lib import tool_fix
|
|
assert tool_fix.case_for(prompt).judge(answer) is expected
|
|
|
|
|
|
def test_no_timer_tool_case_offers_no_timer_tool():
|
|
from m0lib import tool_fix
|
|
names = [t["function"]["name"] for t in tool_fix.case_for("3분 타이머 맞춰 줘.").offered()]
|
|
assert "set_timer" not in names and names
|
|
|
|
|
|
class _LyingClient:
|
|
"""Never calls a tool and always answers with a made-up weather report."""
|
|
|
|
model = "liar"
|
|
|
|
def chat(self, messages, **kwargs):
|
|
if kwargs.get("tools") is None: # planner call
|
|
return {"text": "Reply to the user.", "tool_calls": [], "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
return {"text": "서울은 24도, 맑음입니다. 5분 타이머를 맞췄습니다.", "tool_calls": [], "first_tool": None,
|
|
"ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path):
|
|
from m0lib import tool_fix
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
summary = tool_fix.test_tools_fixed(r, _LyingClient(), tries=1)
|
|
assert not tool_fix.tools_passed(summary)
|
|
tried = []
|
|
llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999)
|
|
assert tried == [llm.CANDIDATES["fast"][0]["model"]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"]
|
|
|
|
|
|
# --- gamemode stage: the one model in each placement the app switches to ----
|
|
|
|
|
|
def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True, only="levels",
|
|
extra_opts=(), stt=None):
|
|
"""Run the stage with Strata replaced; ``gpu(running)`` is nvidia-smi's (used, total), where
|
|
``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None. ``stt`` is a
|
|
_FakeStt whose loaded model adds to the used VRAM."""
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
started = []
|
|
running = {"now": None}
|
|
if stt is not None:
|
|
stt.running = running
|
|
monkeypatch.setattr(llm, "SttLoad", stt.factory)
|
|
base_gpu = gpu
|
|
|
|
def gpu(now):
|
|
used, total = base_gpu(now)
|
|
return used + stt.used_mib(), total
|
|
|
|
def launch(self, preset, context, parallel, vision="on", extra_sets=None, restart=False, timeout=0,
|
|
kv=None, mtp=None):
|
|
spec = {"parallel": parallel, "vision": vision, "sets": list(extra_sets or []), "context": context,
|
|
"kv": kv, "mtp": mtp}
|
|
started.append(spec)
|
|
if stt is not None:
|
|
stt.events.append("llm_start")
|
|
running["now"] = spec if ready(extra_sets or []) else None
|
|
return {"ready": running["now"] is not None}
|
|
|
|
def stop(self):
|
|
running["now"] = None
|
|
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "launch", launch)
|
|
monkeypatch.setattr(llm.Strata, "stop", stop)
|
|
monkeypatch.setattr(llm, "port_in_use", lambda p: running["now"] is not None)
|
|
monkeypatch.setattr(llm, "gpu_memory_mib", lambda: gpu(running["now"]))
|
|
monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["now"]))
|
|
out = tmp_path / "results" / "gamemode.json"
|
|
out.parent.mkdir(parents=True)
|
|
opts = ["--opt", "model=fast", "--opt", f"only={only}"]
|
|
for o in extra_opts:
|
|
opts += ["--opt", o]
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), *opts])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8")), started, running
|
|
|
|
|
|
def _model_mib(spec):
|
|
"""Roughly what the 2026-11-10 game PC showed: 4.4 GiB of weights that never leave the card, the
|
|
expert budget, the conversation memory (int8, 1.43 GiB per 32K; q4_0 half), about 1 GB for the
|
|
MTP drafter and 1.5 GB of buffers. With no expert budget Strata fills the card."""
|
|
experts = next((int(s.split("=")[1]) for s in spec["sets"] if s.startswith("expert-vram=")), None)
|
|
if experts is None:
|
|
return 13400
|
|
kv = 1464 * spec["context"] / 32768 * (0.5 if spec["kv"] == "q4_0" else 1)
|
|
return int(4506 + experts * 1024 + kv + (0 if spec["mtp"] == "off" else 1000) + 1500)
|
|
|
|
|
|
def _vram(running):
|
|
"""16 GB card, 2 GB used by Windows."""
|
|
return (2000 if running is None else 2000 + _model_mib(running)), 16303
|
|
|
|
|
|
def _vram_20g(running):
|
|
"""A 20 GB card, where the smaller game placements do leave the reserve."""
|
|
return _vram(running)[0], 20480
|
|
|
|
|
|
def _vram_big_floor(running):
|
|
"""A card where even the smallest placement leaves the game less than its reserve."""
|
|
return (2000 if running is None else 2000 + _model_mib(running) + 3000), 16303
|
|
|
|
|
|
def test_gamemode_measures_every_placement_and_returns_to_normal(tmp_path, monkeypatch, fake_server):
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_20g)
|
|
assert r["stage"] == "gamemode" and r["status"] == "ok"
|
|
assert [x["sets"] for x in started] == [lv["sets"] for lv in llm.LEVELS]
|
|
# Each placement starts with its own context, conversation memory format and drafter.
|
|
for spec, lv in zip(started, llm.LEVELS):
|
|
assert spec["context"] == lv.get("context", 32768)
|
|
assert (spec["kv"], spec["mtp"]) == (lv.get("kv"), lv.get("mtp"))
|
|
assert started[-1] == started[0] # the last start is the normal placement again
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
for lv in llm.LEVELS:
|
|
assert rows[lv["key"]]["switch_s"] is not None
|
|
assert rows[lv["key"]]["voice_first_sentence"]["n"] > 0
|
|
# The game placements go below the 2 GB expert budget that still left a 16 GB card only 4 GB.
|
|
game = [rows[lv["key"]] for lv in llm.LEVELS if lv["key"].startswith("game_")]
|
|
assert max(x["vram_free_mib"] for x in game) > min(x["vram_free_mib"] for x in game)
|
|
assert min(x["vram_free_mib"] for x in game) > rows["normal"]["vram_free_mib"]
|
|
# On this card a smaller placement does leave the reserve, and the choice is one that does.
|
|
assert r["game_choice"]["vram_free_mib"] >= llm.GAME_RESERVE_MIB
|
|
assert running["now"] is None # the model is not left running after the stage
|
|
|
|
|
|
def test_when_no_placement_leaves_the_reserve_the_result_says_how_much_is_free(tmp_path, monkeypatch,
|
|
fake_server):
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_big_floor)
|
|
assert r["game_choice"] is None
|
|
assert r["status"] == "ok" # every placement was measured; it is the card that is too small
|
|
most = max(x["vram_free_mib"] for x in r["levels"] if x["key"].startswith("game_"))
|
|
assert any(str(most) in n for n in r["notes"])
|
|
|
|
|
|
def test_engine_lines_are_only_those_of_the_latest_start(tmp_path):
|
|
# The engine log is appended across starts: reading it from the top repeated an old start's
|
|
# placement lines for every placement.
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
log = tmp_path / "strata" / "localappdata" / "Strata" / "launch" / "ud-q4_k_xl-32k.engine.log"
|
|
log.parent.mkdir(parents=True)
|
|
log.write_text("strata-q36: hybrid experts: 59 of 256 per layer in VRAM (4.32 GiB)\n", encoding="utf-8")
|
|
s.mark_engine_logs()
|
|
with open(log, "a", encoding="utf-8") as f:
|
|
f.write("strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)\n")
|
|
assert s.engine_lines() == ["strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)"]
|
|
# A log written again from the start (shorter than the mark) is read whole.
|
|
log.write_text("strata-q36: context 8192 cells\n", encoding="utf-8")
|
|
assert s.engine_lines() == ["strata-q36: context 8192 cells"]
|
|
|
|
|
|
def test_placement_memory_format_and_drafter_reach_the_launch_command(tmp_path, monkeypatch):
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--context", "--kv", "--mtp"})
|
|
s.launch(llm.PRESETS["fast"], 8192, parallel=1, kv="q4_0", mtp="off")
|
|
s.launch(llm.PRESETS["fast"], 32768, parallel=1)
|
|
with_opts, plain = ran
|
|
assert with_opts[with_opts.index("--kv") + 1] == "q4_0" and with_opts[with_opts.index("--mtp") + 1] == "off"
|
|
assert with_opts[with_opts.index("--context") + 1] == "8192"
|
|
assert "--kv" not in plain and "--mtp" not in plain
|
|
|
|
|
|
def test_a_placement_that_does_not_start_is_recorded_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
|
|
ready=lambda sets: "expert-vram=0" not in sets)
|
|
assert len(started) == len(llm.LEVELS)
|
|
failed = [lv["key"] for lv in llm.LEVELS if "expert-vram=0" in lv["sets"]]
|
|
row = next(x for x in r["levels"] if x["key"] == failed[0])
|
|
assert row["switch_s"] is None and "decode_tok_s" not in row
|
|
# A placement that never started is a gap in the measurement, not a pass.
|
|
assert r["status"] == "partial"
|
|
assert any(failed[0] in e["where"] for e in r["errors"])
|
|
|
|
|
|
def test_only_the_normal_placement_starting_is_not_ok(tmp_path, monkeypatch, fake_server):
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
|
|
ready=lambda sets: not any(s.startswith("expert-vram=") for s in sets))
|
|
assert r["game_choice"] is None
|
|
assert r["status"] == "partial"
|
|
game = [lv for lv in llm.LEVELS if lv["key"].startswith("game_")]
|
|
assert sum(1 for e in r["errors"] if e["where"].startswith("level game_")) == len(game)
|
|
|
|
|
|
def _row(key, free, tok_s):
|
|
return {"key": key, "vram_free_mib": free, "decode_tok_s": {"median": tok_s}, "switch_s": 50.0}
|
|
|
|
|
|
def test_game_choice_is_the_fastest_placement_that_leaves_the_reserve_free():
|
|
rows = [_row("normal", 900, 82.0), _row("game_experts_6g", 6000, 70.0),
|
|
_row("game_experts_4g", 8100, 60.0), _row("game_experts_2g", 10100, 45.0)]
|
|
choice = llm.pick_game_level(rows, reserve_mib=8000)
|
|
assert choice["key"] == "game_experts_4g" and choice["decode_tok_s"] == 60.0
|
|
|
|
|
|
def test_no_game_choice_when_no_placement_leaves_the_reserve():
|
|
rows = [_row("normal", 900, 82.0), _row("game_experts_2g", 7000, 45.0)]
|
|
assert llm.pick_game_level(rows, reserve_mib=8000) is None
|
|
|
|
|
|
def test_kit_has_a_one_click_runner_for_gamemode():
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert "gamemode" in keys
|
|
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode" in bat and "-Redo" in bat
|
|
|
|
|
|
def test_a_measurement_error_inside_a_placement_marks_the_stage_partial(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "quick_bench", lambda c: {"errors": ["decode: TimeoutError"], "decode_tok_s": {"n": 0}})
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert r["status"] == "partial"
|
|
assert any(e["error"] == "decode: TimeoutError" for e in r["errors"])
|
|
|
|
|
|
def test_gamemode_measures_the_2g_expert_placement_with_a_16k_context(tmp_path, monkeypatch, fake_server):
|
|
# The one placement between "2 GB, 32K" and "0 GB, 16K" that was never measured.
|
|
_, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert any("expert-vram=2" in x["sets"] and x["context"] == 16384 for x in started)
|
|
|
|
|
|
def test_normal_placement_answers_one_request_at_a_time(tmp_path, monkeypatch, fake_server):
|
|
# One request at a time is the default; two only when a chat channel answers every message.
|
|
_, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert started[0]["parallel"] == 1 and started[-1]["parallel"] == 1
|
|
|
|
|
|
# --- gamemode_stt / gamemode_live: the LLM next to the STT that runs with it --
|
|
|
|
|
|
STT_GPU_MIB = 2740 # M0 stt: the model 1300 MiB and 1440 MiB more for ten speakers at once
|
|
|
|
|
|
class _FakeStt:
|
|
"""Stands in for the STT worker process: on cuda its model holds VRAM while it is loaded."""
|
|
|
|
def __init__(self, fail_device=None, broken_at_load=False, finish_result=None):
|
|
self.finish_result = finish_result
|
|
self.events: list[str] = []
|
|
self.devices: list[str] = []
|
|
self.loaded: str | None = None
|
|
self.running: dict = {}
|
|
self.fail_device = fail_device
|
|
self.broken_at_load = broken_at_load
|
|
|
|
def used_mib(self) -> int:
|
|
return STT_GPU_MIB if self.loaded == "cuda" else 0
|
|
|
|
def factory(self, python, work, device, log):
|
|
fake = self
|
|
fake.devices.append(device)
|
|
|
|
class _One:
|
|
def start(self, timeout=900):
|
|
if fake.broken_at_load:
|
|
return {"status": "failed", "errors": ["load: RuntimeError: CUDA failed"]}
|
|
fake.loaded = device
|
|
fake.events.append("stt_loaded")
|
|
return {"status": "loaded", "load_s": 3.0}
|
|
|
|
def go(self):
|
|
assert fake.running.get("now") is not None, "STT load must overlap a running LLM"
|
|
fake.events.append("stt_go")
|
|
|
|
def finish(self, timeout=120):
|
|
fake.events.append("stt_finish")
|
|
if fake.finish_result is not None:
|
|
return fake.finish_result
|
|
errs = ["RuntimeError: CUDA out of memory"] if device == fake.fail_device else []
|
|
return {"status": "done", "batch10_s": {"n": 4, "median": 0.9}, "single_s": {"n": 4, "median": 0.2},
|
|
"errors": errs}
|
|
|
|
def close(self):
|
|
fake.loaded = None
|
|
|
|
return _One()
|
|
|
|
|
|
def test_together_stage_measures_the_llm_with_the_stt_loaded_and_busy(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt()
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["stage"] == "gamemode_stt" and r["status"] == "ok"
|
|
assert len(started) == len(llm.TOGETHER)
|
|
# The STT model is in VRAM before the LLM picks its placement, and its ten-speaker loop runs
|
|
# while the LLM is measured (asserted by _FakeStt.go).
|
|
for i in range(len(llm.TOGETHER)):
|
|
assert stt.events[i * 4:(i + 1) * 4] == ["stt_loaded", "llm_start", "stt_go", "stt_finish"]
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
for lv in llm.TOGETHER:
|
|
row = rows[lv["key"]]
|
|
assert row["stt"]["device"] == lv["stt"]
|
|
assert row["stt"]["batch10_s"]["n"] > 0 and row["decode_tok_s"]["n"] > 0
|
|
# What is left for the game already has the STT taken out when it runs on the GPU.
|
|
_, total = _vram(None)
|
|
assert row["vram_free_mib"] == total - row["vram_peak_mib"]
|
|
# Both places for the STT are measured in the same game placement.
|
|
game = [lv for lv in llm.TOGETHER if lv["key"].startswith("game_")]
|
|
assert {lv["stt"] for lv in game} == {"cuda", "cpu"}
|
|
pairs = [(rows[a["key"]], rows[b["key"]]) for a in game for b in game
|
|
if a["stt"] == "cuda" and b["stt"] == "cpu" and (a["sets"], a.get("context")) == (b["sets"], b.get("context"))]
|
|
assert pairs
|
|
for on_gpu, on_cpu in pairs:
|
|
assert on_cpu["vram_free_mib"] - on_gpu["vram_free_mib"] == STT_GPU_MIB
|
|
# A chat channel that answers every message runs two requests at once: measured with the STT too.
|
|
assert any(lv["parallel"] == 2 for lv in llm.TOGETHER if lv["key"].startswith("normal"))
|
|
assert stt.loaded is None and running["now"] is None # nothing is left running
|
|
|
|
|
|
def test_stt_error_next_to_the_llm_marks_the_together_stage_partial(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt(fail_device="cuda")
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["status"] == "partial"
|
|
assert any("out of memory" in e["error"] for e in r["errors"])
|
|
|
|
|
|
def test_stt_that_does_not_load_skips_that_placement_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
|
|
stt = _FakeStt(broken_at_load=True)
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert started == [] # an LLM measured without its STT would be the wrong number
|
|
assert len(stt.devices) == len(llm.TOGETHER)
|
|
assert r["status"] == "failed"
|
|
|
|
|
|
def test_together_stage_without_the_stt_environment_measures_nothing(tmp_path, monkeypatch, fake_server):
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together")
|
|
assert started == [] and r["status"] == "failed"
|
|
|
|
|
|
def test_live_stage_is_skipped_when_nobody_is_there_to_start_a_game(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "interactive", lambda: False)
|
|
stt = _FakeStt()
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["status"] == "skipped" and started == []
|
|
|
|
|
|
def test_live_stage_measures_game_placements_while_the_game_runs(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "interactive", lambda: True)
|
|
monkeypatch.setattr(llm, "ask", lambda *a, **k: "y")
|
|
monkeypatch.setattr(llm, "wait_enter", lambda *a, **k: "")
|
|
monkeypatch.setattr(llm.time, "sleep", lambda s: None)
|
|
stt = _FakeStt()
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["stage"] == "gamemode_live" and r["status"] == "ok"
|
|
# Only game placements: starting the normal one would take the game's VRAM.
|
|
assert len(started) == len(llm.LIVE) and all(lv["key"].startswith("game_") for lv in llm.LIVE)
|
|
assert {lv["stt"] for lv in llm.LIVE} == {"cuda", "cpu"}
|
|
assert all(x["vram_idle_mib"] is not None for x in r["levels"]) # Windows and the game
|
|
assert running["now"] is None and stt.loaded is None
|
|
|
|
|
|
def _fake_kit_run(tmp_path, monkeypatch, *args):
|
|
import m0
|
|
ran = []
|
|
monkeypatch.setattr(m0, "run_stage", lambda key, *a, **k: ran.append(key))
|
|
monkeypatch.setattr(m0, "pack", lambda work: tmp_path / "x.zip")
|
|
monkeypatch.setattr(m0.subprocess, "run", lambda *a, **k: None)
|
|
monkeypatch.setattr(sys, "argv", ["m0.py", "--work", str(tmp_path / "w"), *args])
|
|
m0.main()
|
|
return ran
|
|
|
|
|
|
def test_full_kit_run_leaves_out_the_stage_that_needs_a_running_game(tmp_path, monkeypatch):
|
|
ran = _fake_kit_run(tmp_path, monkeypatch)
|
|
assert "gamemode_stt" in ran and "gamemode_live" not in ran
|
|
assert _fake_kit_run(tmp_path, monkeypatch, "--only", "gamemode_live", "--redo") == ["gamemode_live"]
|
|
|
|
|
|
def test_stages_that_run_the_stt_next_to_the_llm_get_the_stt_python(tmp_path, monkeypatch):
|
|
import m0
|
|
cmds = []
|
|
monkeypatch.setattr(m0.envs, "ensure_env", lambda name, work, log: Path(f"/venvs/{name}/python"))
|
|
monkeypatch.setattr(m0.subprocess, "run", lambda cmd, **k: cmds.append(cmd) or type("P", (), {"returncode": 0}))
|
|
for key in ("gamemode_stt", "gamemode"):
|
|
m0.run_stage(key, "base", "llm.py", ["model=fast"], tmp_path, {})
|
|
together, alone = cmds
|
|
assert f"stt_python={Path('/venvs/stt/python')}" in together
|
|
assert not any(str(x).startswith("stt_python=") for x in alone)
|
|
|
|
|
|
def test_one_click_runners_cover_the_new_gamemode_stages():
|
|
import m0
|
|
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode,gamemode_stt" in bat
|
|
live = (Path(m0.KIT) / "run_m0_game_live.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode_live" in live and "-Redo" in live
|
|
|
|
|
|
def test_stt_load_loop_waits_for_go_and_runs_until_stop(tmp_path):
|
|
from m0lib import stt
|
|
go, stop = tmp_path / "go", tmp_path / "stop"
|
|
calls = []
|
|
|
|
def batch():
|
|
calls.append("b")
|
|
if not go.exists():
|
|
raise AssertionError("ran before go")
|
|
|
|
def single():
|
|
calls.append("s")
|
|
if calls.count("s") == 3:
|
|
stop.write_text("")
|
|
|
|
go.write_text("")
|
|
res = stt.load_loop(batch, single, go, stop)
|
|
assert res["status"] == "done" and res["batch10_s"]["n"] == 3 and res["single_s"]["n"] == 3
|
|
assert calls == ["b", "s"] * 3
|
|
|
|
|
|
def test_stt_load_loop_keeps_an_out_of_memory_and_stops(tmp_path):
|
|
from m0lib import stt
|
|
go, stop = tmp_path / "go", tmp_path / "stop"
|
|
go.write_text("")
|
|
|
|
def batch():
|
|
raise RuntimeError("CUDA failed with error out of memory")
|
|
|
|
res = stt.load_loop(batch, lambda: None, go, stop)
|
|
assert res["batch10_s"]["n"] == 0 and "out of memory" in res["errors"][0]
|
|
|
|
|
|
def test_stt_load_loop_does_not_run_when_stopped_before_go(tmp_path):
|
|
from m0lib import stt
|
|
stop = tmp_path / "stop"
|
|
stop.write_text("")
|
|
res = stt.load_loop(lambda: 1 / 0, lambda: 1 / 0, tmp_path / "go", stop)
|
|
assert res["status"] == "not_started" and res["errors"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("busy", [
|
|
{"status": "timeout", "errors": []}, # the worker did not stop in time
|
|
{"status": "loaded"}, # it died before writing its timings
|
|
{"status": "not_started", "batch10_s": {"n": 0}, "single_s": {"n": 0}, "errors": []},
|
|
{"status": "done", "batch10_s": {"n": 0}, "single_s": {"n": 3, "median": 0.2}, "errors": []},
|
|
])
|
|
def test_stt_load_without_timings_is_a_gap_not_a_pass(tmp_path, monkeypatch, fake_server, busy):
|
|
stt = _FakeStt(finish_result=busy)
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
|
|
extra_opts=["stt_python=py"], stt=stt)
|
|
assert r["status"] == "partial"
|
|
assert sum(1 for e in r["errors"] if e["where"].endswith(" stt")) == len(llm.TOGETHER)
|
|
|
|
|
|
def test_stt_worker_that_ignores_stop_is_reported_as_timed_out_and_killed(tmp_path):
|
|
s = llm.SttLoad(sys.executable, tmp_path, "cpu", tmp_path / "stt.log")
|
|
s.cmd = [sys.executable, "-c",
|
|
"import json,sys,time; open(sys.argv[1],'w').write(json.dumps({'status':'loaded'})); time.sleep(60)",
|
|
str(s.out)]
|
|
assert s.start(timeout=30)["status"] == "loaded"
|
|
s.go()
|
|
res = s.finish(timeout=1)
|
|
assert res["status"] == "timeout" and res["errors"]
|
|
s.close()
|
|
assert s.proc.poll() is not None
|