The first game PC run showed that a 2 GB expert budget still left a 16 GB card only 4.3 GB free: the weights, the 32K int8 conversation memory, the MTP drafter and buffers take about 8.7 GB on their own. The game placements now also shrink the memory format (q4_0), the context (16K, 8K) and the drafter, with half the CPU workers (no speed cost measured). The engine log is appended across starts, so each placement now reads only its own lines; when no placement leaves the reserve the result names the most free one. The gamemode runner skips the desktop stage, which is already measured. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
622 lines
31 KiB
Python
622 lines
31 KiB
Python
"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
|
|
|
|
The kit runs once on the gaming PC and its results leave the PC, so the
|
|
privacy scrub and the timing client must behave before it is handed over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import common, llm, llm_bench # noqa: E402
|
|
from m0lib.llm_client import ChatClient # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
|
|
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
|
|
out = common.scrub(text)
|
|
assert "minsu" not in out.lower()
|
|
assert "<HOME>" in out and "<USER>" in out
|
|
|
|
|
|
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["ci"])
|
|
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
|
|
|
|
|
|
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
|
|
target = tmp_path / "r.json"
|
|
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
|
|
|
|
|
|
def test_cer_ignores_spacing_and_punctuation():
|
|
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
|
|
assert common.cer("가나다라", "가나다마") == 0.25
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
"""Streams a tool call split over chunks, then text, then usage."""
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
request_sizes: list[int] = []
|
|
|
|
def do_POST(self):
|
|
n = int(self.headers.get("Content-Length", 0))
|
|
self.rfile.read(n)
|
|
_FakeOpenAI.request_sizes.append(n)
|
|
chunks = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
|
|
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
|
|
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
|
|
]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.end_headers()
|
|
for c in chunks:
|
|
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
|
|
self.wfile.flush()
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
|
c = ChatClient(fake_server)
|
|
assert c.model == "fake-model"
|
|
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
|
|
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
|
|
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
|
|
assert a["finish"] == "tool_calls"
|
|
assert a["usage"]["completion_tokens"] == 9
|
|
# The first sentence ends at "요. ", before the whole answer is done.
|
|
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|
|
|
|
|
|
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
|
|
# The fake server always reports 12 prompt tokens. The context test must not
|
|
# trust that blindly and grow each next prompt without limit (it used to
|
|
# reach gigabytes and exhaust RAM).
|
|
_FakeOpenAI.request_sizes.clear()
|
|
largest_target = 24576
|
|
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
|
|
assert _FakeOpenAI.request_sizes
|
|
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
|
|
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
|
|
|
|
|
|
# --- LLM stage: a failed start must not count as measured -------------------
|
|
|
|
|
|
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
|
|
out = tmp_path / "results" / "llm_fast.json"
|
|
out.parent.mkdir(parents=True)
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
|
|
# A failed stage is retried on the next run; "partial" would be skipped.
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
|
|
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
|
|
|
|
|
|
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
|
|
launched = []
|
|
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
|
|
r = _run_llm_stage(tmp_path, monkeypatch)
|
|
assert r["status"] == "failed"
|
|
assert launched == []
|
|
assert "server" not in r
|
|
|
|
|
|
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
|
|
|
|
|
|
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
|
|
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
|
|
# so the engine's own log must reach the results zip, which packs logs/*.log.
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
|
|
logs = tmp_path / "logs"
|
|
logs.mkdir()
|
|
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
|
|
engine_dir.mkdir(parents=True, exist_ok=True)
|
|
home = str(Path.home())
|
|
(engine_dir / "q2_0-32k.engine.log").write_text(
|
|
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
|
|
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
|
|
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
|
|
assert len(kept) == 1
|
|
text = kept[0].read_text(encoding="utf-8")
|
|
assert "out of memory" in text
|
|
assert home not in text
|
|
|
|
|
|
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
|
|
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
|
|
# canonical Q2_0 pack is not a native pack, so the engine stops at start
|
|
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
|
|
for parallel in (1, 2):
|
|
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
|
|
for cmd in ran:
|
|
assert "--drafting" in cmd
|
|
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
|
|
|
|
|
|
def test_virtual_display_structures_match_the_win32_layout():
|
|
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
|
|
import ctypes
|
|
|
|
from m0lib import winvdd
|
|
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
|
|
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
|
|
if ctypes.sizeof(ctypes.c_void_p) == 8:
|
|
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
|
|
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
|
|
|
|
|
|
def test_virtual_display_goes_where_it_only_touches_a_corner():
|
|
from m0lib import winvdd
|
|
# Two side-by-side 1440p monitors, the second one lower than the first.
|
|
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
|
|
x, y = winvdd.corner_position(physical)
|
|
assert (x, y) == (5120, 1640)
|
|
for d in physical:
|
|
left, top, right, bottom = d["rect"]
|
|
# Only a point is shared: no edge of length > 0 with any real monitor.
|
|
assert not (left <= x < right and top <= y < bottom)
|
|
|
|
|
|
def test_virtual_display_remove_sends_the_index_the_driver_expects():
|
|
from m0lib import winvdd
|
|
sent = []
|
|
vdd = object.__new__(winvdd.Vdd)
|
|
vdd.added = [1]
|
|
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
|
|
vdd.remove(1)
|
|
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
|
|
assert vdd.added == []
|
|
|
|
|
|
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert keys.index("vdisplay") < keys.index("tts_melo")
|
|
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
|
|
|
|
|
|
def test_virtual_display_names_are_read_from_utf16_buffers():
|
|
from m0lib import winvdd
|
|
dd = winvdd.DISPLAY_DEVICEW()
|
|
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
|
|
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
|
|
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
|
|
|
|
class _ScriptedClient:
|
|
"""Answers chat calls from a script; records what was sent."""
|
|
|
|
model = "scripted"
|
|
|
|
def __init__(self, answers):
|
|
self.answers = list(answers)
|
|
self.sent = []
|
|
|
|
def chat(self, messages, **kwargs):
|
|
self.sent.append((messages, kwargs))
|
|
text, calls = self.answers.pop(0)
|
|
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_kit_uses_the_production_prompt_rules_and_planner():
|
|
from jarvis import system_prompt
|
|
from m0lib import tool_fix
|
|
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
|
|
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
|
|
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
|
|
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
|
|
# The real jarvis package stays importable next to the kit's private copy.
|
|
import jarvis.reply.planner as real_planner
|
|
assert real_planner.named_tool_steps is not planner.named_tool_steps
|
|
|
|
|
|
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("서울 날씨 어때?")
|
|
client = _ScriptedClient([
|
|
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("서울은 13도이고 비가 옵니다.", []), # answer from the result
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
|
|
assert row["final_ok"] is True
|
|
final_messages = client.sent[-1][0]
|
|
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
|
|
|
|
|
|
def test_model_that_calls_the_tool_is_scored_without_the_guard():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("5분 타이머 맞춰 줘.")
|
|
client = _ScriptedClient([
|
|
("set_timer minutes='5'", []),
|
|
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
|
|
("5분 타이머를 맞췄습니다.", []),
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
|
|
assert row["text_with_call"] is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# "If the fix does not work, change the model": the tools stage tries the
|
|
# next model of the same family until one passes every scored case.
|
|
|
|
def _summary(ok, total=21):
|
|
return {"ok": ok, "model_ok": ok, "total": total, "guard_ran": 0, "text_with_call": 0}
|
|
|
|
|
|
def test_passing_default_tries_no_other_model(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(21), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 999)
|
|
assert tried == [] and r["tools_verdict"]["chosen"] == llm.PRESETS["fast"]["label"]
|
|
|
|
|
|
def test_failing_default_stops_at_the_first_candidate_that_passes(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
results = iter([_summary(20), _summary(21)])
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(18), lambda p: tried.append(p["model"]) or next(results),
|
|
free_gb=lambda: 999)
|
|
assert tried == [c["model"] for c in llm.CANDIDATES["fast"][:2]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][1]["label"]
|
|
|
|
|
|
def test_candidate_without_disk_space_is_skipped_not_downloaded(tmp_path):
|
|
r = common.StageResult("tools_recommended", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "recommended", _summary(18), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 1)
|
|
assert tried == []
|
|
assert r["tools_verdict"]["chosen"] is None
|
|
assert all(c["skipped"] == "disk" for c in r["tools_verdict"]["candidates"])
|
|
|
|
|
|
|
|
def _run_scripted(case_prompt, answers):
|
|
from m0lib import tool_fix
|
|
client = _ScriptedClient(answers)
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
return tool_fix.run_case(client, "system", planner, tool_fix.case_for(case_prompt)), client
|
|
|
|
|
|
def test_right_tool_with_a_made_up_final_answer_fails():
|
|
# The weather tool ran and said 13 degrees; an answer of "24도, 맑음" is still a lie.
|
|
row, _ = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather city='서울'", []),
|
|
("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]),
|
|
("서울은 지금 24도, 맑음입니다.", []),
|
|
])
|
|
assert row["model_ok"] is True and row["final_ok"] is False and row["ok"] is False
|
|
|
|
|
|
def test_claiming_an_action_no_tool_can_do_fails():
|
|
claimed, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불을 껐습니다.", [])])
|
|
honest, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불은 제가 끌 수 없습니다.", [])])
|
|
assert claimed["ok"] is False and honest["ok"] is True
|
|
|
|
|
|
def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine():
|
|
row, client = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather town='서울'", []), # key get_weather does not take
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("지금은 날씨를 확인하지 못했습니다.", []), # answer from the failed result
|
|
])
|
|
assert row["guard_ran"] is False and row["ok"] is False
|
|
assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다."
|
|
last = client.sent[-1][0][-1]
|
|
assert last["role"] == "tool" and "could not be run" in last["content"]
|
|
|
|
|
|
|
|
# Final-answer judging: the reviewer's counterexamples must fail, honest answers pass.
|
|
@pytest.mark.parametrize("prompt,answer,expected", [
|
|
("서울 날씨 어때?", "서울은 지금 13도이고 비가 오고 있습니다.", True),
|
|
("서울 날씨 어때?", "서울은 지금 24도, 맑음입니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도이고 비가 오지 않습니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도, 어제는 20도였고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.9도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 -13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 영하 13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.0도이고 비가 옵니다.", True),
|
|
("부산 지금 비 와?", "부산은 맑고 비가 오지 않아요, 17.5도입니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "영하 13도에 비가 오고 있으니 우산을 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5.5분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "-5분 타이머를 맞췄습니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오고 있으니 우산을 챙기세요.", True),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오지 않으니 우산은 필요 없습니다.", False),
|
|
("부산 지금 비 와?", "부산은 지금 맑고 비가 오지 않아요, 17도입니다.", True),
|
|
("부산 지금 비 와?", "부산은 지금 비가 와요, 우산 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5분 타이머를 맞췄습니다.", True),
|
|
("5분 타이머 맞춰 줘.", "15분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "50분 타이머를 맞췄습니다.", False),
|
|
("10분 뒤에 알려 줘.", "10분 뒤에 알려 드릴게요.", True),
|
|
("10분 뒤에 알려 줘.", "110분 뒤에 알려 드릴게요.", False),
|
|
("거실 불 꺼 줘.", "거실 불은 제가 끌 수 없습니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다.", False),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다. 다시 켜는 기능은 지원하지 않습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "저는 타이머를 맞출 수 없습니다.", True),
|
|
("3분 타이머 맞춰 줘.", "3분 타이머를 맞췄습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머는 지원하지 않지만 3분 뒤에 알려 드릴게요.", False),
|
|
# Real refusals from the gaming-PC run (2026-11-10) that name the missing tool.
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 없습니다.", True),
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 스위치를 조작하거나 스마트 홈 기기를 통해 조절해 주셔야 합니다.", True),
|
|
("3분 타이머 맞춰 줘.", "죄송합니다만, 현재 타이머를 설정할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 알람을 설정해 주시거나 다른 방법을 찾아보시길 바랍니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 끌 수 있는 도구가 없지만 대신 꺼 드렸습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머 도구는 없지만 3분 타이머를 맞췄습니다.", False),
|
|
])
|
|
def test_final_answer_judge(prompt, answer, expected):
|
|
from m0lib import tool_fix
|
|
assert tool_fix.case_for(prompt).judge(answer) is expected
|
|
|
|
|
|
def test_no_timer_tool_case_offers_no_timer_tool():
|
|
from m0lib import tool_fix
|
|
names = [t["function"]["name"] for t in tool_fix.case_for("3분 타이머 맞춰 줘.").offered()]
|
|
assert "set_timer" not in names and names
|
|
|
|
|
|
class _LyingClient:
|
|
"""Never calls a tool and always answers with a made-up weather report."""
|
|
|
|
model = "liar"
|
|
|
|
def chat(self, messages, **kwargs):
|
|
if kwargs.get("tools") is None: # planner call
|
|
return {"text": "Reply to the user.", "tool_calls": [], "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
return {"text": "서울은 24도, 맑음입니다. 5분 타이머를 맞췄습니다.", "tool_calls": [], "first_tool": None,
|
|
"ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path):
|
|
from m0lib import tool_fix
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
summary = tool_fix.test_tools_fixed(r, _LyingClient(), tries=1)
|
|
assert not tool_fix.tools_passed(summary)
|
|
tried = []
|
|
llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999)
|
|
assert tried == [llm.CANDIDATES["fast"][0]["model"]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"]
|
|
|
|
|
|
# --- gamemode stage: the one model in each placement the app switches to ----
|
|
|
|
|
|
def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True):
|
|
"""Run the stage with Strata replaced; ``gpu(running)`` is nvidia-smi's (used, total), where
|
|
``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None."""
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
started = []
|
|
running = {"now": None}
|
|
|
|
def launch(self, preset, context, parallel, vision="on", extra_sets=None, restart=False, timeout=0,
|
|
kv=None, mtp=None):
|
|
spec = {"parallel": parallel, "vision": vision, "sets": list(extra_sets or []), "context": context,
|
|
"kv": kv, "mtp": mtp}
|
|
started.append(spec)
|
|
running["now"] = spec if ready(extra_sets or []) else None
|
|
return {"ready": running["now"] is not None}
|
|
|
|
def stop(self):
|
|
running["now"] = None
|
|
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "launch", launch)
|
|
monkeypatch.setattr(llm.Strata, "stop", stop)
|
|
monkeypatch.setattr(llm, "port_in_use", lambda p: running["now"] is not None)
|
|
monkeypatch.setattr(llm, "gpu_memory_mib", lambda: gpu(running["now"]))
|
|
monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["now"]))
|
|
out = tmp_path / "results" / "gamemode.json"
|
|
out.parent.mkdir(parents=True)
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path),
|
|
"--opt", "model=fast", "--opt", "only=levels"])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8")), started, running
|
|
|
|
|
|
def _model_mib(spec):
|
|
"""Roughly what the 2026-11-10 game PC showed: 4.4 GiB of weights that never leave the card, the
|
|
expert budget, the conversation memory (int8, 1.43 GiB per 32K; q4_0 half), about 1 GB for the
|
|
MTP drafter and 1.5 GB of buffers. With no expert budget Strata fills the card."""
|
|
experts = next((int(s.split("=")[1]) for s in spec["sets"] if s.startswith("expert-vram=")), None)
|
|
if experts is None:
|
|
return 13400
|
|
kv = 1464 * spec["context"] / 32768 * (0.5 if spec["kv"] == "q4_0" else 1)
|
|
return int(4506 + experts * 1024 + kv + (0 if spec["mtp"] == "off" else 1000) + 1500)
|
|
|
|
|
|
def _vram(running):
|
|
"""16 GB card, 2 GB used by Windows."""
|
|
return (2000 if running is None else 2000 + _model_mib(running)), 16303
|
|
|
|
|
|
def _vram_big_floor(running):
|
|
"""A card where even the smallest placement leaves the game less than its reserve."""
|
|
return (2000 if running is None else 2000 + _model_mib(running) + 3000), 16303
|
|
|
|
|
|
def test_gamemode_measures_every_placement_and_returns_to_normal(tmp_path, monkeypatch, fake_server):
|
|
r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert r["stage"] == "gamemode" and r["status"] == "ok"
|
|
assert [x["sets"] for x in started] == [lv["sets"] for lv in llm.LEVELS]
|
|
# Each placement starts with its own context, conversation memory format and drafter.
|
|
for spec, lv in zip(started, llm.LEVELS):
|
|
assert spec["context"] == lv.get("context", 32768)
|
|
assert (spec["kv"], spec["mtp"]) == (lv.get("kv"), lv.get("mtp"))
|
|
assert started[-1] == started[0] # the last start is the normal placement again
|
|
rows = {x["key"]: x for x in r["levels"]}
|
|
for lv in llm.LEVELS:
|
|
assert rows[lv["key"]]["switch_s"] is not None
|
|
assert rows[lv["key"]]["voice_first_sentence"]["n"] > 0
|
|
# The game placements go below the 2 GB expert budget that still left a 16 GB card only 4 GB.
|
|
game = [rows[lv["key"]] for lv in llm.LEVELS if lv["key"].startswith("game_")]
|
|
assert max(x["vram_free_mib"] for x in game) > min(x["vram_free_mib"] for x in game)
|
|
assert min(x["vram_free_mib"] for x in game) > rows["normal"]["vram_free_mib"]
|
|
# On this card a smaller placement does leave the reserve, and the choice is one that does.
|
|
assert r["game_choice"]["vram_free_mib"] >= llm.GAME_RESERVE_MIB
|
|
assert running["now"] is None # the model is not left running after the stage
|
|
|
|
|
|
def test_when_no_placement_leaves_the_reserve_the_result_says_how_much_is_free(tmp_path, monkeypatch,
|
|
fake_server):
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_big_floor)
|
|
assert r["game_choice"] is None
|
|
assert r["status"] == "ok" # every placement was measured; it is the card that is too small
|
|
most = max(x["vram_free_mib"] for x in r["levels"] if x["key"].startswith("game_"))
|
|
assert any(str(most) in n for n in r["notes"])
|
|
|
|
|
|
def test_engine_lines_are_only_those_of_the_latest_start(tmp_path):
|
|
# The engine log is appended across starts: reading it from the top repeated an old start's
|
|
# placement lines for every placement.
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
log = tmp_path / "strata" / "localappdata" / "Strata" / "launch" / "ud-q4_k_xl-32k.engine.log"
|
|
log.parent.mkdir(parents=True)
|
|
log.write_text("strata-q36: hybrid experts: 59 of 256 per layer in VRAM (4.32 GiB)\n", encoding="utf-8")
|
|
s.mark_engine_logs()
|
|
with open(log, "a", encoding="utf-8") as f:
|
|
f.write("strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)\n")
|
|
assert s.engine_lines() == ["strata-q36: hybrid experts: 14 of 256 per layer in VRAM (1.02 GiB)"]
|
|
# A log written again from the start (shorter than the mark) is read whole.
|
|
log.write_text("strata-q36: context 8192 cells\n", encoding="utf-8")
|
|
assert s.engine_lines() == ["strata-q36: context 8192 cells"]
|
|
|
|
|
|
def test_placement_memory_format_and_drafter_reach_the_launch_command(tmp_path, monkeypatch):
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--context", "--kv", "--mtp"})
|
|
s.launch(llm.PRESETS["fast"], 8192, parallel=1, kv="q4_0", mtp="off")
|
|
s.launch(llm.PRESETS["fast"], 32768, parallel=1)
|
|
with_opts, plain = ran
|
|
assert with_opts[with_opts.index("--kv") + 1] == "q4_0" and with_opts[with_opts.index("--mtp") + 1] == "off"
|
|
assert with_opts[with_opts.index("--context") + 1] == "8192"
|
|
assert "--kv" not in plain and "--mtp" not in plain
|
|
|
|
|
|
def test_a_placement_that_does_not_start_is_recorded_and_the_rest_still_run(tmp_path, monkeypatch, fake_server):
|
|
r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
|
|
ready=lambda sets: "expert-vram=1" not in sets)
|
|
assert len(started) == len(llm.LEVELS)
|
|
failed = [lv["key"] for lv in llm.LEVELS if "expert-vram=1" in lv["sets"]]
|
|
row = next(x for x in r["levels"] if x["key"] == failed[0])
|
|
assert row["switch_s"] is None and "decode_tok_s" not in row
|
|
# A placement that never started is a gap in the measurement, not a pass.
|
|
assert r["status"] == "partial"
|
|
assert any(failed[0] in e["where"] for e in r["errors"])
|
|
|
|
|
|
def test_only_the_normal_placement_starting_is_not_ok(tmp_path, monkeypatch, fake_server):
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram,
|
|
ready=lambda sets: not any(s.startswith("expert-vram=") for s in sets))
|
|
assert r["game_choice"] is None
|
|
assert r["status"] == "partial"
|
|
game = [lv for lv in llm.LEVELS if lv["key"].startswith("game_")]
|
|
assert sum(1 for e in r["errors"] if e["where"].startswith("level game_")) == len(game)
|
|
|
|
|
|
def _row(key, free, tok_s):
|
|
return {"key": key, "vram_free_mib": free, "decode_tok_s": {"median": tok_s}, "switch_s": 50.0}
|
|
|
|
|
|
def test_game_choice_is_the_fastest_placement_that_leaves_the_reserve_free():
|
|
rows = [_row("normal", 900, 82.0), _row("game_experts_6g", 6000, 70.0),
|
|
_row("game_experts_4g", 8100, 60.0), _row("game_experts_2g", 10100, 45.0)]
|
|
choice = llm.pick_game_level(rows, reserve_mib=8000)
|
|
assert choice["key"] == "game_experts_4g" and choice["decode_tok_s"] == 60.0
|
|
|
|
|
|
def test_no_game_choice_when_no_placement_leaves_the_reserve():
|
|
rows = [_row("normal", 900, 82.0), _row("game_experts_2g", 7000, 45.0)]
|
|
assert llm.pick_game_level(rows, reserve_mib=8000) is None
|
|
|
|
|
|
def test_kit_has_a_one_click_runner_for_gamemode():
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert "gamemode" in keys
|
|
bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8")
|
|
assert "-Only gamemode" in bat and "-Redo" in bat
|
|
|
|
|
|
def test_a_measurement_error_inside_a_placement_marks_the_stage_partial(tmp_path, monkeypatch, fake_server):
|
|
monkeypatch.setattr(llm, "quick_bench", lambda c: {"errors": ["decode: TimeoutError"], "decode_tok_s": {"n": 0}})
|
|
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram)
|
|
assert r["status"] == "partial"
|
|
assert any(e["error"] == "decode: TimeoutError" for e in r["errors"])
|