"""Behaviour of the M0 measurement kit's pure parts (scripts/m0). The kit runs once on the gaming PC and its results leave the PC, so the privacy scrub and the timing client must behave before it is handed over. """ from __future__ import annotations import json import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0")) from m0lib import common, llm, llm_bench # noqa: E402 from m0lib.llm_client import ChatClient # noqa: E402 pytestmark = pytest.mark.unit def test_scrub_masks_home_folder_and_user_name(monkeypatch): monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True)) text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x" out = common.scrub(text) assert "minsu" not in out.lower() assert "" in out and "" in out def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch): monkeypatch.setattr(common, "_TOKENS", ["ci"]) assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user " def test_written_results_are_scrubbed(tmp_path, monkeypatch): monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"]) target = tmp_path / "r.json" common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]}) data = json.loads(target.read_text(encoding="utf-8")) assert data == {"path": "/work", "who": [""]} def test_cer_ignores_spacing_and_punctuation(): assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0 assert common.cer("가나다라", "가나다마") == 0.25 class _FakeOpenAI(BaseHTTPRequestHandler): """Streams a tool call split over chunks, then text, then usage.""" def log_message(self, *a): pass def do_GET(self): body = json.dumps({"data": [{"id": "fake-model"}]}).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) request_sizes: list[int] = [] def do_POST(self): n = int(self.headers.get("Content-Length", 0)) self.rfile.read(n) _FakeOpenAI.request_sizes.append(n) chunks = [ {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]}, {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]}, {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]}, {"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]}, {"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]}, {"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}}, ] self.send_response(200) self.send_header("Content-Type", "text/event-stream") self.end_headers() for c in chunks: self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode()) self.wfile.flush() self.wfile.write(b"data: [DONE]\n\n") @pytest.fixture() def fake_server(): srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI) t = threading.Thread(target=srv.serve_forever, daemon=True) t.start() yield f"http://127.0.0.1:{srv.server_address[1]}/v1" srv.shutdown() def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server): c = ChatClient(fake_server) assert c.model == "fake-model" a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50) assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}] assert a["text"] == "네, 확인해 볼게요. 잠시만요" assert a["finish"] == "tool_calls" assert a["usage"]["completion_tokens"] == 9 # The first sentence ends at "요. ", before the whole answer is done. assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"] def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server): # The fake server always reports 12 prompt tokens. The context test must not # trust that blindly and grow each next prompt without limit (it used to # reach gigabytes and exhaust RAM). _FakeOpenAI.request_sizes.clear() largest_target = 24576 llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6) assert _FakeOpenAI.request_sizes # Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping. assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2 # --- LLM stage: a failed start must not count as measured ------------------- def _run_llm_stage(tmp_path, monkeypatch) -> dict: out = tmp_path / "results" / "llm_fast.json" out.parent.mkdir(parents=True) monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"]) llm.main() return json.loads(out.read_text(encoding="utf-8")) def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch): # A failed stage is retried on the next run; "partial" would be skipped. monkeypatch.setattr(llm.Strata, "install", lambda self: False) assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed" def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server): port = int(fake_server.rsplit(":", 1)[1].split("/")[0]) monkeypatch.setattr(llm, "PORT", port) monkeypatch.setattr(llm.Strata, "install", lambda self: True) monkeypatch.setattr(llm.Strata, "stop", lambda self: None) monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {}) monkeypatch.setattr(llm.Strata, "status", lambda self: {}) launched = [] monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True}) r = _run_llm_stage(tmp_path, monkeypatch) assert r["status"] == "failed" assert launched == [] assert "server" not in r def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server): port = int(fake_server.rsplit(":", 1)[1].split("/")[0]) monkeypatch.setattr(llm, "PORT", port) monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1) s = llm.Strata(tmp_path, tmp_path / "strata.log") monkeypatch.setattr(s, "supported_flags", lambda: set()) assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch): # Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"), # so the engine's own log must reach the results zip, which packs logs/*.log. monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6) logs = tmp_path / "logs" logs.mkdir() s = llm.Strata(tmp_path, logs / "strata_recommended.log") monkeypatch.setattr(s, "supported_flags", lambda: set()) engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch" engine_dir.mkdir(parents=True, exist_ok=True) home = str(Path.home()) (engine_dir / "q2_0-32k.engine.log").write_text( f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8") assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"] assert len(kept) == 1 text = kept[0].read_text(encoding="utf-8") assert "out of memory" in text assert home not in text