One double-click kit (scripts/m0/run_m0.bat) that measures the gaming PC and packs the results into a single zip: - PC info (GPU, driver, RAM, disks, monitors) - Windows checks with the user: bot browser window hidden from Alt+Tab but listed in Discord's share picker and still drawing while covered; Discord pop-out stream window identification and occlusion; installed games, anti-cheat services and the borderless game detector - MeloTTS, CosyVoice2 and Fun-CosyVoice3 Korean synthesis speed - faster-whisper large-v3-turbo: final/partial latency, CER, wake-word hint, batched decoding for several speakers - Strata LLM (fast and recommended presets): TTFT by context length, decode speed, tool calls, picture reading per quality tier, Korean answers, parallel requests, VRAM give-back and restart/switch timings uv and Python, per-stage venvs, models and Strata all live inside the work folder. Results are scrubbed of user/computer names and home paths. Verified on Linux GPU (LLM client against Ollama, STT, MeloTTS, CosyVoice2/3) and on the Windows 11 CI VM (window checks, game scan, MeloTTS/CosyVoice2/STT installs, Strata install and plan). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
103 lines
4.1 KiB
Python
103 lines
4.1 KiB
Python
"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
|
|
|
|
The kit runs once on the gaming PC and its results leave the PC, so the
|
|
privacy scrub and the timing client must behave before it is handed over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import common # noqa: E402
|
|
from m0lib.llm_client import ChatClient # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
|
|
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
|
|
out = common.scrub(text)
|
|
assert "minsu" not in out.lower()
|
|
assert "<HOME>" in out and "<USER>" in out
|
|
|
|
|
|
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["ci"])
|
|
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
|
|
|
|
|
|
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
|
|
target = tmp_path / "r.json"
|
|
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
|
|
|
|
|
|
def test_cer_ignores_spacing_and_punctuation():
|
|
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
|
|
assert common.cer("가나다라", "가나다마") == 0.25
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
"""Streams a tool call split over chunks, then text, then usage."""
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_POST(self):
|
|
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
|
chunks = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
|
|
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
|
|
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
|
|
]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.end_headers()
|
|
for c in chunks:
|
|
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
|
|
self.wfile.flush()
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
|
c = ChatClient(fake_server)
|
|
assert c.model == "fake-model"
|
|
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
|
|
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
|
|
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
|
|
assert a["finish"] == "tool_calls"
|
|
assert a["usage"]["completion_tokens"] == 9
|
|
# The first sentence ends at "요. ", before the whole answer is done.
|
|
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|