M0 on Strata caught both v2 models skipping a tool they had been given: the recommended model made up "24 degrees, sunny" instead of calling the weather tool, and the fast model said "timer set" without calling the timer tool (3 of 3 each). qwen2.5:7b does the same through the engine. - Prompt: TOOL_HONESTY_RULES in the persona (live facts only from a tool result this turn, actions only claimed after a tool reports success, no answer text next to a tool call), restated at the end of the system message by tool_honesty_reminder() with the turn's real tool names. - Engine: skipped-tool guard. When a native-tool model answers before any tool has run and the plan names a callable tool as a concrete call, the answer is dropped and the plan steps are direct-executed, then the model answers from the results. Only literal, schema-valid steps run (regex fast path, no LLM repair); otherwise the held-back answer is delivered. - Planner: the AVAILABLE TOOLS lines now carry argument names (planner_tool_catalog), so concrete steps use the tool's real keys. - Eval evals/test_tool_skip_confabulation.py; on qwen2.5:7b, 3 runs each, available-tool cases went from 0/12 to 9/12 calling the tool. - M0 kit: tools_fast / tools_recommended stages (run_m0_tools.bat) rerun tool calling before and after the fix on the real models, loading the production system_prompt.py and planner.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
309 lines
14 KiB
Python
309 lines
14 KiB
Python
"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
|
|
|
|
The kit runs once on the gaming PC and its results leave the PC, so the
|
|
privacy scrub and the timing client must behave before it is handed over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import common, llm, llm_bench # noqa: E402
|
|
from m0lib.llm_client import ChatClient # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
|
|
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
|
|
out = common.scrub(text)
|
|
assert "minsu" not in out.lower()
|
|
assert "<HOME>" in out and "<USER>" in out
|
|
|
|
|
|
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["ci"])
|
|
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
|
|
|
|
|
|
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
|
|
target = tmp_path / "r.json"
|
|
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
|
|
|
|
|
|
def test_cer_ignores_spacing_and_punctuation():
|
|
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
|
|
assert common.cer("가나다라", "가나다마") == 0.25
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
"""Streams a tool call split over chunks, then text, then usage."""
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
request_sizes: list[int] = []
|
|
|
|
def do_POST(self):
|
|
n = int(self.headers.get("Content-Length", 0))
|
|
self.rfile.read(n)
|
|
_FakeOpenAI.request_sizes.append(n)
|
|
chunks = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
|
|
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
|
|
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
|
|
]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.end_headers()
|
|
for c in chunks:
|
|
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
|
|
self.wfile.flush()
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
|
c = ChatClient(fake_server)
|
|
assert c.model == "fake-model"
|
|
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
|
|
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
|
|
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
|
|
assert a["finish"] == "tool_calls"
|
|
assert a["usage"]["completion_tokens"] == 9
|
|
# The first sentence ends at "요. ", before the whole answer is done.
|
|
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|
|
|
|
|
|
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
|
|
# The fake server always reports 12 prompt tokens. The context test must not
|
|
# trust that blindly and grow each next prompt without limit (it used to
|
|
# reach gigabytes and exhaust RAM).
|
|
_FakeOpenAI.request_sizes.clear()
|
|
largest_target = 24576
|
|
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
|
|
assert _FakeOpenAI.request_sizes
|
|
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
|
|
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
|
|
|
|
|
|
# --- LLM stage: a failed start must not count as measured -------------------
|
|
|
|
|
|
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
|
|
out = tmp_path / "results" / "llm_fast.json"
|
|
out.parent.mkdir(parents=True)
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
|
|
# A failed stage is retried on the next run; "partial" would be skipped.
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
|
|
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
|
|
|
|
|
|
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
|
|
launched = []
|
|
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
|
|
r = _run_llm_stage(tmp_path, monkeypatch)
|
|
assert r["status"] == "failed"
|
|
assert launched == []
|
|
assert "server" not in r
|
|
|
|
|
|
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
|
|
|
|
|
|
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
|
|
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
|
|
# so the engine's own log must reach the results zip, which packs logs/*.log.
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
|
|
logs = tmp_path / "logs"
|
|
logs.mkdir()
|
|
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
|
|
engine_dir.mkdir(parents=True, exist_ok=True)
|
|
home = str(Path.home())
|
|
(engine_dir / "q2_0-32k.engine.log").write_text(
|
|
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
|
|
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
|
|
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
|
|
assert len(kept) == 1
|
|
text = kept[0].read_text(encoding="utf-8")
|
|
assert "out of memory" in text
|
|
assert home not in text
|
|
|
|
|
|
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
|
|
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
|
|
# canonical Q2_0 pack is not a native pack, so the engine stops at start
|
|
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
|
|
for parallel in (1, 2):
|
|
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
|
|
for cmd in ran:
|
|
assert "--drafting" in cmd
|
|
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
|
|
|
|
|
|
def test_virtual_display_structures_match_the_win32_layout():
|
|
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
|
|
import ctypes
|
|
|
|
from m0lib import winvdd
|
|
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
|
|
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
|
|
if ctypes.sizeof(ctypes.c_void_p) == 8:
|
|
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
|
|
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
|
|
|
|
|
|
def test_virtual_display_goes_where_it_only_touches_a_corner():
|
|
from m0lib import winvdd
|
|
# Two side-by-side 1440p monitors, the second one lower than the first.
|
|
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
|
|
x, y = winvdd.corner_position(physical)
|
|
assert (x, y) == (5120, 1640)
|
|
for d in physical:
|
|
left, top, right, bottom = d["rect"]
|
|
# Only a point is shared: no edge of length > 0 with any real monitor.
|
|
assert not (left <= x < right and top <= y < bottom)
|
|
|
|
|
|
def test_virtual_display_remove_sends_the_index_the_driver_expects():
|
|
from m0lib import winvdd
|
|
sent = []
|
|
vdd = object.__new__(winvdd.Vdd)
|
|
vdd.added = [1]
|
|
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
|
|
vdd.remove(1)
|
|
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
|
|
assert vdd.added == []
|
|
|
|
|
|
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert keys.index("vdisplay") < keys.index("tts_melo")
|
|
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
|
|
|
|
|
|
def test_virtual_display_names_are_read_from_utf16_buffers():
|
|
from m0lib import winvdd
|
|
dd = winvdd.DISPLAY_DEVICEW()
|
|
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
|
|
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
|
|
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
|
|
|
|
class _ScriptedClient:
|
|
"""Answers chat calls from a script; records what was sent."""
|
|
|
|
model = "scripted"
|
|
|
|
def __init__(self, answers):
|
|
self.answers = list(answers)
|
|
self.sent = []
|
|
|
|
def chat(self, messages, **kwargs):
|
|
self.sent.append((messages, kwargs))
|
|
text, calls = self.answers.pop(0)
|
|
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_kit_uses_the_production_prompt_rules_and_planner():
|
|
from jarvis import system_prompt
|
|
from m0lib import tool_fix
|
|
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
|
|
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
|
|
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
|
|
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
|
|
# The real jarvis package stays importable next to the kit's private copy.
|
|
import jarvis.reply.planner as real_planner
|
|
assert real_planner.named_tool_steps is not planner.named_tool_steps
|
|
|
|
|
|
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
|
|
from m0lib import tool_fix
|
|
prompt, want, check, result = tool_fix.FIX_CASES[0] # 서울 날씨
|
|
client = _ScriptedClient([
|
|
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("서울은 13도이고 비가 옵니다.", []), # answer from the result
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, prompt, want, check, result)
|
|
|
|
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
|
|
assert row["final_uses_result"] is True
|
|
final_messages = client.sent[-1][0]
|
|
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
|
|
|
|
|
|
def test_model_that_calls_the_tool_is_scored_without_the_guard():
|
|
from m0lib import tool_fix
|
|
prompt, want, check, result = tool_fix.FIX_CASES[3] # 5분 타이머
|
|
client = _ScriptedClient([
|
|
("set_timer minutes='5'", []),
|
|
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
|
|
("5분 타이머를 맞췄습니다.", []),
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, prompt, want, check, result)
|
|
|
|
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
|
|
assert row["text_with_call"] is False
|