Files
javis_bot/tests/test_m0_kit.py
javis-bot 903e0eb9de fix(reply): stop models answering past an available tool
M0 on Strata caught both v2 models skipping a tool they had been given:
the recommended model made up "24 degrees, sunny" instead of calling the
weather tool, and the fast model said "timer set" without calling the
timer tool (3 of 3 each). qwen2.5:7b does the same through the engine.

- Prompt: TOOL_HONESTY_RULES in the persona (live facts only from a tool
  result this turn, actions only claimed after a tool reports success, no
  answer text next to a tool call), restated at the end of the system
  message by tool_honesty_reminder() with the turn's real tool names.
- Engine: skipped-tool guard. When a native-tool model answers before any
  tool has run and the plan names a callable tool as a concrete call, the
  answer is dropped and the plan steps are direct-executed, then the model
  answers from the results. Only literal, schema-valid steps run (regex
  fast path, no LLM repair); otherwise the held-back answer is delivered.
- Planner: the AVAILABLE TOOLS lines now carry argument names
  (planner_tool_catalog), so concrete steps use the tool's real keys.
- Eval evals/test_tool_skip_confabulation.py; on qwen2.5:7b, 3 runs each,
  available-tool cases went from 0/12 to 9/12 calling the tool.
- M0 kit: tools_fast / tools_recommended stages (run_m0_tools.bat) rerun
  tool calling before and after the fix on the real models, loading the
  production system_prompt.py and planner.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 15:05:40 +09:00

309 lines
14 KiB
Python

"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
The kit runs once on the gaming PC and its results leave the PC, so the
privacy scrub and the timing client must behave before it is handed over.
"""
from __future__ import annotations
import json
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
from m0lib import common, llm, llm_bench # noqa: E402
from m0lib.llm_client import ChatClient # noqa: E402
pytestmark = pytest.mark.unit
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
out = common.scrub(text)
assert "minsu" not in out.lower()
assert "<HOME>" in out and "<USER>" in out
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
monkeypatch.setattr(common, "_TOKENS", ["ci"])
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
target = tmp_path / "r.json"
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
data = json.loads(target.read_text(encoding="utf-8"))
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
def test_cer_ignores_spacing_and_punctuation():
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
assert common.cer("가나다라", "가나다마") == 0.25
class _FakeOpenAI(BaseHTTPRequestHandler):
"""Streams a tool call split over chunks, then text, then usage."""
def log_message(self, *a):
pass
def do_GET(self):
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
request_sizes: list[int] = []
def do_POST(self):
n = int(self.headers.get("Content-Length", 0))
self.rfile.read(n)
_FakeOpenAI.request_sizes.append(n)
chunks = [
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
]
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.end_headers()
for c in chunks:
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
self.wfile.flush()
self.wfile.write(b"data: [DONE]\n\n")
@pytest.fixture()
def fake_server():
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
t = threading.Thread(target=srv.serve_forever, daemon=True)
t.start()
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
srv.shutdown()
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
c = ChatClient(fake_server)
assert c.model == "fake-model"
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
assert a["finish"] == "tool_calls"
assert a["usage"]["completion_tokens"] == 9
# The first sentence ends at "요. ", before the whole answer is done.
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
# The fake server always reports 12 prompt tokens. The context test must not
# trust that blindly and grow each next prompt without limit (it used to
# reach gigabytes and exhaust RAM).
_FakeOpenAI.request_sizes.clear()
largest_target = 24576
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
assert _FakeOpenAI.request_sizes
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
# --- LLM stage: a failed start must not count as measured -------------------
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
out = tmp_path / "results" / "llm_fast.json"
out.parent.mkdir(parents=True)
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
llm.main()
return json.loads(out.read_text(encoding="utf-8"))
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
# A failed stage is retried on the next run; "partial" would be skipped.
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
monkeypatch.setattr(llm, "PORT", port)
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
launched = []
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
r = _run_llm_stage(tmp_path, monkeypatch)
assert r["status"] == "failed"
assert launched == []
assert "server" not in r
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
monkeypatch.setattr(llm, "PORT", port)
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
s = llm.Strata(tmp_path, tmp_path / "strata.log")
monkeypatch.setattr(s, "supported_flags", lambda: set())
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
# so the engine's own log must reach the results zip, which packs logs/*.log.
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
logs = tmp_path / "logs"
logs.mkdir()
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
monkeypatch.setattr(s, "supported_flags", lambda: set())
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
engine_dir.mkdir(parents=True, exist_ok=True)
home = str(Path.home())
(engine_dir / "q2_0-32k.engine.log").write_text(
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
assert len(kept) == 1
text = kept[0].read_text(encoding="utf-8")
assert "out of memory" in text
assert home not in text
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
# canonical Q2_0 pack is not a native pack, so the engine stops at start
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
ran = []
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
s = llm.Strata(tmp_path, tmp_path / "strata.log")
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
for parallel in (1, 2):
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
for cmd in ran:
assert "--drafting" in cmd
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
def test_virtual_display_structures_match_the_win32_layout():
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
import ctypes
from m0lib import winvdd
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
if ctypes.sizeof(ctypes.c_void_p) == 8:
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
def test_virtual_display_goes_where_it_only_touches_a_corner():
from m0lib import winvdd
# Two side-by-side 1440p monitors, the second one lower than the first.
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
x, y = winvdd.corner_position(physical)
assert (x, y) == (5120, 1640)
for d in physical:
left, top, right, bottom = d["rect"]
# Only a point is shared: no edge of length > 0 with any real monitor.
assert not (left <= x < right and top <= y < bottom)
def test_virtual_display_remove_sends_the_index_the_driver_expects():
from m0lib import winvdd
sent = []
vdd = object.__new__(winvdd.Vdd)
vdd.added = [1]
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
vdd.remove(1)
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
assert vdd.added == []
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
import m0
keys = [s[0] for s in m0.STAGES]
assert keys.index("vdisplay") < keys.index("tts_melo")
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
def test_virtual_display_names_are_read_from_utf16_buffers():
from m0lib import winvdd
dd = winvdd.DISPLAY_DEVICEW()
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
# ---------------------------------------------------------------------------
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
class _ScriptedClient:
"""Answers chat calls from a script; records what was sent."""
model = "scripted"
def __init__(self, answers):
self.answers = list(answers)
self.sent = []
def chat(self, messages, **kwargs):
self.sent.append((messages, kwargs))
text, calls = self.answers.pop(0)
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
def test_kit_uses_the_production_prompt_rules_and_planner():
from jarvis import system_prompt
from m0lib import tool_fix
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
# The real jarvis package stays importable next to the kit's private copy.
import jarvis.reply.planner as real_planner
assert real_planner.named_tool_steps is not planner.named_tool_steps
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
from m0lib import tool_fix
prompt, want, check, result = tool_fix.FIX_CASES[0] # 서울 날씨
client = _ScriptedClient([
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
("서울은 24도, 맑음입니다.", []), # model skips the tool
("서울은 13도이고 비가 옵니다.", []), # answer from the result
])
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
row = tool_fix.run_case(client, "system", planner, prompt, want, check, result)
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
assert row["final_uses_result"] is True
final_messages = client.sent[-1][0]
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
def test_model_that_calls_the_tool_is_scored_without_the_guard():
from m0lib import tool_fix
prompt, want, check, result = tool_fix.FIX_CASES[3] # 5분 타이머
client = _ScriptedClient([
("set_timer minutes='5'", []),
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
("5분 타이머를 맞췄습니다.", []),
])
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
row = tool_fix.run_case(client, "system", planner, prompt, want, check, result)
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
assert row["text_with_call"] is False