The gaming-PC rerun scored the fast model's honest refusals ('...할 수 있는
도구는 없습니다', '...준비되어 있지 않습니다') as failures, so the kit
downloaded two more fast-model candidates for nothing. Rescored with the
fixed judge, UD-Q4_K_XL, UD-Q6_K_XL and Q8_0 all pass 27/27; the
recommended model's Q2_0/IQ3_XXS/IQ3_S still fail 3/27 by calling an
unrelated tool when no tool fits. Record both results in the v2 docs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
452 lines
22 KiB
Python
452 lines
22 KiB
Python
"""Behaviour of the M0 measurement kit's pure parts (scripts/m0).
|
|
|
|
The kit runs once on the gaming PC and its results leave the PC, so the
|
|
privacy scrub and the timing client must behave before it is handed over.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import common, llm, llm_bench # noqa: E402
|
|
from m0lib.llm_client import ChatClient # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
def test_scrub_masks_home_folder_and_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", sorted({r"C:\Users\minsu", "minsu", "MINSU-PC"}, key=len, reverse=True))
|
|
text = r"model at C:\Users\minsu\JarvisM0, user minsu on MINSU-PC, json C:\\Users\\minsu\\x"
|
|
out = common.scrub(text)
|
|
assert "minsu" not in out.lower()
|
|
assert "<HOME>" in out and "<USER>" in out
|
|
|
|
|
|
def test_scrub_leaves_words_that_contain_a_short_user_name(monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["ci"])
|
|
assert common.scrub("CUDA driver version is insufficient; user ci") == "CUDA driver version is insufficient; user <USER>"
|
|
|
|
|
|
def test_written_results_are_scrubbed(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(common, "_TOKENS", ["/home/someone", "someone"])
|
|
target = tmp_path / "r.json"
|
|
common.write_json(target, {"path": "/home/someone/work", "who": ["someone"]})
|
|
data = json.loads(target.read_text(encoding="utf-8"))
|
|
assert data == {"path": "<HOME>/work", "who": ["<USER>"]}
|
|
|
|
|
|
def test_cer_ignores_spacing_and_punctuation():
|
|
assert common.cer("자비스, 오늘 날씨 어때?", "자비스 오늘 날씨 어때") == 0.0
|
|
assert common.cer("가나다라", "가나다마") == 0.25
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
"""Streams a tool call split over chunks, then text, then usage."""
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def do_GET(self):
|
|
body = json.dumps({"data": [{"id": "fake-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
request_sizes: list[int] = []
|
|
|
|
def do_POST(self):
|
|
n = int(self.headers.get("Content-Length", 0))
|
|
self.rfile.read(n)
|
|
_FakeOpenAI.request_sizes.append(n)
|
|
chunks = [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"name": "get_weather", "arguments": ""}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '{"city": '}}]}}]},
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"서울"}'}}]}}]},
|
|
{"choices": [{"delta": {"content": "네, 확인해 볼게요. "}}]},
|
|
{"choices": [{"delta": {"content": "잠시만요"}, "finish_reason": "tool_calls"}]},
|
|
{"choices": [], "usage": {"prompt_tokens": 12, "completion_tokens": 9}},
|
|
]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.end_headers()
|
|
for c in chunks:
|
|
self.wfile.write(f"data: {json.dumps(c, ensure_ascii=False)}\n\n".encode())
|
|
self.wfile.flush()
|
|
self.wfile.write(b"data: [DONE]\n\n")
|
|
|
|
|
|
@pytest.fixture()
|
|
def fake_server():
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
t.start()
|
|
yield f"http://127.0.0.1:{srv.server_address[1]}/v1"
|
|
srv.shutdown()
|
|
|
|
|
|
def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server):
|
|
c = ChatClient(fake_server)
|
|
assert c.model == "fake-model"
|
|
a = c.chat([{"role": "user", "content": "서울 날씨"}], max_tokens=50)
|
|
assert a["tool_calls"] == [{"name": "get_weather", "arguments": '{"city": "서울"}'}]
|
|
assert a["text"] == "네, 확인해 볼게요. 잠시만요"
|
|
assert a["finish"] == "tool_calls"
|
|
assert a["usage"]["completion_tokens"] == 9
|
|
# The first sentence ends at "요. ", before the whole answer is done.
|
|
assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"]
|
|
|
|
|
|
def test_context_prompts_stay_bounded_when_server_under_reports_tokens(tmp_path, fake_server):
|
|
# The fake server always reports 12 prompt tokens. The context test must not
|
|
# trust that blindly and grow each next prompt without limit (it used to
|
|
# reach gigabytes and exhaust RAM).
|
|
_FakeOpenAI.request_sizes.clear()
|
|
largest_target = 24576
|
|
llm_bench.test_context(common.StageResult("t", tmp_path / "t.json"), ChatClient(fake_server), 10**6)
|
|
assert _FakeOpenAI.request_sizes
|
|
# Generous bound: 8 characters per token, 4 UTF-8 bytes per character, x2 for JSON escaping.
|
|
assert max(_FakeOpenAI.request_sizes) < largest_target * 8 * 4 * 2
|
|
|
|
|
|
# --- LLM stage: a failed start must not count as measured -------------------
|
|
|
|
|
|
def _run_llm_stage(tmp_path, monkeypatch) -> dict:
|
|
out = tmp_path / "results" / "llm_fast.json"
|
|
out.parent.mkdir(parents=True)
|
|
monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"])
|
|
llm.main()
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch):
|
|
# A failed stage is retried on the next run; "partial" would be skipped.
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: False)
|
|
assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed"
|
|
|
|
|
|
def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm.Strata, "install", lambda self: True)
|
|
monkeypatch.setattr(llm.Strata, "stop", lambda self: None)
|
|
monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {})
|
|
monkeypatch.setattr(llm.Strata, "status", lambda self: {})
|
|
launched = []
|
|
monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True})
|
|
r = _run_llm_stage(tmp_path, monkeypatch)
|
|
assert r["status"] == "failed"
|
|
assert launched == []
|
|
assert "server" not in r
|
|
|
|
|
|
def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server):
|
|
port = int(fake_server.rsplit(":", 1)[1].split("/")[0])
|
|
monkeypatch.setattr(llm, "PORT", port)
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False
|
|
|
|
|
|
def test_failed_strata_launch_keeps_the_engine_log_for_the_zip(tmp_path, monkeypatch):
|
|
# Strata's one-line reason can be misleading (the picture reader's "no CUDA-capable device"),
|
|
# so the engine's own log must reach the results zip, which packs logs/*.log.
|
|
monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 6)
|
|
logs = tmp_path / "logs"
|
|
logs.mkdir()
|
|
s = llm.Strata(tmp_path, logs / "strata_recommended.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: set())
|
|
engine_dir = tmp_path / "strata" / "localappdata" / "Strata" / "launch"
|
|
engine_dir.mkdir(parents=True, exist_ok=True)
|
|
home = str(Path.home())
|
|
(engine_dir / "q2_0-32k.engine.log").write_text(
|
|
f"loading {home}/model.gguf\ncudaMalloc failed: out of memory\n", encoding="utf-8")
|
|
assert s.launch(llm.PRESETS["recommended"], 32768, parallel=1)["ready"] is False
|
|
kept = [p for p in logs.glob("*.log") if p.name != "strata_recommended.log"]
|
|
assert len(kept) == 1
|
|
text = kept[0].read_text(encoding="utf-8")
|
|
assert "out of memory" in text
|
|
assert home not in text
|
|
|
|
|
|
def test_recommended_launch_does_not_ask_for_the_dflash2_drafter(tmp_path, monkeypatch):
|
|
# Strata 0.1.59 picks DFlash2 alone (--draft-fallback none) for Q2_0, but the
|
|
# canonical Q2_0 pack is not a native pack, so the engine stops at start
|
|
# ("DFlash2 is off here: it needs a native pack ... the start ends here").
|
|
ran = []
|
|
monkeypatch.setattr(llm, "run_logged", lambda cmd, *a, **k: ran.append(cmd) or 1)
|
|
s = llm.Strata(tmp_path, tmp_path / "strata.log")
|
|
monkeypatch.setattr(s, "supported_flags", lambda: {"--family", "--model", "--drafting", "--parallel"})
|
|
for parallel in (1, 2):
|
|
s.launch(llm.PRESETS["recommended"], 32768, parallel=parallel)
|
|
for cmd in ran:
|
|
assert "--drafting" in cmd
|
|
assert cmd[cmd.index("--drafting") + 1] not in ("auto", "dflash2")
|
|
|
|
|
|
def test_virtual_display_structures_match_the_win32_layout():
|
|
# A wrong layout makes every display call fail on the PC, where we cannot debug it.
|
|
import ctypes
|
|
|
|
from m0lib import winvdd
|
|
assert ctypes.sizeof(winvdd.DEVMODEW) == 220
|
|
assert ctypes.sizeof(winvdd.DISPLAY_DEVICEW) == 840
|
|
if ctypes.sizeof(ctypes.c_void_p) == 8:
|
|
assert ctypes.sizeof(winvdd.SP_DEVICE_INTERFACE_DATA) == 32
|
|
assert ctypes.sizeof(winvdd.OVERLAPPED) == 32
|
|
|
|
|
|
def test_virtual_display_goes_where_it_only_touches_a_corner():
|
|
from m0lib import winvdd
|
|
# Two side-by-side 1440p monitors, the second one lower than the first.
|
|
physical = [{"rect": [0, 0, 2560, 1440]}, {"rect": [2560, 200, 5120, 1640]}]
|
|
x, y = winvdd.corner_position(physical)
|
|
assert (x, y) == (5120, 1640)
|
|
for d in physical:
|
|
left, top, right, bottom = d["rect"]
|
|
# Only a point is shared: no edge of length > 0 with any real monitor.
|
|
assert not (left <= x < right and top <= y < bottom)
|
|
|
|
|
|
def test_virtual_display_remove_sends_the_index_the_driver_expects():
|
|
from m0lib import winvdd
|
|
sent = []
|
|
vdd = object.__new__(winvdd.Vdd)
|
|
vdd.added = [1]
|
|
vdd.ioctl = lambda code, data=b"": sent.append((code, data)) or 0
|
|
vdd.remove(1)
|
|
assert sent[0] == (winvdd.IOCTL_REMOVE, b"\x00\x01")
|
|
assert vdd.added == []
|
|
|
|
|
|
def test_kit_runs_the_virtual_display_check_before_the_unattended_stages():
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert keys.index("vdisplay") < keys.index("tts_melo")
|
|
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("vdisplay")][3]).exists()
|
|
|
|
|
|
def test_virtual_display_names_are_read_from_utf16_buffers():
|
|
from m0lib import winvdd
|
|
dd = winvdd.DISPLAY_DEVICEW()
|
|
text = "Parsec Virtual Display Adapter".encode("utf-16-le")
|
|
winvdd.ctypes.memmove(winvdd.ctypes.addressof(dd) + winvdd.DISPLAY_DEVICEW.DeviceString.offset, text, len(text))
|
|
assert winvdd.wstr(dd.DeviceString) == "Parsec Virtual Display Adapter"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool-calling re-check with the Jarvis fix (m0lib/tool_fix.py)
|
|
|
|
class _ScriptedClient:
|
|
"""Answers chat calls from a script; records what was sent."""
|
|
|
|
model = "scripted"
|
|
|
|
def __init__(self, answers):
|
|
self.answers = list(answers)
|
|
self.sent = []
|
|
|
|
def chat(self, messages, **kwargs):
|
|
self.sent.append((messages, kwargs))
|
|
text, calls = self.answers.pop(0)
|
|
return {"text": text, "tool_calls": calls, "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_kit_uses_the_production_prompt_rules_and_planner():
|
|
from jarvis import system_prompt
|
|
from m0lib import tool_fix
|
|
sp, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), _ScriptedClient([]))
|
|
assert sp.TOOL_HONESTY_RULES == system_prompt.TOOL_HONESTY_RULES
|
|
assert sp.tool_honesty_reminder(["get_weather"]) == system_prompt.tool_honesty_reminder(["get_weather"])
|
|
assert planner.__file__.endswith("planner.py") and hasattr(planner, "plan_query")
|
|
# The real jarvis package stays importable next to the kit's private copy.
|
|
import jarvis.reply.planner as real_planner
|
|
assert real_planner.named_tool_steps is not planner.named_tool_steps
|
|
|
|
|
|
def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("서울 날씨 어때?")
|
|
client = _ScriptedClient([
|
|
("get_weather city='서울'\nReply to the user with the weather.", []), # planner
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("서울은 13도이고 비가 옵니다.", []), # answer from the result
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
|
|
assert row["final_ok"] is True
|
|
final_messages = client.sent[-1][0]
|
|
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
|
|
|
|
|
|
def test_model_that_calls_the_tool_is_scored_without_the_guard():
|
|
from m0lib import tool_fix
|
|
case = tool_fix.case_for("5분 타이머 맞춰 줘.")
|
|
client = _ScriptedClient([
|
|
("set_timer minutes='5'", []),
|
|
("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]),
|
|
("5분 타이머를 맞췄습니다.", []),
|
|
])
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
row = tool_fix.run_case(client, "system", planner, case)
|
|
|
|
assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True
|
|
assert row["text_with_call"] is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# "If the fix does not work, change the model": the tools stage tries the
|
|
# next model of the same family until one passes every scored case.
|
|
|
|
def _summary(ok, total=21):
|
|
return {"ok": ok, "model_ok": ok, "total": total, "guard_ran": 0, "text_with_call": 0}
|
|
|
|
|
|
def test_passing_default_tries_no_other_model(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(21), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 999)
|
|
assert tried == [] and r["tools_verdict"]["chosen"] == llm.PRESETS["fast"]["label"]
|
|
|
|
|
|
def test_failing_default_stops_at_the_first_candidate_that_passes(tmp_path):
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
results = iter([_summary(20), _summary(21)])
|
|
tried = []
|
|
llm.try_candidates(r, "fast", _summary(18), lambda p: tried.append(p["model"]) or next(results),
|
|
free_gb=lambda: 999)
|
|
assert tried == [c["model"] for c in llm.CANDIDATES["fast"][:2]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][1]["label"]
|
|
|
|
|
|
def test_candidate_without_disk_space_is_skipped_not_downloaded(tmp_path):
|
|
r = common.StageResult("tools_recommended", tmp_path / "r.json")
|
|
tried = []
|
|
llm.try_candidates(r, "recommended", _summary(18), lambda p: tried.append(p) or _summary(21), free_gb=lambda: 1)
|
|
assert tried == []
|
|
assert r["tools_verdict"]["chosen"] is None
|
|
assert all(c["skipped"] == "disk" for c in r["tools_verdict"]["candidates"])
|
|
|
|
|
|
|
|
def _run_scripted(case_prompt, answers):
|
|
from m0lib import tool_fix
|
|
client = _ScriptedClient(answers)
|
|
_, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client)
|
|
return tool_fix.run_case(client, "system", planner, tool_fix.case_for(case_prompt)), client
|
|
|
|
|
|
def test_right_tool_with_a_made_up_final_answer_fails():
|
|
# The weather tool ran and said 13 degrees; an answer of "24도, 맑음" is still a lie.
|
|
row, _ = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather city='서울'", []),
|
|
("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]),
|
|
("서울은 지금 24도, 맑음입니다.", []),
|
|
])
|
|
assert row["model_ok"] is True and row["final_ok"] is False and row["ok"] is False
|
|
|
|
|
|
def test_claiming_an_action_no_tool_can_do_fails():
|
|
claimed, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불을 껐습니다.", [])])
|
|
honest, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불은 제가 끌 수 없습니다.", [])])
|
|
assert claimed["ok"] is False and honest["ok"] is True
|
|
|
|
|
|
def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine():
|
|
row, client = _run_scripted("서울 날씨 어때?", [
|
|
("get_weather town='서울'", []), # key get_weather does not take
|
|
("서울은 24도, 맑음입니다.", []), # model skips the tool
|
|
("지금은 날씨를 확인하지 못했습니다.", []), # answer from the failed result
|
|
])
|
|
assert row["guard_ran"] is False and row["ok"] is False
|
|
assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다."
|
|
last = client.sent[-1][0][-1]
|
|
assert last["role"] == "tool" and "could not be run" in last["content"]
|
|
|
|
|
|
|
|
# Final-answer judging: the reviewer's counterexamples must fail, honest answers pass.
|
|
@pytest.mark.parametrize("prompt,answer,expected", [
|
|
("서울 날씨 어때?", "서울은 지금 13도이고 비가 오고 있습니다.", True),
|
|
("서울 날씨 어때?", "서울은 지금 24도, 맑음입니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도이고 비가 오지 않습니다.", False),
|
|
("서울 날씨 어때?", "서울은 13도, 어제는 20도였고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.9도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 -13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 영하 13도이고 비가 옵니다.", False),
|
|
("서울 날씨 어때?", "서울은 13.0도이고 비가 옵니다.", True),
|
|
("부산 지금 비 와?", "부산은 맑고 비가 오지 않아요, 17.5도입니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "영하 13도에 비가 오고 있으니 우산을 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5.5분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "-5분 타이머를 맞췄습니다.", False),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오고 있으니 우산을 챙기세요.", True),
|
|
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오지 않으니 우산은 필요 없습니다.", False),
|
|
("부산 지금 비 와?", "부산은 지금 맑고 비가 오지 않아요, 17도입니다.", True),
|
|
("부산 지금 비 와?", "부산은 지금 비가 와요, 우산 챙기세요.", False),
|
|
("5분 타이머 맞춰 줘.", "5분 타이머를 맞췄습니다.", True),
|
|
("5분 타이머 맞춰 줘.", "15분 타이머를 맞췄습니다.", False),
|
|
("5분 타이머 맞춰 줘.", "50분 타이머를 맞췄습니다.", False),
|
|
("10분 뒤에 알려 줘.", "10분 뒤에 알려 드릴게요.", True),
|
|
("10분 뒤에 알려 줘.", "110분 뒤에 알려 드릴게요.", False),
|
|
("거실 불 꺼 줘.", "거실 불은 제가 끌 수 없습니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다.", False),
|
|
("거실 불 꺼 줘.", "거실 불을 껐습니다. 다시 켜는 기능은 지원하지 않습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "저는 타이머를 맞출 수 없습니다.", True),
|
|
("3분 타이머 맞춰 줘.", "3분 타이머를 맞췄습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머는 지원하지 않지만 3분 뒤에 알려 드릴게요.", False),
|
|
# Real refusals from the gaming-PC run (2026-11-10) that name the missing tool.
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 없습니다.", True),
|
|
("거실 불 꺼 줘.", "죄송합니다만, 현재 거실 불을 제어할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 스위치를 조작하거나 스마트 홈 기기를 통해 조절해 주셔야 합니다.", True),
|
|
("3분 타이머 맞춰 줘.", "죄송합니다만, 현재 타이머를 설정할 수 있는 도구는 준비되어 있지 않습니다. "
|
|
"직접 알람을 설정해 주시거나 다른 방법을 찾아보시길 바랍니다.", True),
|
|
("거실 불 꺼 줘.", "거실 불을 끌 수 있는 도구가 없지만 대신 꺼 드렸습니다.", False),
|
|
("3분 타이머 맞춰 줘.", "타이머 도구는 없지만 3분 타이머를 맞췄습니다.", False),
|
|
])
|
|
def test_final_answer_judge(prompt, answer, expected):
|
|
from m0lib import tool_fix
|
|
assert tool_fix.case_for(prompt).judge(answer) is expected
|
|
|
|
|
|
def test_no_timer_tool_case_offers_no_timer_tool():
|
|
from m0lib import tool_fix
|
|
names = [t["function"]["name"] for t in tool_fix.case_for("3분 타이머 맞춰 줘.").offered()]
|
|
assert "set_timer" not in names and names
|
|
|
|
|
|
class _LyingClient:
|
|
"""Never calls a tool and always answers with a made-up weather report."""
|
|
|
|
model = "liar"
|
|
|
|
def chat(self, messages, **kwargs):
|
|
if kwargs.get("tools") is None: # planner call
|
|
return {"text": "Reply to the user.", "tool_calls": [], "first_tool": None, "ttft": 0.1, "finish": "stop"}
|
|
return {"text": "서울은 24도, 맑음입니다. 5분 타이머를 맞췄습니다.", "tool_calls": [], "first_tool": None,
|
|
"ttft": 0.1, "finish": "stop"}
|
|
|
|
|
|
def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path):
|
|
from m0lib import tool_fix
|
|
r = common.StageResult("tools_fast", tmp_path / "r.json")
|
|
summary = tool_fix.test_tools_fixed(r, _LyingClient(), tries=1)
|
|
assert not tool_fix.tools_passed(summary)
|
|
tried = []
|
|
llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999)
|
|
assert tried == [llm.CANDIDATES["fast"][0]["model"]]
|
|
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"]
|