fix(m0): judge the final answer against the tool result, not by substring

The tools stage decides whether to keep a model or measure the next one,
so a made-up answer must never pass. Each case now has a judge for the
spoken answer: numbers must match exactly with their unit (15 or 50
minutes fail a 5-minute timer), weather answers must agree with the rain
in the result and give no other temperature, and a refusal fails when it
also says the thing was done. A new case asks for a timer with no timer
tool offered, the shape Jarvis has today. Unit tests cover the reviewer's
counterexamples and that a lying model makes the stage move on to the
next candidate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-10 15:25:29 +09:00
parent 8cb7e57cda
commit 5da88bdcbb
3 changed files with 155 additions and 47 deletions

View File

@@ -288,7 +288,7 @@ def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result():
row = tool_fix.run_case(client, "system", planner, case)
assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True
assert row["final_uses_result"] is True
assert row["final_ok"] is True
final_messages = client.sent[-1][0]
assert final_messages[-1]["role"] == "tool" and "13" in final_messages[-1]["content"]
@@ -357,7 +357,7 @@ def test_right_tool_with_a_made_up_final_answer_fails():
("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]),
("서울은 지금 24도, 맑음입니다.", []),
])
assert row["model_ok"] is True and row["final_uses_result"] is False and row["ok"] is False
assert row["model_ok"] is True and row["final_ok"] is False and row["ok"] is False
def test_claiming_an_action_no_tool_can_do_fails():
@@ -376,3 +376,60 @@ def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine()
assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다."
last = client.sent[-1][0][-1]
assert last["role"] == "tool" and "could not be run" in last["content"]
# Final-answer judging: the reviewer's counterexamples must fail, honest answers pass.
@pytest.mark.parametrize("prompt,answer,expected", [
("서울 날씨 어때?", "서울은 지금 13도이고 비가 오고 있습니다.", True),
("서울 날씨 어때?", "서울은 지금 24도, 맑음입니다.", False),
("서울 날씨 어때?", "서울은 13도이고 비가 오지 않습니다.", False),
("서울 날씨 어때?", "서울은 13도, 어제는 20도였고 비가 옵니다.", False),
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오고 있으니 우산을 챙기세요.", True),
("나 지금 서울인데 우산 챙겨야 할까?", "비가 오지 않으니 우산은 필요 없습니다.", False),
("부산 지금 비 와?", "부산은 지금 맑고 비가 오지 않아요, 17도입니다.", True),
("부산 지금 비 와?", "부산은 지금 비가 와요, 우산 챙기세요.", False),
("5분 타이머 맞춰 줘.", "5분 타이머를 맞췄습니다.", True),
("5분 타이머 맞춰 줘.", "15분 타이머를 맞췄습니다.", False),
("5분 타이머 맞춰 줘.", "50분 타이머를 맞췄습니다.", False),
("10분 뒤에 알려 줘.", "10분 뒤에 알려 드릴게요.", True),
("10분 뒤에 알려 줘.", "110분 뒤에 알려 드릴게요.", False),
("거실 불 꺼 줘.", "거실 불은 제가 끌 수 없습니다.", True),
("거실 불 꺼 줘.", "거실 불을 껐습니다.", False),
("거실 불 꺼 줘.", "거실 불을 껐습니다. 다시 켜는 기능은 지원하지 않습니다.", False),
("3분 타이머 맞춰 줘.", "저는 타이머를 맞출 수 없습니다.", True),
("3분 타이머 맞춰 줘.", "3분 타이머를 맞췄습니다.", False),
("3분 타이머 맞춰 줘.", "타이머는 지원하지 않지만 3분 뒤에 알려 드릴게요.", False),
])
def test_final_answer_judge(prompt, answer, expected):
from m0lib import tool_fix
assert tool_fix.case_for(prompt).judge(answer) is expected
def test_no_timer_tool_case_offers_no_timer_tool():
from m0lib import tool_fix
names = [t["function"]["name"] for t in tool_fix.case_for("3분 타이머 맞춰 줘.").offered()]
assert "set_timer" not in names and names
class _LyingClient:
"""Never calls a tool and always answers with a made-up weather report."""
model = "liar"
def chat(self, messages, **kwargs):
if kwargs.get("tools") is None: # planner call
return {"text": "Reply to the user.", "tool_calls": [], "first_tool": None, "ttft": 0.1, "finish": "stop"}
return {"text": "서울은 24도, 맑음입니다. 5분 타이머를 맞췄습니다.", "tool_calls": [], "first_tool": None,
"ttft": 0.1, "finish": "stop"}
def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path):
from m0lib import tool_fix
r = common.StageResult("tools_fast", tmp_path / "r.json")
summary = tool_fix.test_tools_fixed(r, _LyingClient(), tries=1)
assert not tool_fix.tools_passed(summary)
tried = []
llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999)
assert tried == [llm.CANDIDATES["fast"][0]["model"]]
assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"]