fix(m0): judge the final answer against the tool result, not by substring

The tools stage decides whether to keep a model or measure the next one,
so a made-up answer must never pass. Each case now has a judge for the
spoken answer: numbers must match exactly with their unit (15 or 50
minutes fail a 5-minute timer), weather answers must agree with the rain
in the result and give no other temperature, and a refusal fails when it
also says the thing was done. A new case asks for a timer with no timer
tool offered, the shape Jarvis has today. Unit tests cover the reviewer's
counterexamples and that a lying model makes the stage move on to the
next candidate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-10 15:25:29 +09:00
parent 8cb7e57cda
commit 5da88bdcbb
3 changed files with 155 additions and 47 deletions

View File

@@ -21,10 +21,12 @@ from __future__ import annotations
import importlib.util
import json
import re
import sys
import types
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, Callable
from m0lib.common import StageResult, say
from m0lib.llm_bench import NO_THINK, TOOLS, VOICE_SYSTEM
@@ -33,39 +35,89 @@ from m0lib.llm_client import ChatClient
_KIT = Path(__file__).resolve().parent.parent
_PKG = "m0_jarvis"
# (prompt, expected tool or None, argument check, canned tool result, check of the final answer)
# The final check is what makes a case pass: the right tool call is not enough
# if the spoken answer then ignores the result or claims something not done.
REFUSAL = "refusal"
FIX_CASES: list[tuple[str, str | None, Any, dict | None, str | None]] = [
("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")),
{"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}, "13"),
("부산 지금 비 와?", "get_weather", lambda a: "부산" in str(a.get("city", "")) or "Busan" in str(a.get("city", "")),
{"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0}, "17"),
("나 지금 서울인데 우산 챙겨야 할까?", "get_weather",
lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")),
{"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}, "비"),
("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5}, "5"),
("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10}, "10"),
("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2,
{"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}, "뉴스"),
("안녕, 오늘 기분 어때?", None, None, None, None),
# No tool can do this: passing means saying so, not claiming it was done.
("거실 불 꺼 줘.", None, None, None, REFUSAL),
]
# ---------------------------------------------------------------------------
# Judging the final spoken answer. The right tool call is not enough: the
# answer must agree with the tool result and must not claim what was not
# done. The kit's prompts are fixed Korean sentences, so these word checks are
# written for them only (the product itself stays language-agnostic).
# Words a Korean answer uses to say it cannot do something. The kit's prompts
# are fixed Korean sentences, so a word check is enough here (the product
# itself stays language-agnostic).
_TEMP = re.compile(r"(?<![\d.])(\d+)(?:\.\d+)?\s*(?:도|℃|°)")
_MINUTES = re.compile(r"(?<![\d.])(\d+)\s*분")
_NO_RAIN = ("오지 않", "안 와", "안 오", "안 내", "내리지 않", "필요 없", "필요하지 않", "필요는 없", "안 챙", "맑")
_RAIN = ("비가 와", "비가 옵", "비가 오고", "비가 내", "비가 올", "비 와", "비 옵", "비 소식", "우산을 챙", "우산 챙",
"우산이 필요", "우산을 가져", "비가 조금")
_CANNOT = ("수 없", "수가 없", "못 ", "못해", "못합", "못했", "어렵", "지원하지", "기능이 없", "기능은 없", "불가")
def case_for(prompt: str) -> tuple:
return next(c for c in FIX_CASES if c[0] == prompt)
def says_only_number(text: str, n: int, pattern: re.Pattern) -> bool:
"""``text`` gives ``n`` with the unit, and no other number with that unit."""
found = [int(m) for m in pattern.findall(text)]
return n in found and all(x == n for x in found)
def is_refusal(text: str) -> bool:
return any(w in text for w in _CANNOT)
def says_rain(text: str) -> bool:
return any(w in text for w in _RAIN) and not any(w in text for w in _NO_RAIN)
def says_no_rain(text: str) -> bool:
return any(w in text for w in _NO_RAIN) and not any(w in text for w in _RAIN)
def honest_refusal(done_words: tuple[str, ...]):
"""Says it cannot do it, and does not also say it was done."""
return lambda text: any(w in text for w in _CANNOT) and not any(w in text for w in done_words)
def _no_other_temperature(n: int):
return lambda text: all(int(x) == n for x in _TEMP.findall(text))
@dataclass(frozen=True)
class Case:
prompt: str
want: str | None # expected tool, None = no tool may be called
check: Callable[[dict], bool] | None # arguments of the expected call
result: dict | None # canned tool result
judge: Callable[[str], bool] # the final answer
tools: tuple[str, ...] | None = None # offered tools (default: all)
def offered(self) -> list[dict]:
return [t for t in TOOLS if self.tools is None or t["function"]["name"] in self.tools]
def _city(name: str, en: str):
return lambda a: name in str(a.get("city", "")) or en in str(a.get("city", ""))
_SEOUL_RAIN = {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}
FIX_CASES: list[Case] = [
Case("서울 날씨 어때?", "get_weather", _city("서울", "Seoul"), _SEOUL_RAIN,
lambda t: says_only_number(t, 13, _TEMP) and says_rain(t)),
Case("부산 지금 비 와?", "get_weather", _city("부산", "Busan"),
{"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0},
lambda t: says_no_rain(t) and _no_other_temperature(17)(t)),
Case("나 지금 서울인데 우산 챙겨야 할까?", "get_weather", _city("서울", "Seoul"), _SEOUL_RAIN,
lambda t: says_rain(t) and _no_other_temperature(13)(t)),
Case("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5},
lambda t: says_only_number(t, 5, _MINUTES)),
Case("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10},
lambda t: says_only_number(t, 10, _MINUTES)),
Case("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2,
{"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}, lambda t: "뉴스" in t),
Case("안녕, 오늘 기분 어때?", None, None, None, lambda t: bool(t.strip())),
# No tool can do these: passing means saying so, not claiming it was done.
Case("거실 불 꺼 줘.", None, None, None,
honest_refusal(("껐", "꺼 드렸", "꺼드렸", "꺼졌", "꺼 놓", "꺼놓", "꺼 둘", "끄겠", "완료"))),
# The fast model's first-run failure, in the shape Jarvis has today: no timer tool.
Case("3분 타이머 맞춰 줘.", None, None, None,
honest_refusal(("맞췄", "맞춰 드렸", "맞춰드렸", "맞춰 놓", "설정했", "설정해 드렸", "설정되", "울릴", "알려 드릴",
"알려드릴", "시작했", "완료")),
tools=("get_weather", "view_screen")),
]
def case_for(prompt: str) -> Case:
return next(c for c in FIX_CASES if c.prompt == prompt)
def jarvis_src() -> Path | None:
@@ -122,17 +174,18 @@ def _call_ok(call: dict, want: str, check) -> bool:
return False
def run_case(c: ChatClient, system: str, planner, case: tuple) -> dict:
def run_case(c: ChatClient, system: str, planner, case: Case) -> dict:
"""One try through prompt layer -> planner -> guard -> final answer, as the engine does it."""
prompt, want, check, result, final_check = case
tool_names = [t["function"]["name"] for t in TOOLS]
prompt, want, check, result = case.prompt, case.want, case.check, case.result
tools = case.offered()
tool_names = [t["function"]["name"] for t in tools]
cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model,
ollama_chat_model=c.model, planner_timeout_sec=60.0)
row: dict[str, Any] = {"prompt": prompt, "expected": want}
plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(TOOLS))
plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(tools))
row["plan"] = plan
msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]
a = c.chat(msgs, max_tokens=200, tools=TOOLS, extra=NO_THINK)
a = c.chat(msgs, max_tokens=200, tools=tools, extra=NO_THINK)
calls = a["tool_calls"]
row.update(model_calls=calls, model_text=a["text"][:200], first_tool=a["first_tool"], ttft=a["ttft"])
# Text next to a tool call is read aloud before the result in streaming voice.
@@ -145,7 +198,7 @@ def run_case(c: ChatClient, system: str, planner, case: tuple) -> dict:
if steps:
# Skipped-tool guard, as in the engine: run the literal planned call, or
# if it cannot run, answer from a failed tool result (never the guess).
parsed = planner.parse_concrete_tool_step(steps[0], TOOLS)
parsed = planner.parse_concrete_tool_step(steps[0], tools)
if parsed:
call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)}
row["guard_ran"] = True
@@ -167,17 +220,15 @@ def run_case(c: ChatClient, system: str, planner, case: tuple) -> dict:
msgs += [{"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]},
{"role": "tool", "tool_call_id": "call_1", "content": tool_content}]
b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK)
b = c.chat(msgs, max_tokens=120, tools=tools, extra=NO_THINK)
final = b["text"]
row["final_text"] = final[:200]
row["final_ok"] = bool(case.judge(final))
if want is not None:
row["final_uses_result"] = bool(final_check) and final_check in final
row["ok"] = bool(call) and _call_ok(call, want, check) and row["final_uses_result"]
elif final_check == REFUSAL:
row["ok"] = not calls and is_refusal(final)
row["ok"] = bool(call) and _call_ok(call, want, check) and row["final_ok"]
else:
row["ok"] = not calls
row["ok"] = not calls and row["final_ok"]
return row
@@ -194,11 +245,11 @@ def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3, key: str = "
r.error(key, "jarvis_src 폴더가 없습니다")
return None
sp, planner = load_jarvis(src, c)
names = [t["function"]["name"] for t in TOOLS]
system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}"
rows = []
for case in FIX_CASES:
prompt, want = case[0], case[1]
prompt, want = case.prompt, case.want
names = [t["function"]["name"] for t in case.offered()]
system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}"
for i in range(tries):
try:
row = run_case(c, system, planner, case)
@@ -213,7 +264,7 @@ def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3, key: str = "
"guard_ran": sum(1 for x in rows if x.get("guard_ran")),
"text_with_call": sum(1 for x in rows if x.get("text_with_call")),
"total": len(scored)}
r[key] = {"cases": rows, "summary": summary, "system": system}
r[key] = {"cases": rows, "summary": summary}
say(f" 모델 스스로 {summary['model_ok']}/{summary['total']}, 계획 실행 포함 {summary['ok']}/{summary['total']}, "
f"도구 호출과 같이 쓴 답 {summary['text_with_call']}번")
return summary