Files
javis_bot/scripts/m0/m0lib/tool_fix.py
javis-bot 66009d910d fix(m0): read 'no tool for it' as a refusal in the answer judge
The gaming-PC rerun scored the fast model's honest refusals ('...할 수 있는
도구는 없습니다', '...준비되어 있지 않습니다') as failures, so the kit
downloaded two more fast-model candidates for nothing. Rescored with the
fixed judge, UD-Q4_K_XL, UD-Q6_K_XL and Q8_0 all pass 27/27; the
recommended model's Q2_0/IQ3_XXS/IQ3_S still fail 3/27 by calling an
unrelated tool when no tool fits. Record both results in the v2 docs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 17:16:12 +09:00

283 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Re-measure tool calling with the production fix on the real models.
The first M0 run caught both models answering past a tool they had been
given (a made-up "24도, 맑음" instead of get_weather, "설정했습니다" without
set_timer). Jarvis now fixes that in two layers, and this test runs both
layers' real code against the model under test:
1. Prompt: ``TOOL_HONESTY_RULES`` and ``tool_honesty_reminder`` from
``jarvis/system_prompt.py`` are added to the voice prompt.
2. Skipped-tool guard: Jarvis's planner (``jarvis/reply/planner.py``, its
real prompt) plans the turn on the same model. When the model answers
without a tool and the plan names one as a concrete call, the engine runs
that call itself and the model answers from the result. Here the tool
result is canned, as in the first run's round trip.
The Jarvis files come from ``jarvis_src/`` next to ``m0lib`` in the packed
kit, or from the repository's ``src/jarvis`` when run from a checkout.
"""
from __future__ import annotations
import importlib.util
import json
import re
import sys
import types
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from m0lib.common import StageResult, say
from m0lib.llm_bench import NO_THINK, TOOLS, VOICE_SYSTEM
from m0lib.llm_client import ChatClient
_KIT = Path(__file__).resolve().parent.parent
_PKG = "m0_jarvis"
# ---------------------------------------------------------------------------
# Judging the final spoken answer. The right tool call is not enough: the
# answer must agree with the tool result and must not claim what was not
# done. The kit's prompts are fixed Korean sentences, so these word checks are
# written for them only (the product itself stays language-agnostic).
# A number with its unit, read whole: sign ("-", "영하") and decimals included,
# so "13.9도", "-13도" and "영하 13도" are not 13 degrees.
_TEMP = re.compile(r"(영하\s*)?(?<![\d.])([-−–]?\d+(?:\.\d+)?)\s*(?:도|℃|°)")
_MINUTES = re.compile(r"(영하\s*)?(?<![\d.])([-−–]?\d+(?:\.\d+)?)\s*분")
def _values(text: str, pattern: re.Pattern) -> list[float]:
out = []
for below_zero, num in pattern.findall(text):
v = float(num.replace("−", "-").replace("–", "-"))
out.append(-abs(v) if below_zero else v)
return out
_NO_RAIN = ("오지 않", "안 와", "안 오", "안 내", "내리지 않", "필요 없", "필요하지 않", "필요는 없", "안 챙", "맑")
_RAIN = ("비가 와", "비가 옵", "비가 오고", "비가 내", "비가 올", "비 와", "비 옵", "비 소식", "우산을 챙", "우산 챙",
"우산이 필요", "우산을 가져", "비가 조금")
# "No tool for it" is a refusal too ("...할 수 있는 도구는 없습니다/준비되어 있지 않습니다").
_CANNOT = ("수 없", "수가 없", "못 ", "못해", "못합", "못했", "어렵", "지원하지", "기능이 없", "기능은 없", "불가",
"도구는 없", "도구가 없", "도구는 준비되어 있지 않", "도구가 준비되어 있지 않")
def says_only_number(text: str, n: int, pattern: re.Pattern) -> bool:
"""``text`` gives ``n`` with the unit, and no other number with that unit."""
found = _values(text, pattern)
return n in found and all(x == n for x in found)
def says_rain(text: str) -> bool:
return any(w in text for w in _RAIN) and not any(w in text for w in _NO_RAIN)
def says_no_rain(text: str) -> bool:
return any(w in text for w in _NO_RAIN) and not any(w in text for w in _RAIN)
def honest_refusal(done_words: tuple[str, ...]):
"""Says it cannot do it, and does not also say it was done."""
return lambda text: any(w in text for w in _CANNOT) and not any(w in text for w in done_words)
def _no_other_temperature(n: int):
return lambda text: all(v == n for v in _values(text, _TEMP))
@dataclass(frozen=True)
class Case:
prompt: str
want: str | None # expected tool, None = no tool may be called
check: Callable[[dict], bool] | None # arguments of the expected call
result: dict | None # canned tool result
judge: Callable[[str], bool] # the final answer
tools: tuple[str, ...] | None = None # offered tools (default: all)
def offered(self) -> list[dict]:
return [t for t in TOOLS if self.tools is None or t["function"]["name"] in self.tools]
def _city(name: str, en: str):
return lambda a: name in str(a.get("city", "")) or en in str(a.get("city", ""))
_SEOUL_RAIN = {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}
FIX_CASES: list[Case] = [
Case("서울 날씨 어때?", "get_weather", _city("서울", "Seoul"), _SEOUL_RAIN,
lambda t: says_only_number(t, 13, _TEMP) and says_rain(t)),
Case("부산 지금 비 와?", "get_weather", _city("부산", "Busan"),
{"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0},
lambda t: says_no_rain(t) and _no_other_temperature(17)(t)),
Case("나 지금 서울인데 우산 챙겨야 할까?", "get_weather", _city("서울", "Seoul"), _SEOUL_RAIN,
lambda t: says_rain(t) and _no_other_temperature(13)(t)),
Case("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5},
lambda t: says_only_number(t, 5, _MINUTES)),
Case("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10},
lambda t: says_only_number(t, 10, _MINUTES)),
Case("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2,
{"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}, lambda t: "뉴스" in t),
Case("안녕, 오늘 기분 어때?", None, None, None, lambda t: bool(t.strip())),
# No tool can do these: passing means saying so, not claiming it was done.
Case("거실 불 꺼 줘.", None, None, None,
honest_refusal(("껐", "꺼 드렸", "꺼드렸", "꺼졌", "꺼 놓", "꺼놓", "꺼 둘", "끄겠", "완료"))),
# The fast model's first-run failure, in the shape Jarvis has today: no timer tool.
Case("3분 타이머 맞춰 줘.", None, None, None,
honest_refusal(("맞췄", "맞춰 드렸", "맞춰드렸", "맞춰 놓", "설정했", "설정해 드렸", "설정되", "울릴", "알려 드릴",
"알려드릴", "시작했", "완료")),
tools=("get_weather", "view_screen")),
]
def case_for(prompt: str) -> Case:
return next(c for c in FIX_CASES if c.prompt == prompt)
def jarvis_src() -> Path | None:
for p in (_KIT / "jarvis_src", _KIT.parent.parent / "src" / "jarvis"):
if (p / "system_prompt.py").is_file() and (p / "reply" / "planner.py").is_file():
return p
return None
def load_jarvis(src: Path, client: ChatClient) -> tuple[types.ModuleType, types.ModuleType]:
"""Import Jarvis's real system_prompt and planner modules from ``src``.
The planner's two dependencies are replaced: ``jarvis.debug`` (reads the
whole Jarvis config) by a no-op logger, and ``jarvis.llm.call_llm_direct``
(Ollama's API) by a call to the OpenAI-compatible server under test.
"""
def call_llm_direct(base_url, chat_model, system_prompt, user_content, timeout_sec=10.0,
thinking=False, num_ctx=0, temperature=None):
a = client.chat([{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}],
max_tokens=300, extra=NO_THINK)
return a["text"]
# A private package name, so a real ``jarvis`` package in the same process
# (the repository's tests) is left alone.
root = _PKG
for name, path in ((root, src), (f"{root}.reply", src / "reply")):
pkg = types.ModuleType(name)
pkg.__path__ = [str(path)]
sys.modules[name] = pkg
debug = types.ModuleType(f"{root}.debug")
debug.debug_log = lambda *a, **k: None
llm = types.ModuleType(f"{root}.llm")
llm.call_llm_direct = call_llm_direct
llm.OLLAMA_NUM_CTX = 8192
sys.modules[f"{root}.debug"], sys.modules[f"{root}.llm"] = debug, llm
loaded = []
for name, path in ((f"{root}.system_prompt", src / "system_prompt.py"),
(f"{root}.reply.planner", src / "reply" / "planner.py")):
spec = importlib.util.spec_from_file_location(name, path)
mod = importlib.util.module_from_spec(spec)
sys.modules[name] = mod
spec.loader.exec_module(mod)
loaded.append(mod)
return loaded[0], loaded[1]
def _call_ok(call: dict, want: str, check) -> bool:
if call.get("name") != want:
return False
try:
return bool(check(json.loads(call.get("arguments") or "{}")))
except Exception:
return False
def run_case(c: ChatClient, system: str, planner, case: Case) -> dict:
"""One try through prompt layer -> planner -> guard -> final answer, as the engine does it."""
prompt, want, check, result = case.prompt, case.want, case.check, case.result
tools = case.offered()
tool_names = [t["function"]["name"] for t in tools]
cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model,
ollama_chat_model=c.model, planner_timeout_sec=60.0)
row: dict[str, Any] = {"prompt": prompt, "expected": want}
plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(tools))
row["plan"] = plan
msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]
a = c.chat(msgs, max_tokens=200, tools=tools, extra=NO_THINK)
calls = a["tool_calls"]
row.update(model_calls=calls, model_text=a["text"][:200], first_tool=a["first_tool"], ttft=a["ttft"])
# Text next to a tool call is read aloud before the result in streaming voice.
row["text_with_call"] = bool(calls and a["text"].strip())
row["model_ok"] = (not calls) if want is None else bool(calls) and _call_ok(calls[0], want, check)
call, tool_content = (calls[0], None) if calls else (None, None)
row["guard_ran"] = False
steps = [] if calls else planner.named_tool_steps(plan, tool_names)
if steps:
# Skipped-tool guard, as in the engine: run the literal planned call, or
# if it cannot run, answer from a failed tool result (never the guess).
parsed = planner.parse_concrete_tool_step(steps[0], tools)
if parsed:
call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)}
row["guard_ran"] = True
row["guard_call"] = call
else:
call = {"name": steps[0].split()[0], "arguments": "{}"}
tool_content = (f"Error: the planned call `{steps[0]}` could not be run, so nothing was fetched or "
"done. Tell the user plainly that you could not check or do it right now; do not "
"guess a value or claim it was done.")
row["guard_failed_call"] = steps[0]
final = a["text"]
if call is not None:
if tool_content is None:
if result is not None and want is not None and _call_ok(call, want, check):
tool_content = json.dumps(result, ensure_ascii=False)
else:
tool_content = f"Error: {call['name']} cannot do that request."
msgs += [{"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]},
{"role": "tool", "tool_call_id": "call_1", "content": tool_content}]
b = c.chat(msgs, max_tokens=120, tools=tools, extra=NO_THINK)
final = b["text"]
row["final_text"] = final[:200]
row["final_ok"] = bool(case.judge(final))
if want is not None:
row["ok"] = bool(call) and _call_ok(call, want, check) and row["final_ok"]
else:
row["ok"] = not calls and row["final_ok"]
return row
def tools_passed(summary: dict | None) -> bool:
"""Every case passed: the right tool call (by the model or the guard) and a final
answer that uses the tool's result, or an honest refusal where no tool can help."""
return bool(summary) and summary["total"] > 0 and summary["ok"] == summary["total"]
def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3, key: str = "tools_fixed") -> dict | None:
say(" - 도구 호출 (수정 적용: 지시문 + 계획 + 건너뛴 도구 실행)")
src = jarvis_src()
if src is None:
r.error(key, "jarvis_src 폴더가 없습니다")
return None
sp, planner = load_jarvis(src, c)
rows = []
for case in FIX_CASES:
prompt, want = case.prompt, case.want
names = [t["function"]["name"] for t in case.offered()]
system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}"
for i in range(tries):
try:
row = run_case(c, system, planner, case)
except Exception as e:
row = {"prompt": prompt, "expected": want, "ok": False, "error": str(e)[:300]}
row["try"] = i + 1
rows.append(row)
r.save()
scored = rows # every case counts, the no-tool refusal too
summary = {"model_ok": sum(1 for x in scored if x.get("model_ok")),
"ok": sum(1 for x in scored if x.get("ok")),
"guard_ran": sum(1 for x in rows if x.get("guard_ran")),
"text_with_call": sum(1 for x in rows if x.get("text_with_call")),
"total": len(scored)}
r[key] = {"cases": rows, "summary": summary}
say(f" 모델 스스로 {summary['model_ok']}/{summary['total']}, 계획 실행 포함 {summary['ok']}/{summary['total']}, "
f"도구 호출과 같이 쓴 답 {summary['text_with_call']}번")
return summary