The gaming-PC rerun scored the fast model's honest refusals ('...할 수 있는
도구는 없습니다', '...준비되어 있지 않습니다') as failures, so the kit
downloaded two more fast-model candidates for nothing. Rescored with the
fixed judge, UD-Q4_K_XL, UD-Q6_K_XL and Q8_0 all pass 27/27; the
recommended model's Q2_0/IQ3_XXS/IQ3_S still fail 3/27 by calling an
unrelated tool when no tool fits. Record both results in the v2 docs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
283 lines
14 KiB
Python
283 lines
14 KiB
Python
"""Re-measure tool calling with the production fix on the real models.
|
||
|
||
The first M0 run caught both models answering past a tool they had been
|
||
given (a made-up "24도, 맑음" instead of get_weather, "설정했습니다" without
|
||
set_timer). Jarvis now fixes that in two layers, and this test runs both
|
||
layers' real code against the model under test:
|
||
|
||
1. Prompt: ``TOOL_HONESTY_RULES`` and ``tool_honesty_reminder`` from
|
||
``jarvis/system_prompt.py`` are added to the voice prompt.
|
||
2. Skipped-tool guard: Jarvis's planner (``jarvis/reply/planner.py``, its
|
||
real prompt) plans the turn on the same model. When the model answers
|
||
without a tool and the plan names one as a concrete call, the engine runs
|
||
that call itself and the model answers from the result. Here the tool
|
||
result is canned, as in the first run's round trip.
|
||
|
||
The Jarvis files come from ``jarvis_src/`` next to ``m0lib`` in the packed
|
||
kit, or from the repository's ``src/jarvis`` when run from a checkout.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import importlib.util
|
||
import json
|
||
import re
|
||
import sys
|
||
import types
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any, Callable
|
||
|
||
from m0lib.common import StageResult, say
|
||
from m0lib.llm_bench import NO_THINK, TOOLS, VOICE_SYSTEM
|
||
from m0lib.llm_client import ChatClient
|
||
|
||
_KIT = Path(__file__).resolve().parent.parent
|
||
_PKG = "m0_jarvis"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Judging the final spoken answer. The right tool call is not enough: the
|
||
# answer must agree with the tool result and must not claim what was not
|
||
# done. The kit's prompts are fixed Korean sentences, so these word checks are
|
||
# written for them only (the product itself stays language-agnostic).
|
||
|
||
# A number with its unit, read whole: sign ("-", "영하") and decimals included,
|
||
# so "13.9도", "-13도" and "영하 13도" are not 13 degrees.
|
||
_TEMP = re.compile(r"(영하\s*)?(?<![\d.])([-−–]?\d+(?:\.\d+)?)\s*(?:도|℃|°)")
|
||
_MINUTES = re.compile(r"(영하\s*)?(?<![\d.])([-−–]?\d+(?:\.\d+)?)\s*분")
|
||
|
||
|
||
def _values(text: str, pattern: re.Pattern) -> list[float]:
|
||
out = []
|
||
for below_zero, num in pattern.findall(text):
|
||
v = float(num.replace("−", "-").replace("–", "-"))
|
||
out.append(-abs(v) if below_zero else v)
|
||
return out
|
||
_NO_RAIN = ("오지 않", "안 와", "안 오", "안 내", "내리지 않", "필요 없", "필요하지 않", "필요는 없", "안 챙", "맑")
|
||
_RAIN = ("비가 와", "비가 옵", "비가 오고", "비가 내", "비가 올", "비 와", "비 옵", "비 소식", "우산을 챙", "우산 챙",
|
||
"우산이 필요", "우산을 가져", "비가 조금")
|
||
# "No tool for it" is a refusal too ("...할 수 있는 도구는 없습니다/준비되어 있지 않습니다").
|
||
_CANNOT = ("수 없", "수가 없", "못 ", "못해", "못합", "못했", "어렵", "지원하지", "기능이 없", "기능은 없", "불가",
|
||
"도구는 없", "도구가 없", "도구는 준비되어 있지 않", "도구가 준비되어 있지 않")
|
||
|
||
|
||
def says_only_number(text: str, n: int, pattern: re.Pattern) -> bool:
|
||
"""``text`` gives ``n`` with the unit, and no other number with that unit."""
|
||
found = _values(text, pattern)
|
||
return n in found and all(x == n for x in found)
|
||
|
||
|
||
def says_rain(text: str) -> bool:
|
||
return any(w in text for w in _RAIN) and not any(w in text for w in _NO_RAIN)
|
||
|
||
|
||
def says_no_rain(text: str) -> bool:
|
||
return any(w in text for w in _NO_RAIN) and not any(w in text for w in _RAIN)
|
||
|
||
|
||
def honest_refusal(done_words: tuple[str, ...]):
|
||
"""Says it cannot do it, and does not also say it was done."""
|
||
return lambda text: any(w in text for w in _CANNOT) and not any(w in text for w in done_words)
|
||
|
||
|
||
def _no_other_temperature(n: int):
|
||
return lambda text: all(v == n for v in _values(text, _TEMP))
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Case:
|
||
prompt: str
|
||
want: str | None # expected tool, None = no tool may be called
|
||
check: Callable[[dict], bool] | None # arguments of the expected call
|
||
result: dict | None # canned tool result
|
||
judge: Callable[[str], bool] # the final answer
|
||
tools: tuple[str, ...] | None = None # offered tools (default: all)
|
||
|
||
def offered(self) -> list[dict]:
|
||
return [t for t in TOOLS if self.tools is None or t["function"]["name"] in self.tools]
|
||
|
||
|
||
def _city(name: str, en: str):
|
||
return lambda a: name in str(a.get("city", "")) or en in str(a.get("city", ""))
|
||
|
||
|
||
_SEOUL_RAIN = {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}
|
||
FIX_CASES: list[Case] = [
|
||
Case("서울 날씨 어때?", "get_weather", _city("서울", "Seoul"), _SEOUL_RAIN,
|
||
lambda t: says_only_number(t, 13, _TEMP) and says_rain(t)),
|
||
Case("부산 지금 비 와?", "get_weather", _city("부산", "Busan"),
|
||
{"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0},
|
||
lambda t: says_no_rain(t) and _no_other_temperature(17)(t)),
|
||
Case("나 지금 서울인데 우산 챙겨야 할까?", "get_weather", _city("서울", "Seoul"), _SEOUL_RAIN,
|
||
lambda t: says_rain(t) and _no_other_temperature(13)(t)),
|
||
Case("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5},
|
||
lambda t: says_only_number(t, 5, _MINUTES)),
|
||
Case("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10},
|
||
lambda t: says_only_number(t, 10, _MINUTES)),
|
||
Case("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2,
|
||
{"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}, lambda t: "뉴스" in t),
|
||
Case("안녕, 오늘 기분 어때?", None, None, None, lambda t: bool(t.strip())),
|
||
# No tool can do these: passing means saying so, not claiming it was done.
|
||
Case("거실 불 꺼 줘.", None, None, None,
|
||
honest_refusal(("껐", "꺼 드렸", "꺼드렸", "꺼졌", "꺼 놓", "꺼놓", "꺼 둘", "끄겠", "완료"))),
|
||
# The fast model's first-run failure, in the shape Jarvis has today: no timer tool.
|
||
Case("3분 타이머 맞춰 줘.", None, None, None,
|
||
honest_refusal(("맞췄", "맞춰 드렸", "맞춰드렸", "맞춰 놓", "설정했", "설정해 드렸", "설정되", "울릴", "알려 드릴",
|
||
"알려드릴", "시작했", "완료")),
|
||
tools=("get_weather", "view_screen")),
|
||
]
|
||
|
||
|
||
def case_for(prompt: str) -> Case:
|
||
return next(c for c in FIX_CASES if c.prompt == prompt)
|
||
|
||
|
||
def jarvis_src() -> Path | None:
|
||
for p in (_KIT / "jarvis_src", _KIT.parent.parent / "src" / "jarvis"):
|
||
if (p / "system_prompt.py").is_file() and (p / "reply" / "planner.py").is_file():
|
||
return p
|
||
return None
|
||
|
||
|
||
def load_jarvis(src: Path, client: ChatClient) -> tuple[types.ModuleType, types.ModuleType]:
|
||
"""Import Jarvis's real system_prompt and planner modules from ``src``.
|
||
|
||
The planner's two dependencies are replaced: ``jarvis.debug`` (reads the
|
||
whole Jarvis config) by a no-op logger, and ``jarvis.llm.call_llm_direct``
|
||
(Ollama's API) by a call to the OpenAI-compatible server under test.
|
||
"""
|
||
def call_llm_direct(base_url, chat_model, system_prompt, user_content, timeout_sec=10.0,
|
||
thinking=False, num_ctx=0, temperature=None):
|
||
a = client.chat([{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}],
|
||
max_tokens=300, extra=NO_THINK)
|
||
return a["text"]
|
||
|
||
# A private package name, so a real ``jarvis`` package in the same process
|
||
# (the repository's tests) is left alone.
|
||
root = _PKG
|
||
for name, path in ((root, src), (f"{root}.reply", src / "reply")):
|
||
pkg = types.ModuleType(name)
|
||
pkg.__path__ = [str(path)]
|
||
sys.modules[name] = pkg
|
||
debug = types.ModuleType(f"{root}.debug")
|
||
debug.debug_log = lambda *a, **k: None
|
||
llm = types.ModuleType(f"{root}.llm")
|
||
llm.call_llm_direct = call_llm_direct
|
||
llm.OLLAMA_NUM_CTX = 8192
|
||
sys.modules[f"{root}.debug"], sys.modules[f"{root}.llm"] = debug, llm
|
||
|
||
loaded = []
|
||
for name, path in ((f"{root}.system_prompt", src / "system_prompt.py"),
|
||
(f"{root}.reply.planner", src / "reply" / "planner.py")):
|
||
spec = importlib.util.spec_from_file_location(name, path)
|
||
mod = importlib.util.module_from_spec(spec)
|
||
sys.modules[name] = mod
|
||
spec.loader.exec_module(mod)
|
||
loaded.append(mod)
|
||
return loaded[0], loaded[1]
|
||
|
||
|
||
def _call_ok(call: dict, want: str, check) -> bool:
|
||
if call.get("name") != want:
|
||
return False
|
||
try:
|
||
return bool(check(json.loads(call.get("arguments") or "{}")))
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def run_case(c: ChatClient, system: str, planner, case: Case) -> dict:
|
||
"""One try through prompt layer -> planner -> guard -> final answer, as the engine does it."""
|
||
prompt, want, check, result = case.prompt, case.want, case.check, case.result
|
||
tools = case.offered()
|
||
tool_names = [t["function"]["name"] for t in tools]
|
||
cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model,
|
||
ollama_chat_model=c.model, planner_timeout_sec=60.0)
|
||
row: dict[str, Any] = {"prompt": prompt, "expected": want}
|
||
plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(tools))
|
||
row["plan"] = plan
|
||
msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]
|
||
a = c.chat(msgs, max_tokens=200, tools=tools, extra=NO_THINK)
|
||
calls = a["tool_calls"]
|
||
row.update(model_calls=calls, model_text=a["text"][:200], first_tool=a["first_tool"], ttft=a["ttft"])
|
||
# Text next to a tool call is read aloud before the result in streaming voice.
|
||
row["text_with_call"] = bool(calls and a["text"].strip())
|
||
row["model_ok"] = (not calls) if want is None else bool(calls) and _call_ok(calls[0], want, check)
|
||
|
||
call, tool_content = (calls[0], None) if calls else (None, None)
|
||
row["guard_ran"] = False
|
||
steps = [] if calls else planner.named_tool_steps(plan, tool_names)
|
||
if steps:
|
||
# Skipped-tool guard, as in the engine: run the literal planned call, or
|
||
# if it cannot run, answer from a failed tool result (never the guess).
|
||
parsed = planner.parse_concrete_tool_step(steps[0], tools)
|
||
if parsed:
|
||
call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)}
|
||
row["guard_ran"] = True
|
||
row["guard_call"] = call
|
||
else:
|
||
call = {"name": steps[0].split()[0], "arguments": "{}"}
|
||
tool_content = (f"Error: the planned call `{steps[0]}` could not be run, so nothing was fetched or "
|
||
"done. Tell the user plainly that you could not check or do it right now; do not "
|
||
"guess a value or claim it was done.")
|
||
row["guard_failed_call"] = steps[0]
|
||
|
||
final = a["text"]
|
||
if call is not None:
|
||
if tool_content is None:
|
||
if result is not None and want is not None and _call_ok(call, want, check):
|
||
tool_content = json.dumps(result, ensure_ascii=False)
|
||
else:
|
||
tool_content = f"Error: {call['name']} cannot do that request."
|
||
msgs += [{"role": "assistant", "content": "", "tool_calls": [
|
||
{"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]},
|
||
{"role": "tool", "tool_call_id": "call_1", "content": tool_content}]
|
||
b = c.chat(msgs, max_tokens=120, tools=tools, extra=NO_THINK)
|
||
final = b["text"]
|
||
row["final_text"] = final[:200]
|
||
|
||
row["final_ok"] = bool(case.judge(final))
|
||
if want is not None:
|
||
row["ok"] = bool(call) and _call_ok(call, want, check) and row["final_ok"]
|
||
else:
|
||
row["ok"] = not calls and row["final_ok"]
|
||
return row
|
||
|
||
|
||
def tools_passed(summary: dict | None) -> bool:
|
||
"""Every case passed: the right tool call (by the model or the guard) and a final
|
||
answer that uses the tool's result, or an honest refusal where no tool can help."""
|
||
return bool(summary) and summary["total"] > 0 and summary["ok"] == summary["total"]
|
||
|
||
|
||
def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3, key: str = "tools_fixed") -> dict | None:
|
||
say(" - 도구 호출 (수정 적용: 지시문 + 계획 + 건너뛴 도구 실행)")
|
||
src = jarvis_src()
|
||
if src is None:
|
||
r.error(key, "jarvis_src 폴더가 없습니다")
|
||
return None
|
||
sp, planner = load_jarvis(src, c)
|
||
rows = []
|
||
for case in FIX_CASES:
|
||
prompt, want = case.prompt, case.want
|
||
names = [t["function"]["name"] for t in case.offered()]
|
||
system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}"
|
||
for i in range(tries):
|
||
try:
|
||
row = run_case(c, system, planner, case)
|
||
except Exception as e:
|
||
row = {"prompt": prompt, "expected": want, "ok": False, "error": str(e)[:300]}
|
||
row["try"] = i + 1
|
||
rows.append(row)
|
||
r.save()
|
||
scored = rows # every case counts, the no-tool refusal too
|
||
summary = {"model_ok": sum(1 for x in scored if x.get("model_ok")),
|
||
"ok": sum(1 for x in scored if x.get("ok")),
|
||
"guard_ran": sum(1 for x in rows if x.get("guard_ran")),
|
||
"text_with_call": sum(1 for x in rows if x.get("text_with_call")),
|
||
"total": len(scored)}
|
||
r[key] = {"cases": rows, "summary": summary}
|
||
say(f" 모델 스스로 {summary['model_ok']}/{summary['total']}, 계획 실행 포함 {summary['ok']}/{summary['total']}, "
|
||
f"도구 호출과 같이 쓴 답 {summary['text_with_call']}번")
|
||
return summary
|