"""Re-measure tool calling with the production fix on the real models. The first M0 run caught both models answering past a tool they had been given (a made-up "24도, 맑음" instead of get_weather, "설정했습니다" without set_timer). Jarvis now fixes that in two layers, and this test runs both layers' real code against the model under test: 1. Prompt: ``TOOL_HONESTY_RULES`` and ``tool_honesty_reminder`` from ``jarvis/system_prompt.py`` are added to the voice prompt. 2. Skipped-tool guard: Jarvis's planner (``jarvis/reply/planner.py``, its real prompt) plans the turn on the same model. When the model answers without a tool and the plan names one as a concrete call, the engine runs that call itself and the model answers from the result. Here the tool result is canned, as in the first run's round trip. The Jarvis files come from ``jarvis_src/`` next to ``m0lib`` in the packed kit, or from the repository's ``src/jarvis`` when run from a checkout. """ from __future__ import annotations import importlib.util import json import sys import types from pathlib import Path from typing import Any from m0lib.common import StageResult, say from m0lib.llm_bench import NO_THINK, TOOLS, VOICE_SYSTEM from m0lib.llm_client import ChatClient _KIT = Path(__file__).resolve().parent.parent _PKG = "m0_jarvis" # (prompt, expected tool or None, argument check, canned tool result, check of the final answer) # The final check is what makes a case pass: the right tool call is not enough # if the spoken answer then ignores the result or claims something not done. REFUSAL = "refusal" FIX_CASES: list[tuple[str, str | None, Any, dict | None, str | None]] = [ ("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")), {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}, "13"), ("부산 지금 비 와?", "get_weather", lambda a: "부산" in str(a.get("city", "")) or "Busan" in str(a.get("city", "")), {"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0}, "17"), ("나 지금 서울인데 우산 챙겨야 할까?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")), {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}, "비"), ("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5}, "5"), ("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10}, "10"), ("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2, {"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}, "뉴스"), ("안녕, 오늘 기분 어때?", None, None, None, None), # No tool can do this: passing means saying so, not claiming it was done. ("거실 불 꺼 줘.", None, None, None, REFUSAL), ] # Words a Korean answer uses to say it cannot do something. The kit's prompts # are fixed Korean sentences, so a word check is enough here (the product # itself stays language-agnostic). _CANNOT = ("수 없", "수가 없", "못 ", "못해", "못합", "못했", "어렵", "지원하지", "기능이 없", "기능은 없", "불가") def case_for(prompt: str) -> tuple: return next(c for c in FIX_CASES if c[0] == prompt) def is_refusal(text: str) -> bool: return any(w in text for w in _CANNOT) def jarvis_src() -> Path | None: for p in (_KIT / "jarvis_src", _KIT.parent.parent / "src" / "jarvis"): if (p / "system_prompt.py").is_file() and (p / "reply" / "planner.py").is_file(): return p return None def load_jarvis(src: Path, client: ChatClient) -> tuple[types.ModuleType, types.ModuleType]: """Import Jarvis's real system_prompt and planner modules from ``src``. The planner's two dependencies are replaced: ``jarvis.debug`` (reads the whole Jarvis config) by a no-op logger, and ``jarvis.llm.call_llm_direct`` (Ollama's API) by a call to the OpenAI-compatible server under test. """ def call_llm_direct(base_url, chat_model, system_prompt, user_content, timeout_sec=10.0, thinking=False, num_ctx=0, temperature=None): a = client.chat([{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}], max_tokens=300, extra=NO_THINK) return a["text"] # A private package name, so a real ``jarvis`` package in the same process # (the repository's tests) is left alone. root = _PKG for name, path in ((root, src), (f"{root}.reply", src / "reply")): pkg = types.ModuleType(name) pkg.__path__ = [str(path)] sys.modules[name] = pkg debug = types.ModuleType(f"{root}.debug") debug.debug_log = lambda *a, **k: None llm = types.ModuleType(f"{root}.llm") llm.call_llm_direct = call_llm_direct llm.OLLAMA_NUM_CTX = 8192 sys.modules[f"{root}.debug"], sys.modules[f"{root}.llm"] = debug, llm loaded = [] for name, path in ((f"{root}.system_prompt", src / "system_prompt.py"), (f"{root}.reply.planner", src / "reply" / "planner.py")): spec = importlib.util.spec_from_file_location(name, path) mod = importlib.util.module_from_spec(spec) sys.modules[name] = mod spec.loader.exec_module(mod) loaded.append(mod) return loaded[0], loaded[1] def _call_ok(call: dict, want: str, check) -> bool: if call.get("name") != want: return False try: return bool(check(json.loads(call.get("arguments") or "{}"))) except Exception: return False def run_case(c: ChatClient, system: str, planner, case: tuple) -> dict: """One try through prompt layer -> planner -> guard -> final answer, as the engine does it.""" prompt, want, check, result, final_check = case tool_names = [t["function"]["name"] for t in TOOLS] cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model, ollama_chat_model=c.model, planner_timeout_sec=60.0) row: dict[str, Any] = {"prompt": prompt, "expected": want} plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(TOOLS)) row["plan"] = plan msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}] a = c.chat(msgs, max_tokens=200, tools=TOOLS, extra=NO_THINK) calls = a["tool_calls"] row.update(model_calls=calls, model_text=a["text"][:200], first_tool=a["first_tool"], ttft=a["ttft"]) # Text next to a tool call is read aloud before the result in streaming voice. row["text_with_call"] = bool(calls and a["text"].strip()) row["model_ok"] = (not calls) if want is None else bool(calls) and _call_ok(calls[0], want, check) call, tool_content = (calls[0], None) if calls else (None, None) row["guard_ran"] = False steps = [] if calls else planner.named_tool_steps(plan, tool_names) if steps: # Skipped-tool guard, as in the engine: run the literal planned call, or # if it cannot run, answer from a failed tool result (never the guess). parsed = planner.parse_concrete_tool_step(steps[0], TOOLS) if parsed: call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)} row["guard_ran"] = True row["guard_call"] = call else: call = {"name": steps[0].split()[0], "arguments": "{}"} tool_content = (f"Error: the planned call `{steps[0]}` could not be run, so nothing was fetched or " "done. Tell the user plainly that you could not check or do it right now; do not " "guess a value or claim it was done.") row["guard_failed_call"] = steps[0] final = a["text"] if call is not None: if tool_content is None: if result is not None and want is not None and _call_ok(call, want, check): tool_content = json.dumps(result, ensure_ascii=False) else: tool_content = f"Error: {call['name']} cannot do that request." msgs += [{"role": "assistant", "content": "", "tool_calls": [ {"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]}, {"role": "tool", "tool_call_id": "call_1", "content": tool_content}] b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK) final = b["text"] row["final_text"] = final[:200] if want is not None: row["final_uses_result"] = bool(final_check) and final_check in final row["ok"] = bool(call) and _call_ok(call, want, check) and row["final_uses_result"] elif final_check == REFUSAL: row["ok"] = not calls and is_refusal(final) else: row["ok"] = not calls return row def tools_passed(summary: dict | None) -> bool: """Every case passed: the right tool call (by the model or the guard) and a final answer that uses the tool's result, or an honest refusal where no tool can help.""" return bool(summary) and summary["total"] > 0 and summary["ok"] == summary["total"] def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3, key: str = "tools_fixed") -> dict | None: say(" - 도구 호출 (수정 적용: 지시문 + 계획 + 건너뛴 도구 실행)") src = jarvis_src() if src is None: r.error(key, "jarvis_src 폴더가 없습니다") return None sp, planner = load_jarvis(src, c) names = [t["function"]["name"] for t in TOOLS] system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}" rows = [] for case in FIX_CASES: prompt, want = case[0], case[1] for i in range(tries): try: row = run_case(c, system, planner, case) except Exception as e: row = {"prompt": prompt, "expected": want, "ok": False, "error": str(e)[:300]} row["try"] = i + 1 rows.append(row) r.save() scored = rows # every case counts, the no-tool refusal too summary = {"model_ok": sum(1 for x in scored if x.get("model_ok")), "ok": sum(1 for x in scored if x.get("ok")), "guard_ran": sum(1 for x in rows if x.get("guard_ran")), "text_with_call": sum(1 for x in rows if x.get("text_with_call")), "total": len(scored)} r[key] = {"cases": rows, "summary": summary, "system": system} say(f" 모델 스스로 {summary['model_ok']}/{summary['total']}, 계획 실행 포함 {summary['ok']}/{summary['total']}, " f"도구 호출과 같이 쓴 답 {summary['text_with_call']}번") return summary