Files
javis_bot/scripts/m0/m0lib/tool_fix.py
javis-bot 903e0eb9de fix(reply): stop models answering past an available tool
M0 on Strata caught both v2 models skipping a tool they had been given:
the recommended model made up "24 degrees, sunny" instead of calling the
weather tool, and the fast model said "timer set" without calling the
timer tool (3 of 3 each). qwen2.5:7b does the same through the engine.

- Prompt: TOOL_HONESTY_RULES in the persona (live facts only from a tool
  result this turn, actions only claimed after a tool reports success, no
  answer text next to a tool call), restated at the end of the system
  message by tool_honesty_reminder() with the turn's real tool names.
- Engine: skipped-tool guard. When a native-tool model answers before any
  tool has run and the plan names a callable tool as a concrete call, the
  answer is dropped and the plan steps are direct-executed, then the model
  answers from the results. Only literal, schema-valid steps run (regex
  fast path, no LLM repair); otherwise the held-back answer is delivered.
- Planner: the AVAILABLE TOOLS lines now carry argument names
  (planner_tool_catalog), so concrete steps use the tool's real keys.
- Eval evals/test_tool_skip_confabulation.py; on qwen2.5:7b, 3 runs each,
  available-tool cases went from 0/12 to 9/12 calling the tool.
- M0 kit: tools_fast / tools_recommended stages (run_m0_tools.bat) rerun
  tool calling before and after the fix on the real models, loading the
  production system_prompt.py and planner.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 15:05:40 +09:00

183 lines
8.9 KiB
Python

"""Re-measure tool calling with the production fix on the real models.
The first M0 run caught both models answering past a tool they had been
given (a made-up "24도, 맑음" instead of get_weather, "설정했습니다" without
set_timer). Jarvis now fixes that in two layers, and this test runs both
layers' real code against the model under test:
1. Prompt: ``TOOL_HONESTY_RULES`` and ``tool_honesty_reminder`` from
``jarvis/system_prompt.py`` are added to the voice prompt.
2. Skipped-tool guard: Jarvis's planner (``jarvis/reply/planner.py``, its
real prompt) plans the turn on the same model. When the model answers
without a tool and the plan names one as a concrete call, the engine runs
that call itself and the model answers from the result. Here the tool
result is canned, as in the first run's round trip.
The Jarvis files come from ``jarvis_src/`` next to ``m0lib`` in the packed
kit, or from the repository's ``src/jarvis`` when run from a checkout.
"""
from __future__ import annotations
import importlib.util
import json
import sys
import types
from pathlib import Path
from typing import Any
from m0lib.common import StageResult, say
from m0lib.llm_bench import NO_THINK, TOOLS, VOICE_SYSTEM
from m0lib.llm_client import ChatClient
_KIT = Path(__file__).resolve().parent.parent
_PKG = "m0_jarvis"
# (prompt, expected tool or None, argument check, canned tool result)
FIX_CASES: list[tuple[str, str | None, Any, dict | None]] = [
("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")),
{"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}),
("부산 지금 비 와?", "get_weather", lambda a: "부산" in str(a.get("city", "")) or "Busan" in str(a.get("city", "")),
{"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0}),
("나 지금 서울인데 우산 챙겨야 할까?", "get_weather",
lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")),
{"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}),
("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5}),
("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10}),
("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2,
{"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}),
("안녕, 오늘 기분 어때?", None, None, None),
# No tool can do this: the answer must not say it was done (read the text).
("거실 불 꺼 줘.", None, None, None),
]
# A value from each canned result that a grounded final answer repeats.
_GROUNDING = {"get_weather": "temp_c", "set_timer": "minutes", "view_screen": None}
def jarvis_src() -> Path | None:
for p in (_KIT / "jarvis_src", _KIT.parent.parent / "src" / "jarvis"):
if (p / "system_prompt.py").is_file() and (p / "reply" / "planner.py").is_file():
return p
return None
def load_jarvis(src: Path, client: ChatClient) -> tuple[types.ModuleType, types.ModuleType]:
"""Import Jarvis's real system_prompt and planner modules from ``src``.
The planner's two dependencies are replaced: ``jarvis.debug`` (reads the
whole Jarvis config) by a no-op logger, and ``jarvis.llm.call_llm_direct``
(Ollama's API) by a call to the OpenAI-compatible server under test.
"""
def call_llm_direct(base_url, chat_model, system_prompt, user_content, timeout_sec=10.0,
thinking=False, num_ctx=0, temperature=None):
a = client.chat([{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}],
max_tokens=300, extra=NO_THINK)
return a["text"]
# A private package name, so a real ``jarvis`` package in the same process
# (the repository's tests) is left alone.
root = _PKG
for name, path in ((root, src), (f"{root}.reply", src / "reply")):
pkg = types.ModuleType(name)
pkg.__path__ = [str(path)]
sys.modules[name] = pkg
debug = types.ModuleType(f"{root}.debug")
debug.debug_log = lambda *a, **k: None
llm = types.ModuleType(f"{root}.llm")
llm.call_llm_direct = call_llm_direct
llm.OLLAMA_NUM_CTX = 8192
sys.modules[f"{root}.debug"], sys.modules[f"{root}.llm"] = debug, llm
loaded = []
for name, path in ((f"{root}.system_prompt", src / "system_prompt.py"),
(f"{root}.reply.planner", src / "reply" / "planner.py")):
spec = importlib.util.spec_from_file_location(name, path)
mod = importlib.util.module_from_spec(spec)
sys.modules[name] = mod
spec.loader.exec_module(mod)
loaded.append(mod)
return loaded[0], loaded[1]
def _call_ok(call: dict, want: str, check) -> bool:
if call.get("name") != want:
return False
try:
return bool(check(json.loads(call.get("arguments") or "{}")))
except Exception:
return False
def run_case(c: ChatClient, system: str, planner, prompt: str, want: str | None, check, result: dict | None) -> dict:
"""One try through prompt layer -> planner -> guard -> final answer."""
tool_names = [t["function"]["name"] for t in TOOLS]
cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model,
ollama_chat_model=c.model, planner_timeout_sec=60.0)
row: dict[str, Any] = {"prompt": prompt, "expected": want}
plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(TOOLS))
row["plan"] = plan
msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]
a = c.chat(msgs, max_tokens=200, tools=TOOLS, extra=NO_THINK)
calls = a["tool_calls"]
row.update(model_calls=calls, model_text=a["text"][:200], first_tool=a["first_tool"], ttft=a["ttft"])
# Text next to a tool call is read aloud before the result in streaming voice.
row["text_with_call"] = bool(calls and a["text"].strip())
row["model_ok"] = (not calls) if want is None else bool(calls) and _call_ok(calls[0], want, check)
call = calls[0] if calls else None
row["guard_ran"] = False
if not calls:
steps = planner.named_tool_steps(plan, tool_names)
parsed = planner.parse_concrete_tool_step(steps[0], TOOLS) if steps else None
if parsed:
call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)}
row["guard_ran"] = True
row["guard_call"] = call
if want is None:
row["ok"] = call is None
row["final_text"] = a["text"][:200]
return row
row["ok"] = bool(call) and _call_ok(call, want, check)
if call and result is not None:
msgs += [{"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]},
{"role": "tool", "tool_call_id": "call_1", "content": json.dumps(result, ensure_ascii=False)}]
b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK)
row["final_text"] = b["text"][:200]
key = _GROUNDING.get(call["name"])
row["final_uses_result"] = (str(result.get(key)) in b["text"]) if key and key in result else None
else:
row["final_text"] = a["text"][:200]
return row
def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3) -> None:
say(" - 도구 호출 (수정 적용: 지시문 + 계획 + 건너뛴 도구 실행)")
src = jarvis_src()
if src is None:
r.error("tools_fixed", "jarvis_src 폴더가 없습니다")
return
sp, planner = load_jarvis(src, c)
names = [t["function"]["name"] for t in TOOLS]
system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}"
rows = []
for prompt, want, check, result in FIX_CASES:
for i in range(tries):
try:
row = run_case(c, system, planner, prompt, want, check, result)
except Exception as e:
row = {"prompt": prompt, "expected": want, "ok": False, "error": str(e)[:300]}
row["try"] = i + 1
rows.append(row)
r.save()
scored = [x for x in rows if x["prompt"] != "거실 불 꺼 줘."]
summary = {"model_ok": sum(1 for x in scored if x.get("model_ok")),
"ok": sum(1 for x in scored if x.get("ok")),
"guard_ran": sum(1 for x in rows if x.get("guard_ran")),
"text_with_call": sum(1 for x in rows if x.get("text_with_call")),
"total": len(scored)}
r["tools_fixed"] = {"cases": rows, "summary": summary, "system": system}
say(f" 모델 스스로 {summary['model_ok']}/{summary['total']}, 계획 실행 포함 {summary['ok']}/{summary['total']}, "
f"도구 호출과 같이 쓴 답 {summary['text_with_call']}번")