fix(reply): stop models answering past an available tool
M0 on Strata caught both v2 models skipping a tool they had been given: the recommended model made up "24 degrees, sunny" instead of calling the weather tool, and the fast model said "timer set" without calling the timer tool (3 of 3 each). qwen2.5:7b does the same through the engine. - Prompt: TOOL_HONESTY_RULES in the persona (live facts only from a tool result this turn, actions only claimed after a tool reports success, no answer text next to a tool call), restated at the end of the system message by tool_honesty_reminder() with the turn's real tool names. - Engine: skipped-tool guard. When a native-tool model answers before any tool has run and the plan names a callable tool as a concrete call, the answer is dropped and the plan steps are direct-executed, then the model answers from the results. Only literal, schema-valid steps run (regex fast path, no LLM repair); otherwise the held-back answer is delivered. - Planner: the AVAILABLE TOOLS lines now carry argument names (planner_tool_catalog), so concrete steps use the tool's real keys. - Eval evals/test_tool_skip_confabulation.py; on qwen2.5:7b, 3 runs each, available-tool cases went from 0/12 to 9/12 calling the tool. - M0 kit: tools_fast / tools_recommended stages (run_m0_tools.bat) rerun tool calling before and after the fix on the real models, loading the production system_prompt.py and planner.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -51,6 +51,10 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰
|
||||
.\run_m0.ps1 -Only stt,tts_melo -Redo # 특정 단계만 다시
|
||||
```
|
||||
|
||||
단계 이름: `sysinfo`, `winchecks`, `vdisplay`, `tts_melo`, `tts_cosy2`, `tts_cosy3`, `stt`, `llm_fast`, `llm_recommended`
|
||||
단계 이름: `sysinfo`, `winchecks`, `vdisplay`, `tts_melo`, `tts_cosy2`, `tts_cosy3`, `stt`, `llm_fast`, `llm_recommended`, `tools_fast`, `tools_recommended`
|
||||
|
||||
`tools_fast`, `tools_recommended`는 도구 호출만 다시 잽니다. 처음과 같은 방식(고치기 전)과, 자비스의 수정(지시문 규칙, 계획 단계, 건너뛴 도구 직접 실행)을 넣은 방식을 같은 모델로 비교합니다. 이미 받은 모델을 쓰므로 다운로드는 없고, 모델 하나에 몇 분이면 끝납니다.
|
||||
|
||||
`run_m0_tools.bat`을 더블클릭하면 이 두 단계만 돌고 결과 zip이 만들어집니다. 직접 답하실 부분은 없습니다.
|
||||
|
||||
다 끝나고 필요 없으면 작업 폴더(`\JarvisM0`)를 통째로 지우면 됩니다. Python, 모델, Strata(엔진과 설정 포함)는 모두 그 안에 설치하도록 지정해 두었습니다.
|
||||
|
||||
@@ -36,6 +36,10 @@ STAGES = [
|
||||
("llm_fast", "LLM 빠름 모델 (Qwen3.6-35B-A3B, 약 23GB 받기)", "base", "llm.py", ["model=fast"], 35),
|
||||
("llm_recommended", "LLM 권장 모델 (Qwen3.8-Flash-Next Q2_0, 약 66GB 받기)", "base", "llm.py",
|
||||
["model=recommended"], 80),
|
||||
# Tool calling again with Jarvis's fix (prompt rules + planner + skipped-tool guard).
|
||||
# Reuses the models the two stages above downloaded.
|
||||
("tools_fast", "도구 호출 다시 재기: 빠름 모델", "base", "llm.py", ["model=fast", "only=tools"], 0),
|
||||
("tools_recommended", "도구 호출 다시 재기: 권장 모델", "base", "llm.py", ["model=recommended", "only=tools"], 0),
|
||||
]
|
||||
|
||||
|
||||
@@ -135,7 +139,7 @@ def main() -> int:
|
||||
"errors": [{"where": "disk", "error": f"free {free_gb(work):.0f} GB < {need_gb} GB"}]})
|
||||
continue
|
||||
stage_opts = list(opts)
|
||||
if key.startswith("llm_") and a.llm_url:
|
||||
if key.startswith(("llm_", "tools_")) and a.llm_url:
|
||||
stage_opts.append(f"url={a.llm_url}")
|
||||
try:
|
||||
run_stage(key, venv, script, stage_opts, work, env)
|
||||
|
||||
@@ -4,6 +4,7 @@ Options (--opt key=value):
|
||||
model=fast|recommended which preset to measure (default fast)
|
||||
url=http://.../v1 measure an already running server instead (no Strata management)
|
||||
context=32768 context size to start Strata with
|
||||
only=tools only re-measure tool calling, before and after the Jarvis fix
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,7 +20,9 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
from m0lib.common import IS_WINDOWS, GpuSampler, StageResult, run_logged, say, scrub, stage_args # noqa: E402
|
||||
from m0lib.llm_bench import bench_single, test_decode, test_images, test_parallel, test_vram_shrink, warmup # noqa: E402
|
||||
from m0lib.llm_bench import (bench_single, test_decode, test_images, test_parallel, test_tools, # noqa: E402
|
||||
test_vram_shrink, warmup)
|
||||
from m0lib.tool_fix import test_tools_fixed # noqa: E402
|
||||
from m0lib.llm_client import ChatClient, http_json # noqa: E402
|
||||
|
||||
PRESETS = {
|
||||
@@ -190,16 +193,29 @@ def server_info(base: str) -> dict:
|
||||
return info
|
||||
|
||||
|
||||
def measure_external(r: StageResult, url: str, context: int) -> None:
|
||||
def tools_only(r: StageResult, c: ChatClient) -> None:
|
||||
"""Tool calling as first measured (baseline) and with the Jarvis fix."""
|
||||
for fn in (lambda: warmup(r, c), lambda: test_tools(r, c), lambda: test_tools_fixed(r, c)):
|
||||
try:
|
||||
fn()
|
||||
except Exception as e:
|
||||
r.error("tools", e)
|
||||
r.save()
|
||||
|
||||
|
||||
def measure_external(r: StageResult, url: str, context: int, only: str = "") -> None:
|
||||
c = ChatClient(url)
|
||||
r["server"] = {"url": url, "model": c.model, "info": server_info(c.base)}
|
||||
if only == "tools":
|
||||
tools_only(r, c)
|
||||
return
|
||||
with GpuSampler() as g:
|
||||
bench_single(r, c, context)
|
||||
test_parallel(r, c)
|
||||
r["gpu"] = g.summary()
|
||||
|
||||
|
||||
def measure_strata(r: StageResult, work: Path, which: str, context: int) -> None:
|
||||
def measure_strata(r: StageResult, work: Path, which: str, context: int, only: str = "") -> None:
|
||||
preset = PRESETS[which]
|
||||
r["preset"] = dict(preset, key=which, context=context)
|
||||
s = Strata(work, work / "logs" / f"strata_{which}.log")
|
||||
@@ -227,6 +243,10 @@ def measure_strata(r: StageResult, work: Path, which: str, context: int) -> None
|
||||
base = f"http://127.0.0.1:{PORT}/v1"
|
||||
c = ChatClient(base)
|
||||
r["server"] = {"model": c.model, "info": server_info(base), "status": s.status()}
|
||||
if only == "tools":
|
||||
tools_only(r, c)
|
||||
s.stop()
|
||||
return
|
||||
with GpuSampler() as g:
|
||||
bench_single(r, c, context)
|
||||
r["gpu_single"] = g.summary()
|
||||
@@ -260,12 +280,13 @@ def main() -> int:
|
||||
out, work, opts = stage_args()
|
||||
which = opts.get("model", "fast")
|
||||
context = int(opts.get("context", "32768"))
|
||||
r = StageResult(f"llm_{which}", out)
|
||||
only = opts.get("only", "")
|
||||
r = StageResult(f"{'tools' if only == 'tools' else 'llm'}_{which}", out)
|
||||
try:
|
||||
if opts.get("url"):
|
||||
measure_external(r, opts["url"], context)
|
||||
measure_external(r, opts["url"], context, only)
|
||||
else:
|
||||
measure_strata(r, work, which, context)
|
||||
measure_strata(r, work, which, context, only)
|
||||
except Exception as e:
|
||||
r.error("llm", e)
|
||||
# No server reached means nothing was measured: "failed" so the next run retries it.
|
||||
|
||||
182
scripts/m0/m0lib/tool_fix.py
Normal file
182
scripts/m0/m0lib/tool_fix.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""Re-measure tool calling with the production fix on the real models.
|
||||
|
||||
The first M0 run caught both models answering past a tool they had been
|
||||
given (a made-up "24도, 맑음" instead of get_weather, "설정했습니다" without
|
||||
set_timer). Jarvis now fixes that in two layers, and this test runs both
|
||||
layers' real code against the model under test:
|
||||
|
||||
1. Prompt: ``TOOL_HONESTY_RULES`` and ``tool_honesty_reminder`` from
|
||||
``jarvis/system_prompt.py`` are added to the voice prompt.
|
||||
2. Skipped-tool guard: Jarvis's planner (``jarvis/reply/planner.py``, its
|
||||
real prompt) plans the turn on the same model. When the model answers
|
||||
without a tool and the plan names one as a concrete call, the engine runs
|
||||
that call itself and the model answers from the result. Here the tool
|
||||
result is canned, as in the first run's round trip.
|
||||
|
||||
The Jarvis files come from ``jarvis_src/`` next to ``m0lib`` in the packed
|
||||
kit, or from the repository's ``src/jarvis`` when run from a checkout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from m0lib.common import StageResult, say
|
||||
from m0lib.llm_bench import NO_THINK, TOOLS, VOICE_SYSTEM
|
||||
from m0lib.llm_client import ChatClient
|
||||
|
||||
_KIT = Path(__file__).resolve().parent.parent
|
||||
_PKG = "m0_jarvis"
|
||||
|
||||
# (prompt, expected tool or None, argument check, canned tool result)
|
||||
FIX_CASES: list[tuple[str, str | None, Any, dict | None]] = [
|
||||
("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")),
|
||||
{"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}),
|
||||
("부산 지금 비 와?", "get_weather", lambda a: "부산" in str(a.get("city", "")) or "Busan" in str(a.get("city", "")),
|
||||
{"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0}),
|
||||
("나 지금 서울인데 우산 챙겨야 할까?", "get_weather",
|
||||
lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")),
|
||||
{"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}),
|
||||
("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5}),
|
||||
("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10}),
|
||||
("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2,
|
||||
{"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}),
|
||||
("안녕, 오늘 기분 어때?", None, None, None),
|
||||
# No tool can do this: the answer must not say it was done (read the text).
|
||||
("거실 불 꺼 줘.", None, None, None),
|
||||
]
|
||||
|
||||
# A value from each canned result that a grounded final answer repeats.
|
||||
_GROUNDING = {"get_weather": "temp_c", "set_timer": "minutes", "view_screen": None}
|
||||
|
||||
|
||||
def jarvis_src() -> Path | None:
|
||||
for p in (_KIT / "jarvis_src", _KIT.parent.parent / "src" / "jarvis"):
|
||||
if (p / "system_prompt.py").is_file() and (p / "reply" / "planner.py").is_file():
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def load_jarvis(src: Path, client: ChatClient) -> tuple[types.ModuleType, types.ModuleType]:
|
||||
"""Import Jarvis's real system_prompt and planner modules from ``src``.
|
||||
|
||||
The planner's two dependencies are replaced: ``jarvis.debug`` (reads the
|
||||
whole Jarvis config) by a no-op logger, and ``jarvis.llm.call_llm_direct``
|
||||
(Ollama's API) by a call to the OpenAI-compatible server under test.
|
||||
"""
|
||||
def call_llm_direct(base_url, chat_model, system_prompt, user_content, timeout_sec=10.0,
|
||||
thinking=False, num_ctx=0, temperature=None):
|
||||
a = client.chat([{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}],
|
||||
max_tokens=300, extra=NO_THINK)
|
||||
return a["text"]
|
||||
|
||||
# A private package name, so a real ``jarvis`` package in the same process
|
||||
# (the repository's tests) is left alone.
|
||||
root = _PKG
|
||||
for name, path in ((root, src), (f"{root}.reply", src / "reply")):
|
||||
pkg = types.ModuleType(name)
|
||||
pkg.__path__ = [str(path)]
|
||||
sys.modules[name] = pkg
|
||||
debug = types.ModuleType(f"{root}.debug")
|
||||
debug.debug_log = lambda *a, **k: None
|
||||
llm = types.ModuleType(f"{root}.llm")
|
||||
llm.call_llm_direct = call_llm_direct
|
||||
llm.OLLAMA_NUM_CTX = 8192
|
||||
sys.modules[f"{root}.debug"], sys.modules[f"{root}.llm"] = debug, llm
|
||||
|
||||
loaded = []
|
||||
for name, path in ((f"{root}.system_prompt", src / "system_prompt.py"),
|
||||
(f"{root}.reply.planner", src / "reply" / "planner.py")):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
loaded.append(mod)
|
||||
return loaded[0], loaded[1]
|
||||
|
||||
|
||||
def _call_ok(call: dict, want: str, check) -> bool:
|
||||
if call.get("name") != want:
|
||||
return False
|
||||
try:
|
||||
return bool(check(json.loads(call.get("arguments") or "{}")))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def run_case(c: ChatClient, system: str, planner, prompt: str, want: str | None, check, result: dict | None) -> dict:
|
||||
"""One try through prompt layer -> planner -> guard -> final answer."""
|
||||
tool_names = [t["function"]["name"] for t in TOOLS]
|
||||
cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model,
|
||||
ollama_chat_model=c.model, planner_timeout_sec=60.0)
|
||||
row: dict[str, Any] = {"prompt": prompt, "expected": want}
|
||||
plan = planner.plan_query(cfg, prompt, "", planner.planner_tool_catalog(TOOLS))
|
||||
row["plan"] = plan
|
||||
msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]
|
||||
a = c.chat(msgs, max_tokens=200, tools=TOOLS, extra=NO_THINK)
|
||||
calls = a["tool_calls"]
|
||||
row.update(model_calls=calls, model_text=a["text"][:200], first_tool=a["first_tool"], ttft=a["ttft"])
|
||||
# Text next to a tool call is read aloud before the result in streaming voice.
|
||||
row["text_with_call"] = bool(calls and a["text"].strip())
|
||||
row["model_ok"] = (not calls) if want is None else bool(calls) and _call_ok(calls[0], want, check)
|
||||
|
||||
call = calls[0] if calls else None
|
||||
row["guard_ran"] = False
|
||||
if not calls:
|
||||
steps = planner.named_tool_steps(plan, tool_names)
|
||||
parsed = planner.parse_concrete_tool_step(steps[0], TOOLS) if steps else None
|
||||
if parsed:
|
||||
call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)}
|
||||
row["guard_ran"] = True
|
||||
row["guard_call"] = call
|
||||
if want is None:
|
||||
row["ok"] = call is None
|
||||
row["final_text"] = a["text"][:200]
|
||||
return row
|
||||
row["ok"] = bool(call) and _call_ok(call, want, check)
|
||||
if call and result is not None:
|
||||
msgs += [{"role": "assistant", "content": "", "tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": json.dumps(result, ensure_ascii=False)}]
|
||||
b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK)
|
||||
row["final_text"] = b["text"][:200]
|
||||
key = _GROUNDING.get(call["name"])
|
||||
row["final_uses_result"] = (str(result.get(key)) in b["text"]) if key and key in result else None
|
||||
else:
|
||||
row["final_text"] = a["text"][:200]
|
||||
return row
|
||||
|
||||
|
||||
def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3) -> None:
|
||||
say(" - 도구 호출 (수정 적용: 지시문 + 계획 + 건너뛴 도구 실행)")
|
||||
src = jarvis_src()
|
||||
if src is None:
|
||||
r.error("tools_fixed", "jarvis_src 폴더가 없습니다")
|
||||
return
|
||||
sp, planner = load_jarvis(src, c)
|
||||
names = [t["function"]["name"] for t in TOOLS]
|
||||
system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}"
|
||||
rows = []
|
||||
for prompt, want, check, result in FIX_CASES:
|
||||
for i in range(tries):
|
||||
try:
|
||||
row = run_case(c, system, planner, prompt, want, check, result)
|
||||
except Exception as e:
|
||||
row = {"prompt": prompt, "expected": want, "ok": False, "error": str(e)[:300]}
|
||||
row["try"] = i + 1
|
||||
rows.append(row)
|
||||
r.save()
|
||||
scored = [x for x in rows if x["prompt"] != "거실 불 꺼 줘."]
|
||||
summary = {"model_ok": sum(1 for x in scored if x.get("model_ok")),
|
||||
"ok": sum(1 for x in scored if x.get("ok")),
|
||||
"guard_ran": sum(1 for x in rows if x.get("guard_ran")),
|
||||
"text_with_call": sum(1 for x in rows if x.get("text_with_call")),
|
||||
"total": len(scored)}
|
||||
r["tools_fixed"] = {"cases": rows, "summary": summary, "system": system}
|
||||
say(f" 모델 스스로 {summary['model_ok']}/{summary['total']}, 계획 실행 포함 {summary['ok']}/{summary['total']}, "
|
||||
f"도구 호출과 같이 쓴 답 {summary['text_with_call']}번")
|
||||
5
scripts/m0/run_m0_tools.bat
Normal file
5
scripts/m0/run_m0_tools.bat
Normal file
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
chcp 65001 >nul
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0run_m0.ps1" -Only tools_fast,tools_recommended %*
|
||||
echo.
|
||||
pause
|
||||
Reference in New Issue
Block a user