From 8cb7e57cda287c736ae34cc61a3ee30f37c66516 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Tue, 10 Nov 2026 15:19:30 +0900 Subject: [PATCH] fix(reply): never deliver the tool-less answer when the planned call cannot run When the skipped-tool guard could not run the planned call (not a literal call, or arguments the tool does not take) it delivered the answer it had held back, which was given without the tool and could be a made-up value. The planned call is now recorded as a failed tool result and the model answers from that, so the user hears that it could not check or do it. M0 kit: a case passes only when the final answer uses the tool result (13 degrees, not a made-up 24) or, for the no-tool request, says it cannot do it; the no-tool request now counts. The kit mirrors the engine's failed-result path, and a failing model moves on to the next candidate on these stricter checks. Co-Authored-By: Claude Opus 5.5 --- scripts/m0/README.md | 2 +- scripts/m0/m0lib/tool_fix.py | 99 ++++++++++++++++--------- src/jarvis/reply/engine.py | 42 ++++++++--- src/jarvis/reply/planner.spec.md | 11 ++- src/jarvis/reply/reply.spec.md | 2 +- tests/test_engine_skipped_tool_guard.py | 36 +++++++-- tests/test_m0_kit.py | 44 ++++++++++- 7 files changed, 176 insertions(+), 60 deletions(-) diff --git a/scripts/m0/README.md b/scripts/m0/README.md index 11d6e0b..d6a72ed 100644 --- a/scripts/m0/README.md +++ b/scripts/m0/README.md @@ -55,7 +55,7 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 `tools_fast`, `tools_recommended`는 도구 호출만 다시 잽니다. 처음과 같은 방식(고치기 전)과, 자비스의 수정(지시문 규칙, 계획 단계, 건너뛴 도구 직접 실행)을 넣은 방식을 같은 모델로 비교합니다. 이미 받은 모델을 쓰므로 모델 하나에 몇 분이면 끝납니다. -수정을 넣고도 도구 호출이 하나라도 틀리면, 같은 계열에서 덜 압축된 모델을 받아서 이어서 잽니다. 빠름은 Qwen3.6-35B-A3B UD-Q6_K_XL(약 33GB), 그다음 Q8_0(약 38GB) 순서이고, 권장은 Qwen3.8-Flash-Next IQ3_XXS, 그다음 IQ3_S 순서입니다. 통과하는 모델이 나오면 거기서 멈춥니다. 이때는 내려받느라 몇 시간 걸릴 수 있고, 디스크 공간이 모자란 모델은 건너뜁니다. +통과 기준은 도구를 맞게 부르는 것만이 아닙니다. 마지막 답이 도구 결과(예: 13도)를 그대로 말해야 하고, 할 수 있는 도구가 없는 부탁(거실 불 끄기)에는 못 한다고 답해야 합니다. 수정을 넣고도 이 중 하나라도 틀리면, 같은 계열에서 덜 압축된 모델을 받아서 이어서 잽니다. 빠름은 Qwen3.6-35B-A3B UD-Q6_K_XL(약 33GB), 그다음 Q8_0(약 38GB) 순서이고, 권장은 Qwen3.8-Flash-Next IQ3_XXS, 그다음 IQ3_S 순서입니다. 통과하는 모델이 나오면 거기서 멈춥니다. 이때는 내려받느라 몇 시간 걸릴 수 있고, 디스크 공간이 모자란 모델은 건너뜁니다. `run_m0_tools.bat`을 더블클릭하면 이 두 단계만 돌고 결과 zip이 만들어집니다. 직접 답하실 부분은 없습니다. diff --git a/scripts/m0/m0lib/tool_fix.py b/scripts/m0/m0lib/tool_fix.py index ad76ad4..6255871 100644 --- a/scripts/m0/m0lib/tool_fix.py +++ b/scripts/m0/m0lib/tool_fix.py @@ -33,26 +33,39 @@ from m0lib.llm_client import ChatClient _KIT = Path(__file__).resolve().parent.parent _PKG = "m0_jarvis" -# (prompt, expected tool or None, argument check, canned tool result) -FIX_CASES: list[tuple[str, str | None, Any, dict | None]] = [ +# (prompt, expected tool or None, argument check, canned tool result, check of the final answer) +# The final check is what makes a case pass: the right tool call is not enough +# if the spoken answer then ignores the result or claims something not done. +REFUSAL = "refusal" +FIX_CASES: list[tuple[str, str | None, Any, dict | None, str | None]] = [ ("서울 날씨 어때?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")), - {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}), + {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}, "13"), ("부산 지금 비 와?", "get_weather", lambda a: "부산" in str(a.get("city", "")) or "Busan" in str(a.get("city", "")), - {"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0}), + {"city": "부산", "temp_c": 17, "sky": "맑음", "rain_mm": 0}, "17"), ("나 지금 서울인데 우산 챙겨야 할까?", "get_weather", lambda a: "서울" in str(a.get("city", "")) or "Seoul" in str(a.get("city", "")), - {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}), - ("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5}), - ("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10}), + {"city": "서울", "temp_c": 13, "sky": "흐리고 비", "rain_mm": 2}, "비"), + ("5분 타이머 맞춰 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 5, {"ok": True, "minutes": 5}, "5"), + ("10분 뒤에 알려 줘.", "set_timer", lambda a: int(a.get("minutes", 0)) == 10, {"ok": True, "minutes": 10}, "10"), ("두 번째 모니터에 뭐 떠 있어?", "view_screen", lambda a: int(a.get("monitor", 0)) == 2, - {"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}), - ("안녕, 오늘 기분 어때?", None, None, None), - # No tool can do this: the answer must not say it was done (read the text). - ("거실 불 꺼 줘.", None, None, None), + {"monitor": 2, "seen": "유튜브 재생 화면, 제목 '오늘의 뉴스'"}, "뉴스"), + ("안녕, 오늘 기분 어때?", None, None, None, None), + # No tool can do this: passing means saying so, not claiming it was done. + ("거실 불 꺼 줘.", None, None, None, REFUSAL), ] -# A value from each canned result that a grounded final answer repeats. -_GROUNDING = {"get_weather": "temp_c", "set_timer": "minutes", "view_screen": None} +# Words a Korean answer uses to say it cannot do something. The kit's prompts +# are fixed Korean sentences, so a word check is enough here (the product +# itself stays language-agnostic). +_CANNOT = ("수 없", "수가 없", "못 ", "못해", "못합", "못했", "어렵", "지원하지", "기능이 없", "기능은 없", "불가") + + +def case_for(prompt: str) -> tuple: + return next(c for c in FIX_CASES if c[0] == prompt) + + +def is_refusal(text: str) -> bool: + return any(w in text for w in _CANNOT) def jarvis_src() -> Path | None: @@ -109,8 +122,9 @@ def _call_ok(call: dict, want: str, check) -> bool: return False -def run_case(c: ChatClient, system: str, planner, prompt: str, want: str | None, check, result: dict | None) -> dict: - """One try through prompt layer -> planner -> guard -> final answer.""" +def run_case(c: ChatClient, system: str, planner, case: tuple) -> dict: + """One try through prompt layer -> planner -> guard -> final answer, as the engine does it.""" + prompt, want, check, result, final_check = case tool_names = [t["function"]["name"] for t in TOOLS] cfg = types.SimpleNamespace(planner_enabled=True, ollama_base_url="m0", planner_model=c.model, ollama_chat_model=c.model, planner_timeout_sec=60.0) @@ -125,35 +139,51 @@ def run_case(c: ChatClient, system: str, planner, prompt: str, want: str | None, row["text_with_call"] = bool(calls and a["text"].strip()) row["model_ok"] = (not calls) if want is None else bool(calls) and _call_ok(calls[0], want, check) - call = calls[0] if calls else None + call, tool_content = (calls[0], None) if calls else (None, None) row["guard_ran"] = False - if not calls: - steps = planner.named_tool_steps(plan, tool_names) - parsed = planner.parse_concrete_tool_step(steps[0], TOOLS) if steps else None + steps = [] if calls else planner.named_tool_steps(plan, tool_names) + if steps: + # Skipped-tool guard, as in the engine: run the literal planned call, or + # if it cannot run, answer from a failed tool result (never the guess). + parsed = planner.parse_concrete_tool_step(steps[0], TOOLS) if parsed: call = {"name": parsed[0], "arguments": json.dumps(parsed[1], ensure_ascii=False)} row["guard_ran"] = True row["guard_call"] = call - if want is None: - row["ok"] = call is None - row["final_text"] = a["text"][:200] - return row - row["ok"] = bool(call) and _call_ok(call, want, check) - if call and result is not None: + else: + call = {"name": steps[0].split()[0], "arguments": "{}"} + tool_content = (f"Error: the planned call `{steps[0]}` could not be run, so nothing was fetched or " + "done. Tell the user plainly that you could not check or do it right now; do not " + "guess a value or claim it was done.") + row["guard_failed_call"] = steps[0] + + final = a["text"] + if call is not None: + if tool_content is None: + if result is not None and want is not None and _call_ok(call, want, check): + tool_content = json.dumps(result, ensure_ascii=False) + else: + tool_content = f"Error: {call['name']} cannot do that request." msgs += [{"role": "assistant", "content": "", "tool_calls": [ {"id": "call_1", "type": "function", "function": {"name": call["name"], "arguments": call["arguments"]}}]}, - {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(result, ensure_ascii=False)}] + {"role": "tool", "tool_call_id": "call_1", "content": tool_content}] b = c.chat(msgs, max_tokens=120, tools=TOOLS, extra=NO_THINK) - row["final_text"] = b["text"][:200] - key = _GROUNDING.get(call["name"]) - row["final_uses_result"] = (str(result.get(key)) in b["text"]) if key and key in result else None + final = b["text"] + row["final_text"] = final[:200] + + if want is not None: + row["final_uses_result"] = bool(final_check) and final_check in final + row["ok"] = bool(call) and _call_ok(call, want, check) and row["final_uses_result"] + elif final_check == REFUSAL: + row["ok"] = not calls and is_refusal(final) else: - row["final_text"] = a["text"][:200] + row["ok"] = not calls return row def tools_passed(summary: dict | None) -> bool: - """Every scored case ended with the right tool call (by the model or the guard).""" + """Every case passed: the right tool call (by the model or the guard) and a final + answer that uses the tool's result, or an honest refusal where no tool can help.""" return bool(summary) and summary["total"] > 0 and summary["ok"] == summary["total"] @@ -167,16 +197,17 @@ def test_tools_fixed(r: StageResult, c: ChatClient, tries: int = 3, key: str = " names = [t["function"]["name"] for t in TOOLS] system = f"{VOICE_SYSTEM}\n\n{sp.TOOL_HONESTY_RULES}\n\n{sp.tool_honesty_reminder(names)}" rows = [] - for prompt, want, check, result in FIX_CASES: + for case in FIX_CASES: + prompt, want = case[0], case[1] for i in range(tries): try: - row = run_case(c, system, planner, prompt, want, check, result) + row = run_case(c, system, planner, case) except Exception as e: row = {"prompt": prompt, "expected": want, "ok": False, "error": str(e)[:300]} row["try"] = i + 1 rows.append(row) r.save() - scored = [x for x in rows if x["prompt"] != "거실 불 꺼 줘."] + scored = rows # every case counts, the no-tool refusal too summary = {"model_ok": sum(1 for x in scored if x.get("model_ok")), "ok": sum(1 for x in scored if x.get("ok")), "guard_ran": sum(1 for x in rows if x.get("guard_ran")), diff --git a/src/jarvis/reply/engine.py b/src/jarvis/reply/engine.py index f5d2973..53d985f 100644 --- a/src/jarvis/reply/engine.py +++ b/src/jarvis/reply/engine.py @@ -2261,14 +2261,37 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], continue if _forced_plan_steps and _tool_results_so_far == 0: # The guard could not run the planned tool after all - # (unresolvable step or arguments that do not fit). Fail - # open: deliver the answer it held back rather than ask - # the model again on the same messages. - debug_log("skipped-tool guard: planned tool not runnable, keeping the model answer", "planning") - messages.append(_guard_held_message) - reply = _guard_held_reply - last_candidate_reply = _guard_held_reply - break + # (not a literal call, or arguments the tool does not + # take). The held-back answer was given without the tool, + # so it may be a guess: never deliver it. Record the + # planned call as a failed tool result and let the model + # answer from that, which under the tool honesty rules + # means saying it could not check or do it. + _fail_step = _plan_tool_steps[0] + _fail_name = _fail_step.split()[0] + _fail_id = f"call_plan_{uuid.uuid4().hex[:8]}" + debug_log( + f"skipped-tool guard: planned call not runnable ({_fail_step[:80]!r}); " + "asking the model to answer from a failed tool result", + "planning", + ) + messages.append({ + "role": "assistant", + "content": "", + "tool_calls": [{"id": _fail_id, "type": "function", + "function": {"name": _fail_name, "arguments": {}}}], + }) + messages.append({ + "role": "tool", + "tool_call_id": _fail_id, + "tool_name": _fail_name, + "content": ( + f"Error: the planned call `{_fail_step}` could not be run, so nothing " + "was fetched or done. Tell the user plainly that you could not check " + "or do it right now; do not guess a value or claim it was done." + ), + "tool_failed": True, + }) # Update the system message with fresh context (time/location) before each LLM call # Note: We update the first system message rather than appending a new one because @@ -2757,8 +2780,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], # The model answered without calling the tool the plan chose, so # the answer can only be a guess. Drop it (the synthesis turn must # not anchor on it) and run the planned steps directly. - _guard_held_message = messages.pop() - _guard_held_reply = candidate_reply + messages.pop() _forced_plan_steps = _guard_steps debug_log( f"skipped-tool guard: model answered without the planned tool; " diff --git a/src/jarvis/reply/planner.spec.md b/src/jarvis/reply/planner.spec.md index d003582..34cda79 100644 --- a/src/jarvis/reply/planner.spec.md +++ b/src/jarvis/reply/planner.spec.md @@ -183,10 +183,13 @@ The engine consumes the plan in two phases. against the tool's input schema. An LLM resolver could "repair" a wrong step into a valid but unwanted call (a timer request planned as `setBroadcast duration=5 minutes` was turned into starting the broadcast). -- Fail-open: if the first planned step cannot be run as written, the - engine delivers the answer it held back, without a second chat call. The - guard fires at most once per reply and never in text-tool mode (that mode - already direct-executes every plan). +- If the first planned step cannot be run as written, the held-back answer + is still never delivered (it was given without the tool, so it may be a + guess). The engine records the planned call as a failed tool result + ("could not be run, nothing was fetched or done") and the chat model + answers from that, which under the tool honesty rules means saying it + could not check or do it. The guard fires at most once per reply and never + in text-tool mode (that mode already direct-executes every plan). - In native mode the direct-exec result is a `role=tool` message answering the synthetic assistant `tool_calls` message, and no progress nudge is appended. diff --git a/src/jarvis/reply/reply.spec.md b/src/jarvis/reply/reply.spec.md index aba662a..e3c5f26 100644 --- a/src/jarvis/reply/reply.spec.md +++ b/src/jarvis/reply/reply.spec.md @@ -98,7 +98,7 @@ Design principles enforced by the engine: - When detected, the engine falls back to the standard "I had trouble understanding that request" error reply (model-size-aware). The malformed content is never shown to the user. Skipped-tool guard (native tool calling): - - When a native-tool model answers in prose before any tool has run in this reply, and the plan names a callable tool as a concrete call, the engine discards that answer and direct-executes the plan's tool steps, then the model answers from the results. Only literal, schema-valid steps run (no LLM repair); otherwise the held-back answer is delivered unchanged. See `planner.spec.md` ("Skipped-tool guard"). + - When a native-tool model answers in prose before any tool has run in this reply, and the plan names a callable tool as a concrete call, the engine discards that answer and direct-executes the plan's tool steps, then the model answers from the results. Only literal, schema-valid steps run (no LLM repair). When the planned call cannot run, the held-back answer is still discarded: the planned call is recorded as a failed tool result and the model answers from that (an honest "could not check or do it"). See `planner.spec.md` ("Skipped-tool guard"). Task-list planner (all model sizes, strongest impact on small models): - The planner runs at the **front** of the reply flow (see step 3 above), not after tool selection. By the time the agentic loop starts, the plan already exists, the memory block has either been run or skipped based on the plan's `searchMemory` directive, and the tool allow-list has been derived from the tool names the plan referenced. See `planner.spec.md` for the prompt contract and fail-open semantics. diff --git a/tests/test_engine_skipped_tool_guard.py b/tests/test_engine_skipped_tool_guard.py index b76982e..a178581 100644 --- a/tests/test_engine_skipped_tool_guard.py +++ b/tests/test_engine_skipped_tool_guard.py @@ -136,30 +136,54 @@ def test_single_step_tool_plan_is_still_enforced(mock_config, db, dialogue_memor assert reply == GROUNDED +HONEST_FAILURE = "I could not check that just now." + + +def _failed_tool_result_seen(messages: list[dict]) -> bool: + return any(m.get("role") == "tool" and m.get("tool_failed") for m in messages) + + +def test_unrunnable_plan_never_delivers_the_held_back_guess(mock_config, db, dialogue_memory): + # The weather plan uses a key getWeather does not take, so the guard cannot + # run it. The tool-less "24 degrees and sunny" answer must still not reach + # the user: the model answers again from a failed-tool result. + reply, invoked, seen = _run( + mock_config, db, dialogue_memory, plan=["getWeather town='Seoul'"], + chat_replies=[_content(FABRICATED), _content(HONEST_FAILURE)], + allowed=["getWeather", "stop"], + ) + + assert invoked == [] + assert reply == HONEST_FAILURE + final_turn = seen[-1] + assert not any(FABRICATED in (m.get("content") or "") for m in final_turn) + assert _failed_tool_result_seen(final_turn) + + def test_planned_call_with_arguments_the_tool_does_not_take_is_not_forced(mock_config, db, dialogue_memory): # A weak planner mapped "set a timer" onto setBroadcast with a made-up - # argument. The guard must not trigger that tool; the model's answer stands - # and no second chat call is spent. + # argument. The guard must not trigger that tool; the model is told the + # planned call could not run and answers from that. reply, invoked, seen = _run( mock_config, db, dialogue_memory, plan=["setBroadcast duration='5 minutes'"], - chat_replies=[_content("I have no timer, so I cannot set one.")], + chat_replies=[_content("Timer set for 5 minutes."), _content("I have no timer, so I cannot set one.")], allowed=["setBroadcast", "stop"], ) assert invoked == [] assert reply == "I have no timer, so I cannot set one." - assert len(seen) == 1 + assert _failed_tool_result_seen(seen[-1]) def test_prose_shaped_step_is_not_reinterpreted_into_some_other_call(mock_config, db, dialogue_memory): # "setBroadcast duration=5 minutes" is not a call setBroadcast takes. The # guard must not ask an LLM to repair it (that turned a timer request into - # starting the broadcast); the model's answer stands. + # starting the broadcast). from jarvis.reply import engine as engine_mod with patch.object(engine_mod, "_resolve_plan_step", return_value=("setBroadcast", {"action": "start"})): reply, invoked, seen = _run( mock_config, db, dialogue_memory, plan=["setBroadcast duration=5 minutes"], - chat_replies=[_content("I have no timer, so I cannot set one.")], + chat_replies=[_content("Timer set for 5 minutes."), _content("I have no timer, so I cannot set one.")], allowed=["setBroadcast", "stop"], ) diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index f641400..e88832c 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -278,14 +278,14 @@ def test_kit_uses_the_production_prompt_rules_and_planner(): def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result(): from m0lib import tool_fix - prompt, want, check, result = tool_fix.FIX_CASES[0] # 서울 날씨 + case = tool_fix.case_for("서울 날씨 어때?") client = _ScriptedClient([ ("get_weather city='서울'\nReply to the user with the weather.", []), # planner ("서울은 24도, 맑음입니다.", []), # model skips the tool ("서울은 13도이고 비가 옵니다.", []), # answer from the result ]) _, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client) - row = tool_fix.run_case(client, "system", planner, prompt, want, check, result) + row = tool_fix.run_case(client, "system", planner, case) assert row["model_ok"] is False and row["guard_ran"] is True and row["ok"] is True assert row["final_uses_result"] is True @@ -295,14 +295,14 @@ def test_skipped_tool_is_run_from_the_plan_and_the_answer_uses_its_result(): def test_model_that_calls_the_tool_is_scored_without_the_guard(): from m0lib import tool_fix - prompt, want, check, result = tool_fix.FIX_CASES[3] # 5분 타이머 + case = tool_fix.case_for("5분 타이머 맞춰 줘.") client = _ScriptedClient([ ("set_timer minutes='5'", []), ("", [{"name": "set_timer", "arguments": "{\"minutes\": 5}"}]), ("5분 타이머를 맞췄습니다.", []), ]) _, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client) - row = tool_fix.run_case(client, "system", planner, prompt, want, check, result) + row = tool_fix.run_case(client, "system", planner, case) assert row["model_ok"] is True and row["guard_ran"] is False and row["ok"] is True assert row["text_with_call"] is False @@ -340,3 +340,39 @@ def test_candidate_without_disk_space_is_skipped_not_downloaded(tmp_path): assert tried == [] assert r["tools_verdict"]["chosen"] is None assert all(c["skipped"] == "disk" for c in r["tools_verdict"]["candidates"]) + + + +def _run_scripted(case_prompt, answers): + from m0lib import tool_fix + client = _ScriptedClient(answers) + _, planner = tool_fix.load_jarvis(tool_fix.jarvis_src(), client) + return tool_fix.run_case(client, "system", planner, tool_fix.case_for(case_prompt)), client + + +def test_right_tool_with_a_made_up_final_answer_fails(): + # The weather tool ran and said 13 degrees; an answer of "24도, 맑음" is still a lie. + row, _ = _run_scripted("서울 날씨 어때?", [ + ("get_weather city='서울'", []), + ("", [{"name": "get_weather", "arguments": "{\"city\": \"서울\"}"}]), + ("서울은 지금 24도, 맑음입니다.", []), + ]) + assert row["model_ok"] is True and row["final_uses_result"] is False and row["ok"] is False + + +def test_claiming_an_action_no_tool_can_do_fails(): + claimed, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불을 껐습니다.", [])]) + honest, _ = _run_scripted("거실 불 꺼 줘.", [("Reply to the user.", []), ("거실 불은 제가 끌 수 없습니다.", [])]) + assert claimed["ok"] is False and honest["ok"] is True + + +def test_unrunnable_plan_is_answered_from_a_failed_tool_result_like_the_engine(): + row, client = _run_scripted("서울 날씨 어때?", [ + ("get_weather town='서울'", []), # key get_weather does not take + ("서울은 24도, 맑음입니다.", []), # model skips the tool + ("지금은 날씨를 확인하지 못했습니다.", []), # answer from the failed result + ]) + assert row["guard_ran"] is False and row["ok"] is False + assert row["final_text"] == "지금은 날씨를 확인하지 못했습니다." + last = client.sent[-1][0][-1] + assert last["role"] == "tool" and "could not be run" in last["content"]