Files
javis_bot/tests/test_engine_one_step_turn.py
javis-bot 7da5bca783 fix(reply): a one-step turn whose model skips the tool twice is planned and guarded
After the one nudge, a second answer without the tool no longer goes out
as it is: the turn takes the planned path (the planner runs then, one
extra call only on this failure) and the skipped-tool guard runs the
planned tool, so a made-up "24 degrees" answer cannot reach the user. A
plan that needs no tool (a question back to the user) still lets the
answer go out.

Evals (qwen3:8b, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1, 3 runs):
the five tool-skip/weather/screen/follow-up/router files 72/84 (71/84 on
0.1.20, same known failures); Possessor director-then-filmography 0/3,
also 0/3 on 0.1.20 (qwen3:8b stops after one search; webSearch turns are
still planned).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:09:11 +09:00

146 lines
6.6 KiB
Python

"""One-step tool turns in the v2 brain skip the planner (planner.spec.md, "Intent judge (v2)").
When the router's only pick is a tool that does its job in one call (the weather, a meal log, a
look at the screen), the planner's ordered steps add nothing but a full LLM call before the answer.
The engine then gives the chat model the tool directly. Its skipped-tool guard, which reads the
plan, is replaced for these turns by one nudge: an answer given without the tool is dropped and the
model is asked once to call it.
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
NATIVE_MODEL = "gpt-oss:20b"
FABRICATED = "서울은 지금 24도에 맑습니다."
GROUNDED = "서울은 지금 13도이고 맑습니다."
def _content(text: str) -> dict:
return {"message": {"role": "assistant", "content": text}}
def _native_call(name: str, args: dict) -> dict:
return {"message": {"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": name, "arguments": args}}]}}
def _run(mock_config, db, dialogue_memory, *, routed, chat_replies, plan=("Reply to the user.",)):
from jarvis.memory.history import HistoryContext
from jarvis.reply import engine as engine_mod
from jarvis.tools.types import ToolExecutionResult
mock_config.ollama_chat_model = NATIVE_MODEL
mock_config.evaluator_enabled = False
mock_config.stream_browser = False
seen: list[list[dict]] = []
replies = iter(chat_replies)
def fake_chat(*args, **kwargs):
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
return next(replies)
invoked: list[str] = []
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
invoked.append(tool_name)
return ToolExecutionResult(success=True, reply_text="Seoul: 13C, clear sky.", error_message=None)
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
patch.object(engine_mod, "select_tools", return_value=list(routed)), \
patch.object(engine_mod, "plan_query", return_value=list(plan)) as planner:
reply = engine_mod.run_reply_engine(
db=db, cfg=mock_config, tts=None, text="서울 지금 우산 필요해?",
dialogue_memory=dialogue_memory, history=HistoryContext(), legacy_memory=False,
intent_judge=True)
return reply, invoked, seen, planner.call_count
@pytest.mark.unit
@pytest.mark.parametrize("routed", [["getWeather", "stop"], ["logMeal", "stop"], ["screenView"]])
def test_a_lone_one_step_tool_needs_no_plan(mock_config, db, dialogue_memory, routed):
_, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed,
chat_replies=[_native_call(routed[0], {}), _content(GROUNDED)])
assert planned == 0
@pytest.mark.unit
@pytest.mark.parametrize("routed", [
["webSearch", "stop"], # searches chain (who directed it, what else they made)
["schedule", "stop"], # list, then cancel the one asked about
["getWeather", "webSearch", "stop"], # two tools
], ids=["search", "schedule", "two tools"])
def test_turns_that_may_take_several_steps_keep_the_plan(mock_config, db, dialogue_memory, routed):
_, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed,
chat_replies=[_content(GROUNDED)])
assert planned == 1
@pytest.mark.unit
def test_a_tool_less_answer_on_a_one_step_turn_is_dropped_and_the_tool_called(mock_config, db, dialogue_memory):
reply, invoked, seen, _ = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
chat_replies=[_content(FABRICATED), _native_call("getWeather", {"location": "Seoul"}),
_content(GROUNDED)])
assert invoked == ["getWeather"]
assert reply == GROUNDED
# The made-up answer never reaches a later turn, and the retry names the tool.
assert not any(FABRICATED in (m.get("content") or "") for m in seen[1])
assert "getWeather" in (seen[1][-1].get("content") or "")
@pytest.mark.unit
def test_a_second_made_up_answer_falls_back_to_the_plan_and_its_tool(mock_config, db, dialogue_memory):
# Asked once, the model still answers without the tool: the turn takes the planned path, the
# skipped-tool guard runs the planned call, and no made-up answer reaches the user.
reply, invoked, seen, planned = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
plan=["getWeather location='Seoul'", "Reply to the user with the weather."],
chat_replies=[_content(FABRICATED), _content(FABRICATED), _content(GROUNDED)])
assert planned == 1
assert invoked == ["getWeather"]
assert reply == GROUNDED
assert not any(FABRICATED in (m.get("content") or "") for m in seen[-1])
@pytest.mark.unit
def test_a_question_back_goes_out_when_the_plan_needs_no_tool(mock_config, db, dialogue_memory):
# The late plan judges that no tool is needed (a question back to the user): it is delivered.
question = "어느 도시 날씨를 알려 드릴까요?"
reply, invoked, seen, planned = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
chat_replies=[_content(question), _content(question)])
assert planned == 1
assert invoked == []
assert reply == question
assert len(seen) == 2
@pytest.mark.unit
def test_a_model_that_calls_the_tool_is_not_asked_again(mock_config, db, dialogue_memory):
reply, invoked, seen, _ = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
chat_replies=[_native_call("getWeather", {"location": "Seoul"}), _content(GROUNDED)])
assert invoked == ["getWeather"]
assert reply == GROUNDED
assert len(seen) == 2
@pytest.mark.unit
def test_v1_turns_still_plan_a_one_step_tool(mock_config, db, dialogue_memory):
from jarvis.reply import engine as engine_mod
mock_config.ollama_chat_model = NATIVE_MODEL
mock_config.evaluator_enabled = False
with patch.object(engine_mod, "chat_with_messages", return_value=_content(GROUNDED)), \
patch.object(engine_mod, "select_tools", return_value=["getWeather", "stop"]), \
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]) as planner:
engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="서울 지금 우산 필요해?",
dialogue_memory=dialogue_memory)
assert planner.call_count == 1