After the one nudge, a second answer without the tool no longer goes out as it is: the turn takes the planned path (the planner runs then, one extra call only on this failure) and the skipped-tool guard runs the planned tool, so a made-up "24 degrees" answer cannot reach the user. A plan that needs no tool (a question back to the user) still lets the answer go out. Evals (qwen3:8b, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1, 3 runs): the five tool-skip/weather/screen/follow-up/router files 72/84 (71/84 on 0.1.20, same known failures); Possessor director-then-filmography 0/3, also 0/3 on 0.1.20 (qwen3:8b stops after one search; webSearch turns are still planned). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
146 lines
6.6 KiB
Python
146 lines
6.6 KiB
Python
"""One-step tool turns in the v2 brain skip the planner (planner.spec.md, "Intent judge (v2)").
|
|
|
|
When the router's only pick is a tool that does its job in one call (the weather, a meal log, a
|
|
look at the screen), the planner's ordered steps add nothing but a full LLM call before the answer.
|
|
The engine then gives the chat model the tool directly. Its skipped-tool guard, which reads the
|
|
plan, is replaced for these turns by one nudge: an answer given without the tool is dropped and the
|
|
model is asked once to call it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
|
|
NATIVE_MODEL = "gpt-oss:20b"
|
|
FABRICATED = "서울은 지금 24도에 맑습니다."
|
|
GROUNDED = "서울은 지금 13도이고 맑습니다."
|
|
|
|
|
|
def _content(text: str) -> dict:
|
|
return {"message": {"role": "assistant", "content": text}}
|
|
|
|
|
|
def _native_call(name: str, args: dict) -> dict:
|
|
return {"message": {"role": "assistant", "content": "", "tool_calls": [
|
|
{"id": "call_1", "type": "function", "function": {"name": name, "arguments": args}}]}}
|
|
|
|
|
|
def _run(mock_config, db, dialogue_memory, *, routed, chat_replies, plan=("Reply to the user.",)):
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import engine as engine_mod
|
|
from jarvis.tools.types import ToolExecutionResult
|
|
|
|
mock_config.ollama_chat_model = NATIVE_MODEL
|
|
mock_config.evaluator_enabled = False
|
|
mock_config.stream_browser = False
|
|
seen: list[list[dict]] = []
|
|
replies = iter(chat_replies)
|
|
|
|
def fake_chat(*args, **kwargs):
|
|
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
|
|
return next(replies)
|
|
|
|
invoked: list[str] = []
|
|
|
|
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
|
|
invoked.append(tool_name)
|
|
return ToolExecutionResult(success=True, reply_text="Seoul: 13C, clear sky.", error_message=None)
|
|
|
|
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
|
|
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
|
|
patch.object(engine_mod, "select_tools", return_value=list(routed)), \
|
|
patch.object(engine_mod, "plan_query", return_value=list(plan)) as planner:
|
|
reply = engine_mod.run_reply_engine(
|
|
db=db, cfg=mock_config, tts=None, text="서울 지금 우산 필요해?",
|
|
dialogue_memory=dialogue_memory, history=HistoryContext(), legacy_memory=False,
|
|
intent_judge=True)
|
|
return reply, invoked, seen, planner.call_count
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("routed", [["getWeather", "stop"], ["logMeal", "stop"], ["screenView"]])
|
|
def test_a_lone_one_step_tool_needs_no_plan(mock_config, db, dialogue_memory, routed):
|
|
_, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed,
|
|
chat_replies=[_native_call(routed[0], {}), _content(GROUNDED)])
|
|
assert planned == 0
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("routed", [
|
|
["webSearch", "stop"], # searches chain (who directed it, what else they made)
|
|
["schedule", "stop"], # list, then cancel the one asked about
|
|
["getWeather", "webSearch", "stop"], # two tools
|
|
], ids=["search", "schedule", "two tools"])
|
|
def test_turns_that_may_take_several_steps_keep_the_plan(mock_config, db, dialogue_memory, routed):
|
|
_, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed,
|
|
chat_replies=[_content(GROUNDED)])
|
|
assert planned == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_tool_less_answer_on_a_one_step_turn_is_dropped_and_the_tool_called(mock_config, db, dialogue_memory):
|
|
reply, invoked, seen, _ = _run(
|
|
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
|
|
chat_replies=[_content(FABRICATED), _native_call("getWeather", {"location": "Seoul"}),
|
|
_content(GROUNDED)])
|
|
assert invoked == ["getWeather"]
|
|
assert reply == GROUNDED
|
|
# The made-up answer never reaches a later turn, and the retry names the tool.
|
|
assert not any(FABRICATED in (m.get("content") or "") for m in seen[1])
|
|
assert "getWeather" in (seen[1][-1].get("content") or "")
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_second_made_up_answer_falls_back_to_the_plan_and_its_tool(mock_config, db, dialogue_memory):
|
|
# Asked once, the model still answers without the tool: the turn takes the planned path, the
|
|
# skipped-tool guard runs the planned call, and no made-up answer reaches the user.
|
|
reply, invoked, seen, planned = _run(
|
|
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
|
|
plan=["getWeather location='Seoul'", "Reply to the user with the weather."],
|
|
chat_replies=[_content(FABRICATED), _content(FABRICATED), _content(GROUNDED)])
|
|
assert planned == 1
|
|
assert invoked == ["getWeather"]
|
|
assert reply == GROUNDED
|
|
assert not any(FABRICATED in (m.get("content") or "") for m in seen[-1])
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_question_back_goes_out_when_the_plan_needs_no_tool(mock_config, db, dialogue_memory):
|
|
# The late plan judges that no tool is needed (a question back to the user): it is delivered.
|
|
question = "어느 도시 날씨를 알려 드릴까요?"
|
|
reply, invoked, seen, planned = _run(
|
|
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
|
|
chat_replies=[_content(question), _content(question)])
|
|
assert planned == 1
|
|
assert invoked == []
|
|
assert reply == question
|
|
assert len(seen) == 2
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_model_that_calls_the_tool_is_not_asked_again(mock_config, db, dialogue_memory):
|
|
reply, invoked, seen, _ = _run(
|
|
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
|
|
chat_replies=[_native_call("getWeather", {"location": "Seoul"}), _content(GROUNDED)])
|
|
assert invoked == ["getWeather"]
|
|
assert reply == GROUNDED
|
|
assert len(seen) == 2
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_v1_turns_still_plan_a_one_step_tool(mock_config, db, dialogue_memory):
|
|
from jarvis.reply import engine as engine_mod
|
|
|
|
mock_config.ollama_chat_model = NATIVE_MODEL
|
|
mock_config.evaluator_enabled = False
|
|
with patch.object(engine_mod, "chat_with_messages", return_value=_content(GROUNDED)), \
|
|
patch.object(engine_mod, "select_tools", return_value=["getWeather", "stop"]), \
|
|
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
|
|
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]) as planner:
|
|
engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="서울 지금 우산 필요해?",
|
|
dialogue_memory=dialogue_memory)
|
|
assert planner.call_count == 1
|