Files
javis_bot/tests/test_engine_one_step_turn.py
javis-bot a6932d9b16 perf(reply): v2 turns with one one-step tool skip the planner
When the router's only pick is a tool that does its job in one call
(getWeather, logMeal, fetchMeals, screenView) and the chat model calls
tools natively, the planner no longer runs: on Strata it averaged 2.5 s
per tool turn (2026-11-16 16:45-16:58 engine log, 9 calls) before the
answer could start. Search, browsers, schedule, notes, chat channels and
picks of two or more tools still plan. The rule reads the router's pick,
not the words, so it is language-agnostic.

Without a plan the skipped-tool guard has nothing to run, so these turns
get one nudge instead: an answer given before any tool ran is dropped and
the model is asked to call the tool (or ask for a missing detail).

Evals (qwen3:8b, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1, 3 runs of
tool_skip_confabulation, weather_autoderive_location, screen_view_tool,
followup_supplies_missing_tool_arg, tool_router_implicit): 71/84 before,
72/84 after, same known failures; planner calls 14 -> 3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:00:36 +09:00

131 lines
5.8 KiB
Python

"""One-step tool turns in the v2 brain skip the planner (planner.spec.md, "Intent judge (v2)").
When the router's only pick is a tool that does its job in one call (the weather, a meal log, a
look at the screen), the planner's ordered steps add nothing but a full LLM call before the answer.
The engine then gives the chat model the tool directly. Its skipped-tool guard, which reads the
plan, is replaced for these turns by one nudge: an answer given without the tool is dropped and the
model is asked once to call it.
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
NATIVE_MODEL = "gpt-oss:20b"
FABRICATED = "서울은 지금 24도에 맑습니다."
GROUNDED = "서울은 지금 13도이고 맑습니다."
def _content(text: str) -> dict:
return {"message": {"role": "assistant", "content": text}}
def _native_call(name: str, args: dict) -> dict:
return {"message": {"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": name, "arguments": args}}]}}
def _run(mock_config, db, dialogue_memory, *, routed, chat_replies, plan=("Reply to the user.",)):
from jarvis.memory.history import HistoryContext
from jarvis.reply import engine as engine_mod
from jarvis.tools.types import ToolExecutionResult
mock_config.ollama_chat_model = NATIVE_MODEL
mock_config.evaluator_enabled = False
mock_config.stream_browser = False
seen: list[list[dict]] = []
replies = iter(chat_replies)
def fake_chat(*args, **kwargs):
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
return next(replies)
invoked: list[str] = []
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
invoked.append(tool_name)
return ToolExecutionResult(success=True, reply_text="Seoul: 13C, clear sky.", error_message=None)
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
patch.object(engine_mod, "select_tools", return_value=list(routed)), \
patch.object(engine_mod, "plan_query", return_value=list(plan)) as planner:
reply = engine_mod.run_reply_engine(
db=db, cfg=mock_config, tts=None, text="서울 지금 우산 필요해?",
dialogue_memory=dialogue_memory, history=HistoryContext(), legacy_memory=False,
intent_judge=True)
return reply, invoked, seen, planner.call_count
@pytest.mark.unit
@pytest.mark.parametrize("routed", [["getWeather", "stop"], ["logMeal", "stop"], ["screenView"]])
def test_a_lone_one_step_tool_needs_no_plan(mock_config, db, dialogue_memory, routed):
_, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed,
chat_replies=[_content(GROUNDED)] * 2)
assert planned == 0
@pytest.mark.unit
@pytest.mark.parametrize("routed", [
["webSearch", "stop"], # searches chain (who directed it, what else they made)
["schedule", "stop"], # list, then cancel the one asked about
["getWeather", "webSearch", "stop"], # two tools
], ids=["search", "schedule", "two tools"])
def test_turns_that_may_take_several_steps_keep_the_plan(mock_config, db, dialogue_memory, routed):
_, _, _, planned = _run(mock_config, db, dialogue_memory, routed=routed,
chat_replies=[_content(GROUNDED)])
assert planned == 1
@pytest.mark.unit
def test_a_tool_less_answer_on_a_one_step_turn_is_dropped_and_the_tool_called(mock_config, db, dialogue_memory):
reply, invoked, seen, _ = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
chat_replies=[_content(FABRICATED), _native_call("getWeather", {"location": "Seoul"}),
_content(GROUNDED)])
assert invoked == ["getWeather"]
assert reply == GROUNDED
# The made-up answer never reaches a later turn, and the retry names the tool.
assert not any(FABRICATED in (m.get("content") or "") for m in seen[1])
assert "getWeather" in (seen[1][-1].get("content") or "")
@pytest.mark.unit
def test_the_tool_is_asked_for_only_once(mock_config, db, dialogue_memory):
# A second tool-less answer (for example a question back to the user) is delivered.
question = "어느 도시 날씨를 알려 드릴까요?"
reply, invoked, seen, _ = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
chat_replies=[_content(question), _content(question)])
assert invoked == []
assert reply == question
assert len(seen) == 2
@pytest.mark.unit
def test_a_model_that_calls_the_tool_is_not_asked_again(mock_config, db, dialogue_memory):
reply, invoked, seen, _ = _run(
mock_config, db, dialogue_memory, routed=["getWeather", "stop"],
chat_replies=[_native_call("getWeather", {"location": "Seoul"}), _content(GROUNDED)])
assert invoked == ["getWeather"]
assert reply == GROUNDED
assert len(seen) == 2
@pytest.mark.unit
def test_v1_turns_still_plan_a_one_step_tool(mock_config, db, dialogue_memory):
from jarvis.reply import engine as engine_mod
mock_config.ollama_chat_model = NATIVE_MODEL
mock_config.evaluator_enabled = False
with patch.object(engine_mod, "chat_with_messages", return_value=_content(GROUNDED)), \
patch.object(engine_mod, "select_tools", return_value=["getWeather", "stop"]), \
patch.object(engine_mod, "extract_search_params_for_memory", return_value={"keywords": []}), \
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]) as planner:
engine_mod.run_reply_engine(db=db, cfg=mock_config, tts=None, text="서울 지금 우산 필요해?",
dialogue_memory=dialogue_memory)
assert planner.call_count == 1