When the router's only pick is a tool that does its job in one call (getWeather, logMeal, fetchMeals, screenView) and the chat model calls tools natively, the planner no longer runs: on Strata it averaged 2.5 s per tool turn (2026-11-16 16:45-16:58 engine log, 9 calls) before the answer could start. Search, browsers, schedule, notes, chat channels and picks of two or more tools still plan. The rule reads the router's pick, not the words, so it is language-agnostic. Without a plan the skipped-tool guard has nothing to run, so these turns get one nudge instead: an answer given before any tool ran is dropped and the model is asked to call the tool (or ask for a missing detail). Evals (qwen3:8b, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1, 3 runs of tool_skip_confabulation, weather_autoderive_location, screen_view_tool, followup_supplies_missing_tool_arg, tool_router_implicit): 71/84 before, 72/84 after, same known failures; planner calls 14 -> 3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
5.7 KiB
Python
112 lines
5.7 KiB
Python
"""The v2 brain's compact prompt (voice_worker.spec.md, "History and brain").
|
|
|
|
Strata reads every prompt from the start, so prompt length is reply latency. Behaviour: the compact
|
|
prompt is far shorter than v1's for the same turn, still carries the tool honesty rules and the
|
|
reply-language default, and brings the full tool guidance only on turns that offer a data tool.
|
|
"""
|
|
|
|
from unittest.mock import Mock, patch
|
|
|
|
import pytest
|
|
|
|
from src.jarvis.memory.conversation import DialogueMemory
|
|
from src.jarvis.memory.history import HistoryContext
|
|
from src.jarvis.reply.engine import run_reply_engine
|
|
from src.jarvis.reply.prompts.model_variants import TOOL_CONSTRAINTS_LARGE
|
|
from src.jarvis.system_prompt import COMPACT_ACTION_RULE, TOOL_HONESTY_RULES, default_language_directive
|
|
|
|
from tests.test_ollama_num_predict import _mock_cfg
|
|
|
|
|
|
def _system_prompt(compact, routed):
|
|
cfg = _mock_cfg(512)
|
|
cfg.stream_browser = False
|
|
cfg.tool_selection_strategy = "llm"
|
|
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]), \
|
|
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)), \
|
|
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
|
|
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
|
|
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
|
|
mock_chat.return_value = {"message": {"content": "네."}}
|
|
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="안녕?", dialogue_memory=DialogueMemory(),
|
|
history=HistoryContext(), legacy_memory=False, default_language="Korean",
|
|
compact_prompt=compact)
|
|
messages = mock_chat.call_args.kwargs.get("messages") or mock_chat.call_args.args[2]
|
|
return messages[0]["content"]
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("routed", [[], ["getWeather"]])
|
|
def test_the_compact_prompt_is_far_shorter_and_keeps_the_rules(routed):
|
|
full, compact = _system_prompt(False, routed), _system_prompt(True, routed)
|
|
assert len(compact) < len(full) * 0.6, (len(compact), len(full))
|
|
for prompt in (full, compact):
|
|
assert TOOL_HONESTY_RULES in prompt
|
|
assert default_language_directive("Korean") in prompt
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_full_tool_guidance_comes_only_with_a_data_tool():
|
|
chatty = _system_prompt(True, [])
|
|
assert COMPACT_ACTION_RULE in chatty and TOOL_CONSTRAINTS_LARGE not in chatty
|
|
weather = _system_prompt(True, ["getWeather"])
|
|
assert TOOL_CONSTRAINTS_LARGE in weather and COMPACT_ACTION_RULE not in weather
|
|
|
|
|
|
def _intent_turn(prepared, routed=("getWeather",)):
|
|
cfg = _mock_cfg(512)
|
|
cfg.stream_browser = False
|
|
cfg.tool_selection_strategy = "llm"
|
|
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]) as planner, \
|
|
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)) as router, \
|
|
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
|
|
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
|
|
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
|
|
mock_chat.return_value = {"message": {"content": "네."}}
|
|
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="날씨 어때?", dialogue_memory=DialogueMemory(),
|
|
history=HistoryContext(), legacy_memory=False, intent_judge=True,
|
|
prepared_tools=prepared)
|
|
tools = [t["function"]["name"] for t in (mock_chat.call_args.kwargs.get("tools") or [])]
|
|
return planner.call_count, router.call_count, tools
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_the_intent_judge_plans_only_turns_that_need_a_tool():
|
|
# No tool needed: one router call, then the answer.
|
|
assert _intent_turn(None, routed=())[:2] == (0, 1)
|
|
# A tool turn that may take several steps: the planner orders them.
|
|
planner, router, tools = _intent_turn(None, routed=("webSearch",))
|
|
assert (planner, router) == (1, 1) and "webSearch" in tools
|
|
# A lone one-step tool needs no plan (tests/test_engine_one_step_turn.py).
|
|
planner, router, tools = _intent_turn(None)
|
|
assert (planner, router) == (0, 1) and "getWeather" in tools
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_prepared_pick_replaces_the_router_call():
|
|
planner, router, tools = _intent_turn([])
|
|
assert (planner, router) == (0, 0)
|
|
planner, router, tools = _intent_turn(["webSearch"])
|
|
assert (planner, router) == (1, 0) and "webSearch" in tools and "getWeather" not in tools
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("stream", [False, True])
|
|
def test_screen_share_tools_are_offered_only_in_screen_share_mode(stream):
|
|
cfg = _mock_cfg(512)
|
|
cfg.stream_browser = stream
|
|
cfg.tool_selection_strategy = "llm"
|
|
with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \
|
|
patch("src.jarvis.reply.engine.select_tools", return_value=[]) as router, \
|
|
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
|
|
patch("src.jarvis.reply.engine._maybe_deterministic_site_search", return_value=None), \
|
|
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
|
|
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
|
|
mock_chat.return_value = {"message": {"content": "네."}}
|
|
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="피곤하다", dialogue_memory=DialogueMemory(),
|
|
history=HistoryContext(), legacy_memory=False)
|
|
offered = set(router.call_args.kwargs["builtin_tools"])
|
|
screen = {"controlBrowser", "browseAndPlay", "setBroadcast"}
|
|
assert (screen <= offered) if stream else not (screen & offered)
|
|
assert "getWeather" in offered
|