Files
javis_bot/tests/test_engine_compact_prompt.py
javis-bot a6932d9b16 perf(reply): v2 turns with one one-step tool skip the planner
When the router's only pick is a tool that does its job in one call
(getWeather, logMeal, fetchMeals, screenView) and the chat model calls
tools natively, the planner no longer runs: on Strata it averaged 2.5 s
per tool turn (2026-11-16 16:45-16:58 engine log, 9 calls) before the
answer could start. Search, browsers, schedule, notes, chat channels and
picks of two or more tools still plan. The rule reads the router's pick,
not the words, so it is language-agnostic.

Without a plan the skipped-tool guard has nothing to run, so these turns
get one nudge instead: an answer given before any tool ran is dropped and
the model is asked to call the tool (or ask for a missing detail).

Evals (qwen3:8b, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1, 3 runs of
tool_skip_confabulation, weather_autoderive_location, screen_view_tool,
followup_supplies_missing_tool_arg, tool_router_implicit): 71/84 before,
72/84 after, same known failures; planner calls 14 -> 3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:00:36 +09:00

112 lines
5.7 KiB
Python

"""The v2 brain's compact prompt (voice_worker.spec.md, "History and brain").
Strata reads every prompt from the start, so prompt length is reply latency. Behaviour: the compact
prompt is far shorter than v1's for the same turn, still carries the tool honesty rules and the
reply-language default, and brings the full tool guidance only on turns that offer a data tool.
"""
from unittest.mock import Mock, patch
import pytest
from src.jarvis.memory.conversation import DialogueMemory
from src.jarvis.memory.history import HistoryContext
from src.jarvis.reply.engine import run_reply_engine
from src.jarvis.reply.prompts.model_variants import TOOL_CONSTRAINTS_LARGE
from src.jarvis.system_prompt import COMPACT_ACTION_RULE, TOOL_HONESTY_RULES, default_language_directive
from tests.test_ollama_num_predict import _mock_cfg
def _system_prompt(compact, routed):
cfg = _mock_cfg(512)
cfg.stream_browser = False
cfg.tool_selection_strategy = "llm"
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]), \
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)), \
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
mock_chat.return_value = {"message": {"content": "네."}}
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="안녕?", dialogue_memory=DialogueMemory(),
history=HistoryContext(), legacy_memory=False, default_language="Korean",
compact_prompt=compact)
messages = mock_chat.call_args.kwargs.get("messages") or mock_chat.call_args.args[2]
return messages[0]["content"]
@pytest.mark.unit
@pytest.mark.parametrize("routed", [[], ["getWeather"]])
def test_the_compact_prompt_is_far_shorter_and_keeps_the_rules(routed):
full, compact = _system_prompt(False, routed), _system_prompt(True, routed)
assert len(compact) < len(full) * 0.6, (len(compact), len(full))
for prompt in (full, compact):
assert TOOL_HONESTY_RULES in prompt
assert default_language_directive("Korean") in prompt
@pytest.mark.unit
def test_the_full_tool_guidance_comes_only_with_a_data_tool():
chatty = _system_prompt(True, [])
assert COMPACT_ACTION_RULE in chatty and TOOL_CONSTRAINTS_LARGE not in chatty
weather = _system_prompt(True, ["getWeather"])
assert TOOL_CONSTRAINTS_LARGE in weather and COMPACT_ACTION_RULE not in weather
def _intent_turn(prepared, routed=("getWeather",)):
cfg = _mock_cfg(512)
cfg.stream_browser = False
cfg.tool_selection_strategy = "llm"
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]) as planner, \
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)) as router, \
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
mock_chat.return_value = {"message": {"content": "네."}}
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="날씨 어때?", dialogue_memory=DialogueMemory(),
history=HistoryContext(), legacy_memory=False, intent_judge=True,
prepared_tools=prepared)
tools = [t["function"]["name"] for t in (mock_chat.call_args.kwargs.get("tools") or [])]
return planner.call_count, router.call_count, tools
@pytest.mark.unit
def test_the_intent_judge_plans_only_turns_that_need_a_tool():
# No tool needed: one router call, then the answer.
assert _intent_turn(None, routed=())[:2] == (0, 1)
# A tool turn that may take several steps: the planner orders them.
planner, router, tools = _intent_turn(None, routed=("webSearch",))
assert (planner, router) == (1, 1) and "webSearch" in tools
# A lone one-step tool needs no plan (tests/test_engine_one_step_turn.py).
planner, router, tools = _intent_turn(None)
assert (planner, router) == (0, 1) and "getWeather" in tools
@pytest.mark.unit
def test_a_prepared_pick_replaces_the_router_call():
planner, router, tools = _intent_turn([])
assert (planner, router) == (0, 0)
planner, router, tools = _intent_turn(["webSearch"])
assert (planner, router) == (1, 0) and "webSearch" in tools and "getWeather" not in tools
@pytest.mark.unit
@pytest.mark.parametrize("stream", [False, True])
def test_screen_share_tools_are_offered_only_in_screen_share_mode(stream):
cfg = _mock_cfg(512)
cfg.stream_browser = stream
cfg.tool_selection_strategy = "llm"
with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \
patch("src.jarvis.reply.engine.select_tools", return_value=[]) as router, \
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
patch("src.jarvis.reply.engine._maybe_deterministic_site_search", return_value=None), \
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
mock_chat.return_value = {"message": {"content": "네."}}
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="피곤하다", dialogue_memory=DialogueMemory(),
history=HistoryContext(), legacy_memory=False)
offered = set(router.call_args.kwargs["builtin_tools"])
screen = {"controlBrowser", "browseAndPlay", "setBroadcast"}
assert (screen <= offered) if stream else not (screen & offered)
assert "getWeather" in offered