Files
javis_bot/tests/test_engine_compact_prompt.py
javis-bot 4d0ae91e8a perf(reply): v2 tool turns get the tool guidance in short, and a shorter reply-language default
Strata reads every reply prompt from the start in 2,048-token steps. The v2
compact prompt carried v1's full tool guidance and constraints (about 900
Qwen3.6 tokens) on every turn with a data tool, and the reply-language
default (about 120 tokens) twice.

COMPACT_TOOL_GUIDANCE keeps the same rules (act instead of refusing, facts
only from the result, look up named things, auto-derived and self-contained
arguments) in about a quarter of the tokens, without English examples; the
reply-language default says the same in half. v1's prompts are unchanged.

Measured on Strata (Qwen3.6, 20 lines of recent conversation): a weather
turn's prompt 3,436 -> 2,722 tokens, a search turn's second call
4,096 -> 3,383. Evals (compact prompt + intent judge, Qwen3.6): 24/24
before, 30/30 twice after (weather location, follow-up arguments, tool-skip
confabulation, search language, reply language, Possessor fidelity, reply
length, multi-turn context).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 21:13:08 +09:00

129 lines
6.6 KiB
Python

"""The v2 brain's compact prompt (voice_worker.spec.md, "History and brain").
Strata reads every prompt from the start, so prompt length is reply latency. Behaviour: the compact
prompt is far shorter than v1's for the same turn, still carries the tool honesty rules and the
reply-language default, and brings the (short) tool guidance only on turns that offer a data tool.
"""
from unittest.mock import Mock, patch
import pytest
from src.jarvis.memory.conversation import DialogueMemory
from src.jarvis.memory.history import HistoryContext
from src.jarvis.reply.engine import run_reply_engine
from src.jarvis.reply.prompts.model_variants import TOOL_CONSTRAINTS_LARGE
from src.jarvis.system_prompt import (
COMPACT_ACTION_RULE, COMPACT_TOOL_GUIDANCE, TOOL_HONESTY_RULES, default_language_directive,
)
from tests.test_ollama_num_predict import _mock_cfg
def _system_prompt(compact, routed):
cfg = _mock_cfg(512)
cfg.stream_browser = False
cfg.tool_selection_strategy = "llm"
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]), \
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)), \
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
mock_chat.return_value = {"message": {"content": "네."}}
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="안녕?", dialogue_memory=DialogueMemory(),
history=HistoryContext(), legacy_memory=False, default_language="Korean",
compact_prompt=compact)
messages = mock_chat.call_args.kwargs.get("messages") or mock_chat.call_args.args[2]
return messages[0]["content"]
@pytest.mark.unit
@pytest.mark.parametrize("routed", [[], ["getWeather"]])
def test_the_compact_prompt_is_far_shorter_and_keeps_the_rules(routed):
full, compact = _system_prompt(False, routed), _system_prompt(True, routed)
assert len(compact) < len(full) * 0.6, (len(compact), len(full))
for prompt in (full, compact):
assert TOOL_HONESTY_RULES in prompt
assert default_language_directive("Korean") in prompt
@pytest.mark.unit
def test_the_tool_guidance_comes_only_with_a_data_tool():
chatty = _system_prompt(True, [])
assert COMPACT_ACTION_RULE in chatty and COMPACT_TOOL_GUIDANCE not in chatty
weather = _system_prompt(True, ["getWeather"])
assert COMPACT_TOOL_GUIDANCE in weather and COMPACT_ACTION_RULE not in weather
@pytest.mark.unit
def test_a_data_tool_turn_gets_the_short_tool_guidance_not_v1s():
# v1 keeps its full guidance; the compact prompt says the same rules in a fraction of it.
assert TOOL_CONSTRAINTS_LARGE in _system_prompt(False, ["getWeather"])
weather = _system_prompt(True, ["getWeather"])
assert TOOL_CONSTRAINTS_LARGE not in weather
assert len(COMPACT_TOOL_GUIDANCE) < len(TOOL_CONSTRAINTS_LARGE) / 2
def _intent_turn(prepared, routed=("getWeather",), first=None):
cfg = _mock_cfg(512)
cfg.stream_browser = False
cfg.tool_selection_strategy = "llm"
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]) as planner, \
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)) as router, \
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
answer = {"message": {"content": "네."}}
mock_chat.side_effect = [first, answer] if first else None
mock_chat.return_value = answer
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="날씨 어때?", dialogue_memory=DialogueMemory(),
history=HistoryContext(), legacy_memory=False, intent_judge=True,
prepared_tools=prepared)
tools = [t["function"]["name"] for t in (mock_chat.call_args.kwargs.get("tools") or [])]
return planner.call_count, router.call_count, tools
@pytest.mark.unit
def test_the_intent_judge_plans_only_turns_that_need_a_tool():
# No tool needed: one router call, then the answer.
assert _intent_turn(None, routed=())[:2] == (0, 1)
# A tool turn that may take several steps: the planner orders them.
planner, router, tools = _intent_turn(None, routed=("webSearch",))
assert (planner, router) == (1, 1) and "webSearch" in tools
# A lone one-step tool the model calls needs no plan (tests/test_engine_one_step_turn.py).
weather_call = {"message": {"content": "", "tool_calls": [
{"id": "c1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}}
with patch("src.jarvis.reply.engine.run_tool_with_retries",
return_value=Mock(success=True, reply_text="13C", error_message=None, images=None)):
planner, router, tools = _intent_turn(None, first=weather_call)
assert (planner, router) == (0, 1) and "getWeather" in tools
@pytest.mark.unit
def test_a_prepared_pick_replaces_the_router_call():
planner, router, tools = _intent_turn([])
assert (planner, router) == (0, 0)
planner, router, tools = _intent_turn(["webSearch"])
assert (planner, router) == (1, 0) and "webSearch" in tools and "getWeather" not in tools
@pytest.mark.unit
@pytest.mark.parametrize("stream", [False, True])
def test_screen_share_tools_are_offered_only_in_screen_share_mode(stream):
cfg = _mock_cfg(512)
cfg.stream_browser = stream
cfg.tool_selection_strategy = "llm"
with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \
patch("src.jarvis.reply.engine.select_tools", return_value=[]) as router, \
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
patch("src.jarvis.reply.engine._maybe_deterministic_site_search", return_value=None), \
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
mock_chat.return_value = {"message": {"content": "네."}}
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="피곤하다", dialogue_memory=DialogueMemory(),
history=HistoryContext(), legacy_memory=False)
offered = set(router.call_args.kwargs["builtin_tools"])
screen = {"controlBrowser", "browseAndPlay", "setBroadcast"}
assert (screen <= offered) if stream else not (screen & offered)
assert "getWeather" in offered