feat(voice_worker): a short voice prompt and one judgement before each v2 answer
- compact_prompt: the v2 persona in a tenth of v1's words, answers as short as possible, the full tool guidance only on turns with a data tool. A greeting's reply prompt went from about 4,500 to 1,400 tokens on Strata. - intent_judge: the tool router judges whether the turn needs tools; the planner runs only when it does. prepare_intent runs the pick ahead on a partial transcript (worker message prepare), and the reply reuses it. - Without stream_browser the screen-share tools are not offered. - The time context carries the 12-hour form (10:37 was read as pm). - A <tool_call> block written as the reply is parsed as a call, not spoken. Evals (Qwen3.6 on Strata, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1): 221 passed, 4 failed (known failures); new evals/test_voice_reply_length.py (v2 answers 234 chars against v1's 416) and test_reply_language_default.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
108
tests/test_engine_compact_prompt.py
Normal file
108
tests/test_engine_compact_prompt.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""The v2 brain's compact prompt (voice_worker.spec.md, "History and brain").
|
||||
|
||||
Strata reads every prompt from the start, so prompt length is reply latency. Behaviour: the compact
|
||||
prompt is far shorter than v1's for the same turn, still carries the tool honesty rules and the
|
||||
reply-language default, and brings the full tool guidance only on turns that offer a data tool.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.jarvis.memory.conversation import DialogueMemory
|
||||
from src.jarvis.memory.history import HistoryContext
|
||||
from src.jarvis.reply.engine import run_reply_engine
|
||||
from src.jarvis.reply.prompts.model_variants import TOOL_CONSTRAINTS_LARGE
|
||||
from src.jarvis.system_prompt import COMPACT_ACTION_RULE, TOOL_HONESTY_RULES, default_language_directive
|
||||
|
||||
from tests.test_ollama_num_predict import _mock_cfg
|
||||
|
||||
|
||||
def _system_prompt(compact, routed):
|
||||
cfg = _mock_cfg(512)
|
||||
cfg.stream_browser = False
|
||||
cfg.tool_selection_strategy = "llm"
|
||||
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]), \
|
||||
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)), \
|
||||
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
|
||||
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
|
||||
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
|
||||
mock_chat.return_value = {"message": {"content": "네."}}
|
||||
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="안녕?", dialogue_memory=DialogueMemory(),
|
||||
history=HistoryContext(), legacy_memory=False, default_language="Korean",
|
||||
compact_prompt=compact)
|
||||
messages = mock_chat.call_args.kwargs.get("messages") or mock_chat.call_args.args[2]
|
||||
return messages[0]["content"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("routed", [[], ["getWeather"]])
|
||||
def test_the_compact_prompt_is_far_shorter_and_keeps_the_rules(routed):
|
||||
full, compact = _system_prompt(False, routed), _system_prompt(True, routed)
|
||||
assert len(compact) < len(full) * 0.6, (len(compact), len(full))
|
||||
for prompt in (full, compact):
|
||||
assert TOOL_HONESTY_RULES in prompt
|
||||
assert default_language_directive("Korean") in prompt
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_full_tool_guidance_comes_only_with_a_data_tool():
|
||||
chatty = _system_prompt(True, [])
|
||||
assert COMPACT_ACTION_RULE in chatty and TOOL_CONSTRAINTS_LARGE not in chatty
|
||||
weather = _system_prompt(True, ["getWeather"])
|
||||
assert TOOL_CONSTRAINTS_LARGE in weather and COMPACT_ACTION_RULE not in weather
|
||||
|
||||
|
||||
def _intent_turn(prepared, routed=("getWeather",)):
|
||||
cfg = _mock_cfg(512)
|
||||
cfg.stream_browser = False
|
||||
cfg.tool_selection_strategy = "llm"
|
||||
with patch("src.jarvis.reply.engine.plan_query", return_value=["Reply to the user."]) as planner, \
|
||||
patch("src.jarvis.reply.engine.select_tools", return_value=list(routed)) as router, \
|
||||
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
|
||||
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
|
||||
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
|
||||
mock_chat.return_value = {"message": {"content": "네."}}
|
||||
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="날씨 어때?", dialogue_memory=DialogueMemory(),
|
||||
history=HistoryContext(), legacy_memory=False, intent_judge=True,
|
||||
prepared_tools=prepared)
|
||||
tools = [t["function"]["name"] for t in (mock_chat.call_args.kwargs.get("tools") or [])]
|
||||
return planner.call_count, router.call_count, tools
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_intent_judge_plans_only_turns_that_need_a_tool():
|
||||
# No tool needed: one router call, then the answer.
|
||||
assert _intent_turn(None, routed=())[:2] == (0, 1)
|
||||
# A tool turn: the planner orders the steps.
|
||||
planner, router, tools = _intent_turn(None)
|
||||
assert (planner, router) == (1, 1) and "getWeather" in tools
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_prepared_pick_replaces_the_router_call():
|
||||
planner, router, tools = _intent_turn([])
|
||||
assert (planner, router) == (0, 0)
|
||||
planner, router, tools = _intent_turn(["webSearch"])
|
||||
assert (planner, router) == (1, 0) and "webSearch" in tools and "getWeather" not in tools
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_screen_share_tools_are_offered_only_in_screen_share_mode(stream):
|
||||
cfg = _mock_cfg(512)
|
||||
cfg.stream_browser = stream
|
||||
cfg.tool_selection_strategy = "llm"
|
||||
with patch("src.jarvis.reply.engine.plan_query", return_value=[]), \
|
||||
patch("src.jarvis.reply.engine.select_tools", return_value=[]) as router, \
|
||||
patch("src.jarvis.reply.engine._extra_config", lambda key, default="": default), \
|
||||
patch("src.jarvis.reply.engine._maybe_deterministic_site_search", return_value=None), \
|
||||
patch("src.jarvis.reply.engine.extract_text_from_response", return_value="네."), \
|
||||
patch("src.jarvis.reply.engine.chat_with_messages") as mock_chat:
|
||||
mock_chat.return_value = {"message": {"content": "네."}}
|
||||
run_reply_engine(db=Mock(), cfg=cfg, tts=None, text="피곤하다", dialogue_memory=DialogueMemory(),
|
||||
history=HistoryContext(), legacy_memory=False)
|
||||
offered = set(router.call_args.kwargs["builtin_tools"])
|
||||
screen = {"controlBrowser", "browseAndPlay", "setBroadcast"}
|
||||
assert (screen <= offered) if stream else not (screen & offered)
|
||||
assert "getWeather" in offered
|
||||
@@ -297,3 +297,21 @@ class TestTextToolCallGuidancePrompt:
|
||||
"an example would reinforce the exact behaviour we want to "
|
||||
"stop."
|
||||
)
|
||||
|
||||
|
||||
class TestToolCallBlockAsText:
|
||||
"""`<tool_call>` blocks written as the reply (Qwen XML or Hermes JSON) are tool calls, not
|
||||
speech, also for a tool that is not on offer (the engine then says it is not available)."""
|
||||
|
||||
def test_qwen_xml(self):
|
||||
content = ("<tool_call>\n<function=setTimer>\n<parameter=duration>\n5 minutes\n</parameter>\n"
|
||||
"</function>\n</tool_call>")
|
||||
name, args, call_id = _extract(content)
|
||||
assert name == "setTimer" and args == {"duration": "5 minutes"} and call_id
|
||||
|
||||
def test_hermes_json(self):
|
||||
name, args, _ = _extract('<tool_call>\n{"name": "webSearch", "arguments": {"search_query": "x"}}\n</tool_call>')
|
||||
assert name == "webSearch" and args == {"search_query": "x"}
|
||||
|
||||
def test_prose_mentioning_tags_is_not_a_call(self):
|
||||
assert _extract("I would call <tool_call> if I could.")[0] is None
|
||||
|
||||
@@ -372,9 +372,11 @@ def test_the_brain_answers_in_korean_by_default_even_with_a_v1_english_voice(tmp
|
||||
seen = {}
|
||||
|
||||
def fake_engine(db, cfg, tts, text, mem, language=None, history=None, legacy_memory=True,
|
||||
default_language=None):
|
||||
default_language=None, compact_prompt=False, **kw):
|
||||
seen["directive"] = reply_language_directive(None, cfg.tts_engine, default_language)
|
||||
seen["language"] = language
|
||||
seen["compact"] = compact_prompt
|
||||
seen["stream_browser"] = cfg.stream_browser
|
||||
return "ok"
|
||||
|
||||
import jarvis.reply.engine as engine
|
||||
@@ -382,6 +384,8 @@ def test_the_brain_answers_in_korean_by_default_even_with_a_v1_english_voice(tmp
|
||||
brain_mod.JarvisBrain().reply("100", "what time is it?", HistoryContext())
|
||||
assert seen["directive"] == default_language_directive("Korean")
|
||||
assert seen["language"] == "ko"
|
||||
# The short voice prompt, and no v1 screen-share browser (it needs v1's Node bot).
|
||||
assert seen["compact"] is True and seen["stream_browser"] is False
|
||||
# Without a default the brain answers in the asker's language.
|
||||
brain_mod.JarvisBrain(language=None, reply_language=None).reply("100", "q", HistoryContext())
|
||||
assert seen["directive"] is None and seen["language"] is None
|
||||
@@ -966,3 +970,138 @@ def test_a_chat_question_the_app_never_reports_counts_as_not_posted(store):
|
||||
actions, results = run_delete_then_yes(store, dict(via="chat", speak=False), no_report,
|
||||
delivery_timeout_s=0.1)
|
||||
assert actions == [] and results == [("지워", True), ("응, 지워", False)]
|
||||
|
||||
|
||||
class PreparingBrain(FakeBrain):
|
||||
"""Each prepare waits until the test lets it finish."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
import threading
|
||||
self.prepared = []
|
||||
self.release = threading.Event()
|
||||
|
||||
def prepare(self, user, text, chat):
|
||||
self.prepared.append((user, text, chat))
|
||||
self.release.wait(5)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_prepare_runs_one_at_a_time_per_user_and_only_the_newest_different_text_waits(store):
|
||||
brain = PreparingBrain()
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await send(ws, type="prepare", user="100", text="자비스 오늘")
|
||||
await asyncio.sleep(0.2)
|
||||
await send(ws, type="prepare", user="100", text="자비스 오늘 서울")
|
||||
await send(ws, type="prepare", user="100", text="자비스 오늘 서울 날씨 어때")
|
||||
await send(ws, type="prepare", user="200", text="자비스 안녕")
|
||||
await asyncio.sleep(0.2)
|
||||
assert sorted(brain.prepared) == [("100", "자비스 오늘", False), ("200", "자비스 안녕", False)]
|
||||
brain.release.set()
|
||||
await asyncio.sleep(0.3)
|
||||
# About the same as the one just prepared: not prepared again.
|
||||
await send(ws, type="prepare", user="100", text="자비스 오늘 서울 날씨 어때?")
|
||||
await asyncio.sleep(0.3)
|
||||
await send(ws, type="reply", id="r1", user="100", text="x", via="voice", speak=False)
|
||||
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
|
||||
seen = asyncio.run(with_worker(body, store=store, brain=brain))
|
||||
assert [t for u, t, _ in brain.prepared if u == "100"] == ["자비스 오늘", "자비스 오늘 서울 날씨 어때"]
|
||||
assert not [m for m in seen if isinstance(m, dict) and m.get("type") == "error"], "nothing is sent back"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_brain_without_prepare_ignores_it(store):
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await send(ws, type="prepare", user="100", text="자비스 안녕")
|
||||
await send(ws, type="reply", id="r1", user="100", text="hi", via="voice", speak=False)
|
||||
return await recv_until(ws, lambda m: isinstance(m, dict) and m.get("type") == "reply_text")
|
||||
|
||||
seen = asyncio.run(with_worker(body, store=store, brain=FakeBrain()))
|
||||
assert not [m for m in seen if isinstance(m, dict) and m.get("type") == "error"]
|
||||
|
||||
|
||||
def _jarvis_brain(tmp_path, monkeypatch, plans):
|
||||
"""A real JarvisBrain with the engine's judge and reply replaced: ``plans`` maps text to a pick."""
|
||||
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
|
||||
from voice_worker import brain as brain_mod
|
||||
import jarvis.reply.engine as engine
|
||||
|
||||
cfg = tmp_path / "config.json"
|
||||
cfg.write_text(json.dumps({"db_path": str(tmp_path / "jarvis.db")}))
|
||||
monkeypatch.setenv("JARVIS_CONFIG_PATH", str(cfg))
|
||||
seen = {"judged": [], "prepared": []}
|
||||
|
||||
def fake_prepare(cfg, text, mem, chat_available):
|
||||
seen["judged"].append((text, chat_available))
|
||||
return plans.get(text)
|
||||
|
||||
def fake_engine(db, cfg, tts, text, mem, prepared_tools=None, intent_judge=False, **kw):
|
||||
seen["prepared"].append(prepared_tools)
|
||||
seen["intent_judge"] = intent_judge
|
||||
mem.add_message("user", text)
|
||||
mem.add_message("assistant", "ok")
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(engine, "prepare_intent", fake_prepare)
|
||||
monkeypatch.setattr(engine, "run_reply_engine", fake_engine)
|
||||
return brain_mod.JarvisBrain(), seen
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_reply_uses_the_pick_prepared_for_about_the_same_text(tmp_path, monkeypatch):
|
||||
from jarvis.memory.history import HistoryContext
|
||||
|
||||
plan = ["getWeather"]
|
||||
b, seen = _jarvis_brain(tmp_path, monkeypatch, {"자비스 오늘 서울 날씨 어": plan})
|
||||
b.prepare("100", "자비스 오늘 서울 날씨 어", True)
|
||||
assert seen["judged"] == [("자비스 오늘 서울 날씨 어", True)]
|
||||
b.reply("100", "자비스, 오늘 서울 날씨 어때?", HistoryContext())
|
||||
assert seen["prepared"] == [plan] and seen["intent_judge"] is True
|
||||
# Used once: the next reply judges for itself.
|
||||
b.reply("100", "자비스, 오늘 서울 날씨 어때?", HistoryContext())
|
||||
assert seen["prepared"][-1] is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_prepared_pick_is_not_used_for_other_words_another_person_or_a_changed_dialogue(tmp_path, monkeypatch):
|
||||
from jarvis.memory.history import HistoryContext
|
||||
|
||||
plan = ["webSearch"]
|
||||
b, seen = _jarvis_brain(tmp_path, monkeypatch, {"자비스 오늘 서울 날씨 어때": plan})
|
||||
b.prepare("100", "자비스 오늘 서울 날씨 어때", False)
|
||||
b.reply("100", "자비스, 내일 회의 몇 시지?", HistoryContext())
|
||||
b.prepare("100", "자비스 오늘 서울 날씨 어때", False)
|
||||
b.reply("200", "자비스 오늘 서울 날씨 어때", HistoryContext())
|
||||
assert seen["prepared"] == [None, None]
|
||||
b.prepare("100", "자비스 오늘 서울 날씨 어때", False)
|
||||
b._memory("100").add_message("user", "something said meanwhile")
|
||||
b.reply("100", "자비스 오늘 서울 날씨 어때", HistoryContext())
|
||||
assert seen["prepared"][-1] is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_reply_waits_for_a_pick_still_being_prepared_for_its_text(tmp_path, monkeypatch):
|
||||
import threading
|
||||
import time as _time
|
||||
from jarvis.memory.history import HistoryContext
|
||||
|
||||
plan = [] # "no tools" is a pick too
|
||||
b, seen = _jarvis_brain(tmp_path, monkeypatch, {})
|
||||
import jarvis.reply.engine as engine
|
||||
|
||||
def slow_prepare(cfg, text, mem, chat_available):
|
||||
_time.sleep(0.4)
|
||||
return plan
|
||||
monkeypatch.setattr(engine, "prepare_intent", slow_prepare)
|
||||
t = threading.Thread(target=b.prepare, args=("100", "자비스 날씨 어때", False))
|
||||
t.start()
|
||||
_time.sleep(0.1)
|
||||
b.reply("100", "자비스 날씨 어때?", HistoryContext())
|
||||
t.join()
|
||||
assert seen["prepared"] == [plan]
|
||||
|
||||
Reference in New Issue
Block a user