- compact_prompt: the v2 persona in a tenth of v1's words, answers as short as possible, the full tool guidance only on turns with a data tool. A greeting's reply prompt went from about 4,500 to 1,400 tokens on Strata. - intent_judge: the tool router judges whether the turn needs tools; the planner runs only when it does. prepare_intent runs the pick ahead on a partial transcript (worker message prepare), and the reply reuses it. - Without stream_browser the screen-share tools are not offered. - The time context carries the 12-hour form (10:37 was read as pm). - A <tool_call> block written as the reply is parsed as a call, not spoken. Evals (Qwen3.6 on Strata, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1): 221 passed, 4 failed (known failures); new evals/test_voice_reply_length.py (v2 answers 234 chars against v1's 416) and test_reply_language_default.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69 lines
3.2 KiB
Python
69 lines
3.2 KiB
Python
"""
|
|
Eval: the v2 voice prompt keeps answers short (user request 2026-11-12: "답변은 최대한 짧게").
|
|
|
|
The v2 brain runs the reply engine with ``compact_prompt=True`` (system_prompt.build_voice_persona)
|
|
and its intent judge. Every answer here must be short to speak: at most 70 characters (about four
|
|
seconds of speech) and at most two sentences (a joke may need a question and its punchline),
|
|
measured by script. The comparison runs the same questions with v1's prompt:
|
|
the v2 answers must be shorter in total, so the prompt (not the model) is what makes them short.
|
|
|
|
Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_voice_reply_length.py -v
|
|
"""
|
|
|
|
import re
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run
|
|
|
|
QUESTIONS = [
|
|
"자비스, 하늘은 왜 파래?",
|
|
"자비스, 고마워",
|
|
"자비스, 오늘 하루 어땠어?",
|
|
"자비스, 피곤하다",
|
|
"자비스, 물은 몇 도에서 끓어?",
|
|
"자비스, 대한민국 수도가 어디야?",
|
|
"자비스, 재밌는 얘기 하나 해 줘",
|
|
]
|
|
MAX_CHARS = 70
|
|
# A sentence end: . ! ? (or their full-width forms) followed by more text.
|
|
_SENTENCE_BREAK = re.compile(r"[.!?。!?](?=\s*\S)")
|
|
|
|
|
|
def _reply(mock_config, eval_db, eval_dialogue_memory, query, v2, monkeypatch):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.conversation import DialogueMemory
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
mock_config.tts_engine = "melo"
|
|
mock_config.stream_browser = False
|
|
capture = ToolCallCapture()
|
|
with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {})), \
|
|
patch("jarvis.reply.engine._extra_config", lambda key, default="": default):
|
|
return run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=DialogueMemory(), legacy_memory=False,
|
|
default_language="Korean", compact_prompt=v2, intent_judge=v2) or ""
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestVoiceReplyLength:
|
|
def test_v2_answers_are_short_and_shorter_than_v1(self, mock_config, eval_db,
|
|
eval_dialogue_memory, monkeypatch):
|
|
v2 = [_reply(mock_config, eval_db, eval_dialogue_memory, q, True, monkeypatch) for q in QUESTIONS]
|
|
v1 = [_reply(mock_config, eval_db, eval_dialogue_memory, q, False, monkeypatch) for q in QUESTIONS]
|
|
for q, a, b in zip(QUESTIONS, v2, v1):
|
|
print(f"\n {q}\n v2 ({len(a)}): {a}\n v1 ({len(b)}): {b}")
|
|
too_long = [(q, a) for q, a in zip(QUESTIONS, v2)
|
|
if len(a) > MAX_CHARS or len(_SENTENCE_BREAK.findall(a.strip())) > 1]
|
|
for q, a in zip(QUESTIONS, v2):
|
|
assert_not_fallback_reply(a, context=q)
|
|
print(f"\n total chars v2={sum(map(len, v2))} v1={sum(map(len, v1))}")
|
|
assert not too_long, f"not short: {too_long}"
|
|
assert sum(map(len, v2)) < sum(map(len, v1))
|