Files
javis_bot/evals/test_voice_reply_length.py
javis-bot 46e4e75d05 feat(voice_worker): a short voice prompt and one judgement before each v2 answer
- compact_prompt: the v2 persona in a tenth of v1's words, answers as short as
  possible, the full tool guidance only on turns with a data tool. A greeting's
  reply prompt went from about 4,500 to 1,400 tokens on Strata.
- intent_judge: the tool router judges whether the turn needs tools; the
  planner runs only when it does. prepare_intent runs the pick ahead on a
  partial transcript (worker message prepare), and the reply reuses it.
- Without stream_browser the screen-share tools are not offered.
- The time context carries the 12-hour form (10:37 was read as pm).
- A <tool_call> block written as the reply is parsed as a call, not spoken.

Evals (Qwen3.6 on Strata, EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1):
221 passed, 4 failed (known failures); new evals/test_voice_reply_length.py
(v2 answers 234 chars against v1's 416) and test_reply_language_default.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 12:25:15 +09:00

69 lines
3.2 KiB
Python

"""
Eval: the v2 voice prompt keeps answers short (user request 2026-11-12: "답변은 최대한 짧게").
The v2 brain runs the reply engine with ``compact_prompt=True`` (system_prompt.build_voice_persona)
and its intent judge. Every answer here must be short to speak: at most 70 characters (about four
seconds of speech) and at most two sentences (a joke may need a question and its punchline),
measured by script. The comparison runs the same questions with v1's prompt:
the v2 answers must be shorter in total, so the prompt (not the model) is what makes them short.
Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_voice_reply_length.py -v
"""
import re
from unittest.mock import patch
import pytest
from conftest import requires_judge_llm
from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run
QUESTIONS = [
"자비스, 하늘은 왜 파래?",
"자비스, 고마워",
"자비스, 오늘 하루 어땠어?",
"자비스, 피곤하다",
"자비스, 물은 몇 도에서 끓어?",
"자비스, 대한민국 수도가 어디야?",
"자비스, 재밌는 얘기 하나 해 줘",
]
MAX_CHARS = 70
# A sentence end: . ! ? (or their full-width forms) followed by more text.
_SENTENCE_BREAK = re.compile(r"[.!?。!?](?=\s*\S)")
def _reply(mock_config, eval_db, eval_dialogue_memory, query, v2, monkeypatch):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.conversation import DialogueMemory
from jarvis.reply.engine import run_reply_engine
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.tts_engine = "melo"
mock_config.stream_browser = False
capture = ToolCallCapture()
with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {})), \
patch("jarvis.reply.engine._extra_config", lambda key, default="": default):
return run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=DialogueMemory(), legacy_memory=False,
default_language="Korean", compact_prompt=v2, intent_judge=v2) or ""
@pytest.mark.eval
@requires_judge_llm
class TestVoiceReplyLength:
def test_v2_answers_are_short_and_shorter_than_v1(self, mock_config, eval_db,
eval_dialogue_memory, monkeypatch):
v2 = [_reply(mock_config, eval_db, eval_dialogue_memory, q, True, monkeypatch) for q in QUESTIONS]
v1 = [_reply(mock_config, eval_db, eval_dialogue_memory, q, False, monkeypatch) for q in QUESTIONS]
for q, a, b in zip(QUESTIONS, v2, v1):
print(f"\n {q}\n v2 ({len(a)}): {a}\n v1 ({len(b)}): {b}")
too_long = [(q, a) for q, a in zip(QUESTIONS, v2)
if len(a) > MAX_CHARS or len(_SENTENCE_BREAK.findall(a.strip())) > 1]
for q, a in zip(QUESTIONS, v2):
assert_not_fallback_reply(a, context=q)
print(f"\n total chars v2={sum(map(len, v2))} v1={sum(map(len, v1))}")
assert not too_long, f"not short: {too_long}"
assert sum(map(len, v2)) < sum(map(len, v1))