""" Eval: the v2 voice prompt keeps answers short (user request 2026-11-12: "답변은 최대한 짧게"). The v2 brain runs the reply engine with ``compact_prompt=True`` (system_prompt.build_voice_persona) and its intent judge. Every answer here must be short to speak: at most 70 characters (about four seconds of speech) and at most two sentences (a joke may need a question and its punchline), measured by script. The comparison runs the same questions with v1's prompt: the v2 answers must be shorter in total, so the prompt (not the model) is what makes them short. Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_voice_reply_length.py -v """ import re from unittest.mock import patch import pytest from conftest import requires_judge_llm from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run QUESTIONS = [ "자비스, 하늘은 왜 파래?", "자비스, 고마워", "자비스, 오늘 하루 어땠어?", "자비스, 피곤하다", "자비스, 물은 몇 도에서 끓어?", "자비스, 대한민국 수도가 어디야?", "자비스, 재밌는 얘기 하나 해 줘", ] MAX_CHARS = 70 # A sentence end: . ! ? (or their full-width forms) followed by more text. _SENTENCE_BREAK = re.compile(r"[.!?。!?](?=\s*\S)") def _reply(mock_config, eval_db, eval_dialogue_memory, query, v2, monkeypatch): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.memory.conversation import DialogueMemory from jarvis.reply.engine import run_reply_engine monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_config.tts_engine = "melo" mock_config.stream_browser = False capture = ToolCallCapture() with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {})), \ patch("jarvis.reply.engine._extra_config", lambda key, default="": default): return run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=DialogueMemory(), legacy_memory=False, default_language="Korean", compact_prompt=v2, intent_judge=v2) or "" @pytest.mark.eval @requires_judge_llm class TestVoiceReplyLength: def test_v2_answers_are_short_and_shorter_than_v1(self, mock_config, eval_db, eval_dialogue_memory, monkeypatch): v2 = [_reply(mock_config, eval_db, eval_dialogue_memory, q, True, monkeypatch) for q in QUESTIONS] v1 = [_reply(mock_config, eval_db, eval_dialogue_memory, q, False, monkeypatch) for q in QUESTIONS] for q, a, b in zip(QUESTIONS, v2, v1): print(f"\n {q}\n v2 ({len(a)}): {a}\n v1 ({len(b)}): {b}") too_long = [(q, a) for q, a in zip(QUESTIONS, v2) if len(a) > MAX_CHARS or len(_SENTENCE_BREAK.findall(a.strip())) > 1] for q, a in zip(QUESTIONS, v2): assert_not_fallback_reply(a, context=q) print(f"\n total chars v2={sum(map(len, v2))} v1={sum(map(len, v1))}") assert not too_long, f"not short: {too_long}" assert sum(map(len, v2)) < sum(map(len, v1))