""" Eval: the v2 brain replies in its default language (Korean) unless the user asks for another. The v2 voice worker's brain passes ``default_language="Korean"`` to run_reply_engine (voice_worker.spec.md, "History and brain"; system_prompt.default_language_directive). Unlike the ``output_language`` lock it gives way when the user explicitly asks for a language. Checked by script: a Korean reply is mostly Hangul, an English one has no Hangul. The control runs the same English question without the default, which replies in English: the directive makes the difference. Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_reply_language_default.py -v """ import re from unittest.mock import patch import pytest from conftest import requires_judge_llm from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run _HANGUL = re.compile(r"[가-힣]") _LATIN = re.compile(r"[A-Za-z]") def _share_hangul(text: str) -> float: h, l = len(_HANGUL.findall(text)), len(_LATIN.findall(text)) return h / (h + l) if h + l else 0.0 CASES = [ # (id, query, expected language) ("english-question", "Why is the sky blue?", "ko"), ("english-small-talk", "Hey, how is your day going?", "ko"), ("korean-question", "하늘은 왜 파래?", "ko"), ("asks-english", "영어로 대답해 줘. 하늘은 왜 파래?", "en"), ("asks-english-in-english", "Please answer in English: why is the sky blue?", "en"), ] def _run(mock_config, eval_db, eval_dialogue_memory, query, default_language, monkeypatch): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.reply.engine import run_reply_engine monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_config.tts_engine = "melo" # as the v2 brain runs capture = ToolCallCapture() with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {})), \ patch("jarvis.reply.engine._extra_config", lambda key, default="": default): return run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=eval_dialogue_memory, legacy_memory=False, default_language=default_language) @pytest.mark.eval @requires_judge_llm class TestReplyLanguageDefault: @pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES]) def test_replies_in_korean_unless_asked_otherwise(self, mock_config, eval_db, eval_dialogue_memory, monkeypatch, case): cid, query, want = case reply = _run(mock_config, eval_db, eval_dialogue_memory, query, "Korean", monkeypatch) or "" print(f"\n Reply language [{cid}]: {reply[:200]}") assert_not_fallback_reply(reply, context=cid) share = _share_hangul(reply) if want == "ko": assert share >= 0.8, f"[{cid}] not a Korean reply ({share:.2f}): {reply!r}" else: assert share == 0.0 and _LATIN.search(reply), f"[{cid}] not an English reply: {reply!r}" def test_without_the_default_an_english_question_gets_english(self, mock_config, eval_db, eval_dialogue_memory, monkeypatch): """Control: the directive, not something else in the prompt, makes the reply Korean.""" reply = _run(mock_config, eval_db, eval_dialogue_memory, "Why is the sky blue?", None, monkeypatch) or "" print(f"\n Reply language control: {reply[:200]}") assert _share_hangul(reply) == 0.0, reply