The brain passes default_language (worker --reply-language, default Korean) to the reply engine; unlike the output_language lock it gives way when the user explicitly asks for a language. Tools get the ko code (--tool-language). Eval evals/test_reply_language_default.py (Qwen3.6 on Strata): 6/6, English questions answered in Korean, an explicit English request in English, and the control without the default answers English in English. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
78 lines
3.6 KiB
Python
78 lines
3.6 KiB
Python
"""
|
|
Eval: the v2 brain replies in its default language (Korean) unless the user asks for another.
|
|
|
|
The v2 voice worker's brain passes ``default_language="Korean"`` to run_reply_engine
|
|
(voice_worker.spec.md, "History and brain"; system_prompt.default_language_directive). Unlike the
|
|
``output_language`` lock it gives way when the user explicitly asks for a language. Checked by
|
|
script: a Korean reply is mostly Hangul, an English one has no Hangul. The control runs the same
|
|
English question without the default, which replies in English: the directive makes the difference.
|
|
|
|
Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_reply_language_default.py -v
|
|
"""
|
|
|
|
import re
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run
|
|
|
|
_HANGUL = re.compile(r"[가-힣]")
|
|
_LATIN = re.compile(r"[A-Za-z]")
|
|
|
|
|
|
def _share_hangul(text: str) -> float:
|
|
h, l = len(_HANGUL.findall(text)), len(_LATIN.findall(text))
|
|
return h / (h + l) if h + l else 0.0
|
|
|
|
|
|
CASES = [
|
|
# (id, query, expected language)
|
|
("english-question", "Why is the sky blue?", "ko"),
|
|
("english-small-talk", "Hey, how is your day going?", "ko"),
|
|
("korean-question", "하늘은 왜 파래?", "ko"),
|
|
("asks-english", "영어로 대답해 줘. 하늘은 왜 파래?", "en"),
|
|
("asks-english-in-english", "Please answer in English: why is the sky blue?", "en"),
|
|
]
|
|
|
|
|
|
def _run(mock_config, eval_db, eval_dialogue_memory, query, default_language, monkeypatch):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
mock_config.tts_engine = "melo" # as the v2 brain runs
|
|
capture = ToolCallCapture()
|
|
with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {})), \
|
|
patch("jarvis.reply.engine._extra_config", lambda key, default="": default):
|
|
return run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, legacy_memory=False,
|
|
default_language=default_language)
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestReplyLanguageDefault:
|
|
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
|
|
def test_replies_in_korean_unless_asked_otherwise(self, mock_config, eval_db, eval_dialogue_memory,
|
|
monkeypatch, case):
|
|
cid, query, want = case
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, query, "Korean", monkeypatch) or ""
|
|
print(f"\n Reply language [{cid}]: {reply[:200]}")
|
|
assert_not_fallback_reply(reply, context=cid)
|
|
share = _share_hangul(reply)
|
|
if want == "ko":
|
|
assert share >= 0.8, f"[{cid}] not a Korean reply ({share:.2f}): {reply!r}"
|
|
else:
|
|
assert share == 0.0 and _LATIN.search(reply), f"[{cid}] not an English reply: {reply!r}"
|
|
|
|
def test_without_the_default_an_english_question_gets_english(self, mock_config, eval_db,
|
|
eval_dialogue_memory, monkeypatch):
|
|
"""Control: the directive, not something else in the prompt, makes the reply Korean."""
|
|
reply = _run(mock_config, eval_db, eval_dialogue_memory, "Why is the sky blue?", None, monkeypatch) or ""
|
|
print(f"\n Reply language control: {reply[:200]}")
|
|
assert _share_hangul(reply) == 0.0, reply
|