Files
javis_bot/evals/test_reply_language_default.py
javis-bot 7623d6d002 feat(voice_worker): the v2 brain answers in Korean unless asked for another language
The brain passes default_language (worker --reply-language, default Korean) to
the reply engine; unlike the output_language lock it gives way when the user
explicitly asks for a language. Tools get the ko code (--tool-language).

Eval evals/test_reply_language_default.py (Qwen3.6 on Strata): 6/6, English
questions answered in Korean, an explicit English request in English, and the
control without the default answers English in English.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 10:34:12 +09:00

78 lines
3.6 KiB
Python

"""
Eval: the v2 brain replies in its default language (Korean) unless the user asks for another.
The v2 voice worker's brain passes ``default_language="Korean"`` to run_reply_engine
(voice_worker.spec.md, "History and brain"; system_prompt.default_language_directive). Unlike the
``output_language`` lock it gives way when the user explicitly asks for a language. Checked by
script: a Korean reply is mostly Hangul, an English one has no Hangul. The control runs the same
English question without the default, which replies in English: the directive makes the difference.
Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_reply_language_default.py -v
"""
import re
from unittest.mock import patch
import pytest
from conftest import requires_judge_llm
from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run
_HANGUL = re.compile(r"[가-힣]")
_LATIN = re.compile(r"[A-Za-z]")
def _share_hangul(text: str) -> float:
h, l = len(_HANGUL.findall(text)), len(_LATIN.findall(text))
return h / (h + l) if h + l else 0.0
CASES = [
# (id, query, expected language)
("english-question", "Why is the sky blue?", "ko"),
("english-small-talk", "Hey, how is your day going?", "ko"),
("korean-question", "하늘은 왜 파래?", "ko"),
("asks-english", "영어로 대답해 줘. 하늘은 왜 파래?", "en"),
("asks-english-in-english", "Please answer in English: why is the sky blue?", "en"),
]
def _run(mock_config, eval_db, eval_dialogue_memory, query, default_language, monkeypatch):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.reply.engine import run_reply_engine
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.tts_engine = "melo" # as the v2 brain runs
capture = ToolCallCapture()
with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {})), \
patch("jarvis.reply.engine._extra_config", lambda key, default="": default):
return run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, legacy_memory=False,
default_language=default_language)
@pytest.mark.eval
@requires_judge_llm
class TestReplyLanguageDefault:
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
def test_replies_in_korean_unless_asked_otherwise(self, mock_config, eval_db, eval_dialogue_memory,
monkeypatch, case):
cid, query, want = case
reply = _run(mock_config, eval_db, eval_dialogue_memory, query, "Korean", monkeypatch) or ""
print(f"\n Reply language [{cid}]: {reply[:200]}")
assert_not_fallback_reply(reply, context=cid)
share = _share_hangul(reply)
if want == "ko":
assert share >= 0.8, f"[{cid}] not a Korean reply ({share:.2f}): {reply!r}"
else:
assert share == 0.0 and _LATIN.search(reply), f"[{cid}] not an English reply: {reply!r}"
def test_without_the_default_an_english_question_gets_english(self, mock_config, eval_db,
eval_dialogue_memory, monkeypatch):
"""Control: the directive, not something else in the prompt, makes the reply Korean."""
reply = _run(mock_config, eval_db, eval_dialogue_memory, "Why is the sky blue?", None, monkeypatch) or ""
print(f"\n Reply language control: {reply[:200]}")
assert _share_hangul(reply) == 0.0, reply