Files
javis_bot/evals/test_search_in_users_language.py
javis-bot d31d75f18f fix(planner): plan in the query's own writing system only
Live 2026-11-18 '자비 사과랑 배 중에 뭐가 더 달아' was searched as
'사과와 배 중哪个가 더 단가요'. The rule and the user message's closing line
now say to use only the query's letters.

Eval test_search_in_users_language.py, new case fruit (qwen3.6-35b-a3b,
game PC): Han in the search 7/7 before; rule only 3/3, closing line only
3/3, both 0/6; whole file 12/12.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 02:22:17 +09:00

68 lines
3.5 KiB
Python

"""
Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get.
Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to
watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in
English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact
prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is
a fake that records its query. Checked by script: the query has Hangul.
Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v
"""
import re
from unittest.mock import patch
import pytest
from conftest import requires_judge_llm
from helpers import ToolCallCapture, create_mock_tool_run
CASES = [
("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"),
("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"),
("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"),
# Live 2026-11-18: the search was "사과와 배 중哪个가 더 단가요" (another script mixed in).
("fruit", "자비 사과랑 배 중에 뭐가 더 달아 검색해줘"),
]
def _foreign_letters(text: str, reference: str) -> str:
"""Letters of ``text`` whose writing system is neither in ``reference`` nor Latin."""
import unicodedata
def script(ch):
return unicodedata.name(ch, "").split(" ")[0]
keep = {script(c) for c in reference if c.isalpha()} | {"LATIN"}
return "".join(c for c in text if c.isalpha() and script(c) not in keep)
@pytest.mark.eval
@requires_judge_llm
class TestSearchInUsersLanguage:
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory,
monkeypatch, case):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply.engine import run_reply_engine
cid, query = case
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.location_name = "대한민국"
capture = ToolCallCapture()
result = "Search results: (fake) 1. 결과 하나 2. 결과 둘"
with patch("jarvis.reply.engine.run_tool_with_retries",
side_effect=create_mock_tool_run(capture, {"webSearch": result})):
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean", compact_prompt=True)
searches = [str(c["args"].get("search_query") or c["args"].get("query") or "")
for c in capture.calls if c["name"] == "webSearch"]
print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}")
assert searches, f"[{cid}] no web search: {capture.tool_names()}"
assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}"
assert not any(_foreign_letters(q, query) for q in searches), f"[{cid}] another script mixed in: {searches}"