Live 2026-11-18 '자비 사과랑 배 중에 뭐가 더 달아' was searched as '사과와 배 중哪个가 더 단가요'. The rule and the user message's closing line now say to use only the query's letters. Eval test_search_in_users_language.py, new case fruit (qwen3.6-35b-a3b, game PC): Han in the search 7/7 before; rule only 3/3, closing line only 3/3, both 0/6; whole file 12/12. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
68 lines
3.5 KiB
Python
68 lines
3.5 KiB
Python
"""
|
|
Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get.
|
|
|
|
Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to
|
|
watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in
|
|
English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact
|
|
prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is
|
|
a fake that records its query. Checked by script: the query has Hangul.
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v
|
|
"""
|
|
|
|
import re
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import ToolCallCapture, create_mock_tool_run
|
|
|
|
CASES = [
|
|
("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"),
|
|
("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"),
|
|
("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"),
|
|
# Live 2026-11-18: the search was "사과와 배 중哪个가 더 단가요" (another script mixed in).
|
|
("fruit", "자비 사과랑 배 중에 뭐가 더 달아 검색해줘"),
|
|
]
|
|
|
|
|
|
def _foreign_letters(text: str, reference: str) -> str:
|
|
"""Letters of ``text`` whose writing system is neither in ``reference`` nor Latin."""
|
|
import unicodedata
|
|
|
|
def script(ch):
|
|
return unicodedata.name(ch, "").split(" ")[0]
|
|
keep = {script(c) for c in reference if c.isalpha()} | {"LATIN"}
|
|
return "".join(c for c in text if c.isalpha() and script(c) not in keep)
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestSearchInUsersLanguage:
|
|
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
|
|
def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory,
|
|
monkeypatch, case):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
cid, query = case
|
|
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
mock_config.location_name = "대한민국"
|
|
capture = ToolCallCapture()
|
|
result = "Search results: (fake) 1. 결과 하나 2. 결과 둘"
|
|
with patch("jarvis.reply.engine.run_tool_with_retries",
|
|
side_effect=create_mock_tool_run(capture, {"webSearch": result})):
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean", compact_prompt=True)
|
|
searches = [str(c["args"].get("search_query") or c["args"].get("query") or "")
|
|
for c in capture.calls if c["name"] == "webSearch"]
|
|
print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}")
|
|
assert searches, f"[{cid}] no web search: {capture.tool_names()}"
|
|
assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}"
|
|
assert not any(_foreign_letters(q, query) for q in searches), f"[{cid}] another script mixed in: {searches}"
|