""" Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get. Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is a fake that records its query. Checked by script: the query has Hangul. Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v """ import re from unittest.mock import patch import pytest from conftest import requires_judge_llm from helpers import ToolCallCapture, create_mock_tool_run CASES = [ ("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"), ("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"), ("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"), # Live 2026-11-18: the search was "사과와 배 중哪个가 더 단가요" (another script mixed in). ("fruit", "자비 사과랑 배 중에 뭐가 더 달아 검색해줘"), ] def _foreign_letters(text: str, reference: str) -> str: """Letters of ``text`` whose writing system is neither in ``reference`` nor Latin.""" import unicodedata def script(ch): return unicodedata.name(ch, "").split(" ")[0] keep = {script(c) for c in reference if c.isalpha()} | {"LATIN"} return "".join(c for c in text if c.isalpha() and script(c) not in keep) @pytest.mark.eval @requires_judge_llm class TestSearchInUsersLanguage: @pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES]) def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory, monkeypatch, case): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.memory.history import HistoryContext from jarvis.reply.engine import run_reply_engine cid, query = case monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_config.location_name = "대한민국" capture = ToolCallCapture() result = "Search results: (fake) 1. 결과 하나 2. 결과 둘" with patch("jarvis.reply.engine.run_tool_with_retries", side_effect=create_mock_tool_run(capture, {"webSearch": result})): reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=eval_dialogue_memory, history=HistoryContext(), legacy_memory=False, default_language="Korean", compact_prompt=True) searches = [str(c["args"].get("search_query") or c["args"].get("query") or "") for c in capture.calls if c["name"] == "webSearch"] print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}") assert searches, f"[{cid}] no web search: {capture.tool_names()}" assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}" assert not any(_foreign_letters(q, query) for q in searches), f"[{cid}] another script mixed in: {searches}"