DuckDuckGo now serves only its bot-check page (game PC and host alike), so every search fell to Wikipedia full-text search, which picked unrelated pages (박진영 for songs for a rainy day, 대한민국 for a graphics card ranking). Live 2026-11-18 answers took 9-15 s and were wrong. - Exa's free MCP endpoint (no key) is the first provider after the live-broadcast Chrome; it returns page text, so no fetch is needed. - Gemini runs only when Exa found nothing and a CLI login (oauth) or key (apikey) is present; without a login the CLI is no longer started (about 2 s per search). - Wikipedia is used only when the page title is named in the query; otherwise the honest nothing-found envelope. - Letters of a script the user did not use are dropped from the query (the model searched '사과와 배 중哪个가 더 단가요'). - DuckDuckGo removed; spec, docs and tests follow the new order. - weather: any forecast failure (not only requests errors) tries met.no. Real model (qwen3.6-35b-a3b, game PC), real network: graphics card ranking, apples vs pears, rainy-day genres and black holes all answered correctly in 3.7-5.2 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
66 lines
3.3 KiB
Python
66 lines
3.3 KiB
Python
"""
|
|
Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get.
|
|
|
|
Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to
|
|
watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in
|
|
English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact
|
|
prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is
|
|
a fake that records its query. Checked by script: the query has Hangul.
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v
|
|
"""
|
|
|
|
import re
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import ToolCallCapture, create_mock_tool_run
|
|
|
|
CASES = [
|
|
("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"),
|
|
("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"),
|
|
("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"),
|
|
]
|
|
|
|
|
|
def _foreign_letters(text: str, reference: str) -> str:
|
|
"""Letters of ``text`` whose writing system is neither in ``reference`` nor Latin."""
|
|
import unicodedata
|
|
|
|
def script(ch):
|
|
return unicodedata.name(ch, "").split(" ")[0]
|
|
keep = {script(c) for c in reference if c.isalpha()} | {"LATIN"}
|
|
return "".join(c for c in text if c.isalpha() and script(c) not in keep)
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestSearchInUsersLanguage:
|
|
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
|
|
def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory,
|
|
monkeypatch, case):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
cid, query = case
|
|
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
mock_config.location_name = "대한민국"
|
|
capture = ToolCallCapture()
|
|
result = "Search results: (fake) 1. 결과 하나 2. 결과 둘"
|
|
with patch("jarvis.reply.engine.run_tool_with_retries",
|
|
side_effect=create_mock_tool_run(capture, {"webSearch": result})):
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean", compact_prompt=True)
|
|
searches = [str(c["args"].get("search_query") or c["args"].get("query") or "")
|
|
for c in capture.calls if c["name"] == "webSearch"]
|
|
print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}")
|
|
assert searches, f"[{cid}] no web search: {capture.tool_names()}"
|
|
assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}"
|
|
assert not any(_foreign_letters(q, query) for q in searches), f"[{cid}] another script mixed in: {searches}"
|