- botBrowser `play`: searches YouTube for the words as said, opens the first video (not an ad, without the playlist) and starts it, in one call (an address given opens that page); `type` also takes `query`. Live 0.1.19: the model stopped on YouTube's home page or the results, a retry was refused as a third call, and the answer still said the video played. - `brain.location` (기본 위치, default 대한민국) goes to the brain as `location_name`: the time line says it without GeoIP (no database on the game PC), and getWeather without a place uses it. - Planner and webSearch keep search words in the user's language (never translated into English), so recommendations fit where the user is. Evals (qwen3:8b, 3 runs): play ending on a playing video found with the words as said 0/6 -> 5/6; Korean searches for Korean requests 6/9 -> 8/9. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
55 lines
2.8 KiB
Python
55 lines
2.8 KiB
Python
"""
|
|
Eval: a Korean request searches the web in Korean, so the results are the ones people in Korea get.
|
|
|
|
Live 2026-11-16 (0.1.19): "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘" searched "best Netflix dramas to
|
|
watch now 2026" and recommended American shows; "엔비디아 rtx 5050 그래픽카드 가격 검색해 줘" searched in
|
|
English and answered with an Amazon US price. The reply engine runs as the v2 brain runs it (compact
|
|
prompt, Korean default, the place set in the settings: brain.location, default 대한민국); webSearch is
|
|
a fake that records its query. Checked by script: the query has Hangul.
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen3:8b python -m pytest evals/test_search_in_users_language.py -v
|
|
"""
|
|
|
|
import re
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import ToolCallCapture, create_mock_tool_run
|
|
|
|
CASES = [
|
|
("netflix", "요즘 볼 만한 넷플릭스 드라마 세 개만 추천해 줘"),
|
|
("price", "엔비디아 RTX 5050 그래픽카드 가격 검색해 줘"),
|
|
("restaurants", "이번 주말에 갈 만한 맛집 좀 찾아 줘"),
|
|
]
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestSearchInUsersLanguage:
|
|
@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES])
|
|
def test_a_korean_request_searches_in_korean(self, mock_config, eval_db, eval_dialogue_memory,
|
|
monkeypatch, case):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
cid, query = case
|
|
monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False)
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
mock_config.location_name = "대한민국"
|
|
capture = ToolCallCapture()
|
|
result = "Search results: (fake) 1. 결과 하나 2. 결과 둘"
|
|
with patch("jarvis.reply.engine.run_tool_with_retries",
|
|
side_effect=create_mock_tool_run(capture, {"webSearch": result})):
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean", compact_prompt=True)
|
|
searches = [str(c["args"].get("search_query") or c["args"].get("query") or "")
|
|
for c in capture.calls if c["name"] == "webSearch"]
|
|
print(f"\n search language [{JUDGE_MODEL}] {cid}: searches={searches} reply={(reply or '')[:120]!r}")
|
|
assert searches, f"[{cid}] no web search: {capture.tool_names()}"
|
|
assert all(re.search(r"[가-힣]", q) for q in searches), f"[{cid}] searched in another language: {searches}"
|