DuckDuckGo now serves only its bot-check page (game PC and host alike), so every search fell to Wikipedia full-text search, which picked unrelated pages (박진영 for songs for a rainy day, 대한민국 for a graphics card ranking). Live 2026-11-18 answers took 9-15 s and were wrong. - Exa's free MCP endpoint (no key) is the first provider after the live-broadcast Chrome; it returns page text, so no fetch is needed. - Gemini runs only when Exa found nothing and a CLI login (oauth) or key (apikey) is present; without a login the CLI is no longer started (about 2 s per search). - Wikipedia is used only when the page title is named in the query; otherwise the honest nothing-found envelope. - Letters of a script the user did not use are dropped from the query (the model searched '사과와 배 중哪个가 더 단가요'). - DuckDuckGo removed; spec, docs and tests follow the new order. - weather: any forecast failure (not only requests errors) tries met.no. Real model (qwen3.6-35b-a3b, game PC), real network: graphics card ranking, apples vs pears, rainy-day genres and black holes all answered correctly in 3.7-5.2 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
139 lines
4.7 KiB
Python
139 lines
4.7 KiB
Python
"""Behaviour tests for webSearch provider routing.
|
|
|
|
Routing contract (docs/stream_browser_modes.md, web_search.spec.md):
|
|
- master flag cfg.stream_browser (env STREAM_BROWSER) = broadcast capability
|
|
- context.broadcasting = live screen-share state for the turn (set by the bot)
|
|
|
|
master live(ctx) -> backend
|
|
------- --------- -------
|
|
True True on-screen Chrome (browser_search)
|
|
True None Chrome (no signal -> fall back to master)
|
|
True False Exa
|
|
False True/None Exa (broadcast disabled, never Chrome)
|
|
|
|
Gemini only runs when Exa found nothing, and the CLI only with a Google login.
|
|
"""
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import jarvis.tools.builtin.realtime_search as rs
|
|
import jarvis.tools.builtin.web_search as ws
|
|
from jarvis.tools.base import ToolContext
|
|
from jarvis.tools.builtin.web_search import WebSearchTool
|
|
|
|
EXA_HIT = [("Seoul weather", "https://exa.test/seoul", "EXA_TEXT sunny, 21C")]
|
|
|
|
|
|
def _ctx(stream_browser, broadcasting):
|
|
cfg = SimpleNamespace(
|
|
web_search_enabled=True,
|
|
stream_browser=stream_browser,
|
|
gemini_auth="oauth",
|
|
gemini_api_key="",
|
|
gemini_model="gemini-2.0-flash",
|
|
wikipedia_fallback_enabled=False,
|
|
)
|
|
return ToolContext(
|
|
db=None,
|
|
cfg=cfg,
|
|
system_prompt="",
|
|
original_prompt="",
|
|
redacted_text="",
|
|
max_retries=0,
|
|
user_print=lambda *a, **k: None,
|
|
language=None,
|
|
broadcasting=broadcasting,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def backends(monkeypatch):
|
|
"""Fake every backend; returns the list of backends that ran, in order."""
|
|
ran = []
|
|
|
|
def fake(name, result):
|
|
def call(*a, **k):
|
|
ran.append(name)
|
|
return result() if callable(result) else result
|
|
return call
|
|
|
|
state = SimpleNamespace(exa=EXA_HIT, login=True, ran=ran)
|
|
monkeypatch.setattr(rs, "browser_search", fake("browser", "BROWSER_RESULT"))
|
|
monkeypatch.setattr(rs, "gemini_cli_search", fake("gemini", "GEMINI_RESULT"))
|
|
monkeypatch.setattr(rs, "gemini_search", fake("gemini", "GEMINI_RESULT"))
|
|
monkeypatch.setattr(rs, "gemini_oauth_ready", lambda: state.login)
|
|
monkeypatch.setattr(ws, "_exa_search", fake("exa", lambda: state.exa))
|
|
monkeypatch.setattr(ws, "_wikipedia_summary", fake("wikipedia", None))
|
|
return state
|
|
|
|
|
|
def _route(stream_browser, broadcasting, query="현재 서울 날씨"):
|
|
res = WebSearchTool().run({"search_query": query}, _ctx(stream_browser, broadcasting))
|
|
if res.reply_text == "BROWSER_RESULT":
|
|
return "browser"
|
|
if res.reply_text == "GEMINI_RESULT":
|
|
return "gemini"
|
|
if "EXA_TEXT" in res.reply_text:
|
|
return "exa"
|
|
return "fallthrough"
|
|
|
|
|
|
def test_master_on_live_on_uses_browser(backends):
|
|
assert _route(True, True) == "browser"
|
|
assert backends.ran == ["browser"]
|
|
|
|
|
|
def test_master_on_live_off_uses_exa(backends):
|
|
assert _route(True, False) == "exa"
|
|
assert "browser" not in backends.ran
|
|
|
|
|
|
def test_master_on_no_signal_defaults_to_browser(backends):
|
|
# None = no live signal (evals / text entry / older bot) -> behave as master
|
|
assert _route(True, None) == "browser"
|
|
|
|
|
|
def test_master_off_never_uses_browser_even_if_live(backends):
|
|
# STREAM_BROWSER=false disables broadcast entirely.
|
|
assert _route(False, True) == "exa"
|
|
assert "browser" not in backends.ran
|
|
|
|
|
|
def test_master_off_no_signal_uses_exa(backends):
|
|
assert _route(False, None) == "exa"
|
|
|
|
|
|
def test_browser_failure_while_live_falls_through_to_exa(backends, monkeypatch):
|
|
monkeypatch.setattr(rs, "browser_search", lambda *a, **k: None)
|
|
assert _route(True, True) == "exa"
|
|
|
|
|
|
def test_exa_results_mean_gemini_never_runs(backends):
|
|
assert _route(False, None) == "exa"
|
|
assert "gemini" not in backends.ran
|
|
|
|
|
|
def test_exa_empty_with_login_uses_gemini_cli(backends):
|
|
backends.exa = []
|
|
assert _route(False, None) == "gemini"
|
|
assert backends.ran == ["exa", "gemini"]
|
|
|
|
|
|
def test_exa_empty_without_login_never_starts_gemini_cli(backends):
|
|
# No Google login: the CLI would only fail, so it is not started at all.
|
|
backends.exa = []
|
|
backends.login = False
|
|
assert _route(False, None) == "fallthrough"
|
|
assert "gemini" not in backends.ran
|
|
|
|
|
|
def test_a_query_given_as_query_instead_of_search_query_still_searches(backends, monkeypatch):
|
|
# Live 2026-11-16: the planner wrote "webSearch query='...'" and the model passed `query`;
|
|
# both searches failed with "Please provide a search query".
|
|
seen = []
|
|
monkeypatch.setattr(ws, "_exa_search", lambda q, *a, **k: seen.append(q) or EXA_HIT)
|
|
res = WebSearchTool().run({"query": "안강 산책 코스"}, _ctx(False, None))
|
|
assert "EXA_TEXT" in res.reply_text and seen == ["안강 산책 코스"]
|