Files
javis_bot/tests/test_web_search_broadcast_routing.py
javis-bot 2238ddbe42 feat(search): search with Exa, Gemini only with a login, title-guarded Wikipedia
DuckDuckGo now serves only its bot-check page (game PC and host alike), so
every search fell to Wikipedia full-text search, which picked unrelated
pages (박진영 for songs for a rainy day, 대한민국 for a graphics card
ranking). Live 2026-11-18 answers took 9-15 s and were wrong.

- Exa's free MCP endpoint (no key) is the first provider after the
  live-broadcast Chrome; it returns page text, so no fetch is needed.
- Gemini runs only when Exa found nothing and a CLI login (oauth) or key
  (apikey) is present; without a login the CLI is no longer started
  (about 2 s per search).
- Wikipedia is used only when the page title is named in the query;
  otherwise the honest nothing-found envelope.
- Letters of a script the user did not use are dropped from the query
  (the model searched '사과와 배 중哪个가 더 단가요').
- DuckDuckGo removed; spec, docs and tests follow the new order.
- weather: any forecast failure (not only requests errors) tries met.no.

Real model (qwen3.6-35b-a3b, game PC), real network: graphics card
ranking, apples vs pears, rainy-day genres and black holes all answered
correctly in 3.7-5.2 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 02:11:28 +09:00

139 lines
4.7 KiB
Python

"""Behaviour tests for webSearch provider routing.
Routing contract (docs/stream_browser_modes.md, web_search.spec.md):
- master flag cfg.stream_browser (env STREAM_BROWSER) = broadcast capability
- context.broadcasting = live screen-share state for the turn (set by the bot)
master live(ctx) -> backend
------- --------- -------
True True on-screen Chrome (browser_search)
True None Chrome (no signal -> fall back to master)
True False Exa
False True/None Exa (broadcast disabled, never Chrome)
Gemini only runs when Exa found nothing, and the CLI only with a Google login.
"""
from types import SimpleNamespace
import pytest
import jarvis.tools.builtin.realtime_search as rs
import jarvis.tools.builtin.web_search as ws
from jarvis.tools.base import ToolContext
from jarvis.tools.builtin.web_search import WebSearchTool
EXA_HIT = [("Seoul weather", "https://exa.test/seoul", "EXA_TEXT sunny, 21C")]
def _ctx(stream_browser, broadcasting):
cfg = SimpleNamespace(
web_search_enabled=True,
stream_browser=stream_browser,
gemini_auth="oauth",
gemini_api_key="",
gemini_model="gemini-2.0-flash",
wikipedia_fallback_enabled=False,
)
return ToolContext(
db=None,
cfg=cfg,
system_prompt="",
original_prompt="",
redacted_text="",
max_retries=0,
user_print=lambda *a, **k: None,
language=None,
broadcasting=broadcasting,
)
@pytest.fixture
def backends(monkeypatch):
"""Fake every backend; returns the list of backends that ran, in order."""
ran = []
def fake(name, result):
def call(*a, **k):
ran.append(name)
return result() if callable(result) else result
return call
state = SimpleNamespace(exa=EXA_HIT, login=True, ran=ran)
monkeypatch.setattr(rs, "browser_search", fake("browser", "BROWSER_RESULT"))
monkeypatch.setattr(rs, "gemini_cli_search", fake("gemini", "GEMINI_RESULT"))
monkeypatch.setattr(rs, "gemini_search", fake("gemini", "GEMINI_RESULT"))
monkeypatch.setattr(rs, "gemini_oauth_ready", lambda: state.login)
monkeypatch.setattr(ws, "_exa_search", fake("exa", lambda: state.exa))
monkeypatch.setattr(ws, "_wikipedia_summary", fake("wikipedia", None))
return state
def _route(stream_browser, broadcasting, query="현재 서울 날씨"):
res = WebSearchTool().run({"search_query": query}, _ctx(stream_browser, broadcasting))
if res.reply_text == "BROWSER_RESULT":
return "browser"
if res.reply_text == "GEMINI_RESULT":
return "gemini"
if "EXA_TEXT" in res.reply_text:
return "exa"
return "fallthrough"
def test_master_on_live_on_uses_browser(backends):
assert _route(True, True) == "browser"
assert backends.ran == ["browser"]
def test_master_on_live_off_uses_exa(backends):
assert _route(True, False) == "exa"
assert "browser" not in backends.ran
def test_master_on_no_signal_defaults_to_browser(backends):
# None = no live signal (evals / text entry / older bot) -> behave as master
assert _route(True, None) == "browser"
def test_master_off_never_uses_browser_even_if_live(backends):
# STREAM_BROWSER=false disables broadcast entirely.
assert _route(False, True) == "exa"
assert "browser" not in backends.ran
def test_master_off_no_signal_uses_exa(backends):
assert _route(False, None) == "exa"
def test_browser_failure_while_live_falls_through_to_exa(backends, monkeypatch):
monkeypatch.setattr(rs, "browser_search", lambda *a, **k: None)
assert _route(True, True) == "exa"
def test_exa_results_mean_gemini_never_runs(backends):
assert _route(False, None) == "exa"
assert "gemini" not in backends.ran
def test_exa_empty_with_login_uses_gemini_cli(backends):
backends.exa = []
assert _route(False, None) == "gemini"
assert backends.ran == ["exa", "gemini"]
def test_exa_empty_without_login_never_starts_gemini_cli(backends):
# No Google login: the CLI would only fail, so it is not started at all.
backends.exa = []
backends.login = False
assert _route(False, None) == "fallthrough"
assert "gemini" not in backends.ran
def test_a_query_given_as_query_instead_of_search_query_still_searches(backends, monkeypatch):
# Live 2026-11-16: the planner wrote "webSearch query='...'" and the model passed `query`;
# both searches failed with "Please provide a search query".
seen = []
monkeypatch.setattr(ws, "_exa_search", lambda q, *a, **k: seen.append(q) or EXA_HIT)
res = WebSearchTool().run({"query": "안강 산책 코스"}, _ctx(False, None))
assert "EXA_TEXT" in res.reply_text and seen == ["안강 산책 코스"]