DuckDuckGo now serves only its bot-check page (game PC and host alike), so every search fell to Wikipedia full-text search, which picked unrelated pages (박진영 for songs for a rainy day, 대한민국 for a graphics card ranking). Live 2026-11-18 answers took 9-15 s and were wrong. - Exa's free MCP endpoint (no key) is the first provider after the live-broadcast Chrome; it returns page text, so no fetch is needed. - Gemini runs only when Exa found nothing and a CLI login (oauth) or key (apikey) is present; without a login the CLI is no longer started (about 2 s per search). - Wikipedia is used only when the page title is named in the query; otherwise the honest nothing-found envelope. - Letters of a script the user did not use are dropped from the query (the model searched '사과와 배 중哪个가 더 단가요'). - DuckDuckGo removed; spec, docs and tests follow the new order. - weather: any forecast failure (not only requests errors) tries met.no. Real model (qwen3.6-35b-a3b, game PC), real network: graphics card ranking, apples vs pears, rainy-day genres and black holes all answered correctly in 3.7-5.2 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
91 lines
3.3 KiB
Python
91 lines
3.3 KiB
Python
"""
|
|
Regression eval: the provider chain rescues a failed primary search honestly.
|
|
|
|
Exa is the primary provider. When it is down or finds nothing (and no Gemini
|
|
login or key is set up), Wikipedia is the last resort; when that misses too,
|
|
an honest "found nothing" envelope forbids facts so the reply LLM cannot
|
|
confabulate from its priors.
|
|
|
|
This file is behavioural, not judge-driven: it exercises the real
|
|
`WebSearchTool.run` against a mocked network and asserts the observable
|
|
outcome — the rescued content lands in the untrusted-extract fence and no
|
|
anti-confabulation envelope fires when a rescue succeeded.
|
|
|
|
Run: .venv/bin/python -m pytest evals/test_web_search_fallback.py -v
|
|
"""
|
|
|
|
from unittest.mock import Mock, patch
|
|
|
|
import pytest
|
|
import requests
|
|
|
|
from jarvis.tools.base import ToolContext
|
|
from jarvis.tools.builtin.web_search import WebSearchTool
|
|
|
|
|
|
def _make_ctx(cfg_overrides=None):
|
|
cfg = Mock()
|
|
cfg.web_search_enabled = True
|
|
cfg.voice_debug = False
|
|
cfg.stream_browser = False
|
|
cfg.gemini_auth = "api_key"
|
|
cfg.gemini_api_key = ""
|
|
cfg.brave_search_api_key = ""
|
|
cfg.wikipedia_fallback_enabled = True
|
|
for k, v in (cfg_overrides or {}).items():
|
|
setattr(cfg, k, v)
|
|
ctx = Mock(spec=ToolContext)
|
|
ctx.user_print = Mock()
|
|
ctx.cfg = cfg
|
|
ctx.language = "en"
|
|
return ctx
|
|
|
|
|
|
# Exa unreachable: the real `_exa_search` runs and finds nothing.
|
|
def _exa_down():
|
|
return patch(
|
|
"jarvis.tools.builtin.web_search.requests.post",
|
|
side_effect=requests.exceptions.ConnectionError("exa down"),
|
|
)
|
|
|
|
|
|
@pytest.mark.eval
|
|
class TestFallbackChainRescuesFailedSearch:
|
|
"""Exa down + Wikipedia fallback = honest rescue, not confabulation."""
|
|
|
|
@_exa_down()
|
|
@patch("jarvis.tools.builtin.web_search._wikipedia_summary")
|
|
def test_wikipedia_rescues_when_exa_is_down(self, mock_wiki, _post):
|
|
mock_wiki.return_value = (
|
|
"Possessor",
|
|
"https://en.wikipedia.org/wiki/Possessor",
|
|
"Possessor is a 2020 psychological body-horror film.",
|
|
)
|
|
|
|
result = WebSearchTool().run({"search_query": "possessor movie"}, _make_ctx())
|
|
|
|
assert result.success is True
|
|
# Rescued content must be inside the untrusted fence.
|
|
assert "<<<BEGIN UNTRUSTED WEB EXTRACT>>>" in result.reply_text
|
|
assert "psychological body-horror" in result.reply_text
|
|
# The honest envelope must NOT fire — the chain rescued the query.
|
|
lowered = result.reply_text.lower()
|
|
assert "found nothing usable" not in lowered
|
|
assert "you have failed" not in lowered
|
|
# Provenance line list matches the rescue source.
|
|
assert "en.wikipedia.org" in result.reply_text
|
|
|
|
@_exa_down()
|
|
@patch("jarvis.tools.builtin.web_search._wikipedia_summary", return_value=None)
|
|
def test_honest_envelope_when_all_providers_fail(self, _wiki, _post):
|
|
"""Exa down, no Gemini, no Brave key, Wikipedia miss → honest envelope."""
|
|
result = WebSearchTool().run({"search_query": "obscure thing"}, _make_ctx())
|
|
|
|
assert result.success is True
|
|
lowered = result.reply_text.lower()
|
|
assert "found nothing usable" in lowered
|
|
assert "you have failed" in lowered
|
|
assert "two short sentences" in lowered
|
|
# Must not pretend there were results.
|
|
assert "<<<BEGIN UNTRUSTED WEB EXTRACT>>>" not in result.reply_text
|