""" Regression eval: the provider chain rescues a failed primary search honestly. Exa is the primary provider. When it is down or finds nothing (and no Brave key is set up), Wikipedia is the last resort; when that misses too, an honest "found nothing" envelope forbids facts so the reply LLM cannot confabulate from its priors. This file is behavioural, not judge-driven: it exercises the real `WebSearchTool.run` against a mocked network and asserts the observable outcome — the rescued content lands in the untrusted-extract fence and no anti-confabulation envelope fires when a rescue succeeded. Run: .venv/bin/python -m pytest evals/test_web_search_fallback.py -v """ from unittest.mock import Mock, patch import pytest import requests from jarvis.tools.base import ToolContext from jarvis.tools.builtin.web_search import WebSearchTool def _make_ctx(cfg_overrides=None): cfg = Mock() cfg.web_search_enabled = True cfg.voice_debug = False cfg.stream_browser = False cfg.brave_search_api_key = "" cfg.wikipedia_fallback_enabled = True for k, v in (cfg_overrides or {}).items(): setattr(cfg, k, v) ctx = Mock(spec=ToolContext) ctx.user_print = Mock() ctx.cfg = cfg ctx.language = "en" return ctx # Exa unreachable: the real `_exa_search` runs and finds nothing. def _exa_down(): return patch( "jarvis.tools.builtin.web_search.requests.post", side_effect=requests.exceptions.ConnectionError("exa down"), ) @pytest.mark.eval class TestFallbackChainRescuesFailedSearch: """Exa down + Wikipedia fallback = honest rescue, not confabulation.""" @_exa_down() @patch("jarvis.tools.builtin.web_search._wikipedia_summary") def test_wikipedia_rescues_when_exa_is_down(self, mock_wiki, _post): mock_wiki.return_value = ( "Possessor", "https://en.wikipedia.org/wiki/Possessor", "Possessor is a 2020 psychological body-horror film.", ) result = WebSearchTool().run({"search_query": "possessor movie"}, _make_ctx()) assert result.success is True # Rescued content must be inside the untrusted fence. assert "<<>>" in result.reply_text assert "psychological body-horror" in result.reply_text # The honest envelope must NOT fire — the chain rescued the query. lowered = result.reply_text.lower() assert "found nothing usable" not in lowered assert "you have failed" not in lowered # Provenance line list matches the rescue source. assert "en.wikipedia.org" in result.reply_text @_exa_down() @patch("jarvis.tools.builtin.web_search._wikipedia_summary", return_value=None) def test_honest_envelope_when_all_providers_fail(self, _wiki, _post): """Exa down, no Brave key, Wikipedia miss → honest envelope.""" result = WebSearchTool().run({"search_query": "obscure thing"}, _make_ctx()) assert result.success is True lowered = result.reply_text.lower() assert "found nothing usable" in lowered assert "you have failed" in lowered assert "two short sentences" in lowered # Must not pretend there were results. assert "<<>>" not in result.reply_text