Files
javis_bot/evals/test_web_search_fallback.py
javis-bot af1276cf6b refactor(search): remove the Gemini search backend
The user asked to drop it: nobody had the Gemini CLI login or a key set up,
and Exa then Wikipedia cover the voice use. webSearch is now live Chrome,
Exa, Brave (opt-in), Wikipedia. Removes the CLI/REST backends, the
GEMINI_* settings, the Docker login mount and checks, and updates the specs
and docs/llm_contexts.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 09:24:11 +09:00

89 lines
3.3 KiB
Python

"""
Regression eval: the provider chain rescues a failed primary search honestly.
Exa is the primary provider. When it is down or finds nothing (and no Brave
key is set up), Wikipedia is the last resort; when that misses too,
an honest "found nothing" envelope forbids facts so the reply LLM cannot
confabulate from its priors.
This file is behavioural, not judge-driven: it exercises the real
`WebSearchTool.run` against a mocked network and asserts the observable
outcome — the rescued content lands in the untrusted-extract fence and no
anti-confabulation envelope fires when a rescue succeeded.
Run: .venv/bin/python -m pytest evals/test_web_search_fallback.py -v
"""
from unittest.mock import Mock, patch
import pytest
import requests
from jarvis.tools.base import ToolContext
from jarvis.tools.builtin.web_search import WebSearchTool
def _make_ctx(cfg_overrides=None):
cfg = Mock()
cfg.web_search_enabled = True
cfg.voice_debug = False
cfg.stream_browser = False
cfg.brave_search_api_key = ""
cfg.wikipedia_fallback_enabled = True
for k, v in (cfg_overrides or {}).items():
setattr(cfg, k, v)
ctx = Mock(spec=ToolContext)
ctx.user_print = Mock()
ctx.cfg = cfg
ctx.language = "en"
return ctx
# Exa unreachable: the real `_exa_search` runs and finds nothing.
def _exa_down():
return patch(
"jarvis.tools.builtin.web_search.requests.post",
side_effect=requests.exceptions.ConnectionError("exa down"),
)
@pytest.mark.eval
class TestFallbackChainRescuesFailedSearch:
"""Exa down + Wikipedia fallback = honest rescue, not confabulation."""
@_exa_down()
@patch("jarvis.tools.builtin.web_search._wikipedia_summary")
def test_wikipedia_rescues_when_exa_is_down(self, mock_wiki, _post):
mock_wiki.return_value = (
"Possessor",
"https://en.wikipedia.org/wiki/Possessor",
"Possessor is a 2020 psychological body-horror film.",
)
result = WebSearchTool().run({"search_query": "possessor movie"}, _make_ctx())
assert result.success is True
# Rescued content must be inside the untrusted fence.
assert "<<<BEGIN UNTRUSTED WEB EXTRACT>>>" in result.reply_text
assert "psychological body-horror" in result.reply_text
# The honest envelope must NOT fire — the chain rescued the query.
lowered = result.reply_text.lower()
assert "found nothing usable" not in lowered
assert "you have failed" not in lowered
# Provenance line list matches the rescue source.
assert "en.wikipedia.org" in result.reply_text
@_exa_down()
@patch("jarvis.tools.builtin.web_search._wikipedia_summary", return_value=None)
def test_honest_envelope_when_all_providers_fail(self, _wiki, _post):
"""Exa down, no Brave key, Wikipedia miss → honest envelope."""
result = WebSearchTool().run({"search_query": "obscure thing"}, _make_ctx())
assert result.success is True
lowered = result.reply_text.lower()
assert "found nothing usable" in lowered
assert "you have failed" in lowered
assert "two short sentences" in lowered
# Must not pretend there were results.
assert "<<<BEGIN UNTRUSTED WEB EXTRACT>>>" not in result.reply_text