Files
javis_bot/tests/test_eval_helpers_backend.py
javis-bot 69a884b182 test(evals): run the evals against any server named by EVAL_JUDGE_BASE_URL, including /v1
Ten eval files pinned their reply engine to http://localhost:11434 and the
judge only spoke Ollama's /api/tags and /api/chat, so pointing the suite at
Strata (OpenAI-compatible /v1 only) skipped or broke most evals.

- MockConfig and every eval config read EVAL_JUDGE_BASE_URL.
- The judge availability check reads /v1/models and the judge call goes
  through jarvis.llm when the URL ends in /v1; the Ollama path is unchanged.
- run_evals.sh probes /models for a /v1 URL.

The intent-judge and listener evals stay pinned to gemma4:e2b on local Ollama.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 18:49:08 +09:00

91 lines
3.1 KiB
Python

"""The eval suite talks to whichever LLM server EVAL_JUDGE_BASE_URL names.
Strata (and any OpenAI-compatible server) only serves ``/v1``; a base URL
ending in ``/v1`` must work for the judge availability check, the judge
call, and the default config every eval builds its reply engine from.
These tests run the helpers against a small local fake of the OpenAI API.
"""
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import json
import sys
import threading
import pytest
_ROOT = Path(__file__).resolve().parent.parent
_EVALS = _ROOT / "evals"
if str(_EVALS) not in sys.path:
sys.path.insert(0, str(_EVALS))
import helpers # noqa: E402
class _FakeOpenAI(BaseHTTPRequestHandler):
models = ["qwen-test"]
seen_bodies: list = []
def log_message(self, *args): # keep test output quiet
pass
def _send(self, code, payload):
raw = json.dumps(payload).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def do_GET(self):
if self.path == "/v1/models":
self._send(200, {"object": "list", "data": [{"id": m} for m in self.models]})
else:
self._send(404, {"error": "not found"})
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
if self.path == "/v1/chat/completions":
type(self).seen_bodies.append(body)
self._send(200, {"choices": [{"message": {"role": "assistant", "content": "OVERALL: PASS"}}]})
else:
self._send(404, {"error": "not found"})
@pytest.fixture
def openai_server(monkeypatch):
_FakeOpenAI.seen_bodies = []
server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
url = f"http://127.0.0.1:{server.server_address[1]}/v1"
monkeypatch.setattr(helpers, "JUDGE_BASE_URL", url)
try:
yield url
finally:
server.shutdown()
server.server_close()
def test_judge_is_available_when_the_v1_server_lists_the_model(openai_server, monkeypatch):
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
assert helpers.is_judge_llm_available() is True
def test_judge_is_unavailable_when_the_v1_server_lacks_the_model(openai_server, monkeypatch):
monkeypatch.setattr(helpers, "JUDGE_MODEL", "gemma4:e2b")
assert helpers.is_judge_llm_available() is False
def test_judge_call_goes_through_chat_completions(openai_server, monkeypatch):
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
reply = helpers.call_judge_llm("judge system", "judge user")
assert reply == "OVERALL: PASS"
sent = _FakeOpenAI.seen_bodies[-1]
assert sent["model"] == "qwen-test"
assert [m["content"] for m in sent["messages"]] == ["judge system", "judge user"]
def test_default_eval_config_points_at_the_judge_server(openai_server):
assert helpers.MockConfig().ollama_base_url == openai_server