"""The eval suite talks to whichever LLM server EVAL_JUDGE_BASE_URL names. Strata (and any OpenAI-compatible server) only serves ``/v1``; a base URL ending in ``/v1`` must work for the judge availability check, the judge call, and the default config every eval builds its reply engine from. These tests run the helpers against a small local fake of the OpenAI API. """ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import json import sys import threading import pytest _ROOT = Path(__file__).resolve().parent.parent _EVALS = _ROOT / "evals" if str(_EVALS) not in sys.path: sys.path.insert(0, str(_EVALS)) import helpers # noqa: E402 class _FakeOpenAI(BaseHTTPRequestHandler): models = ["qwen-test"] seen_bodies: list = [] def log_message(self, *args): # keep test output quiet pass def _send(self, code, payload): raw = json.dumps(payload).encode() self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(raw))) self.end_headers() self.wfile.write(raw) def do_GET(self): if self.path == "/v1/models": self._send(200, {"object": "list", "data": [{"id": m} for m in self.models]}) else: self._send(404, {"error": "not found"}) def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) if self.path == "/v1/chat/completions": type(self).seen_bodies.append(body) self._send(200, {"choices": [{"message": {"role": "assistant", "content": "OVERALL: PASS"}}]}) else: self._send(404, {"error": "not found"}) @pytest.fixture def openai_server(monkeypatch): _FakeOpenAI.seen_bodies = [] server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() url = f"http://127.0.0.1:{server.server_address[1]}/v1" monkeypatch.setattr(helpers, "JUDGE_BASE_URL", url) try: yield url finally: server.shutdown() server.server_close() def test_judge_is_available_when_the_v1_server_lists_the_model(openai_server, monkeypatch): monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test") assert helpers.is_judge_llm_available() is True def test_judge_is_unavailable_when_the_v1_server_lacks_the_model(openai_server, monkeypatch): monkeypatch.setattr(helpers, "JUDGE_MODEL", "gemma4:e2b") assert helpers.is_judge_llm_available() is False def test_judge_call_goes_through_chat_completions(openai_server, monkeypatch): monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test") reply = helpers.call_judge_llm("judge system", "judge user") assert reply == "OVERALL: PASS" sent = _FakeOpenAI.seen_bodies[-1] assert sent["model"] == "qwen-test" assert [m["content"] for m in sent["messages"]] == ["judge system", "judge user"] def test_default_eval_config_points_at_the_judge_server(openai_server): assert helpers.MockConfig().ollama_base_url == openai_server