Ten eval files pinned their reply engine to http://localhost:11434 and the judge only spoke Ollama's /api/tags and /api/chat, so pointing the suite at Strata (OpenAI-compatible /v1 only) skipped or broke most evals. - MockConfig and every eval config read EVAL_JUDGE_BASE_URL. - The judge availability check reads /v1/models and the judge call goes through jarvis.llm when the URL ends in /v1; the Ollama path is unchanged. - run_evals.sh probes /models for a /v1 URL. The intent-judge and listener evals stay pinned to gemma4:e2b on local Ollama. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
91 lines
3.1 KiB
Python
91 lines
3.1 KiB
Python
"""The eval suite talks to whichever LLM server EVAL_JUDGE_BASE_URL names.
|
|
|
|
Strata (and any OpenAI-compatible server) only serves ``/v1``; a base URL
|
|
ending in ``/v1`` must work for the judge availability check, the judge
|
|
call, and the default config every eval builds its reply engine from.
|
|
These tests run the helpers against a small local fake of the OpenAI API.
|
|
"""
|
|
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
import json
|
|
import sys
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
_ROOT = Path(__file__).resolve().parent.parent
|
|
_EVALS = _ROOT / "evals"
|
|
if str(_EVALS) not in sys.path:
|
|
sys.path.insert(0, str(_EVALS))
|
|
|
|
import helpers # noqa: E402
|
|
|
|
|
|
class _FakeOpenAI(BaseHTTPRequestHandler):
|
|
models = ["qwen-test"]
|
|
seen_bodies: list = []
|
|
|
|
def log_message(self, *args): # keep test output quiet
|
|
pass
|
|
|
|
def _send(self, code, payload):
|
|
raw = json.dumps(payload).encode()
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(raw)))
|
|
self.end_headers()
|
|
self.wfile.write(raw)
|
|
|
|
def do_GET(self):
|
|
if self.path == "/v1/models":
|
|
self._send(200, {"object": "list", "data": [{"id": m} for m in self.models]})
|
|
else:
|
|
self._send(404, {"error": "not found"})
|
|
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
|
if self.path == "/v1/chat/completions":
|
|
type(self).seen_bodies.append(body)
|
|
self._send(200, {"choices": [{"message": {"role": "assistant", "content": "OVERALL: PASS"}}]})
|
|
else:
|
|
self._send(404, {"error": "not found"})
|
|
|
|
|
|
@pytest.fixture
|
|
def openai_server(monkeypatch):
|
|
_FakeOpenAI.seen_bodies = []
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
url = f"http://127.0.0.1:{server.server_address[1]}/v1"
|
|
monkeypatch.setattr(helpers, "JUDGE_BASE_URL", url)
|
|
try:
|
|
yield url
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
|
|
|
|
def test_judge_is_available_when_the_v1_server_lists_the_model(openai_server, monkeypatch):
|
|
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
|
|
assert helpers.is_judge_llm_available() is True
|
|
|
|
|
|
def test_judge_is_unavailable_when_the_v1_server_lacks_the_model(openai_server, monkeypatch):
|
|
monkeypatch.setattr(helpers, "JUDGE_MODEL", "gemma4:e2b")
|
|
assert helpers.is_judge_llm_available() is False
|
|
|
|
|
|
def test_judge_call_goes_through_chat_completions(openai_server, monkeypatch):
|
|
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
|
|
reply = helpers.call_judge_llm("judge system", "judge user")
|
|
assert reply == "OVERALL: PASS"
|
|
sent = _FakeOpenAI.seen_bodies[-1]
|
|
assert sent["model"] == "qwen-test"
|
|
assert [m["content"] for m in sent["messages"]] == ["judge system", "judge user"]
|
|
|
|
|
|
def test_default_eval_config_points_at_the_judge_server(openai_server):
|
|
assert helpers.MockConfig().ollama_base_url == openai_server
|