test(evals): run the evals against any server named by EVAL_JUDGE_BASE_URL, including /v1

Ten eval files pinned their reply engine to http://localhost:11434 and the
judge only spoke Ollama's /api/tags and /api/chat, so pointing the suite at
Strata (OpenAI-compatible /v1 only) skipped or broke most evals.

- MockConfig and every eval config read EVAL_JUDGE_BASE_URL.
- The judge availability check reads /v1/models and the judge call goes
  through jarvis.llm when the URL ends in /v1; the Ollama path is unchanged.
- run_evals.sh probes /models for a /v1 URL.

The intent-judge and listener evals stay pinned to gemma4:e2b on local Ollama.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-11 18:49:08 +09:00
parent a45e198df6
commit 69a884b182
13 changed files with 167 additions and 50 deletions

View File

@@ -23,6 +23,8 @@ import os
# were only testing the 20B tier. Defaulting to the small tier is the
# cheapest way to stop that happening again.
JUDGE_MODEL = os.environ.get("EVAL_JUDGE_MODEL", "gemma4:e2b")
# A base URL ending in ``/v1`` (Strata, llama.cpp, Ollama's /v1) selects the
# OpenAI-compatible API, the same switch the app uses in ``jarvis.llm``.
JUDGE_BASE_URL = os.environ.get("EVAL_JUDGE_BASE_URL", "http://localhost:11434")
@@ -242,7 +244,8 @@ def create_mock_tool_run(
@dataclass
class MockConfig:
"""Minimal config object for eval tests."""
ollama_base_url: str = "http://localhost:11434"
# Read at construction so EVAL_JUDGE_BASE_URL reaches every eval.
ollama_base_url: str = field(default_factory=lambda: JUDGE_BASE_URL)
ollama_chat_model: str = "gemma4:e2b"
ollama_embed_model: str = "nomic-embed-text"
db_path: str = ":memory:"
@@ -398,16 +401,21 @@ class JudgeVerdict:
def is_judge_llm_available() -> bool:
"""Check if the judge LLM is available and the model exists."""
import requests
from jarvis.llm import is_openai_compatible
base = JUDGE_BASE_URL.rstrip('/')
try:
# First check if Ollama is running
resp = requests.get(f"{JUDGE_BASE_URL.rstrip('/')}/api/tags", timeout=2)
if resp.status_code != 200:
return False
# Check if the judge model is available
data = resp.json()
models = data.get("models", [])
model_names = [m.get("name", "").split(":")[0] for m in models]
if is_openai_compatible(base):
resp = requests.get(f"{base}/models", timeout=2)
if resp.status_code != 200:
return False
model_names = [m.get("id", "") for m in resp.json().get("data", [])]
else:
# First check if Ollama is running
resp = requests.get(f"{base}/api/tags", timeout=2)
if resp.status_code != 200:
return False
models = resp.json().get("models", [])
model_names = [m.get("name", "").split(":")[0] for m in models]
# Check if our judge model (or a variant) is available
judge_base = JUDGE_MODEL.split(":")[0]
@@ -419,13 +427,23 @@ def is_judge_llm_available() -> bool:
def call_judge_llm(system_prompt: str, user_prompt: str, timeout_sec: float = 120.0) -> Optional[str]:
"""Call the judge LLM with a prompt."""
import requests
from jarvis.llm import chat_with_messages, is_openai_compatible
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]
if is_openai_compatible(JUDGE_BASE_URL):
data = chat_with_messages(JUDGE_BASE_URL, JUDGE_MODEL, messages, timeout_sec=timeout_sec)
if isinstance(data, dict) and isinstance(data.get("message"), dict):
return data["message"].get("content", "")
print("⚠️ Judge LLM call failed: no reply")
return None
payload = {
"model": JUDGE_MODEL,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
],
"messages": messages,
"stream": False,
"options": {"num_ctx": 4096},
}

View File

@@ -17,6 +17,7 @@ from unittest.mock import patch
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
MockConfig, ToolCallCapture,
create_mock_llm_response, create_tool_call,
create_mock_tool_run,
@@ -428,7 +429,7 @@ class TestMemoryEnrichment:
from jarvis.reply.enrichment import extract_search_params_for_memory
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
result = extract_search_params_for_memory(
@@ -466,7 +467,7 @@ class TestMemoryEnrichment:
from jarvis.reply.enrichment import extract_search_params_for_memory
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
context_hint = (
@@ -627,7 +628,7 @@ class TestLiveEndToEnd:
query = "how's the weather this week?"
test_location = "London, England, United Kingdom"
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
def mock_get_location(**kwargs):
@@ -670,7 +671,7 @@ class TestLiveEndToEnd:
query = "what news from today might interest me?"
capture = ToolCallCapture()
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Provide enrichment context so LLM has user interests available
@@ -768,7 +769,7 @@ class TestLiveEndToEnd:
capture = ToolCallCapture()
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_enrichment_context = [
@@ -902,7 +903,7 @@ class TestHelpfulness:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -955,7 +956,7 @@ class TestHelpfulness:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -1024,7 +1025,7 @@ class TestHelpfulness:
from jarvis.reply.prompts import detect_model_size, ModelSize
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
is_small = detect_model_size(JUDGE_MODEL) == ModelSize.SMALL
@@ -1120,7 +1121,7 @@ class TestHelpfulness:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Graph enrichment is opt-in via this setting; MockConfig defaults it off.
mock_config.memory_enrichment_source = "all"
@@ -1254,7 +1255,7 @@ class TestHelpfulness:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.memory_enrichment_source = "all"
@@ -1315,7 +1316,7 @@ class TestHelpfulness:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.memory_enrichment_source = "all"

View File

@@ -35,7 +35,7 @@ import pytest
from unittest.mock import patch
from conftest import requires_judge_llm
from helpers import ToolCallCapture, create_mock_tool_run
from helpers import ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL
# Diary context carried from a prior session about the movie Possessor.
@@ -86,7 +86,7 @@ class TestContextSwitchTools:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Location enabled so getWeather's auto-derive path would succeed
# if the model actually calls it.

View File

@@ -27,6 +27,7 @@ import pytest
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
ToolCallCapture,
assert_not_fallback_reply,
seed_diary_summaries,
@@ -86,7 +87,7 @@ class TestDiarySuppliesMissingToolArg:
):
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Geoip disabled — the only way the model gets a location is from
# diary recall.

View File

@@ -22,6 +22,7 @@ from unittest.mock import patch
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
JUDGE_MODEL,
ToolCallCapture,
assert_not_fallback_reply,
@@ -106,7 +107,7 @@ MOCK_MADRID_LIVE = (
def _configure(mock_config):
"""Pin the eval to the live small model with the evaluator enabled."""
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Evaluator on (default None for SMALL already enables it, but be explicit
# so failures are unambiguous if the model-size detection changes).

View File

@@ -34,6 +34,7 @@ import pytest
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
ToolCallCapture,
assert_not_fallback_reply,
JUDGE_MODEL,
@@ -104,7 +105,7 @@ class TestFollowupSuppliesMissingToolArg:
):
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Geoip disabled — the only way the model gets a location is
# from the user supplying one on turn 2.

View File

@@ -19,6 +19,7 @@ import pytest
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
ToolCallCapture,
assert_not_fallback_reply,
JUDGE_MODEL,
@@ -69,7 +70,7 @@ class TestGraphSuppliesMissingToolArg:
):
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Geoip disabled — the only way the model gets a location is from
# the warm profile loaded out of the graph.

View File

@@ -11,7 +11,7 @@ import pytest
from unittest.mock import patch
from conftest import requires_judge_llm
from helpers import MockConfig, ToolCallCapture, create_mock_tool_run
from helpers import MockConfig, ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL
def _assert_no_tools(capture, query, is_small, model_name):
@@ -69,7 +69,7 @@ class TestGreetingNoToolsLive:
from helpers import JUDGE_MODEL
# Use the judge model (which may be small or large)
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Small models may fail this test due to limited reasoning capacity
@@ -113,7 +113,7 @@ class TestGreetingNoToolsLive:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
is_small = _is_small_model(JUDGE_MODEL)
@@ -162,7 +162,7 @@ class TestGreetingNoToolsLive:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
is_small = _is_small_model(JUDGE_MODEL)
@@ -219,7 +219,7 @@ class TestGreetingNoToolsLive:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
is_small = _is_small_model(JUDGE_MODEL)
@@ -295,7 +295,7 @@ class TestGreetingNoToolsLive:
from helpers import JUDGE_MODEL
query = "what's the weather today"
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()

View File

@@ -19,6 +19,7 @@ from unittest.mock import patch
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
MockConfig, ToolCallCapture,
create_mock_tool_run,
JUDGE_MODEL,
@@ -92,7 +93,7 @@ class TestTopicSwitching:
"""
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -163,7 +164,7 @@ class TestTopicSwitching:
"""
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -238,7 +239,7 @@ class TestFollowUpContext:
"""
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -341,7 +342,7 @@ class TestSelfContainedToolArguments:
"""
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -431,7 +432,7 @@ class TestMultiTurnExtended:
"""
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()

View File

@@ -28,7 +28,7 @@ import pytest
from unittest.mock import MagicMock, patch
from conftest import requires_judge_llm
from helpers import ToolCallCapture, create_mock_tool_run
from helpers import ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL
def _fake_graph_nodes():
@@ -154,7 +154,7 @@ class TestPossessorFieldRepro:
from jarvis.reply.engine import run_reply_engine
from helpers import JUDGE_MODEL
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -325,7 +325,7 @@ class TestPossessorFieldRepro:
" Link: https://www.amazon.co.uk/Possessor-Andrea-Riseborough/dp/B08MXZDZCB\n"
)
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -500,7 +500,7 @@ class TestPossessorFieldRepro:
" Link: https://www.channel4.com/programmes/possessor\n"
)
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
@@ -643,7 +643,7 @@ class TestPossessorFieldRepro:
"and Christopher Abbott."
)
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
# Force digest ON regardless of model-size auto-detection so this
# case runs the digest path deterministically.

View File

@@ -37,6 +37,7 @@ import pytest
from conftest import requires_judge_llm
from helpers import (
JUDGE_BASE_URL,
ToolCallCapture,
assert_not_fallback_reply,
create_mock_tool_run,
@@ -87,7 +88,7 @@ def _run_weather_query(mock_config, eval_db, eval_dialogue_memory, query: str):
from helpers import JUDGE_MODEL
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = "http://localhost:11434"
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
mock_config.location_enabled = True

View File

@@ -12,7 +12,7 @@
#
# Environment variables:
# EVAL_JUDGE_MODEL - Model to use for LLM-as-judge (default: gpt-oss:20b)
# EVAL_JUDGE_BASE_URL - Ollama base URL (default: http://localhost:11434)
# EVAL_JUDGE_BASE_URL - LLM server URL (default: http://localhost:11434); ending in /v1 uses the OpenAI-compatible API (Strata)
# EVAL_REPEAT_COUNT - Number of times to run each test (default: 1; use 3 when tuning prompts to surface flakiness)
set -e
@@ -35,7 +35,9 @@ echo ""
# Check if Ollama is available
OLLAMA_AVAILABLE=false
OLLAMA_URL="${EVAL_JUDGE_BASE_URL:-http://localhost:11434}"
if curl -s "${OLLAMA_URL}/api/tags" > /dev/null 2>&1; then
PROBE_PATH="/api/tags"
case "${OLLAMA_URL%/}" in */v1) PROBE_PATH="/models" ;; esac
if curl -s "${OLLAMA_URL%/}${PROBE_PATH}" > /dev/null 2>&1; then
OLLAMA_AVAILABLE=true
echo " ✅ Ollama detected at ${OLLAMA_URL}"
else

View File

@@ -0,0 +1,90 @@
"""The eval suite talks to whichever LLM server EVAL_JUDGE_BASE_URL names.
Strata (and any OpenAI-compatible server) only serves ``/v1``; a base URL
ending in ``/v1`` must work for the judge availability check, the judge
call, and the default config every eval builds its reply engine from.
These tests run the helpers against a small local fake of the OpenAI API.
"""
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import json
import sys
import threading
import pytest
_ROOT = Path(__file__).resolve().parent.parent
_EVALS = _ROOT / "evals"
if str(_EVALS) not in sys.path:
sys.path.insert(0, str(_EVALS))
import helpers # noqa: E402
class _FakeOpenAI(BaseHTTPRequestHandler):
models = ["qwen-test"]
seen_bodies: list = []
def log_message(self, *args): # keep test output quiet
pass
def _send(self, code, payload):
raw = json.dumps(payload).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def do_GET(self):
if self.path == "/v1/models":
self._send(200, {"object": "list", "data": [{"id": m} for m in self.models]})
else:
self._send(404, {"error": "not found"})
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
if self.path == "/v1/chat/completions":
type(self).seen_bodies.append(body)
self._send(200, {"choices": [{"message": {"role": "assistant", "content": "OVERALL: PASS"}}]})
else:
self._send(404, {"error": "not found"})
@pytest.fixture
def openai_server(monkeypatch):
_FakeOpenAI.seen_bodies = []
server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
url = f"http://127.0.0.1:{server.server_address[1]}/v1"
monkeypatch.setattr(helpers, "JUDGE_BASE_URL", url)
try:
yield url
finally:
server.shutdown()
server.server_close()
def test_judge_is_available_when_the_v1_server_lists_the_model(openai_server, monkeypatch):
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
assert helpers.is_judge_llm_available() is True
def test_judge_is_unavailable_when_the_v1_server_lacks_the_model(openai_server, monkeypatch):
monkeypatch.setattr(helpers, "JUDGE_MODEL", "gemma4:e2b")
assert helpers.is_judge_llm_available() is False
def test_judge_call_goes_through_chat_completions(openai_server, monkeypatch):
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
reply = helpers.call_judge_llm("judge system", "judge user")
assert reply == "OVERALL: PASS"
sent = _FakeOpenAI.seen_bodies[-1]
assert sent["model"] == "qwen-test"
assert [m["content"] for m in sent["messages"]] == ["judge system", "judge user"]
def test_default_eval_config_points_at_the_judge_server(openai_server):
assert helpers.MockConfig().ollama_base_url == openai_server