diff --git a/evals/helpers.py b/evals/helpers.py index 0b37d9e..dcb4630 100644 --- a/evals/helpers.py +++ b/evals/helpers.py @@ -23,6 +23,8 @@ import os # were only testing the 20B tier. Defaulting to the small tier is the # cheapest way to stop that happening again. JUDGE_MODEL = os.environ.get("EVAL_JUDGE_MODEL", "gemma4:e2b") +# A base URL ending in ``/v1`` (Strata, llama.cpp, Ollama's /v1) selects the +# OpenAI-compatible API, the same switch the app uses in ``jarvis.llm``. JUDGE_BASE_URL = os.environ.get("EVAL_JUDGE_BASE_URL", "http://localhost:11434") @@ -242,7 +244,8 @@ def create_mock_tool_run( @dataclass class MockConfig: """Minimal config object for eval tests.""" - ollama_base_url: str = "http://localhost:11434" + # Read at construction so EVAL_JUDGE_BASE_URL reaches every eval. + ollama_base_url: str = field(default_factory=lambda: JUDGE_BASE_URL) ollama_chat_model: str = "gemma4:e2b" ollama_embed_model: str = "nomic-embed-text" db_path: str = ":memory:" @@ -398,16 +401,21 @@ class JudgeVerdict: def is_judge_llm_available() -> bool: """Check if the judge LLM is available and the model exists.""" import requests + from jarvis.llm import is_openai_compatible + base = JUDGE_BASE_URL.rstrip('/') try: - # First check if Ollama is running - resp = requests.get(f"{JUDGE_BASE_URL.rstrip('/')}/api/tags", timeout=2) - if resp.status_code != 200: - return False - - # Check if the judge model is available - data = resp.json() - models = data.get("models", []) - model_names = [m.get("name", "").split(":")[0] for m in models] + if is_openai_compatible(base): + resp = requests.get(f"{base}/models", timeout=2) + if resp.status_code != 200: + return False + model_names = [m.get("id", "") for m in resp.json().get("data", [])] + else: + # First check if Ollama is running + resp = requests.get(f"{base}/api/tags", timeout=2) + if resp.status_code != 200: + return False + models = resp.json().get("models", []) + model_names = [m.get("name", "").split(":")[0] for m in models] # Check if our judge model (or a variant) is available judge_base = JUDGE_MODEL.split(":")[0] @@ -419,13 +427,23 @@ def is_judge_llm_available() -> bool: def call_judge_llm(system_prompt: str, user_prompt: str, timeout_sec: float = 120.0) -> Optional[str]: """Call the judge LLM with a prompt.""" import requests + from jarvis.llm import chat_with_messages, is_openai_compatible + + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt} + ] + + if is_openai_compatible(JUDGE_BASE_URL): + data = chat_with_messages(JUDGE_BASE_URL, JUDGE_MODEL, messages, timeout_sec=timeout_sec) + if isinstance(data, dict) and isinstance(data.get("message"), dict): + return data["message"].get("content", "") + print("⚠️ Judge LLM call failed: no reply") + return None payload = { "model": JUDGE_MODEL, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt} - ], + "messages": messages, "stream": False, "options": {"num_ctx": 4096}, } diff --git a/evals/test_agent_behavior.py b/evals/test_agent_behavior.py index 1bec5d5..c1f2f03 100644 --- a/evals/test_agent_behavior.py +++ b/evals/test_agent_behavior.py @@ -17,6 +17,7 @@ from unittest.mock import patch from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, MockConfig, ToolCallCapture, create_mock_llm_response, create_tool_call, create_mock_tool_run, @@ -428,7 +429,7 @@ class TestMemoryEnrichment: from jarvis.reply.enrichment import extract_search_params_for_memory from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL result = extract_search_params_for_memory( @@ -466,7 +467,7 @@ class TestMemoryEnrichment: from jarvis.reply.enrichment import extract_search_params_for_memory from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL context_hint = ( @@ -627,7 +628,7 @@ class TestLiveEndToEnd: query = "how's the weather this week?" test_location = "London, England, United Kingdom" - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL def mock_get_location(**kwargs): @@ -670,7 +671,7 @@ class TestLiveEndToEnd: query = "what news from today might interest me?" capture = ToolCallCapture() - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Provide enrichment context so LLM has user interests available @@ -768,7 +769,7 @@ class TestLiveEndToEnd: capture = ToolCallCapture() - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_enrichment_context = [ @@ -902,7 +903,7 @@ class TestHelpfulness: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -955,7 +956,7 @@ class TestHelpfulness: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -1024,7 +1025,7 @@ class TestHelpfulness: from jarvis.reply.prompts import detect_model_size, ModelSize from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL is_small = detect_model_size(JUDGE_MODEL) == ModelSize.SMALL @@ -1120,7 +1121,7 @@ class TestHelpfulness: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Graph enrichment is opt-in via this setting; MockConfig defaults it off. mock_config.memory_enrichment_source = "all" @@ -1254,7 +1255,7 @@ class TestHelpfulness: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_config.memory_enrichment_source = "all" @@ -1315,7 +1316,7 @@ class TestHelpfulness: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_config.memory_enrichment_source = "all" diff --git a/evals/test_context_switch_tools.py b/evals/test_context_switch_tools.py index 42a47bc..751014c 100644 --- a/evals/test_context_switch_tools.py +++ b/evals/test_context_switch_tools.py @@ -35,7 +35,7 @@ import pytest from unittest.mock import patch from conftest import requires_judge_llm -from helpers import ToolCallCapture, create_mock_tool_run +from helpers import ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL # Diary context carried from a prior session about the movie Possessor. @@ -86,7 +86,7 @@ class TestContextSwitchTools: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Location enabled so getWeather's auto-derive path would succeed # if the model actually calls it. diff --git a/evals/test_diary_supplies_missing_tool_arg.py b/evals/test_diary_supplies_missing_tool_arg.py index b31d41e..924eea6 100644 --- a/evals/test_diary_supplies_missing_tool_arg.py +++ b/evals/test_diary_supplies_missing_tool_arg.py @@ -27,6 +27,7 @@ import pytest from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, ToolCallCapture, assert_not_fallback_reply, seed_diary_summaries, @@ -86,7 +87,7 @@ class TestDiarySuppliesMissingToolArg: ): from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Geoip disabled — the only way the model gets a location is from # diary recall. diff --git a/evals/test_evaluator_loop.py b/evals/test_evaluator_loop.py index 7855ff5..87e5058 100644 --- a/evals/test_evaluator_loop.py +++ b/evals/test_evaluator_loop.py @@ -22,6 +22,7 @@ from unittest.mock import patch from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, JUDGE_MODEL, ToolCallCapture, assert_not_fallback_reply, @@ -106,7 +107,7 @@ MOCK_MADRID_LIVE = ( def _configure(mock_config): """Pin the eval to the live small model with the evaluator enabled.""" - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Evaluator on (default None for SMALL already enables it, but be explicit # so failures are unambiguous if the model-size detection changes). diff --git a/evals/test_followup_supplies_missing_tool_arg.py b/evals/test_followup_supplies_missing_tool_arg.py index 4c0d4ae..9de7f42 100644 --- a/evals/test_followup_supplies_missing_tool_arg.py +++ b/evals/test_followup_supplies_missing_tool_arg.py @@ -34,6 +34,7 @@ import pytest from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, ToolCallCapture, assert_not_fallback_reply, JUDGE_MODEL, @@ -104,7 +105,7 @@ class TestFollowupSuppliesMissingToolArg: ): from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Geoip disabled — the only way the model gets a location is # from the user supplying one on turn 2. diff --git a/evals/test_graph_supplies_missing_tool_arg.py b/evals/test_graph_supplies_missing_tool_arg.py index 2c47728..a408309 100644 --- a/evals/test_graph_supplies_missing_tool_arg.py +++ b/evals/test_graph_supplies_missing_tool_arg.py @@ -19,6 +19,7 @@ import pytest from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, ToolCallCapture, assert_not_fallback_reply, JUDGE_MODEL, @@ -69,7 +70,7 @@ class TestGraphSuppliesMissingToolArg: ): from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Geoip disabled — the only way the model gets a location is from # the warm profile loaded out of the graph. diff --git a/evals/test_greeting_no_tools.py b/evals/test_greeting_no_tools.py index 88e64aa..529021a 100644 --- a/evals/test_greeting_no_tools.py +++ b/evals/test_greeting_no_tools.py @@ -11,7 +11,7 @@ import pytest from unittest.mock import patch from conftest import requires_judge_llm -from helpers import MockConfig, ToolCallCapture, create_mock_tool_run +from helpers import MockConfig, ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL def _assert_no_tools(capture, query, is_small, model_name): @@ -69,7 +69,7 @@ class TestGreetingNoToolsLive: from helpers import JUDGE_MODEL # Use the judge model (which may be small or large) - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Small models may fail this test due to limited reasoning capacity @@ -113,7 +113,7 @@ class TestGreetingNoToolsLive: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL is_small = _is_small_model(JUDGE_MODEL) @@ -162,7 +162,7 @@ class TestGreetingNoToolsLive: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL is_small = _is_small_model(JUDGE_MODEL) @@ -219,7 +219,7 @@ class TestGreetingNoToolsLive: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL is_small = _is_small_model(JUDGE_MODEL) @@ -295,7 +295,7 @@ class TestGreetingNoToolsLive: from helpers import JUDGE_MODEL query = "what's the weather today" - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() diff --git a/evals/test_multi_turn_context.py b/evals/test_multi_turn_context.py index 2b75576..89e9d8d 100644 --- a/evals/test_multi_turn_context.py +++ b/evals/test_multi_turn_context.py @@ -19,6 +19,7 @@ from unittest.mock import patch from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, MockConfig, ToolCallCapture, create_mock_tool_run, JUDGE_MODEL, @@ -92,7 +93,7 @@ class TestTopicSwitching: """ from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -163,7 +164,7 @@ class TestTopicSwitching: """ from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -238,7 +239,7 @@ class TestFollowUpContext: """ from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -341,7 +342,7 @@ class TestSelfContainedToolArguments: """ from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -431,7 +432,7 @@ class TestMultiTurnExtended: """ from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() diff --git a/evals/test_possessor_field_repro.py b/evals/test_possessor_field_repro.py index e96ac43..c054c0f 100644 --- a/evals/test_possessor_field_repro.py +++ b/evals/test_possessor_field_repro.py @@ -28,7 +28,7 @@ import pytest from unittest.mock import MagicMock, patch from conftest import requires_judge_llm -from helpers import ToolCallCapture, create_mock_tool_run +from helpers import ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL def _fake_graph_nodes(): @@ -154,7 +154,7 @@ class TestPossessorFieldRepro: from jarvis.reply.engine import run_reply_engine from helpers import JUDGE_MODEL - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -325,7 +325,7 @@ class TestPossessorFieldRepro: " Link: https://www.amazon.co.uk/Possessor-Andrea-Riseborough/dp/B08MXZDZCB\n" ) - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -500,7 +500,7 @@ class TestPossessorFieldRepro: " Link: https://www.channel4.com/programmes/possessor\n" ) - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL capture = ToolCallCapture() @@ -643,7 +643,7 @@ class TestPossessorFieldRepro: "and Christopher Abbott." ) - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL # Force digest ON regardless of model-size auto-detection so this # case runs the digest path deterministically. diff --git a/evals/test_weather_autoderive_location.py b/evals/test_weather_autoderive_location.py index 5f5ee9e..e8c1a7e 100644 --- a/evals/test_weather_autoderive_location.py +++ b/evals/test_weather_autoderive_location.py @@ -37,6 +37,7 @@ import pytest from conftest import requires_judge_llm from helpers import ( + JUDGE_BASE_URL, ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run, @@ -87,7 +88,7 @@ def _run_weather_query(mock_config, eval_db, eval_dialogue_memory, query: str): from helpers import JUDGE_MODEL from jarvis.reply.engine import run_reply_engine - mock_config.ollama_base_url = "http://localhost:11434" + mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL mock_config.location_enabled = True diff --git a/scripts/run_evals.sh b/scripts/run_evals.sh index f54cee4..536847e 100755 --- a/scripts/run_evals.sh +++ b/scripts/run_evals.sh @@ -12,7 +12,7 @@ # # Environment variables: # EVAL_JUDGE_MODEL - Model to use for LLM-as-judge (default: gpt-oss:20b) -# EVAL_JUDGE_BASE_URL - Ollama base URL (default: http://localhost:11434) +# EVAL_JUDGE_BASE_URL - LLM server URL (default: http://localhost:11434); ending in /v1 uses the OpenAI-compatible API (Strata) # EVAL_REPEAT_COUNT - Number of times to run each test (default: 1; use 3 when tuning prompts to surface flakiness) set -e @@ -35,7 +35,9 @@ echo "" # Check if Ollama is available OLLAMA_AVAILABLE=false OLLAMA_URL="${EVAL_JUDGE_BASE_URL:-http://localhost:11434}" -if curl -s "${OLLAMA_URL}/api/tags" > /dev/null 2>&1; then +PROBE_PATH="/api/tags" +case "${OLLAMA_URL%/}" in */v1) PROBE_PATH="/models" ;; esac +if curl -s "${OLLAMA_URL%/}${PROBE_PATH}" > /dev/null 2>&1; then OLLAMA_AVAILABLE=true echo " ✅ Ollama detected at ${OLLAMA_URL}" else diff --git a/tests/test_eval_helpers_backend.py b/tests/test_eval_helpers_backend.py new file mode 100644 index 0000000..c6e873d --- /dev/null +++ b/tests/test_eval_helpers_backend.py @@ -0,0 +1,90 @@ +"""The eval suite talks to whichever LLM server EVAL_JUDGE_BASE_URL names. + +Strata (and any OpenAI-compatible server) only serves ``/v1``; a base URL +ending in ``/v1`` must work for the judge availability check, the judge +call, and the default config every eval builds its reply engine from. +These tests run the helpers against a small local fake of the OpenAI API. +""" + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +import json +import sys +import threading + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent +_EVALS = _ROOT / "evals" +if str(_EVALS) not in sys.path: + sys.path.insert(0, str(_EVALS)) + +import helpers # noqa: E402 + + +class _FakeOpenAI(BaseHTTPRequestHandler): + models = ["qwen-test"] + seen_bodies: list = [] + + def log_message(self, *args): # keep test output quiet + pass + + def _send(self, code, payload): + raw = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_GET(self): + if self.path == "/v1/models": + self._send(200, {"object": "list", "data": [{"id": m} for m in self.models]}) + else: + self._send(404, {"error": "not found"}) + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if self.path == "/v1/chat/completions": + type(self).seen_bodies.append(body) + self._send(200, {"choices": [{"message": {"role": "assistant", "content": "OVERALL: PASS"}}]}) + else: + self._send(404, {"error": "not found"}) + + +@pytest.fixture +def openai_server(monkeypatch): + _FakeOpenAI.seen_bodies = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + url = f"http://127.0.0.1:{server.server_address[1]}/v1" + monkeypatch.setattr(helpers, "JUDGE_BASE_URL", url) + try: + yield url + finally: + server.shutdown() + server.server_close() + + +def test_judge_is_available_when_the_v1_server_lists_the_model(openai_server, monkeypatch): + monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test") + assert helpers.is_judge_llm_available() is True + + +def test_judge_is_unavailable_when_the_v1_server_lacks_the_model(openai_server, monkeypatch): + monkeypatch.setattr(helpers, "JUDGE_MODEL", "gemma4:e2b") + assert helpers.is_judge_llm_available() is False + + +def test_judge_call_goes_through_chat_completions(openai_server, monkeypatch): + monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test") + reply = helpers.call_judge_llm("judge system", "judge user") + assert reply == "OVERALL: PASS" + sent = _FakeOpenAI.seen_bodies[-1] + assert sent["model"] == "qwen-test" + assert [m["content"] for m in sent["messages"]] == ["judge system", "judge user"] + + +def test_default_eval_config_points_at_the_judge_server(openai_server): + assert helpers.MockConfig().ollama_base_url == openai_server