test(evals): run the evals against any server named by EVAL_JUDGE_BASE_URL, including /v1
Ten eval files pinned their reply engine to http://localhost:11434 and the judge only spoke Ollama's /api/tags and /api/chat, so pointing the suite at Strata (OpenAI-compatible /v1 only) skipped or broke most evals. - MockConfig and every eval config read EVAL_JUDGE_BASE_URL. - The judge availability check reads /v1/models and the judge call goes through jarvis.llm when the URL ends in /v1; the Ollama path is unchanged. - run_evals.sh probes /models for a /v1 URL. The intent-judge and listener evals stay pinned to gemma4:e2b on local Ollama. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,6 +23,8 @@ import os
|
||||
# were only testing the 20B tier. Defaulting to the small tier is the
|
||||
# cheapest way to stop that happening again.
|
||||
JUDGE_MODEL = os.environ.get("EVAL_JUDGE_MODEL", "gemma4:e2b")
|
||||
# A base URL ending in ``/v1`` (Strata, llama.cpp, Ollama's /v1) selects the
|
||||
# OpenAI-compatible API, the same switch the app uses in ``jarvis.llm``.
|
||||
JUDGE_BASE_URL = os.environ.get("EVAL_JUDGE_BASE_URL", "http://localhost:11434")
|
||||
|
||||
|
||||
@@ -242,7 +244,8 @@ def create_mock_tool_run(
|
||||
@dataclass
|
||||
class MockConfig:
|
||||
"""Minimal config object for eval tests."""
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
# Read at construction so EVAL_JUDGE_BASE_URL reaches every eval.
|
||||
ollama_base_url: str = field(default_factory=lambda: JUDGE_BASE_URL)
|
||||
ollama_chat_model: str = "gemma4:e2b"
|
||||
ollama_embed_model: str = "nomic-embed-text"
|
||||
db_path: str = ":memory:"
|
||||
@@ -398,16 +401,21 @@ class JudgeVerdict:
|
||||
def is_judge_llm_available() -> bool:
|
||||
"""Check if the judge LLM is available and the model exists."""
|
||||
import requests
|
||||
from jarvis.llm import is_openai_compatible
|
||||
base = JUDGE_BASE_URL.rstrip('/')
|
||||
try:
|
||||
# First check if Ollama is running
|
||||
resp = requests.get(f"{JUDGE_BASE_URL.rstrip('/')}/api/tags", timeout=2)
|
||||
if resp.status_code != 200:
|
||||
return False
|
||||
|
||||
# Check if the judge model is available
|
||||
data = resp.json()
|
||||
models = data.get("models", [])
|
||||
model_names = [m.get("name", "").split(":")[0] for m in models]
|
||||
if is_openai_compatible(base):
|
||||
resp = requests.get(f"{base}/models", timeout=2)
|
||||
if resp.status_code != 200:
|
||||
return False
|
||||
model_names = [m.get("id", "") for m in resp.json().get("data", [])]
|
||||
else:
|
||||
# First check if Ollama is running
|
||||
resp = requests.get(f"{base}/api/tags", timeout=2)
|
||||
if resp.status_code != 200:
|
||||
return False
|
||||
models = resp.json().get("models", [])
|
||||
model_names = [m.get("name", "").split(":")[0] for m in models]
|
||||
|
||||
# Check if our judge model (or a variant) is available
|
||||
judge_base = JUDGE_MODEL.split(":")[0]
|
||||
@@ -419,13 +427,23 @@ def is_judge_llm_available() -> bool:
|
||||
def call_judge_llm(system_prompt: str, user_prompt: str, timeout_sec: float = 120.0) -> Optional[str]:
|
||||
"""Call the judge LLM with a prompt."""
|
||||
import requests
|
||||
from jarvis.llm import chat_with_messages, is_openai_compatible
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
]
|
||||
|
||||
if is_openai_compatible(JUDGE_BASE_URL):
|
||||
data = chat_with_messages(JUDGE_BASE_URL, JUDGE_MODEL, messages, timeout_sec=timeout_sec)
|
||||
if isinstance(data, dict) and isinstance(data.get("message"), dict):
|
||||
return data["message"].get("content", "")
|
||||
print("⚠️ Judge LLM call failed: no reply")
|
||||
return None
|
||||
|
||||
payload = {
|
||||
"model": JUDGE_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"num_ctx": 4096},
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ from unittest.mock import patch
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
MockConfig, ToolCallCapture,
|
||||
create_mock_llm_response, create_tool_call,
|
||||
create_mock_tool_run,
|
||||
@@ -428,7 +429,7 @@ class TestMemoryEnrichment:
|
||||
from jarvis.reply.enrichment import extract_search_params_for_memory
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
result = extract_search_params_for_memory(
|
||||
@@ -466,7 +467,7 @@ class TestMemoryEnrichment:
|
||||
from jarvis.reply.enrichment import extract_search_params_for_memory
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
context_hint = (
|
||||
@@ -627,7 +628,7 @@ class TestLiveEndToEnd:
|
||||
query = "how's the weather this week?"
|
||||
test_location = "London, England, United Kingdom"
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
def mock_get_location(**kwargs):
|
||||
@@ -670,7 +671,7 @@ class TestLiveEndToEnd:
|
||||
query = "what news from today might interest me?"
|
||||
capture = ToolCallCapture()
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
# Provide enrichment context so LLM has user interests available
|
||||
@@ -768,7 +769,7 @@ class TestLiveEndToEnd:
|
||||
|
||||
capture = ToolCallCapture()
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
mock_enrichment_context = [
|
||||
@@ -902,7 +903,7 @@ class TestHelpfulness:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -955,7 +956,7 @@ class TestHelpfulness:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -1024,7 +1025,7 @@ class TestHelpfulness:
|
||||
from jarvis.reply.prompts import detect_model_size, ModelSize
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
is_small = detect_model_size(JUDGE_MODEL) == ModelSize.SMALL
|
||||
@@ -1120,7 +1121,7 @@ class TestHelpfulness:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Graph enrichment is opt-in via this setting; MockConfig defaults it off.
|
||||
mock_config.memory_enrichment_source = "all"
|
||||
@@ -1254,7 +1255,7 @@ class TestHelpfulness:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
mock_config.memory_enrichment_source = "all"
|
||||
|
||||
@@ -1315,7 +1316,7 @@ class TestHelpfulness:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
mock_config.memory_enrichment_source = "all"
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import ToolCallCapture, create_mock_tool_run
|
||||
from helpers import ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL
|
||||
|
||||
|
||||
# Diary context carried from a prior session about the movie Possessor.
|
||||
@@ -86,7 +86,7 @@ class TestContextSwitchTools:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Location enabled so getWeather's auto-derive path would succeed
|
||||
# if the model actually calls it.
|
||||
|
||||
@@ -27,6 +27,7 @@ import pytest
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
ToolCallCapture,
|
||||
assert_not_fallback_reply,
|
||||
seed_diary_summaries,
|
||||
@@ -86,7 +87,7 @@ class TestDiarySuppliesMissingToolArg:
|
||||
):
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Geoip disabled — the only way the model gets a location is from
|
||||
# diary recall.
|
||||
|
||||
@@ -22,6 +22,7 @@ from unittest.mock import patch
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
JUDGE_MODEL,
|
||||
ToolCallCapture,
|
||||
assert_not_fallback_reply,
|
||||
@@ -106,7 +107,7 @@ MOCK_MADRID_LIVE = (
|
||||
|
||||
def _configure(mock_config):
|
||||
"""Pin the eval to the live small model with the evaluator enabled."""
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Evaluator on (default None for SMALL already enables it, but be explicit
|
||||
# so failures are unambiguous if the model-size detection changes).
|
||||
|
||||
@@ -34,6 +34,7 @@ import pytest
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
ToolCallCapture,
|
||||
assert_not_fallback_reply,
|
||||
JUDGE_MODEL,
|
||||
@@ -104,7 +105,7 @@ class TestFollowupSuppliesMissingToolArg:
|
||||
):
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Geoip disabled — the only way the model gets a location is
|
||||
# from the user supplying one on turn 2.
|
||||
|
||||
@@ -19,6 +19,7 @@ import pytest
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
ToolCallCapture,
|
||||
assert_not_fallback_reply,
|
||||
JUDGE_MODEL,
|
||||
@@ -69,7 +70,7 @@ class TestGraphSuppliesMissingToolArg:
|
||||
):
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Geoip disabled — the only way the model gets a location is from
|
||||
# the warm profile loaded out of the graph.
|
||||
|
||||
@@ -11,7 +11,7 @@ import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import MockConfig, ToolCallCapture, create_mock_tool_run
|
||||
from helpers import MockConfig, ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL
|
||||
|
||||
|
||||
def _assert_no_tools(capture, query, is_small, model_name):
|
||||
@@ -69,7 +69,7 @@ class TestGreetingNoToolsLive:
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
# Use the judge model (which may be small or large)
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
# Small models may fail this test due to limited reasoning capacity
|
||||
@@ -113,7 +113,7 @@ class TestGreetingNoToolsLive:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
is_small = _is_small_model(JUDGE_MODEL)
|
||||
@@ -162,7 +162,7 @@ class TestGreetingNoToolsLive:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
is_small = _is_small_model(JUDGE_MODEL)
|
||||
|
||||
@@ -219,7 +219,7 @@ class TestGreetingNoToolsLive:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
is_small = _is_small_model(JUDGE_MODEL)
|
||||
|
||||
@@ -295,7 +295,7 @@ class TestGreetingNoToolsLive:
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
query = "what's the weather today"
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
|
||||
@@ -19,6 +19,7 @@ from unittest.mock import patch
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
MockConfig, ToolCallCapture,
|
||||
create_mock_tool_run,
|
||||
JUDGE_MODEL,
|
||||
@@ -92,7 +93,7 @@ class TestTopicSwitching:
|
||||
"""
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -163,7 +164,7 @@ class TestTopicSwitching:
|
||||
"""
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -238,7 +239,7 @@ class TestFollowUpContext:
|
||||
"""
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -341,7 +342,7 @@ class TestSelfContainedToolArguments:
|
||||
"""
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -431,7 +432,7 @@ class TestMultiTurnExtended:
|
||||
"""
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
|
||||
@@ -28,7 +28,7 @@ import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import ToolCallCapture, create_mock_tool_run
|
||||
from helpers import ToolCallCapture, create_mock_tool_run, JUDGE_BASE_URL
|
||||
|
||||
|
||||
def _fake_graph_nodes():
|
||||
@@ -154,7 +154,7 @@ class TestPossessorFieldRepro:
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
from helpers import JUDGE_MODEL
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
|
||||
capture = ToolCallCapture()
|
||||
@@ -325,7 +325,7 @@ class TestPossessorFieldRepro:
|
||||
" Link: https://www.amazon.co.uk/Possessor-Andrea-Riseborough/dp/B08MXZDZCB\n"
|
||||
)
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
capture = ToolCallCapture()
|
||||
|
||||
@@ -500,7 +500,7 @@ class TestPossessorFieldRepro:
|
||||
" Link: https://www.channel4.com/programmes/possessor\n"
|
||||
)
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
capture = ToolCallCapture()
|
||||
|
||||
@@ -643,7 +643,7 @@ class TestPossessorFieldRepro:
|
||||
"and Christopher Abbott."
|
||||
)
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
# Force digest ON regardless of model-size auto-detection so this
|
||||
# case runs the digest path deterministically.
|
||||
|
||||
@@ -37,6 +37,7 @@ import pytest
|
||||
|
||||
from conftest import requires_judge_llm
|
||||
from helpers import (
|
||||
JUDGE_BASE_URL,
|
||||
ToolCallCapture,
|
||||
assert_not_fallback_reply,
|
||||
create_mock_tool_run,
|
||||
@@ -87,7 +88,7 @@ def _run_weather_query(mock_config, eval_db, eval_dialogue_memory, query: str):
|
||||
from helpers import JUDGE_MODEL
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
|
||||
mock_config.ollama_base_url = "http://localhost:11434"
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
mock_config.location_enabled = True
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#
|
||||
# Environment variables:
|
||||
# EVAL_JUDGE_MODEL - Model to use for LLM-as-judge (default: gpt-oss:20b)
|
||||
# EVAL_JUDGE_BASE_URL - Ollama base URL (default: http://localhost:11434)
|
||||
# EVAL_JUDGE_BASE_URL - LLM server URL (default: http://localhost:11434); ending in /v1 uses the OpenAI-compatible API (Strata)
|
||||
# EVAL_REPEAT_COUNT - Number of times to run each test (default: 1; use 3 when tuning prompts to surface flakiness)
|
||||
|
||||
set -e
|
||||
@@ -35,7 +35,9 @@ echo ""
|
||||
# Check if Ollama is available
|
||||
OLLAMA_AVAILABLE=false
|
||||
OLLAMA_URL="${EVAL_JUDGE_BASE_URL:-http://localhost:11434}"
|
||||
if curl -s "${OLLAMA_URL}/api/tags" > /dev/null 2>&1; then
|
||||
PROBE_PATH="/api/tags"
|
||||
case "${OLLAMA_URL%/}" in */v1) PROBE_PATH="/models" ;; esac
|
||||
if curl -s "${OLLAMA_URL%/}${PROBE_PATH}" > /dev/null 2>&1; then
|
||||
OLLAMA_AVAILABLE=true
|
||||
echo " ✅ Ollama detected at ${OLLAMA_URL}"
|
||||
else
|
||||
|
||||
90
tests/test_eval_helpers_backend.py
Normal file
90
tests/test_eval_helpers_backend.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""The eval suite talks to whichever LLM server EVAL_JUDGE_BASE_URL names.
|
||||
|
||||
Strata (and any OpenAI-compatible server) only serves ``/v1``; a base URL
|
||||
ending in ``/v1`` must work for the judge availability check, the judge
|
||||
call, and the default config every eval builds its reply engine from.
|
||||
These tests run the helpers against a small local fake of the OpenAI API.
|
||||
"""
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
_EVALS = _ROOT / "evals"
|
||||
if str(_EVALS) not in sys.path:
|
||||
sys.path.insert(0, str(_EVALS))
|
||||
|
||||
import helpers # noqa: E402
|
||||
|
||||
|
||||
class _FakeOpenAI(BaseHTTPRequestHandler):
|
||||
models = ["qwen-test"]
|
||||
seen_bodies: list = []
|
||||
|
||||
def log_message(self, *args): # keep test output quiet
|
||||
pass
|
||||
|
||||
def _send(self, code, payload):
|
||||
raw = json.dumps(payload).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/v1/models":
|
||||
self._send(200, {"object": "list", "data": [{"id": m} for m in self.models]})
|
||||
else:
|
||||
self._send(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
||||
if self.path == "/v1/chat/completions":
|
||||
type(self).seen_bodies.append(body)
|
||||
self._send(200, {"choices": [{"message": {"role": "assistant", "content": "OVERALL: PASS"}}]})
|
||||
else:
|
||||
self._send(404, {"error": "not found"})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_server(monkeypatch):
|
||||
_FakeOpenAI.seen_bodies = []
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), _FakeOpenAI)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
url = f"http://127.0.0.1:{server.server_address[1]}/v1"
|
||||
monkeypatch.setattr(helpers, "JUDGE_BASE_URL", url)
|
||||
try:
|
||||
yield url
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_judge_is_available_when_the_v1_server_lists_the_model(openai_server, monkeypatch):
|
||||
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
|
||||
assert helpers.is_judge_llm_available() is True
|
||||
|
||||
|
||||
def test_judge_is_unavailable_when_the_v1_server_lacks_the_model(openai_server, monkeypatch):
|
||||
monkeypatch.setattr(helpers, "JUDGE_MODEL", "gemma4:e2b")
|
||||
assert helpers.is_judge_llm_available() is False
|
||||
|
||||
|
||||
def test_judge_call_goes_through_chat_completions(openai_server, monkeypatch):
|
||||
monkeypatch.setattr(helpers, "JUDGE_MODEL", "qwen-test")
|
||||
reply = helpers.call_judge_llm("judge system", "judge user")
|
||||
assert reply == "OVERALL: PASS"
|
||||
sent = _FakeOpenAI.seen_bodies[-1]
|
||||
assert sent["model"] == "qwen-test"
|
||||
assert [m["content"] for m in sent["messages"]] == ["judge system", "judge user"]
|
||||
|
||||
|
||||
def test_default_eval_config_points_at_the_judge_server(openai_server):
|
||||
assert helpers.MockConfig().ollama_base_url == openai_server
|
||||
Reference in New Issue
Block a user