getWeather takes days_ahead (days from today, up to 10) and days (a span, 2 for a weekend). Inside Korea with the app's KMA key: 단기예보 for today to 3 days ahead (morning and afternoon weather, lowest and highest, highest chance of rain), then 중기기온 and 중기육상 for the nearest of 32 KMA cities and its region. Days KMA does not give, and every KMA failure, come from open-meteo's daily forecast. days_ahead 0 with more days gives the weather now, then each day. Eval: evals/test_weather_forecast_days.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
3.5 KiB
Python
79 lines
3.5 KiB
Python
"""
|
|
Eval: a question about a later day reaches getWeather as that day (days_ahead/days).
|
|
|
|
Before getWeather took days_ahead/days (2026-11-18), "내일 날씨 어때?" was answered
|
|
with the weather now. The model counts the days from today's date in the context:
|
|
tomorrow is 1, this weekend is the coming Saturday with 2 days, and a plain
|
|
question about the weather stays the weather now (no days_ahead or 0).
|
|
|
|
Runs like the v2 brain (compact prompt, intent judge, Korean replies).
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen3:8b ./scripts/run_evals.sh weather_forecast_days
|
|
"""
|
|
|
|
from datetime import date
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import JUDGE_BASE_URL, ToolCallCapture, create_mock_tool_run
|
|
|
|
|
|
def _ask(mock_config, eval_db, eval_dialogue_memory, query):
|
|
from helpers import JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
capture = ToolCallCapture()
|
|
turn_state.reset()
|
|
try:
|
|
with patch('jarvis.reply.engine.run_tool_with_retries', side_effect=create_mock_tool_run(capture, {
|
|
"getWeather": "서울 날씨는 맑음, 최저 -1도, 최고 4도, 강수확률 0%입니다.",
|
|
})), patch('jarvis.reply.engine.get_location_context_with_timezone',
|
|
return_value=("Location: Seoul, South Korea", None)):
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean",
|
|
compact_prompt=True, intent_judge=True)
|
|
finally:
|
|
turn_state.reset()
|
|
args = capture.get_args("getWeather")
|
|
print(f"\n forecast days [{JUDGE_MODEL}] {query!r}: tools={capture.tool_names()} "
|
|
f"args={args} reply={(reply or '')[:160]!r}")
|
|
return args
|
|
|
|
|
|
def _num(args, name):
|
|
try:
|
|
return int(float((args or {}).get(name) or 0))
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestWeatherForecastDays:
|
|
@pytest.mark.parametrize("query", ["내일 날씨 어때?", "what's the weather tomorrow?"])
|
|
def test_tomorrow_is_one_day_ahead(self, mock_config, eval_db, eval_dialogue_memory, query):
|
|
args = _ask(mock_config, eval_db, eval_dialogue_memory, query)
|
|
assert args is not None, "getWeather was not called"
|
|
assert _num(args, "days_ahead") == 1, args
|
|
|
|
def test_the_weekend_is_the_coming_saturday_and_sunday(self, mock_config, eval_db, eval_dialogue_memory):
|
|
today = date.today()
|
|
if today.weekday() >= 5:
|
|
pytest.skip("on a weekend day 'this weekend' has two readings")
|
|
args = _ask(mock_config, eval_db, eval_dialogue_memory, "이번 주말 날씨 어때?")
|
|
assert args is not None, "getWeather was not called"
|
|
assert _num(args, "days_ahead") == 5 - today.weekday(), args
|
|
assert _num(args, "days") >= 2, args
|
|
|
|
def test_the_weather_now_stays_now(self, mock_config, eval_db, eval_dialogue_memory):
|
|
args = _ask(mock_config, eval_db, eval_dialogue_memory, "지금 날씨 어때?")
|
|
assert args is not None, "getWeather was not called"
|
|
assert _num(args, "days_ahead") == 0 and _num(args, "days") <= 1, args
|