Files
javis_bot/evals/test_weather_forecast_days.py
javis-bot 17927b6450 feat(weather): KMA forecasts for a later day (days_ahead, days)
getWeather takes days_ahead (days from today, up to 10) and days (a span,
2 for a weekend). Inside Korea with the app's KMA key: 단기예보 for today
to 3 days ahead (morning and afternoon weather, lowest and highest, highest
chance of rain), then 중기기온 and 중기육상 for the nearest of 32 KMA cities
and its region. Days KMA does not give, and every KMA failure, come from
open-meteo's daily forecast. days_ahead 0 with more days gives the weather
now, then each day. Eval: evals/test_weather_forecast_days.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 10:57:12 +09:00

79 lines
3.5 KiB
Python

"""
Eval: a question about a later day reaches getWeather as that day (days_ahead/days).
Before getWeather took days_ahead/days (2026-11-18), "내일 날씨 어때?" was answered
with the weather now. The model counts the days from today's date in the context:
tomorrow is 1, this weekend is the coming Saturday with 2 days, and a plain
question about the weather stays the weather now (no days_ahead or 0).
Runs like the v2 brain (compact prompt, intent judge, Korean replies).
Run: EVAL_JUDGE_MODEL=qwen3:8b ./scripts/run_evals.sh weather_forecast_days
"""
from datetime import date
from unittest.mock import patch
import pytest
from conftest import requires_judge_llm
from helpers import JUDGE_BASE_URL, ToolCallCapture, create_mock_tool_run
def _ask(mock_config, eval_db, eval_dialogue_memory, query):
from helpers import JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
turn_state.reset()
try:
with patch('jarvis.reply.engine.run_tool_with_retries', side_effect=create_mock_tool_run(capture, {
"getWeather": "서울 날씨는 맑음, 최저 -1도, 최고 4도, 강수확률 0%입니다.",
})), patch('jarvis.reply.engine.get_location_context_with_timezone',
return_value=("Location: Seoul, South Korea", None)):
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean",
compact_prompt=True, intent_judge=True)
finally:
turn_state.reset()
args = capture.get_args("getWeather")
print(f"\n forecast days [{JUDGE_MODEL}] {query!r}: tools={capture.tool_names()} "
f"args={args} reply={(reply or '')[:160]!r}")
return args
def _num(args, name):
try:
return int(float((args or {}).get(name) or 0))
except (TypeError, ValueError):
return 0
@pytest.mark.eval
@requires_judge_llm
class TestWeatherForecastDays:
@pytest.mark.parametrize("query", ["내일 날씨 어때?", "what's the weather tomorrow?"])
def test_tomorrow_is_one_day_ahead(self, mock_config, eval_db, eval_dialogue_memory, query):
args = _ask(mock_config, eval_db, eval_dialogue_memory, query)
assert args is not None, "getWeather was not called"
assert _num(args, "days_ahead") == 1, args
def test_the_weekend_is_the_coming_saturday_and_sunday(self, mock_config, eval_db, eval_dialogue_memory):
today = date.today()
if today.weekday() >= 5:
pytest.skip("on a weekend day 'this weekend' has two readings")
args = _ask(mock_config, eval_db, eval_dialogue_memory, "이번 주말 날씨 어때?")
assert args is not None, "getWeather was not called"
assert _num(args, "days_ahead") == 5 - today.weekday(), args
assert _num(args, "days") >= 2, args
def test_the_weather_now_stays_now(self, mock_config, eval_db, eval_dialogue_memory):
args = _ask(mock_config, eval_db, eval_dialogue_memory, "지금 날씨 어때?")
assert args is not None, "getWeather was not called"
assert _num(args, "days_ahead") == 0 and _num(args, "days") <= 1, args