""" Eval: the model uses the v2 schedule tool for reminders, timers and scheduled tasks instead of claiming it set them (src/jarvis/tools/builtin/schedule.spec.md). Before the tool existed the fast model answered "5분 타이머 맞춰 줘" with "설정했습니다" and nothing was set (docs/v2/decisions.md, 도구 호출). The schedule backend is a fake that keeps what it is given; the tool itself runs for real. Cases: - a timer in minutes is a reminder about that many minutes from now, - "tomorrow at 7" is a one-off reminder at 07:00 tomorrow, local time, - "every weekday at 9, tell me the weather" is a weekday task at 09:00, - asking what is scheduled lists, and adds nothing, - with the v2 brain's reply language (Korean) the reminder text is Korean too: it is delivered as it is. Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_schedule_tool.py -v (v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1) """ import time from datetime import datetime, timedelta import pytest from conftest import requires_judge_llm class FakeSchedules: def __init__(self): self.acts = [] def act(self, action, **kw): self.acts.append((action, kw)) if action == "add": return {"ok": True, "item": {"id": str(len(self.acts)), "next": kw["at"], **kw}} if action == "list": return {"ok": True, "items": [{"id": "1", "kind": "remind", "text": "회의", "repeat": "none", "next": time.time() + 3600}]} return {"ok": True, "item": {"id": kw.get("schedule"), "kind": "remind", "text": "회의", "repeat": "none", "next": time.time() + 3600}} def adds(self): return [kw for a, kw in self.acts if a == "add"] def _run(mock_config, eval_db, eval_dialogue_memory, query): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.memory.history import HistoryContext from jarvis.reply import turn_state from jarvis.reply.engine import run_reply_engine mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL sched = FakeSchedules() turn_state.reset() turn_state.set_user("100") turn_state.set_schedule(sched) try: reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=eval_dialogue_memory, history=HistoryContext(), legacy_memory=False, default_language="Korean") finally: turn_state.reset() print(f"\n schedule [{JUDGE_MODEL}] {query!r}: acts={sched.acts} reply={(reply or '')[:200]!r}") return sched, reply @pytest.mark.eval @requires_judge_llm class TestScheduleTool: def test_a_timer_is_scheduled_not_claimed(self, mock_config, eval_db, eval_dialogue_memory): before = time.time() sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "5분 타이머 맞춰 줘") adds = sched.adds() assert adds, sched.acts assert before + 4 * 60 <= adds[0]["at"] <= time.time() + 6 * 60, adds assert adds[0]["repeat"] == "none" text = adds[0]["text"] assert any("\uac00" <= c <= "\ud7a3" for c in text) and not any("\u4e00" <= c <= "\u9fff" for c in text), text def test_a_reminder_tomorrow_at_seven(self, mock_config, eval_db, eval_dialogue_memory): sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "내일 아침 7시에 회의 있다고 알려 줘") adds = sched.adds() assert adds, sched.acts want = (datetime.now() + timedelta(days=1)).replace(hour=7, minute=0, second=0, microsecond=0) assert adds[0]["at"] == want.timestamp(), (datetime.fromtimestamp(adds[0]["at"]), want) assert adds[0]["kind"] == "remind" and adds[0]["repeat"] == "none" def test_a_weekday_weather_task(self, mock_config, eval_db, eval_dialogue_memory): sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "평일마다 아침 9시에 서울 날씨 알려 줘") adds = sched.adds() assert adds, sched.acts at = datetime.fromtimestamp(adds[0]["at"]) assert (at.hour, at.minute) == (9, 0), at assert adds[0]["kind"] == "task" and adds[0]["repeat"] == "weekdays", adds def test_asking_what_is_scheduled_lists(self, mock_config, eval_db, eval_dialogue_memory): sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "내가 예약해 둔 알림 뭐 있어?") assert any(a == "list" for a, _ in sched.acts), sched.acts assert not sched.adds(), sched.acts