Files
javis_bot/evals/test_schedule_tool.py
javis-bot ea2d3ba958 feat: reminders and scheduled tasks (schedule tool, app store and delivery)
The brain gets a schedule tool (add, list, delete the asker's own reminders,
timers and tasks) offered only with a schedule backend. The voice worker asks
the app through schedule_action and runs fired tasks as reply via task with
read-only tools, no MCP and a fresh short-term memory. The app keeps the
schedules in config/schedules.json, fires them every 5 s, speaks them when the
person is in Jarvis's voice channel and otherwise posts in the chat channel
they asked in (or schedule.chatChannel for voice). A feature switch and a
예약 screen list and delete everyone's schedules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 16:27:42 +09:00

100 lines
4.5 KiB
Python

"""
Eval: the model uses the v2 schedule tool for reminders, timers and scheduled tasks instead of
claiming it set them (src/jarvis/tools/builtin/schedule.spec.md).
Before the tool existed the fast model answered "5분 타이머 맞춰 줘" with "설정했습니다" and nothing
was set (docs/v2/decisions.md, 도구 호출). The schedule backend is a fake that keeps what it is
given; the tool itself runs for real. Cases:
- a timer in minutes is a reminder about that many minutes from now,
- "tomorrow at 7" is a one-off reminder at 07:00 tomorrow, local time,
- "every weekday at 9, tell me the weather" is a weekday task at 09:00,
- asking what is scheduled lists, and adds nothing,
- with the v2 brain's reply language (Korean) the reminder text is Korean too: it is delivered as it is.
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_schedule_tool.py -v
(v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
"""
import time
from datetime import datetime, timedelta
import pytest
from conftest import requires_judge_llm
class FakeSchedules:
def __init__(self):
self.acts = []
def act(self, action, **kw):
self.acts.append((action, kw))
if action == "add":
return {"ok": True, "item": {"id": str(len(self.acts)), "next": kw["at"], **kw}}
if action == "list":
return {"ok": True, "items": [{"id": "1", "kind": "remind", "text": "회의", "repeat": "none",
"next": time.time() + 3600}]}
return {"ok": True, "item": {"id": kw.get("schedule"), "kind": "remind", "text": "회의",
"repeat": "none", "next": time.time() + 3600}}
def adds(self):
return [kw for a, kw in self.acts if a == "add"]
def _run(mock_config, eval_db, eval_dialogue_memory, query):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
sched = FakeSchedules()
turn_state.reset()
turn_state.set_user("100")
turn_state.set_schedule(sched)
try:
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean")
finally:
turn_state.reset()
print(f"\n schedule [{JUDGE_MODEL}] {query!r}: acts={sched.acts} reply={(reply or '')[:200]!r}")
return sched, reply
@pytest.mark.eval
@requires_judge_llm
class TestScheduleTool:
def test_a_timer_is_scheduled_not_claimed(self, mock_config, eval_db, eval_dialogue_memory):
before = time.time()
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "5분 타이머 맞춰 줘")
adds = sched.adds()
assert adds, sched.acts
assert before + 4 * 60 <= adds[0]["at"] <= time.time() + 6 * 60, adds
assert adds[0]["repeat"] == "none"
text = adds[0]["text"]
assert any("\uac00" <= c <= "\ud7a3" for c in text) and not any("\u4e00" <= c <= "\u9fff" for c in text), text
def test_a_reminder_tomorrow_at_seven(self, mock_config, eval_db, eval_dialogue_memory):
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "내일 아침 7시에 회의 있다고 알려 줘")
adds = sched.adds()
assert adds, sched.acts
want = (datetime.now() + timedelta(days=1)).replace(hour=7, minute=0, second=0, microsecond=0)
assert adds[0]["at"] == want.timestamp(), (datetime.fromtimestamp(adds[0]["at"]), want)
assert adds[0]["kind"] == "remind" and adds[0]["repeat"] == "none"
def test_a_weekday_weather_task(self, mock_config, eval_db, eval_dialogue_memory):
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "평일마다 아침 9시에 서울 날씨 알려 줘")
adds = sched.adds()
assert adds, sched.acts
at = datetime.fromtimestamp(adds[0]["at"])
assert (at.hour, at.minute) == (9, 0), at
assert adds[0]["kind"] == "task" and adds[0]["repeat"] == "weekdays", adds
def test_asking_what_is_scheduled_lists(self, mock_config, eval_db, eval_dialogue_memory):
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "내가 예약해 둔 알림 뭐 있어?")
assert any(a == "list" for a, _ in sched.acts), sched.acts
assert not sched.adds(), sched.acts