The brain gets a schedule tool (add, list, delete the asker's own reminders, timers and tasks) offered only with a schedule backend. The voice worker asks the app through schedule_action and runs fired tasks as reply via task with read-only tools, no MCP and a fresh short-term memory. The app keeps the schedules in config/schedules.json, fires them every 5 s, speaks them when the person is in Jarvis's voice channel and otherwise posts in the chat channel they asked in (or schedule.chatChannel for voice). A feature switch and a 예약 screen list and delete everyone's schedules. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
100 lines
4.5 KiB
Python
100 lines
4.5 KiB
Python
"""
|
|
Eval: the model uses the v2 schedule tool for reminders, timers and scheduled tasks instead of
|
|
claiming it set them (src/jarvis/tools/builtin/schedule.spec.md).
|
|
|
|
Before the tool existed the fast model answered "5분 타이머 맞춰 줘" with "설정했습니다" and nothing
|
|
was set (docs/v2/decisions.md, 도구 호출). The schedule backend is a fake that keeps what it is
|
|
given; the tool itself runs for real. Cases:
|
|
|
|
- a timer in minutes is a reminder about that many minutes from now,
|
|
- "tomorrow at 7" is a one-off reminder at 07:00 tomorrow, local time,
|
|
- "every weekday at 9, tell me the weather" is a weekday task at 09:00,
|
|
- asking what is scheduled lists, and adds nothing,
|
|
- with the v2 brain's reply language (Korean) the reminder text is Korean too: it is delivered as it is.
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_schedule_tool.py -v
|
|
(v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
|
|
"""
|
|
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
|
|
|
|
class FakeSchedules:
|
|
def __init__(self):
|
|
self.acts = []
|
|
|
|
def act(self, action, **kw):
|
|
self.acts.append((action, kw))
|
|
if action == "add":
|
|
return {"ok": True, "item": {"id": str(len(self.acts)), "next": kw["at"], **kw}}
|
|
if action == "list":
|
|
return {"ok": True, "items": [{"id": "1", "kind": "remind", "text": "회의", "repeat": "none",
|
|
"next": time.time() + 3600}]}
|
|
return {"ok": True, "item": {"id": kw.get("schedule"), "kind": "remind", "text": "회의",
|
|
"repeat": "none", "next": time.time() + 3600}}
|
|
|
|
def adds(self):
|
|
return [kw for a, kw in self.acts if a == "add"]
|
|
|
|
|
|
def _run(mock_config, eval_db, eval_dialogue_memory, query):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
sched = FakeSchedules()
|
|
turn_state.reset()
|
|
turn_state.set_user("100")
|
|
turn_state.set_schedule(sched)
|
|
try:
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean")
|
|
finally:
|
|
turn_state.reset()
|
|
print(f"\n schedule [{JUDGE_MODEL}] {query!r}: acts={sched.acts} reply={(reply or '')[:200]!r}")
|
|
return sched, reply
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestScheduleTool:
|
|
def test_a_timer_is_scheduled_not_claimed(self, mock_config, eval_db, eval_dialogue_memory):
|
|
before = time.time()
|
|
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "5분 타이머 맞춰 줘")
|
|
adds = sched.adds()
|
|
assert adds, sched.acts
|
|
assert before + 4 * 60 <= adds[0]["at"] <= time.time() + 6 * 60, adds
|
|
assert adds[0]["repeat"] == "none"
|
|
text = adds[0]["text"]
|
|
assert any("\uac00" <= c <= "\ud7a3" for c in text) and not any("\u4e00" <= c <= "\u9fff" for c in text), text
|
|
|
|
def test_a_reminder_tomorrow_at_seven(self, mock_config, eval_db, eval_dialogue_memory):
|
|
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "내일 아침 7시에 회의 있다고 알려 줘")
|
|
adds = sched.adds()
|
|
assert adds, sched.acts
|
|
want = (datetime.now() + timedelta(days=1)).replace(hour=7, minute=0, second=0, microsecond=0)
|
|
assert adds[0]["at"] == want.timestamp(), (datetime.fromtimestamp(adds[0]["at"]), want)
|
|
assert adds[0]["kind"] == "remind" and adds[0]["repeat"] == "none"
|
|
|
|
def test_a_weekday_weather_task(self, mock_config, eval_db, eval_dialogue_memory):
|
|
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "평일마다 아침 9시에 서울 날씨 알려 줘")
|
|
adds = sched.adds()
|
|
assert adds, sched.acts
|
|
at = datetime.fromtimestamp(adds[0]["at"])
|
|
assert (at.hour, at.minute) == (9, 0), at
|
|
assert adds[0]["kind"] == "task" and adds[0]["repeat"] == "weekdays", adds
|
|
|
|
def test_asking_what_is_scheduled_lists(self, mock_config, eval_db, eval_dialogue_memory):
|
|
sched, _ = _run(mock_config, eval_db, eval_dialogue_memory, "내가 예약해 둔 알림 뭐 있어?")
|
|
assert any(a == "list" for a, _ in sched.acts), sched.acts
|
|
assert not sched.adds(), sched.acts
|