Files
javis_bot/evals/test_personal_notes.py
javis-bot 59c3f33c2c test(notes): prepare the pick with the app's offer and require it to be reused
The partial-transcript eval now offers chat, schedule, notes and screen as
voice_worker/brain.py does, sets those backends for the reply, and fails
when the intent judge made no pick or the pick lacks personalNotes, so it
cannot pass by re-routing the final text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 02:30:49 +09:00

112 lines
5.5 KiB
Python

"""
Eval: per-person standing instructions (src/jarvis/tools/builtin/personal_notes.spec.md).
- A saved instruction reaches the answer: with the note "end every answer with the word
Roger." the answer to an unrelated question ends with "Roger" (and without the note it does not,
so the check measures the note, not the model's habits).
- Asking for something to apply from now on saves a note; a one-off question does not.
The notes backend is a fake that keeps what it is given; the tool runs for real.
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_personal_notes.py -v
(v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
"""
import re
import pytest
from conftest import requires_judge_llm
class FakeNotes:
def __init__(self):
self.acts = []
def act(self, action, **kw):
self.acts.append((action, kw))
if action == "add":
return {"ok": True, "item": {"id": "1", "text": kw["text"]}}
if action == "list":
return {"ok": True, "items": []}
return {"ok": True, "item": {"id": kw.get("note"), "text": ""}}
def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None, partial=None):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.memory.history import HistoryContext
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
backend = FakeNotes()
kw = {}
if partial is not None:
# The v2 brain's path: the intent judge picks on a partial transcript while the person
# still speaks, and the reply reuses that pick when the final text is close to it.
from jarvis.reply.engine import prepare_intent
from voice_worker.brain import similar
assert similar(partial, query)
# The same offer as the app (voice_worker/brain.py): chat, schedule, notes and screen.
prepared = prepare_intent(mock_config, partial, eval_dialogue_memory, True,
schedule_available=True, notes_available=True, screen_available=True)
print(f"\n notes [{JUDGE_MODEL}] prepared on {partial!r}: {prepared}")
assert prepared is not None, "the intent judge made no pick, so nothing would be reused"
assert "personalNotes" in prepared, prepared
kw = {"intent_judge": True, "prepared_tools": prepared}
turn_state.reset()
turn_state.set_user("100")
turn_state.set_notes(backend)
if partial is not None:
from test_chat_channels_tool import FakeChat
from test_schedule_tool import FakeSchedules
from test_screen_view_tool import FakeScreen
turn_state.set_chat(FakeChat())
turn_state.set_schedule(FakeSchedules())
turn_state.set_screen(FakeScreen())
try:
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
legacy_memory=False, default_language="Korean", person_notes=notes, **kw)
finally:
turn_state.reset()
print(f"\n notes [{JUDGE_MODEL}] {query!r} notes={notes}: acts={backend.acts} reply={(reply or '')[:200]!r}")
return backend, reply or ""
def _ends_with_roger(reply: str) -> bool:
return re.search(r"roger\W*$", reply.strip(), re.IGNORECASE) is not None
@pytest.mark.eval
@requires_judge_llm
class TestPersonalNotes:
def test_a_saved_instruction_shapes_the_answer(self, mock_config, eval_db, eval_dialogue_memory):
_, reply = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?",
notes=["End every answer with the word Roger."])
assert _ends_with_roger(reply), reply
def test_without_the_note_the_answer_does_not_end_that_way(self, mock_config, eval_db, eval_dialogue_memory):
_, reply = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?")
assert not _ends_with_roger(reply), reply
# The second phrasing is from the field (2026-11-18): no note was saved but the answer said
# "from now on I will answer briefly".
@pytest.mark.parametrize("query", ["앞으로 나한테는 반말로 대답해", "자비스 앞으로 나한테는 짧게 대답해줘"])
def test_asking_for_it_from_now_on_saves_a_note(self, mock_config, eval_db, eval_dialogue_memory, query):
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, query)
adds = [kw for a, kw in backend.acts if a == "add"]
assert adds and adds[0]["text"], backend.acts
def test_the_pick_made_while_still_speaking_saves_the_note(self, mock_config, eval_db,
eval_dialogue_memory):
# Live 2026-11-18 the reply ran on the pick prepared from the partial transcript.
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "자비스 앞으로 나한테는 짧게 대답해줘",
partial="자비스 앞으로 나한테는 짧게 대답")
adds = [kw for a, kw in backend.acts if a == "add"]
assert adds and adds[0]["text"], backend.acts
def test_a_one_off_question_saves_nothing(self, mock_config, eval_db, eval_dialogue_memory):
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?")
assert not any(a == "add" for a, _ in backend.acts), backend.acts