The partial-transcript eval now offers chat, schedule, notes and screen as voice_worker/brain.py does, sets those backends for the reply, and fails when the intent judge made no pick or the pick lacks personalNotes, so it cannot pass by re-routing the final text. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
5.5 KiB
Python
112 lines
5.5 KiB
Python
"""
|
|
Eval: per-person standing instructions (src/jarvis/tools/builtin/personal_notes.spec.md).
|
|
|
|
- A saved instruction reaches the answer: with the note "end every answer with the word
|
|
Roger." the answer to an unrelated question ends with "Roger" (and without the note it does not,
|
|
so the check measures the note, not the model's habits).
|
|
- Asking for something to apply from now on saves a note; a one-off question does not.
|
|
|
|
The notes backend is a fake that keeps what it is given; the tool runs for real.
|
|
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_personal_notes.py -v
|
|
(v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1)
|
|
"""
|
|
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
|
|
|
|
class FakeNotes:
|
|
def __init__(self):
|
|
self.acts = []
|
|
|
|
def act(self, action, **kw):
|
|
self.acts.append((action, kw))
|
|
if action == "add":
|
|
return {"ok": True, "item": {"id": "1", "text": kw["text"]}}
|
|
if action == "list":
|
|
return {"ok": True, "items": []}
|
|
return {"ok": True, "item": {"id": kw.get("note"), "text": ""}}
|
|
|
|
|
|
def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None, partial=None):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import turn_state
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
backend = FakeNotes()
|
|
kw = {}
|
|
if partial is not None:
|
|
# The v2 brain's path: the intent judge picks on a partial transcript while the person
|
|
# still speaks, and the reply reuses that pick when the final text is close to it.
|
|
from jarvis.reply.engine import prepare_intent
|
|
from voice_worker.brain import similar
|
|
assert similar(partial, query)
|
|
# The same offer as the app (voice_worker/brain.py): chat, schedule, notes and screen.
|
|
prepared = prepare_intent(mock_config, partial, eval_dialogue_memory, True,
|
|
schedule_available=True, notes_available=True, screen_available=True)
|
|
print(f"\n notes [{JUDGE_MODEL}] prepared on {partial!r}: {prepared}")
|
|
assert prepared is not None, "the intent judge made no pick, so nothing would be reused"
|
|
assert "personalNotes" in prepared, prepared
|
|
kw = {"intent_judge": True, "prepared_tools": prepared}
|
|
turn_state.reset()
|
|
turn_state.set_user("100")
|
|
turn_state.set_notes(backend)
|
|
if partial is not None:
|
|
from test_chat_channels_tool import FakeChat
|
|
from test_schedule_tool import FakeSchedules
|
|
from test_screen_view_tool import FakeScreen
|
|
turn_state.set_chat(FakeChat())
|
|
turn_state.set_schedule(FakeSchedules())
|
|
turn_state.set_screen(FakeScreen())
|
|
try:
|
|
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
|
legacy_memory=False, default_language="Korean", person_notes=notes, **kw)
|
|
finally:
|
|
turn_state.reset()
|
|
print(f"\n notes [{JUDGE_MODEL}] {query!r} notes={notes}: acts={backend.acts} reply={(reply or '')[:200]!r}")
|
|
return backend, reply or ""
|
|
|
|
|
|
def _ends_with_roger(reply: str) -> bool:
|
|
return re.search(r"roger\W*$", reply.strip(), re.IGNORECASE) is not None
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestPersonalNotes:
|
|
def test_a_saved_instruction_shapes_the_answer(self, mock_config, eval_db, eval_dialogue_memory):
|
|
_, reply = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?",
|
|
notes=["End every answer with the word Roger."])
|
|
assert _ends_with_roger(reply), reply
|
|
|
|
def test_without_the_note_the_answer_does_not_end_that_way(self, mock_config, eval_db, eval_dialogue_memory):
|
|
_, reply = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?")
|
|
assert not _ends_with_roger(reply), reply
|
|
|
|
# The second phrasing is from the field (2026-11-18): no note was saved but the answer said
|
|
# "from now on I will answer briefly".
|
|
@pytest.mark.parametrize("query", ["앞으로 나한테는 반말로 대답해", "자비스 앞으로 나한테는 짧게 대답해줘"])
|
|
def test_asking_for_it_from_now_on_saves_a_note(self, mock_config, eval_db, eval_dialogue_memory, query):
|
|
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, query)
|
|
adds = [kw for a, kw in backend.acts if a == "add"]
|
|
assert adds and adds[0]["text"], backend.acts
|
|
|
|
def test_the_pick_made_while_still_speaking_saves_the_note(self, mock_config, eval_db,
|
|
eval_dialogue_memory):
|
|
# Live 2026-11-18 the reply ran on the pick prepared from the partial transcript.
|
|
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "자비스 앞으로 나한테는 짧게 대답해줘",
|
|
partial="자비스 앞으로 나한테는 짧게 대답")
|
|
adds = [kw for a, kw in backend.acts if a == "add"]
|
|
assert adds and adds[0]["text"], backend.acts
|
|
|
|
def test_a_one_off_question_saves_nothing(self, mock_config, eval_db, eval_dialogue_memory):
|
|
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?")
|
|
assert not any(a == "add" for a, _ in backend.acts), backend.acts
|