""" Eval: per-person standing instructions (src/jarvis/tools/builtin/personal_notes.spec.md). - A saved instruction reaches the answer: with the note "end every answer with the word Roger." the answer to an unrelated question ends with "Roger" (and without the note it does not, so the check measures the note, not the model's habits). - Asking for something to apply from now on saves a note; a one-off question does not. The notes backend is a fake that keeps what it is given; the tool runs for real. Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_personal_notes.py -v (v2 brain setup: EVAL_COMPACT_PROMPT=1 EVAL_INTENT_JUDGE=1) """ import re import pytest from conftest import requires_judge_llm class FakeNotes: def __init__(self): self.acts = [] def act(self, action, **kw): self.acts.append((action, kw)) if action == "add": return {"ok": True, "item": {"id": "1", "text": kw["text"]}} if action == "list": return {"ok": True, "items": []} return {"ok": True, "item": {"id": kw.get("note"), "text": ""}} def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.memory.history import HistoryContext from jarvis.reply import turn_state from jarvis.reply.engine import run_reply_engine mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL backend = FakeNotes() turn_state.reset() turn_state.set_user("100") turn_state.set_notes(backend) try: reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=eval_dialogue_memory, history=HistoryContext(), legacy_memory=False, default_language="Korean", person_notes=notes) finally: turn_state.reset() print(f"\n notes [{JUDGE_MODEL}] {query!r} notes={notes}: acts={backend.acts} reply={(reply or '')[:200]!r}") return backend, reply or "" def _ends_with_roger(reply: str) -> bool: return re.search(r"roger\W*$", reply.strip(), re.IGNORECASE) is not None @pytest.mark.eval @requires_judge_llm class TestPersonalNotes: def test_a_saved_instruction_shapes_the_answer(self, mock_config, eval_db, eval_dialogue_memory): _, reply = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?", notes=["End every answer with the word Roger."]) assert _ends_with_roger(reply), reply def test_without_the_note_the_answer_does_not_end_that_way(self, mock_config, eval_db, eval_dialogue_memory): _, reply = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?") assert not _ends_with_roger(reply), reply def test_asking_for_it_from_now_on_saves_a_note(self, mock_config, eval_db, eval_dialogue_memory): backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "앞으로 나한테는 반말로 대답해") adds = [kw for a, kw in backend.acts if a == "add"] assert adds and adds[0]["text"], backend.acts def test_a_one_off_question_saves_nothing(self, mock_config, eval_db, eval_dialogue_memory): backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?") assert not any(a == "add" for a, _ in backend.acts), backend.acts