From 64dceb816e8dbc2aae58d439c175e72770105256 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Wed, 18 Nov 2026 02:18:12 +0900 Subject: [PATCH] test(notes): eval the pick prepared from the partial transcript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v2 brain reuses the intent judge's pick made while the person still speaks. Eval: partial '자비스 앞으로 나한테는 짧게 대답' -> final request saves the note. Old description 0/3, current 3/3 (qwen3.6-35b-a3b, game PC); whole file 18/18. Co-Authored-By: Claude Opus 5.5 --- evals/test_personal_notes.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/evals/test_personal_notes.py b/evals/test_personal_notes.py index 2dc7237..cd89de2 100644 --- a/evals/test_personal_notes.py +++ b/evals/test_personal_notes.py @@ -31,7 +31,7 @@ class FakeNotes: return {"ok": True, "item": {"id": kw.get("note"), "text": ""}} -def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None): +def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None, partial=None): from helpers import JUDGE_BASE_URL, JUDGE_MODEL from jarvis.memory.history import HistoryContext from jarvis.reply import turn_state @@ -40,13 +40,22 @@ def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None): mock_config.ollama_base_url = JUDGE_BASE_URL mock_config.ollama_chat_model = JUDGE_MODEL backend = FakeNotes() + kw = {} + if partial is not None: + # The v2 brain's path: the intent judge picks on a partial transcript while the person + # still speaks, and the reply reuses that pick when the final text is close to it. + from jarvis.reply.engine import prepare_intent + from voice_worker.brain import similar + assert similar(partial, query) + kw = {"intent_judge": True, "prepared_tools": prepare_intent( + mock_config, partial, eval_dialogue_memory, False, notes_available=True)} turn_state.reset() turn_state.set_user("100") turn_state.set_notes(backend) try: reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, dialogue_memory=eval_dialogue_memory, history=HistoryContext(), - legacy_memory=False, default_language="Korean", person_notes=notes) + legacy_memory=False, default_language="Korean", person_notes=notes, **kw) finally: turn_state.reset() print(f"\n notes [{JUDGE_MODEL}] {query!r} notes={notes}: acts={backend.acts} reply={(reply or '')[:200]!r}") @@ -77,6 +86,14 @@ class TestPersonalNotes: adds = [kw for a, kw in backend.acts if a == "add"] assert adds and adds[0]["text"], backend.acts + def test_the_pick_made_while_still_speaking_saves_the_note(self, mock_config, eval_db, + eval_dialogue_memory): + # Live 2026-11-18 the reply ran on the pick prepared from the partial transcript. + backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "자비스 앞으로 나한테는 짧게 대답해줘", + partial="자비스 앞으로 나한테는 짧게 대답") + adds = [kw for a, kw in backend.acts if a == "add"] + assert adds and adds[0]["text"], backend.acts + def test_a_one_off_question_saves_nothing(self, mock_config, eval_db, eval_dialogue_memory): backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?") assert not any(a == "add" for a, _ in backend.acts), backend.acts