test(notes): eval the pick prepared from the partial transcript
The v2 brain reuses the intent judge's pick made while the person still speaks. Eval: partial '자비스 앞으로 나한테는 짧게 대답' -> final request saves the note. Old description 0/3, current 3/3 (qwen3.6-35b-a3b, game PC); whole file 18/18. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -31,7 +31,7 @@ class FakeNotes:
|
||||
return {"ok": True, "item": {"id": kw.get("note"), "text": ""}}
|
||||
|
||||
|
||||
def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None):
|
||||
def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None, partial=None):
|
||||
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
||||
from jarvis.memory.history import HistoryContext
|
||||
from jarvis.reply import turn_state
|
||||
@@ -40,13 +40,22 @@ def _run(mock_config, eval_db, eval_dialogue_memory, query, notes=None):
|
||||
mock_config.ollama_base_url = JUDGE_BASE_URL
|
||||
mock_config.ollama_chat_model = JUDGE_MODEL
|
||||
backend = FakeNotes()
|
||||
kw = {}
|
||||
if partial is not None:
|
||||
# The v2 brain's path: the intent judge picks on a partial transcript while the person
|
||||
# still speaks, and the reply reuses that pick when the final text is close to it.
|
||||
from jarvis.reply.engine import prepare_intent
|
||||
from voice_worker.brain import similar
|
||||
assert similar(partial, query)
|
||||
kw = {"intent_judge": True, "prepared_tools": prepare_intent(
|
||||
mock_config, partial, eval_dialogue_memory, False, notes_available=True)}
|
||||
turn_state.reset()
|
||||
turn_state.set_user("100")
|
||||
turn_state.set_notes(backend)
|
||||
try:
|
||||
reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query,
|
||||
dialogue_memory=eval_dialogue_memory, history=HistoryContext(),
|
||||
legacy_memory=False, default_language="Korean", person_notes=notes)
|
||||
legacy_memory=False, default_language="Korean", person_notes=notes, **kw)
|
||||
finally:
|
||||
turn_state.reset()
|
||||
print(f"\n notes [{JUDGE_MODEL}] {query!r} notes={notes}: acts={backend.acts} reply={(reply or '')[:200]!r}")
|
||||
@@ -77,6 +86,14 @@ class TestPersonalNotes:
|
||||
adds = [kw for a, kw in backend.acts if a == "add"]
|
||||
assert adds and adds[0]["text"], backend.acts
|
||||
|
||||
def test_the_pick_made_while_still_speaking_saves_the_note(self, mock_config, eval_db,
|
||||
eval_dialogue_memory):
|
||||
# Live 2026-11-18 the reply ran on the pick prepared from the partial transcript.
|
||||
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "자비스 앞으로 나한테는 짧게 대답해줘",
|
||||
partial="자비스 앞으로 나한테는 짧게 대답")
|
||||
adds = [kw for a, kw in backend.acts if a == "add"]
|
||||
assert adds and adds[0]["text"], backend.acts
|
||||
|
||||
def test_a_one_off_question_saves_nothing(self, mock_config, eval_db, eval_dialogue_memory):
|
||||
backend, _ = _run(mock_config, eval_db, eval_dialogue_memory, "고양이는 하루에 몇 시간 자?")
|
||||
assert not any(a == "add" for a, _ in backend.acts), backend.acts
|
||||
|
||||
Reference in New Issue
Block a user