- A final is stored under the channel, speaker name and time fixed when its end was decided. Decoding takes a while; a move, rename or late joiner meanwhile used to change where and when it was said, and who heard it. - At start the worker closes presence a killed run left open (HistoryStore.close_open_presence), so a person who is gone no longer hears new speech after a restart. - run_reply_engine(legacy_memory=False) skips v1's single-user memory (warm profile, extractor, diary, graph, digest); the v2 brain uses it so another user's profile or diary never reaches the prompt. - While a brain reply runs, stdout and stderr are muted (also helper threads such as the web fetch pool); the worker logs through voice_worker/log.py to the original stderr. - Live check: Ollama's /v1 fixes the context at 2,048 tokens and cut the prompt in earlier runs; with an 8k tag the chat-record question was answered 4 of 4 and nothing reached the output. Noted in llm_contexts and the eval. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
5.0 KiB
Python
110 lines
5.0 KiB
Python
"""
|
|
Eval: the model answers from the v2 history store's context block.
|
|
|
|
The v2 brain host passes a HistoryContext (src/jarvis/memory/history.spec.md)
|
|
to run_reply_engine: the recent window and the earlier records that match the
|
|
question, already limited to what the asker may see. This eval checks the
|
|
model actually uses a matching earlier record to answer a question about a
|
|
past conversation, and that the block is what makes the difference (the same
|
|
question without the block cannot be answered from anything else).
|
|
|
|
The reply must answer in its own words, not read a record line out (TTS would
|
|
speak the stamp). The fact is a day of the week that the model cannot guess, so the check is a
|
|
plain substring on the reply (in either language, since the reply language is a
|
|
setting).
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_history_context_recall.py -v
|
|
"""
|
|
|
|
import re
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run
|
|
|
|
CASES = [
|
|
{
|
|
"id": "korean",
|
|
"query": "내 고양이 병원 예약 언제라고 했지?",
|
|
"recent": [
|
|
"[2026-11-11 12:01] Bob: 오늘 저녁 뭐 먹지",
|
|
"[2026-11-11 12:02] Jarvis: 김치찌개는 어떠세요?",
|
|
],
|
|
"matches": ["[2026-11-08 20:10] Alice: 고양이 병원 예약은 목요일 오후 세 시야"],
|
|
# The day in either language: the eval config may lock the reply language.
|
|
"expect": ["목요일", "thursday"],
|
|
},
|
|
{
|
|
"id": "english",
|
|
"query": "When did I say the cat's vet appointment was?",
|
|
"recent": ["[2026-11-11 12:01] Bob: what should we eat tonight"],
|
|
"matches": ["[2026-11-08 20:10] Alice: the cat's vet appointment is on Thursday at 3pm"],
|
|
"expect": ["thursday"],
|
|
},
|
|
{
|
|
# Field run 2026-11-11 (qwen2.5:7b through the voice worker): a chat channel record from
|
|
# someone else, asked about by the user. The reply opened by reading the record line out,
|
|
# stamp and all, which TTS would speak. That run went through Ollama's /v1, which fixes the
|
|
# context at 2,048 tokens and cut the system prompt; with the full prompt (8k) the same
|
|
# question was answered in own words 4 of 4 times. Kept as a guard.
|
|
"id": "korean-chat-record",
|
|
"query": "이번 주 모임 언제 어디라고 했지?",
|
|
"recent": [],
|
|
"matches": ["[2026-11-11 11:55] 민수: 이번 주 모임은 토요일 오후 네 시에 강남역이야"],
|
|
"expect": ["토요일", "saturday"],
|
|
},
|
|
]
|
|
|
|
# A record line read out: any "[YYYY-MM-DD HH:MM]" stamp in the reply.
|
|
_STAMP = re.compile(r"\[\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}\]")
|
|
|
|
|
|
def _run(mock_config, eval_db, eval_dialogue_memory, query, history):
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
from jarvis.reply.engine import run_reply_engine
|
|
|
|
mock_config.ollama_base_url = JUDGE_BASE_URL
|
|
mock_config.ollama_chat_model = JUDGE_MODEL
|
|
capture = ToolCallCapture()
|
|
with patch(
|
|
"jarvis.reply.engine.run_tool_with_retries",
|
|
side_effect=create_mock_tool_run(capture, {}),
|
|
):
|
|
return run_reply_engine(
|
|
db=eval_db, cfg=mock_config, tts=None, text=query,
|
|
dialogue_memory=eval_dialogue_memory, history=history,
|
|
)
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
class TestHistoryContextRecall:
|
|
@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
|
|
def test_answers_from_an_earlier_record(self, mock_config, eval_db, eval_dialogue_memory, case):
|
|
from helpers import JUDGE_MODEL
|
|
from jarvis.memory.history import HistoryContext
|
|
|
|
history = HistoryContext(recent=case["recent"], matches=case["matches"])
|
|
response = _run(mock_config, eval_db, eval_dialogue_memory, case["query"], history)
|
|
print(f"\n History recall [{case['id']}] ({JUDGE_MODEL}): {(response or '')[:300]}")
|
|
assert_not_fallback_reply(response, context=case["id"])
|
|
lowered = (response or "").lower()
|
|
assert any(e in lowered for e in case["expect"]), (
|
|
f"[{case['id']}] reply did not use the earlier record: {response!r}"
|
|
)
|
|
assert not _STAMP.search(response or ""), (
|
|
f"[{case['id']}] reply read a record line out instead of answering: {response!r}"
|
|
)
|
|
|
|
@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
|
|
def test_without_the_block_the_fact_is_unknown(self, mock_config, eval_db, eval_dialogue_memory, case):
|
|
"""Control: proves the fact comes from the block, not from elsewhere in the prompt."""
|
|
response = _run(mock_config, eval_db, eval_dialogue_memory, case["query"], None)
|
|
print(f"\n History control [{case['id']}]: {(response or '')[:300]}")
|
|
lowered = (response or "").lower()
|
|
assert not any(e in lowered for e in case["expect"]), (
|
|
f"[{case['id']}] the fact appeared without the history block: {response!r}"
|
|
)
|