Files
javis_bot/evals/test_history_context_recall.py
javis-bot b3dcd12ec9 fix(voice_worker): keep history and the brain's output scoped to who was there
- A final is stored under the channel, speaker name and time fixed when its
  end was decided. Decoding takes a while; a move, rename or late joiner
  meanwhile used to change where and when it was said, and who heard it.
- At start the worker closes presence a killed run left open
  (HistoryStore.close_open_presence), so a person who is gone no longer
  hears new speech after a restart.
- run_reply_engine(legacy_memory=False) skips v1's single-user memory (warm
  profile, extractor, diary, graph, digest); the v2 brain uses it so another
  user's profile or diary never reaches the prompt.
- While a brain reply runs, stdout and stderr are muted (also helper threads
  such as the web fetch pool); the worker logs through voice_worker/log.py to
  the original stderr.
- Live check: Ollama's /v1 fixes the context at 2,048 tokens and cut the
  prompt in earlier runs; with an 8k tag the chat-record question was answered
  4 of 4 and nothing reached the output. Noted in llm_contexts and the eval.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 13:28:25 +09:00

110 lines
5.0 KiB
Python

"""
Eval: the model answers from the v2 history store's context block.
The v2 brain host passes a HistoryContext (src/jarvis/memory/history.spec.md)
to run_reply_engine: the recent window and the earlier records that match the
question, already limited to what the asker may see. This eval checks the
model actually uses a matching earlier record to answer a question about a
past conversation, and that the block is what makes the difference (the same
question without the block cannot be answered from anything else).
The reply must answer in its own words, not read a record line out (TTS would
speak the stamp). The fact is a day of the week that the model cannot guess, so the check is a
plain substring on the reply (in either language, since the reply language is a
setting).
Run: EVAL_JUDGE_MODEL=qwen2.5:7b python -m pytest evals/test_history_context_recall.py -v
"""
import re
from unittest.mock import patch
import pytest
from conftest import requires_judge_llm
from helpers import ToolCallCapture, assert_not_fallback_reply, create_mock_tool_run
CASES = [
{
"id": "korean",
"query": "내 고양이 병원 예약 언제라고 했지?",
"recent": [
"[2026-11-11 12:01] Bob: 오늘 저녁 뭐 먹지",
"[2026-11-11 12:02] Jarvis: 김치찌개는 어떠세요?",
],
"matches": ["[2026-11-08 20:10] Alice: 고양이 병원 예약은 목요일 오후 세 시야"],
# The day in either language: the eval config may lock the reply language.
"expect": ["목요일", "thursday"],
},
{
"id": "english",
"query": "When did I say the cat's vet appointment was?",
"recent": ["[2026-11-11 12:01] Bob: what should we eat tonight"],
"matches": ["[2026-11-08 20:10] Alice: the cat's vet appointment is on Thursday at 3pm"],
"expect": ["thursday"],
},
{
# Field run 2026-11-11 (qwen2.5:7b through the voice worker): a chat channel record from
# someone else, asked about by the user. The reply opened by reading the record line out,
# stamp and all, which TTS would speak. That run went through Ollama's /v1, which fixes the
# context at 2,048 tokens and cut the system prompt; with the full prompt (8k) the same
# question was answered in own words 4 of 4 times. Kept as a guard.
"id": "korean-chat-record",
"query": "이번 주 모임 언제 어디라고 했지?",
"recent": [],
"matches": ["[2026-11-11 11:55] 민수: 이번 주 모임은 토요일 오후 네 시에 강남역이야"],
"expect": ["토요일", "saturday"],
},
]
# A record line read out: any "[YYYY-MM-DD HH:MM]" stamp in the reply.
_STAMP = re.compile(r"\[\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}\]")
def _run(mock_config, eval_db, eval_dialogue_memory, query, history):
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
from jarvis.reply.engine import run_reply_engine
mock_config.ollama_base_url = JUDGE_BASE_URL
mock_config.ollama_chat_model = JUDGE_MODEL
capture = ToolCallCapture()
with patch(
"jarvis.reply.engine.run_tool_with_retries",
side_effect=create_mock_tool_run(capture, {}),
):
return run_reply_engine(
db=eval_db, cfg=mock_config, tts=None, text=query,
dialogue_memory=eval_dialogue_memory, history=history,
)
@pytest.mark.eval
@requires_judge_llm
class TestHistoryContextRecall:
@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
def test_answers_from_an_earlier_record(self, mock_config, eval_db, eval_dialogue_memory, case):
from helpers import JUDGE_MODEL
from jarvis.memory.history import HistoryContext
history = HistoryContext(recent=case["recent"], matches=case["matches"])
response = _run(mock_config, eval_db, eval_dialogue_memory, case["query"], history)
print(f"\n History recall [{case['id']}] ({JUDGE_MODEL}): {(response or '')[:300]}")
assert_not_fallback_reply(response, context=case["id"])
lowered = (response or "").lower()
assert any(e in lowered for e in case["expect"]), (
f"[{case['id']}] reply did not use the earlier record: {response!r}"
)
assert not _STAMP.search(response or ""), (
f"[{case['id']}] reply read a record line out instead of answering: {response!r}"
)
@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
def test_without_the_block_the_fact_is_unknown(self, mock_config, eval_db, eval_dialogue_memory, case):
"""Control: proves the fact comes from the block, not from elsewhere in the prompt."""
response = _run(mock_config, eval_db, eval_dialogue_memory, case["query"], None)
print(f"\n History control [{case['id']}]: {(response or '')[:300]}")
lowered = (response or "").lower()
assert not any(e in lowered for e in case["expect"]), (
f"[{case['id']}] the fact appeared without the history block: {response!r}"
)