- Search scored only the newest 5,000 matching records, so an old record behind enough newer chatter was never found. Both passes (word rarity counts and the score) now run in SQLite over every record in scope and only the best rows come back. - context() excluded the recent window from search and then cut that window for the budget, so a relevant recent record could vanish. Search now covers the recent window, matches keep their half of the budget, and a matched record is removed from the recent block so it appears once. - Matching used SQLite's ASCII-only lower() against Python's lowercasing. Stored text gets a folded column (NFKC + casefold) and query words are folded the same way, so case is ignored in every script. - Prompt label "Records related to this question" (matches may now be recent); eval re-run 3x on qwen2.5:7b, all pass. Spec, memory.md and roadmap updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
228 lines
9.6 KiB
Python
228 lines
9.6 KiB
Python
"""Conversation history store (src/jarvis/memory/history.spec.md).
|
||
|
||
Behaviours: what each asker may see (own records, chat channels, others only
|
||
while co-present), recent window, language-agnostic search, the size budget,
|
||
chat edits and deletes, per-person view and delete, and retention.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
|
||
import pytest
|
||
|
||
from jarvis.memory.history import HistoryStore, Scope
|
||
|
||
VC = "vc-1"
|
||
VC2 = "vc-2"
|
||
CHAT = "chat-1"
|
||
BOT = "bot"
|
||
T0 = 1_800_000_000.0
|
||
|
||
|
||
@pytest.fixture
|
||
def store(tmp_path):
|
||
s = HistoryStore(str(tmp_path / "history.sqlite"))
|
||
yield s
|
||
s.close()
|
||
|
||
|
||
def texts(records):
|
||
return [r.text for r in records]
|
||
|
||
|
||
def test_own_records_and_answers_to_them_are_always_in_scope(store):
|
||
store.add_voice(VC, "a", "Alice", "내일 아침 일곱 시에 깨워 줘", T0)
|
||
store.add_reply(VC, "a", "일곱 시에 알람을 맞췄습니다", T0 + 1)
|
||
store.add_voice(VC, "b", "Bob", "나는 피자 먹을래", T0 + 2)
|
||
store.add_reply(VC, "b", "피자 좋네요", T0 + 3)
|
||
|
||
off = Scope(chat=False, others=False)
|
||
got = texts(store.recent("a", now=T0 + 10, scope=off))
|
||
assert got == ["내일 아침 일곱 시에 깨워 줘", "일곱 시에 알람을 맞췄습니다"]
|
||
|
||
|
||
def test_others_are_heard_only_while_the_asker_was_in_the_same_channel(store):
|
||
store.add_voice(VC, "b", "Bob", "before alice came", T0)
|
||
store.presence_join("a", VC, T0 + 10)
|
||
store.add_voice(VC, "b", "Bob", "while alice was here", T0 + 20)
|
||
store.add_reply(VC, "b", "answer to bob while alice was here", T0 + 21)
|
||
store.add_voice(VC2, "c", "Carol", "other channel at the same time", T0 + 22)
|
||
store.presence_leave("a", VC, T0 + 30)
|
||
store.add_voice(VC, "b", "Bob", "after alice left", T0 + 40)
|
||
|
||
got = texts(store.recent("a", now=T0 + 50, scope=Scope()))
|
||
assert got == ["while alice was here", "answer to bob while alice was here"]
|
||
|
||
no_others = texts(store.recent("a", now=T0 + 50, scope=Scope(others=False)))
|
||
assert no_others == []
|
||
|
||
|
||
def test_leaving_the_channel_closes_everyone_s_presence(store):
|
||
store.presence_join("a", VC, T0)
|
||
store.presence_close_channel(VC, T0 + 10)
|
||
store.add_voice(VC, "b", "Bob", "said after jarvis left", T0 + 20)
|
||
assert store.recent("a", now=T0 + 30, scope=Scope()) == []
|
||
|
||
|
||
def test_chat_channel_messages_follow_the_chat_setting(store):
|
||
store.add_chat(CHAT, "m1", "b", "Bob", "오늘 회의는 3시", T0)
|
||
store.add_chat(CHAT, "m2", BOT, "Jarvis", "알겠습니다", T0 + 1, to_id="a")
|
||
|
||
assert texts(store.recent("a", now=T0 + 5, scope=Scope())) == ["오늘 회의는 3시", "알겠습니다"]
|
||
# With chat off, only Jarvis's answer to the asker stays (own record).
|
||
assert texts(store.recent("a", now=T0 + 5, scope=Scope(chat=False))) == ["알겠습니다"]
|
||
|
||
|
||
def test_chat_edits_replace_and_deletes_remove(store):
|
||
store.add_chat(CHAT, "m1", "b", "Bob", "first text", T0)
|
||
store.edit_chat(CHAT, "m1", "edited text")
|
||
assert texts(store.recent("a", now=T0 + 1, scope=Scope())) == ["edited text"]
|
||
store.delete_chat(CHAT, "m1")
|
||
assert store.recent("a", now=T0 + 1, scope=Scope()) == []
|
||
|
||
|
||
def test_scope_off_and_since(store):
|
||
store.add_voice(VC, "a", "Alice", "old words", T0)
|
||
store.add_voice(VC, "a", "Alice", "new words", T0 + 100)
|
||
assert store.recent("a", now=T0 + 101, scope=Scope(off=True)) == []
|
||
assert texts(store.search("a", "words", scope=Scope(since=T0 + 50))) == ["new words"]
|
||
|
||
|
||
def test_recent_window_keeps_the_newest_within_the_time_and_count_limits(store):
|
||
for i in range(30):
|
||
store.add_voice(VC, "a", "Alice", f"line {i}", T0 + i * 60)
|
||
now = T0 + 29 * 60
|
||
got = texts(store.recent("a", now=now, scope=Scope(), minutes=30, max_records=20))
|
||
assert got == [f"line {i}" for i in range(10, 30)]
|
||
got = texts(store.recent("a", now=now, scope=Scope(), minutes=5, max_records=20))
|
||
assert got == [f"line {i}" for i in range(24, 30)]
|
||
|
||
|
||
def test_empty_text_is_not_stored(store):
|
||
store.add_voice(VC, "a", "Alice", " ", T0)
|
||
assert store.recent("a", now=T0 + 1, scope=Scope()) == []
|
||
|
||
|
||
def test_search_finds_a_stem_with_a_different_ending(store):
|
||
store.add_voice(VC, "a", "Alice", "어제 인터스텔라 영화 봤어", T0)
|
||
store.add_voice(VC, "a", "Alice", "점심은 김치찌개", T0 + 1)
|
||
store.add_voice(VC, "a", "Alice", "We watched the match", T0 + 2)
|
||
|
||
assert texts(store.search("a", "영화를 다시 보고 싶어", scope=Scope())) == ["어제 인터스텔라 영화 봤어"]
|
||
assert texts(store.search("a", "watching", scope=Scope())) == ["We watched the match"]
|
||
|
||
|
||
def test_search_prefers_rare_words_and_respects_scope(store):
|
||
for i in range(10):
|
||
store.add_voice(VC, "a", "Alice", f"오늘 날씨 이야기 {i}", T0 + i)
|
||
store.add_voice(VC, "a", "Alice", "오늘 고양이 병원 예약", T0 + 50)
|
||
store.add_voice(VC, "z", "Zed", "고양이 사진 보여 줄게", T0 + 60) # Alice was never there
|
||
|
||
got = store.search("a", "오늘 고양이 병원", scope=Scope(), limit=3)
|
||
assert got[0].text == "오늘 고양이 병원 예약"
|
||
assert all(r.author_id == "a" for r in got)
|
||
|
||
|
||
def test_search_skips_excluded_records(store):
|
||
rid = store.add_voice(VC, "a", "Alice", "pasta recipe", T0)
|
||
store.add_voice(VC, "a", "Alice", "pasta shopping", T0 + 1)
|
||
got = store.search("a", "pasta", scope=Scope(), exclude={rid})
|
||
assert texts(got) == ["pasta shopping"]
|
||
|
||
|
||
def test_search_of_nothing_finds_nothing(store):
|
||
store.add_voice(VC, "a", "Alice", "anything", T0)
|
||
assert store.search("a", "?! .", scope=Scope()) == []
|
||
|
||
|
||
def test_context_formats_lines_and_keeps_under_budget(store):
|
||
store.add_voice(VC, "a", "Alice", "고양이 병원은 금요일", T0)
|
||
for i in range(40):
|
||
store.add_voice(VC, "a", "Alice", f"잡담 {i} " + "가" * 60, T0 + 1000 + i)
|
||
|
||
ctx = store.context("a", "고양이 병원 언제였지", now=T0 + 1040, scope=Scope(), budget_tokens=200)
|
||
assert any("고양이 병원은 금요일" in line for line in ctx.matches)
|
||
assert all(line.startswith("[") and "Alice: " in line for line in ctx.recent + ctx.matches)
|
||
total = sum(len(line.encode("utf-8")) for line in ctx.recent + ctx.matches) / 3
|
||
assert total <= 200
|
||
# The newest recent line survives the cut; the oldest go first.
|
||
assert ctx.recent and ctx.recent[-1].endswith("잡담 39 " + "가" * 60)
|
||
|
||
|
||
def test_context_drops_matches_before_recent_lines(store):
|
||
store.add_voice(VC, "a", "Alice", "old match about cats " + "x" * 300, T0)
|
||
store.add_voice(VC, "a", "Alice", "recent line", T0 + 5000)
|
||
ctx = store.context("a", "cats", now=T0 + 5001, scope=Scope(), budget_tokens=20)
|
||
assert ctx.matches == []
|
||
assert ctx.recent and ctx.recent[-1].endswith("recent line")
|
||
|
||
|
||
def test_per_person_view_and_delete(store):
|
||
store.presence_join("a", VC, T0)
|
||
store.add_voice(VC, "a", "Alice", "alice speaks", T0 + 1)
|
||
store.add_reply(VC, "a", "answer to alice", T0 + 2)
|
||
store.add_voice(VC, "b", "Bob", "bob speaks", T0 + 3)
|
||
|
||
assert texts(store.records_of("a")) == ["alice speaks", "answer to alice"]
|
||
store.forget_person("a")
|
||
assert store.records_of("a") == []
|
||
assert texts(store.records_of("b")) == ["bob speaks"]
|
||
# Presence went too, so Bob's words are no longer in Alice's scope.
|
||
assert store.recent("a", now=T0 + 10, scope=Scope()) == []
|
||
|
||
|
||
def test_retention_removes_old_records(store):
|
||
store.add_voice(VC, "a", "Alice", "old", T0)
|
||
store.add_voice(VC, "a", "Alice", "new", T0 + 100)
|
||
store.purge_older_than(T0 + 50)
|
||
assert texts(store.records_of("a")) == ["new"]
|
||
|
||
|
||
def test_records_survive_reopening(tmp_path):
|
||
path = str(tmp_path / "h.sqlite")
|
||
s = HistoryStore(path)
|
||
s.add_voice(VC, "a", "Alice", "persisted", T0)
|
||
s.close()
|
||
s = HistoryStore(path)
|
||
assert texts(s.records_of("a")) == ["persisted"]
|
||
s.close()
|
||
|
||
|
||
def test_search_over_a_large_history_stays_fast(store):
|
||
for i in range(20_000):
|
||
store.add_voice(VC, "a", "Alice", f"메시지 번호 {i} 내용 {i % 97}", T0 + i)
|
||
store.add_voice(VC, "a", "Alice", "희귀한 단어 오로라", T0 + 60_000)
|
||
t = time.perf_counter()
|
||
got = store.search("a", "오로라 봤던 거", scope=Scope())
|
||
elapsed = time.perf_counter() - t
|
||
assert got and got[0].text == "희귀한 단어 오로라"
|
||
assert elapsed < 1.0
|
||
|
||
|
||
def test_search_covers_the_whole_history_not_just_the_newest_records(store):
|
||
store.add_voice(VC, "a", "Alice", "오늘 고양이 병원 예약은 목요일", T0)
|
||
for i in range(6000):
|
||
store.add_voice(VC, "a", "Alice", f"오늘 점심 이야기 {i}", T0 + 10 + i)
|
||
got = store.search("a", "오늘 고양이 병원 예약 언제", scope=Scope(), limit=3)
|
||
assert got and got[0].text == "오늘 고양이 병원 예약은 목요일"
|
||
|
||
|
||
def test_a_relevant_recent_record_survives_the_budget(store):
|
||
now = T0 + 1000
|
||
store.add_voice(VC, "a", "Alice", "고양이 병원 예약은 목요일", now - 600)
|
||
for i in range(19):
|
||
store.add_voice(VC, "a", "Alice", f"잡담 {i} " + "가" * 400, now - 500 + i)
|
||
ctx = store.context("a", "고양이 병원 예약 언제", now=now, scope=Scope())
|
||
lines = ctx.recent + ctx.matches
|
||
assert sum(1 for x in lines if "고양이 병원 예약은 목요일" in x) == 1
|
||
total = sum(len(x.encode("utf-8")) for x in lines) / 3
|
||
assert total <= 4000
|
||
|
||
|
||
def test_search_folds_case_the_same_way_for_every_script(store):
|
||
store.add_voice(VC, "a", "Alice", "Поездка в МОСКВА завтра", T0)
|
||
store.add_voice(VC, "a", "Alice", "Straße frei", T0 + 1)
|
||
assert texts(store.search("a", "москва", scope=Scope())) == ["Поездка в МОСКВА завтра"]
|
||
assert texts(store.search("a", "STRASSE", scope=Scope())) == ["Straße frei"]
|