- A final is stored under the channel, speaker name and time fixed when its end was decided. Decoding takes a while; a move, rename or late joiner meanwhile used to change where and when it was said, and who heard it. - At start the worker closes presence a killed run left open (HistoryStore.close_open_presence), so a person who is gone no longer hears new speech after a restart. - run_reply_engine(legacy_memory=False) skips v1's single-user memory (warm profile, extractor, diary, graph, digest); the v2 brain uses it so another user's profile or diary never reaches the prompt. - While a brain reply runs, stdout and stderr are muted (also helper threads such as the web fetch pool); the worker logs through voice_worker/log.py to the original stderr. - Live check: Ollama's /v1 fixes the context at 2,048 tokens and cut the prompt in earlier runs; with an 8k tag the chat-record question was answered 4 of 4 and nothing reached the output. Noted in llm_contexts and the eval. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
329 lines
14 KiB
Python
329 lines
14 KiB
Python
"""Conversation history store (v2): voice transcripts, Jarvis's answers and
|
|
registered chat channel messages, searched per asker. Contract:
|
|
``history.spec.md`` next to this file."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import re
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import unicodedata
|
|
from dataclasses import dataclass, field
|
|
from typing import Iterable, List, Optional
|
|
|
|
from ..debug import debug_log
|
|
|
|
_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS history (
|
|
id INTEGER PRIMARY KEY,
|
|
ts REAL NOT NULL,
|
|
kind TEXT NOT NULL, -- voice | reply | chat
|
|
channel_id TEXT NOT NULL,
|
|
author_id TEXT NOT NULL,
|
|
author_name TEXT NOT NULL,
|
|
to_id TEXT,
|
|
message_id TEXT,
|
|
text TEXT NOT NULL,
|
|
norm TEXT NOT NULL -- text folded for search (_fold)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS history_ts ON history(ts);
|
|
CREATE INDEX IF NOT EXISTS history_author ON history(author_id);
|
|
CREATE INDEX IF NOT EXISTS history_to ON history(to_id);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS history_message ON history(channel_id, message_id)
|
|
WHERE message_id IS NOT NULL;
|
|
CREATE TABLE IF NOT EXISTS presence (
|
|
id INTEGER PRIMARY KEY,
|
|
user_id TEXT NOT NULL,
|
|
channel_id TEXT NOT NULL,
|
|
joined REAL NOT NULL,
|
|
left_at REAL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS presence_user ON presence(user_id, channel_id);
|
|
"""
|
|
|
|
_WORD = re.compile(r"\w{2,}", re.UNICODE)
|
|
_BYTES_PER_TOKEN = 3
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Scope:
|
|
"""What an asker may see besides their own records (``history.spec.md``)."""
|
|
|
|
chat: bool = True
|
|
others: bool = True
|
|
since: Optional[float] = None
|
|
off: bool = False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Record:
|
|
id: int
|
|
ts: float
|
|
kind: str
|
|
channel_id: str
|
|
author_id: str
|
|
author_name: str
|
|
to_id: Optional[str]
|
|
message_id: Optional[str]
|
|
text: str
|
|
|
|
|
|
@dataclass
|
|
class HistoryContext:
|
|
"""Lines for the prompt: the recent window and the search matches, kept apart for labelling."""
|
|
|
|
recent: List[str] = field(default_factory=list)
|
|
matches: List[str] = field(default_factory=list)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""One Unicode folding for stored text and query words alike (NFKC, then case folding),
|
|
so matching does not depend on SQLite's ASCII-only lower()."""
|
|
return unicodedata.normalize("NFKC", text).casefold()
|
|
|
|
|
|
def _word_patterns(query: str) -> List[List[str]]:
|
|
"""For each query word, its accepted forms, longest first (the word, then shorter prefixes)."""
|
|
out: List[List[str]] = []
|
|
seen = set()
|
|
for w in _WORD.findall(_fold(query)):
|
|
if w in seen:
|
|
continue
|
|
seen.add(w)
|
|
shortest = max(2, math.ceil(len(w) * 0.6))
|
|
out.append([w[:n] for n in range(len(w), shortest - 1, -1)])
|
|
return out
|
|
|
|
|
|
_COLUMNS = "h.id, h.ts, h.kind, h.channel_id, h.author_id, h.author_name, h.to_id, h.message_id, h.text"
|
|
|
|
|
|
def _record(row: sqlite3.Row) -> Record:
|
|
return Record(**{k: row[k] for k in Record.__dataclass_fields__})
|
|
|
|
|
|
def _tokens(line: str) -> float:
|
|
return len(line.encode("utf-8")) / _BYTES_PER_TOKEN
|
|
|
|
|
|
class HistoryStore:
|
|
def __init__(self, path: str, assistant_name: str = "Jarvis") -> None:
|
|
self.assistant_name = assistant_name
|
|
self._lock = threading.Lock()
|
|
self._conn = sqlite3.connect(path, check_same_thread=False)
|
|
self._conn.row_factory = sqlite3.Row
|
|
self._conn.execute("PRAGMA journal_mode=WAL")
|
|
self._conn.execute("PRAGMA synchronous=NORMAL")
|
|
self._conn.executescript(_SCHEMA)
|
|
self._conn.commit()
|
|
|
|
def close(self) -> None:
|
|
with self._lock:
|
|
self._conn.close()
|
|
|
|
# -- writes -------------------------------------------------------------
|
|
|
|
def _insert(self, kind: str, channel_id: str, author_id: str, author_name: str, text: str,
|
|
ts: float, to_id: Optional[str] = None, message_id: Optional[str] = None) -> Optional[int]:
|
|
text = (text or "").strip()
|
|
if not text:
|
|
return None
|
|
with self._lock:
|
|
cur = self._conn.execute(
|
|
"INSERT OR REPLACE INTO history(ts, kind, channel_id, author_id, author_name, to_id, message_id,"
|
|
" text, norm) VALUES (?,?,?,?,?,?,?,?,?)",
|
|
(ts, kind, channel_id, author_id, author_name, to_id, message_id, text, _fold(text)),
|
|
)
|
|
self._conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
def add_voice(self, channel_id: str, speaker_id: str, speaker_name: str, text: str, ts: float) -> Optional[int]:
|
|
return self._insert("voice", channel_id, speaker_id, speaker_name, text, ts)
|
|
|
|
def add_reply(self, channel_id: str, to_id: str, text: str, ts: float) -> Optional[int]:
|
|
return self._insert("reply", channel_id, "", self.assistant_name, text, ts, to_id=to_id)
|
|
|
|
def add_chat(self, channel_id: str, message_id: str, author_id: str, author_name: str, text: str,
|
|
ts: float, to_id: Optional[str] = None) -> Optional[int]:
|
|
return self._insert("chat", channel_id, author_id, author_name, text, ts, to_id=to_id, message_id=message_id)
|
|
|
|
def edit_chat(self, channel_id: str, message_id: str, text: str) -> None:
|
|
text = (text or "").strip()
|
|
if not text:
|
|
self.delete_chat(channel_id, message_id)
|
|
return
|
|
with self._lock:
|
|
self._conn.execute("UPDATE history SET text=?, norm=? WHERE channel_id=? AND message_id=?",
|
|
(text, _fold(text), channel_id, message_id))
|
|
self._conn.commit()
|
|
|
|
def delete_chat(self, channel_id: str, message_id: str) -> None:
|
|
with self._lock:
|
|
self._conn.execute("DELETE FROM history WHERE channel_id=? AND message_id=?", (channel_id, message_id))
|
|
self._conn.commit()
|
|
|
|
def presence_join(self, user_id: str, channel_id: str, ts: float) -> None:
|
|
with self._lock:
|
|
open_row = self._conn.execute(
|
|
"SELECT 1 FROM presence WHERE user_id=? AND channel_id=? AND left_at IS NULL",
|
|
(user_id, channel_id)).fetchone()
|
|
if open_row is None:
|
|
self._conn.execute("INSERT INTO presence(user_id, channel_id, joined) VALUES (?,?,?)",
|
|
(user_id, channel_id, ts))
|
|
self._conn.commit()
|
|
|
|
def presence_leave(self, user_id: str, channel_id: str, ts: float) -> None:
|
|
with self._lock:
|
|
self._conn.execute(
|
|
"UPDATE presence SET left_at=? WHERE user_id=? AND channel_id=? AND left_at IS NULL",
|
|
(ts, user_id, channel_id))
|
|
self._conn.commit()
|
|
|
|
def close_open_presence(self, ts: float) -> int:
|
|
"""Close every open interval (at start, after a run that ended without closing them)."""
|
|
with self._lock:
|
|
n = self._conn.execute("UPDATE presence SET left_at=? WHERE left_at IS NULL", (ts,)).rowcount
|
|
self._conn.commit()
|
|
return n
|
|
|
|
def presence_close_channel(self, channel_id: str, ts: float) -> None:
|
|
with self._lock:
|
|
self._conn.execute("UPDATE presence SET left_at=? WHERE channel_id=? AND left_at IS NULL",
|
|
(ts, channel_id))
|
|
self._conn.commit()
|
|
|
|
# -- maintenance --------------------------------------------------------
|
|
|
|
def records_of(self, person: str) -> List[Record]:
|
|
with self._lock:
|
|
rows = self._conn.execute(
|
|
f"SELECT {_COLUMNS} FROM history h WHERE author_id=? OR to_id=? ORDER BY ts, id",
|
|
(person, person)).fetchall()
|
|
return [_record(r) for r in rows]
|
|
|
|
def forget_person(self, person: str) -> None:
|
|
with self._lock:
|
|
n = self._conn.execute("DELETE FROM history WHERE author_id=? OR to_id=?", (person, person)).rowcount
|
|
self._conn.execute("DELETE FROM presence WHERE user_id=?", (person,))
|
|
self._conn.commit()
|
|
debug_log(f"history: forgot a person ({n} records)", "memory")
|
|
|
|
def purge_older_than(self, ts: float) -> None:
|
|
with self._lock:
|
|
n = self._conn.execute("DELETE FROM history WHERE ts < ?", (ts,)).rowcount
|
|
self._conn.execute("DELETE FROM presence WHERE left_at IS NOT NULL AND left_at < ?", (ts,))
|
|
self._conn.commit()
|
|
debug_log(f"history: retention removed {n} records", "memory")
|
|
|
|
# -- reads --------------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _scope_sql(asker: str, scope: Scope) -> tuple[str, list]:
|
|
clauses = ["h.author_id = ? OR h.to_id = ?"]
|
|
params: list = [asker, asker]
|
|
if scope.chat:
|
|
clauses.append("h.kind = 'chat'")
|
|
if scope.others:
|
|
clauses.append(
|
|
"h.kind IN ('voice','reply') AND EXISTS (SELECT 1 FROM presence p WHERE p.user_id = ?"
|
|
" AND p.channel_id = h.channel_id AND p.joined <= h.ts AND (p.left_at IS NULL OR p.left_at >= h.ts))")
|
|
params.append(asker)
|
|
sql = "(" + " OR ".join(f"({c})" for c in clauses) + ")"
|
|
if scope.since is not None:
|
|
sql += " AND h.ts >= ?"
|
|
params.append(scope.since)
|
|
return sql, params
|
|
|
|
def recent(self, asker: str, now: float, scope: Scope, minutes: float = 30,
|
|
max_records: int = 20) -> List[Record]:
|
|
if scope.off or max_records <= 0:
|
|
return []
|
|
where, params = self._scope_sql(asker, scope)
|
|
with self._lock:
|
|
rows = self._conn.execute(
|
|
f"SELECT {_COLUMNS} FROM history h WHERE {where} AND h.ts >= ? AND h.ts <= ?"
|
|
" ORDER BY h.ts DESC, h.id DESC LIMIT ?",
|
|
(*params, now - minutes * 60, now, max_records)).fetchall()
|
|
return [_record(r) for r in reversed(rows)]
|
|
|
|
def _scored_search(self, asker: str, query: str, scope: Scope, limit: int,
|
|
exclude: Iterable[int]) -> List[tuple[float, Record]]:
|
|
"""Score every record in scope (no recency cut before scoring) and return the best ``limit``.
|
|
|
|
Two passes in SQLite: the first counts, per query word, how many records in scope match
|
|
it (for the rarity weight); the second scores and ranks in SQL so only ``limit`` rows
|
|
come back to Python.
|
|
"""
|
|
words = _word_patterns(query)
|
|
if scope.off or not words or limit <= 0:
|
|
return []
|
|
where, params = self._scope_sql(asker, scope)
|
|
skip = sorted(set(exclude))
|
|
skip_sql = f" AND h.id NOT IN ({','.join('?' * len(skip))})" if skip else ""
|
|
any_sql = " OR ".join("instr(h.norm, ?) > 0" for forms in words for _ in forms)
|
|
any_params = [f for forms in words for f in forms]
|
|
|
|
def word_hit(forms: List[str]) -> tuple[str, list]:
|
|
# Fraction of the word matched: the longest accepted form found in the record.
|
|
whens = " ".join("WHEN instr(h.norm, ?) > 0 THEN ?" for _ in forms)
|
|
args: list = []
|
|
for f in forms:
|
|
args += [f, len(f) / len(forms[0])]
|
|
return f"(CASE {whens} ELSE 0 END)", args
|
|
|
|
t0 = time.perf_counter()
|
|
with self._lock:
|
|
counts_sql = ", ".join(f"SUM({word_hit(forms)[0]} > 0)" for forms in words)
|
|
counts_args = [a for forms in words for a in word_hit(forms)[1]]
|
|
row = self._conn.execute(
|
|
f"SELECT COUNT(*), {counts_sql} FROM history h WHERE {where} AND ({any_sql}){skip_sql}",
|
|
(*counts_args, *params, *any_params, *skip)).fetchone()
|
|
n = row[0] or 0
|
|
if not n:
|
|
return []
|
|
weights = [math.log(1 + n / max(1, row[i + 1] or 0)) for i in range(len(words))]
|
|
score_parts, score_args = [], []
|
|
for w, forms in zip(weights, words):
|
|
expr, args = word_hit(forms)
|
|
score_parts.append(f"? * {expr}")
|
|
score_args += [w, *args]
|
|
rows = self._conn.execute(
|
|
f"SELECT {_COLUMNS}, ({' + '.join(score_parts)}) AS score FROM history h"
|
|
f" WHERE {where} AND ({any_sql}){skip_sql}"
|
|
" ORDER BY score DESC, h.ts DESC, h.id DESC LIMIT ?",
|
|
(*score_args, *params, *any_params, *skip, limit)).fetchall()
|
|
debug_log(f"history search: {n} candidates, {(time.perf_counter() - t0) * 1000:.0f} ms", "memory")
|
|
return [(r["score"], _record(r)) for r in rows if r["score"] > 0]
|
|
|
|
def search(self, asker: str, query: str, scope: Scope, limit: int = 5,
|
|
exclude: Iterable[int] = ()) -> List[Record]:
|
|
return [rec for _, rec in self._scored_search(asker, query, scope, limit, exclude)]
|
|
|
|
@staticmethod
|
|
def format_line(rec: Record) -> str:
|
|
stamp = time.strftime("%Y-%m-%d %H:%M", time.localtime(rec.ts))
|
|
return f"[{stamp}] {rec.author_name}: {rec.text}"
|
|
|
|
def context(self, asker: str, query: str, now: float, scope: Scope, minutes: float = 30,
|
|
max_records: int = 20, limit: int = 5, budget_tokens: float = 4000) -> HistoryContext:
|
|
recent = self.recent(asker, now, scope, minutes=minutes, max_records=max_records)
|
|
# Search the whole scope, recent records included: a recent record the question is
|
|
# about must keep its place under the matches' share of the budget.
|
|
matches = self.search(asker, query, scope, limit=limit) # best first
|
|
# Matches get at most half the budget so a long recent window cannot crowd out the
|
|
# records the question is about; the recent window gets the rest.
|
|
while matches and sum(_tokens(self.format_line(r)) for r in matches) > budget_tokens / 2:
|
|
matches.pop()
|
|
kept = {r.id for r in matches}
|
|
recent = [r for r in recent if r.id not in kept] # each record appears once
|
|
total = sum(_tokens(self.format_line(r)) for r in recent + matches)
|
|
while recent and total > budget_tokens:
|
|
total -= _tokens(self.format_line(recent.pop(0)))
|
|
matches.sort(key=lambda r: (r.ts, r.id)) # oldest first reads like a timeline
|
|
debug_log(f"history context: {len(recent)} recent, {len(matches)} matches, "
|
|
f"~{total:.0f} tokens", "memory")
|
|
return HistoryContext(recent=[self.format_line(r) for r in recent],
|
|
matches=[self.format_line(r) for r in matches])
|