diff --git a/src/jarvis/memory/history.py b/src/jarvis/memory/history.py index 8f026d8..f30a0de 100644 --- a/src/jarvis/memory/history.py +++ b/src/jarvis/memory/history.py @@ -4,6 +4,7 @@ registered chat channel messages, searched per asker. Contract: from __future__ import annotations +import json import math import re import sqlite3 @@ -41,6 +42,19 @@ CREATE TABLE IF NOT EXISTS presence ( left_at REAL ); CREATE INDEX IF NOT EXISTS presence_user ON presence(user_id, channel_id); +CREATE TABLE IF NOT EXISTS traces ( + id INTEGER PRIMARY KEY, + ts REAL NOT NULL, -- when the answer was ready + via TEXT NOT NULL, -- voice | chat | task + channel_id TEXT, -- where it was asked; none for a task + user_id TEXT NOT NULL, + question TEXT NOT NULL, + answer TEXT NOT NULL, -- empty when the brain gave none + ms INTEGER NOT NULL, + steps TEXT NOT NULL -- JSON list (history.spec.md, "Traces") +); +CREATE INDEX IF NOT EXISTS traces_ts ON traces(ts); +CREATE INDEX IF NOT EXISTS traces_user ON traces(user_id); """ _WORD = re.compile(r"\w{2,}", re.UNICODE) @@ -70,6 +84,21 @@ class Record: text: str +@dataclass(frozen=True) +class Trace: + """How one answer was made: never read back into a prompt (``history.spec.md``, "Traces").""" + + id: int + ts: float + via: str + channel_id: Optional[str] + user_id: str + question: str + answer: str + ms: int + steps: list + + @dataclass class HistoryContext: """Lines for the prompt: the recent window and the search matches, kept apart for labelling.""" @@ -196,6 +225,29 @@ class HistoryStore: # -- maintenance -------------------------------------------------------- + def add_trace(self, via: str, channel_id: Optional[str], user_id: str, question: str, answer: str, + steps: list, ms: int, ts: float) -> int: + with self._lock: + cur = self._conn.execute( + "INSERT INTO traces(ts, via, channel_id, user_id, question, answer, ms, steps) VALUES (?,?,?,?,?,?,?,?)", + (ts, via, channel_id, user_id, question, answer or "", int(ms), + json.dumps(steps, ensure_ascii=False, default=str))) + self._conn.commit() + return int(cur.lastrowid) + + def traces_of(self, person: str) -> List[Trace]: + with self._lock: + rows = self._conn.execute( + "SELECT id, ts, via, channel_id, user_id, question, answer, ms, steps FROM traces" + " WHERE user_id=? ORDER BY ts, id", (person,)).fetchall() + return [Trace(**{**dict(r), "steps": json.loads(r["steps"])}) for r in rows] + + def purge_traces_older_than(self, ts: float) -> int: + with self._lock: + n = self._conn.execute("DELETE FROM traces WHERE ts < ?", (ts,)).rowcount + self._conn.commit() + return n + def records_of(self, person: str) -> List[Record]: with self._lock: rows = self._conn.execute( @@ -207,6 +259,7 @@ class HistoryStore: with self._lock: n = self._conn.execute("DELETE FROM history WHERE author_id=? OR to_id=?", (person, person)).rowcount self._conn.execute("DELETE FROM presence WHERE user_id=?", (person,)) + self._conn.execute("DELETE FROM traces WHERE user_id=?", (person,)) self._conn.commit() debug_log(f"history: forgot a person ({n} records)", "memory") @@ -214,6 +267,7 @@ class HistoryStore: with self._lock: n = self._conn.execute("DELETE FROM history WHERE ts < ?", (ts,)).rowcount self._conn.execute("DELETE FROM presence WHERE left_at IS NOT NULL AND left_at < ?", (ts,)) + self._conn.execute("DELETE FROM traces WHERE ts < ?", (ts,)) self._conn.commit() debug_log(f"history: retention removed {n} records", "memory") diff --git a/src/jarvis/reply/engine.py b/src/jarvis/reply/engine.py index 9567b2a..217d7c9 100644 --- a/src/jarvis/reply/engine.py +++ b/src/jarvis/reply/engine.py @@ -27,6 +27,7 @@ from .enrichment import ( digest_tool_result_for_query, digest_loop_for_max_turns, ) +from . import turn_state as _turn_state from .prompt_dump import dump_reply_turn, is_enabled as _prompt_dump_enabled, new_session_id from .prompts import ModelSize, detect_model_size, get_system_prompts from .compound_query import split_compound_query @@ -61,6 +62,22 @@ if TYPE_CHECKING: # ── Helpers ───────────────────────────────────────────────────────────────── +# How much of a tool result and of the model's thinking a trace keeps (history.spec.md, "Traces"). +TRACE_RESULT_CHARS = 2000 +TRACE_THINKING_CHARS = 8000 + + +def _clip(text: str, limit: int) -> str: + return text if len(text) <= limit else text[:limit] + "…" + + +def _trace_tool(name: str, args, result) -> None: + """One executed tool call in the turn's trace: name, arguments, outcome and the start of its result.""" + text = result.reply_text or result.error_message or "" + _turn_state.trace("tool", name=name, args=args or {}, ok=bool(result.success), + result=_clip(str(text), TRACE_RESULT_CHARS)) + + def _indent_text(text: str, prefix: str = " ") -> str: return f"\n{prefix}".join(text.splitlines()) @@ -1419,6 +1436,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], routed_tools = [t for t in routed_tools if t != "webSearch"] debug_log("screen-share: site named — dropping webSearch so controlBrowser wins", "tools") + _turn_state.trace("tools", names=list(routed_tools)) _planner_schema = generate_tools_json_schema(routed_tools, mcp_tools) _planner_tool_catalog = planner_tool_catalog(_planner_schema or []) @@ -1438,6 +1456,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], debug_log(f"planner step failed (non-fatal): {_plan_exc}", "planning") action_plan = [] if action_plan: + _turn_state.trace("plan", steps=list(action_plan)) _plan_preview = " | ".join(s[:50] for s in action_plan) print( f" 🗺️ Plan: {len(action_plan)} step(s) — {_plan_preview}", @@ -2389,6 +2408,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], max_retries=1, language=language, ) + _trace_tool(_name, _args, _plan_result) if _plan_result.reply_text: _plan_text = _maybe_digest_tool_result( cfg=cfg, @@ -2602,6 +2622,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], # Always show thinking if present, regardless of content if thinking: + _turn_state.trace("think", text=_clip(str(thinking), TRACE_THINKING_CHARS)) debug_log(f" 💭 LLM thinking: '{thinking[:300]}{'...' if len(thinking) > 300 else ''}'", "planning") # Extract tool call if present @@ -2647,6 +2668,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], # Check if tool is not allowed - respond with tool error if tool_name not in allowed_tools: debug_log(f" ⚠️ tool not allowed: {tool_name}", "planning") + _turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="not_allowed") print(f" ⚠️ Tool '{tool_name}' not in allow-list", flush=True) # Use tool response instead of system message to maintain native tool calling compatibility messages.append({ @@ -2694,6 +2716,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], if signature in recent_tool_signatures: debug_log(f" ⚠️ Duplicate {tool_name} call - returning cached guidance", "planning") + _turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="duplicate") if use_text_tools: messages.append({"role": "user", "content": f"[Tool: {tool_name}] You already called this tool with these arguments. Use the results from the previous tool call to answer the user."}) else: @@ -2713,6 +2736,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], ) if duplicate_tool_count >= 2: debug_log(f" ⚠️ Too many {tool_name} calls ({duplicate_tool_count}) - returning guidance", "planning") + _turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="duplicate") if use_text_tools: messages.append({"role": "user", "content": f"[Tool: {tool_name}] You have already called this tool {duplicate_tool_count} times. Use the results from those calls to answer the user's question."}) else: @@ -2731,6 +2755,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any], max_retries=1, language=language, ) + _trace_tool(tool_name, tool_args, result) # Handle stop tool - end conversation without response if result.reply_text == STOP_SIGNAL: diff --git a/src/jarvis/reply/turn_state.py b/src/jarvis/reply/turn_state.py index 8b2e14f..26c7fa7 100644 --- a/src/jarvis/reply/turn_state.py +++ b/src/jarvis/reply/turn_state.py @@ -31,6 +31,9 @@ notes : Optional[object] screen : Optional[object] v2: the app's screen backend for the asker (screen_view.spec.md). None means the screenView tool is not offered. +trace : Optional[list] + v2: where the engine writes down how it made this answer (history.spec.md, "Traces"): the + tools it was given, the plan, the model's thinking and each tool call. None records nothing. """ from __future__ import annotations @@ -50,6 +53,22 @@ def reset() -> None: _state.schedule = None _state.notes = None _state.screen = None + _state.trace = None + + +def set_trace(steps: Optional[list]) -> None: + _state.trace = steps + + +def get_trace() -> Optional[list]: + return getattr(_state, "trace", None) + + +def trace(kind: str, **data) -> None: + """Add one step to this turn's trace, if one is kept.""" + steps = get_trace() + if steps is not None: + steps.append({"kind": kind, **data}) def set_screen(backend) -> None: diff --git a/src/voice_worker/brain.py b/src/voice_worker/brain.py index 85f5414..1031c2a 100644 --- a/src/voice_worker/brain.py +++ b/src/voice_worker/brain.py @@ -19,7 +19,7 @@ from jarvis.memory.history import HistoryContext, Scope class Brain(Protocol): def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None, task: bool = False, notes: Optional[list] = None, notes_backend=None, - screen=None) -> Optional[str]: + screen=None, trace: Optional[list] = None) -> Optional[str]: """Blocking; called on a worker thread, one call per person at a time (a voice and a chat reply for different people may overlap). ``chat``: the chat backend for the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels. @@ -27,7 +27,8 @@ class Brain(Protocol): ``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``). ``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the - screenView tool (screen_view.spec.md).""" + screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of + this answer to (history.spec.md, "Traces").""" def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]: """Blocking: was ``text`` said to the assistant (called ``names``)? None when it could not tell. @@ -296,7 +297,7 @@ class JarvisBrain: def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None, task: bool = False, notes: Optional[list] = None, notes_backend=None, - screen=None) -> Optional[str]: + screen=None, trace: Optional[list] = None) -> Optional[str]: from jarvis.memory.conversation import DialogueMemory from jarvis.reply import turn_state from jarvis.reply.engine import run_reply_engine @@ -317,6 +318,7 @@ class JarvisBrain: turn_state.set_schedule(None if task else schedule) turn_state.set_notes(None if task else notes_backend) turn_state.set_screen(None if task else screen) + turn_state.set_trace(trace) try: return run_reply_engine(self.db, cfg, None, text, memory, language=self.language, history=history, legacy_memory=False, diff --git a/src/voice_worker/server.py b/src/voice_worker/server.py index 33e690b..3276ff4 100644 --- a/src/voice_worker/server.py +++ b/src/voice_worker/server.py @@ -51,6 +51,8 @@ class _LlmPaused(Exception): # The asker's standing instructions the app sends with a reply (personal_notes.spec.md). MAX_NOTES = 20 MAX_NOTE_CHARS = 300 +# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts. +TRACE_KEEP_DAYS = 30 class VoiceWorker: @@ -116,6 +118,9 @@ class VoiceWorker: n = self.history.close_open_presence(time.time()) if n: debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker") + n = self.history.purge_traces_older_than(time.time() - TRACE_KEEP_DAYS * 86400) + if n: + debug_log(f"removed {n} answer traces older than {TRACE_KEEP_DAYS} days", "voice_worker") if self.transcriber is not None: self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held) self.scheduler.start() @@ -603,6 +608,23 @@ class Session: waiting[user] = waiting.get(user, 0) + 1 asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes)) + @staticmethod + def _trace_start(history, notes: list | None) -> list: + """The first step of an answer's trace: the records and notes it was given.""" + return [{"kind": "records", "recent": list(history.recent), "matches": list(history.matches), + "notes": len(notes or [])}] + + def _trace_end(self, via: str, channel: str | None, user: str, question: str, answer: str, + steps: list, began: float) -> None: + """Keep how this answer was made (history.spec.md, "Traces"); never stops the answer.""" + if self.worker.history is None: + return + try: + self.worker.history.add_trace(via, channel, user, question, answer, steps, + ms=round((time.time() - began) * 1000), ts=time.time()) + except Exception as e: # noqa: BLE001 - a trace is a view, not part of the answer + debug_log(f"trace not kept: {type(e).__name__}", "llm") + async def _task(self, rid: str, user: str, text: str, notes: list | None = None) -> None: """A scheduled task (schedule.spec.md, "Tasks"): answered in the chat lane with the read-only tools, its own short-term memory and the person's history scope. It is not a turn of their @@ -622,12 +644,14 @@ class Session: from jarvis.memory.history import HistoryContext history = HistoryContext() try: + steps = self._trace_start(history, notes) async with self._llm(): answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True, - notes=notes or []) + notes=notes or [], trace=steps) except Exception as e: # the brain must never take the session down debug_log(f"brain task {rid} failed: {type(e).__name__}", "llm") - answer = (answer or "").strip() + answer = (answer or "").strip() + self._trace_end("task", None, user, text, answer, steps, now) self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer}) def _on_llm(self, msg: dict) -> None: @@ -834,13 +858,15 @@ class Session: schedule = AppBridge(self, asyncio.get_running_loop(), "schedule", *origin) notes_backend = AppBridge(self, asyncio.get_running_loop(), "note", *origin) screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin) + steps = self._trace_start(history, notes) try: async with self._llm(rid if speak else None): w.answering += 1 try: answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat, schedule=schedule, notes=notes or [], - notes_backend=notes_backend, screen=screen) + notes_backend=notes_backend, screen=screen, + trace=steps) finally: w.answering -= 1 if not w.answering and w.scheduler: @@ -848,7 +874,8 @@ class Session: except Exception as e: # the brain must never take the session down debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm") answer = None - answer = (answer or "").strip() + answer = (answer or "").strip() + self._trace_end(via, channel if via == "voice" else chat_channel, user, text, answer, steps, now) if not answer and rid in self.tts_requests: self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer}) diff --git a/tests/test_engine_trace.py b/tests/test_engine_trace.py new file mode 100644 index 0000000..9ba4364 --- /dev/null +++ b/tests/test_engine_trace.py @@ -0,0 +1,59 @@ +"""The reply engine writes down how it made an answer (history.spec.md, "Traces"). + +With a trace list set on the turn state, a reply records the tools it was given, the plan, the +model's thinking and each tool call with its outcome. Without one nothing is recorded. +""" + +from unittest.mock import Mock, patch + +import pytest + +from src.jarvis.memory.conversation import DialogueMemory +from src.jarvis.reply import turn_state +from src.jarvis.reply.engine import run_reply_engine + +from tests.test_engine_tool_carryover import _mock_cfg + + +def _run(trace): + tool = Mock(reply_text="서울: 맑음, 12도", error_message=None, success=True) + replies = [ + {"message": {"content": "", "thinking": "날씨를 찾아봐야겠다", "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": "webSearch", "arguments": {"query": "서울 날씨"}}}]}}, + {"message": {"content": "서울은 맑고 12도입니다."}}, + ] + turn_state.reset() + turn_state.set_trace(trace) + try: + with patch("src.jarvis.reply.engine.plan_query", return_value=["Search the web", "Reply to the user."]), \ + patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \ + patch("src.jarvis.reply.engine.run_tool_with_retries", return_value=tool), \ + patch("src.jarvis.reply.engine.extract_text_from_response", + side_effect=lambda r: r["message"]["content"]), \ + patch("src.jarvis.reply.engine.chat_with_messages", side_effect=replies): + return run_reply_engine(db=Mock(), cfg=_mock_cfg(), tts=None, text="서울 날씨 어때", + dialogue_memory=DialogueMemory()) + finally: + turn_state.reset() + + +@pytest.mark.unit +def test_a_reply_records_its_tools_plan_thinking_and_tool_calls(): + trace = [] + assert _run(trace) + kinds = [s["kind"] for s in trace] + assert kinds.index("tools") < kinds.index("plan") < kinds.index("think") < kinds.index("tool") + tools = next(s for s in trace if s["kind"] == "tools") + assert "webSearch" in tools["names"] + assert next(s for s in trace if s["kind"] == "plan")["steps"] == ["Search the web", "Reply to the user."] + assert next(s for s in trace if s["kind"] == "think")["text"] == "날씨를 찾아봐야겠다" + call = next(s for s in trace if s["kind"] == "tool") + assert (call["name"], call["args"], call["ok"], call["result"]) == \ + ("webSearch", {"query": "서울 날씨"}, True, "서울: 맑음, 12도") + + +@pytest.mark.unit +def test_without_a_trace_nothing_is_recorded(): + assert _run(None) + assert turn_state.get_trace() is None diff --git a/tests/test_history_store.py b/tests/test_history_store.py index 08bf283..8d66ddf 100644 --- a/tests/test_history_store.py +++ b/tests/test_history_store.py @@ -238,3 +238,26 @@ def test_chat_tool_reads_and_searches_one_channels_messages(store): assert sorted(texts(store.search_chat(["c1", "c2"], "회의"))) == ["회의 취소", "회의는 3시"] assert store.search_chat([], "회의") == [] assert store.chat_channel_of("m3") == "c2" and store.chat_channel_of("nope") is None + + +def test_how_an_answer_was_made_is_kept_per_person_and_never_searched(store): + steps = [{"kind": "tools", "names": ["getWeather"]}, + {"kind": "tool", "name": "getWeather", "args": {"location": "서울"}, "ok": True, "result": "맑음"}] + store.add_trace("voice", VC, "a", "서울 날씨 어때", "맑습니다", steps, ms=1200, ts=T0) + store.add_trace("chat", CHAT, "b", "안녕", "", [], ms=50, ts=T0 + 1) + [t] = store.traces_of("a") + assert (t.via, t.channel_id, t.question, t.answer, t.ms, t.steps) == ("voice", VC, "서울 날씨 어때", "맑습니다", 1200, steps) + # A trace is no record: the brain never reads it back as history. + assert store.search("a", "날씨", scope=Scope()) == [] + assert store.recent("a", T0 + 2, scope=Scope()) == [] + store.forget_person("a") + assert store.traces_of("a") == [] and len(store.traces_of("b")) == 1 + + +def test_traces_have_their_own_retention_and_follow_the_history_retention(store): + store.add_trace("voice", VC, "a", "옛 질문", "옛 답", [], ms=1, ts=T0) + store.add_trace("voice", VC, "a", "새 질문", "새 답", [], ms=1, ts=T0 + 100) + store.purge_traces_older_than(T0 + 50) + assert [t.question for t in store.traces_of("a")] == ["새 질문"] + store.purge_older_than(T0 + 200) + assert store.traces_of("a") == [] diff --git a/tests/test_voice_worker_brain.py b/tests/test_voice_worker_brain.py index c4e08a2..61fe5bd 100644 --- a/tests/test_voice_worker_brain.py +++ b/tests/test_voice_worker_brain.py @@ -29,8 +29,10 @@ class FakeBrain: self.calls = [] def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None, - screen=None): + screen=None, trace=None): self.calls.append((user, text, history)) + if trace is not None: + trace.append({"kind": "tools", "names": ["getWeather"]}) self.chat, self.schedule, self.task = chat, schedule, task self.notes, self.notes_backend, self.screen = notes, notes_backend, screen return self.answer(user, text) @@ -211,6 +213,37 @@ def test_a_failed_brain_answer_is_reported(store, answer): assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True} +@pytest.mark.unit +def test_each_answer_keeps_how_it_was_made_with_the_records_it_was_given(store): + async def body(url): + async with connect(url) as ws: + await hello(ws, stt={"partials": False}) + await send(ws, type="voice_join", channel="vc1") + await send(ws, type="member_join", user="100", name="Alice") + await say(ws, "100", 0.9) + await send(ws, type="reply", id="r1", user="100", text="what did I say", via="voice", speak=False, + notes=["반말로 해"]) + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1") + await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False, channel="ch1") + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1") + await send(ws, type="reply", id="t1", user="100", text="task", via="task") + await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "t1") + + brain = FakeBrain(lambda u, t: None if t == "hi" else f"answer for {t}") + asyncio.run(with_worker(body, store=store, brain=brain)) + voice, chat, task = store.traces_of("100") + assert (voice.via, voice.channel_id, voice.question, voice.answer) == \ + ("voice", "vc1", "what did I say", "answer for what did I say") + assert voice.ms >= 0 + records = voice.steps[0] + assert records["kind"] == "records" and records["notes"] == 1 + assert any("Alice: level 0.9" in x for x in records["recent"]) + assert voice.steps[1:] == [{"kind": "tools", "names": ["getWeather"]}] + # A failed answer is kept too, with no answer; a task has no channel. + assert (chat.via, chat.channel_id, chat.answer) == ("chat", "ch1", "") + assert (task.via, task.channel_id, task.answer) == ("task", None, "answer for task") + + @pytest.mark.unit def test_without_a_brain_reply_is_an_error(): async def body(url): @@ -300,6 +333,21 @@ def test_presence_left_open_by_a_previous_run_is_closed_at_start(store): assert store.recent("old", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == [] +@pytest.mark.unit +def test_traces_older_than_thirty_days_are_removed_at_start(store): + import time + now = time.time() + store.add_trace("voice", "vc1", "100", "old", "a", [], ms=1, ts=now - 31 * 86400) + store.add_trace("voice", "vc1", "100", "new", "a", [], ms=1, ts=now - 29 * 86400) + + async def body(url): + async with connect(url) as ws: + await hello(ws) + + asyncio.run(with_worker(body, store=store)) + assert [t.question for t in store.traces_of("100")] == ["new"] + + @pytest.mark.unit def test_the_brain_prints_no_text_from_its_thread(tmp_path, monkeypatch, capfd): pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)