feat(brain): keep how each answer was made (records, tools, plan, thinking, tool calls)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-16 03:00:10 +09:00
parent 2a140d8490
commit a20ebe4eae
8 changed files with 265 additions and 8 deletions

View File

@@ -4,6 +4,7 @@ registered chat channel messages, searched per asker. Contract:
from __future__ import annotations
import json
import math
import re
import sqlite3
@@ -41,6 +42,19 @@ CREATE TABLE IF NOT EXISTS presence (
left_at REAL
);
CREATE INDEX IF NOT EXISTS presence_user ON presence(user_id, channel_id);
CREATE TABLE IF NOT EXISTS traces (
id INTEGER PRIMARY KEY,
ts REAL NOT NULL, -- when the answer was ready
via TEXT NOT NULL, -- voice | chat | task
channel_id TEXT, -- where it was asked; none for a task
user_id TEXT NOT NULL,
question TEXT NOT NULL,
answer TEXT NOT NULL, -- empty when the brain gave none
ms INTEGER NOT NULL,
steps TEXT NOT NULL -- JSON list (history.spec.md, "Traces")
);
CREATE INDEX IF NOT EXISTS traces_ts ON traces(ts);
CREATE INDEX IF NOT EXISTS traces_user ON traces(user_id);
"""
_WORD = re.compile(r"\w{2,}", re.UNICODE)
@@ -70,6 +84,21 @@ class Record:
text: str
@dataclass(frozen=True)
class Trace:
"""How one answer was made: never read back into a prompt (``history.spec.md``, "Traces")."""
id: int
ts: float
via: str
channel_id: Optional[str]
user_id: str
question: str
answer: str
ms: int
steps: list
@dataclass
class HistoryContext:
"""Lines for the prompt: the recent window and the search matches, kept apart for labelling."""
@@ -196,6 +225,29 @@ class HistoryStore:
# -- maintenance --------------------------------------------------------
def add_trace(self, via: str, channel_id: Optional[str], user_id: str, question: str, answer: str,
steps: list, ms: int, ts: float) -> int:
with self._lock:
cur = self._conn.execute(
"INSERT INTO traces(ts, via, channel_id, user_id, question, answer, ms, steps) VALUES (?,?,?,?,?,?,?,?)",
(ts, via, channel_id, user_id, question, answer or "", int(ms),
json.dumps(steps, ensure_ascii=False, default=str)))
self._conn.commit()
return int(cur.lastrowid)
def traces_of(self, person: str) -> List[Trace]:
with self._lock:
rows = self._conn.execute(
"SELECT id, ts, via, channel_id, user_id, question, answer, ms, steps FROM traces"
" WHERE user_id=? ORDER BY ts, id", (person,)).fetchall()
return [Trace(**{**dict(r), "steps": json.loads(r["steps"])}) for r in rows]
def purge_traces_older_than(self, ts: float) -> int:
with self._lock:
n = self._conn.execute("DELETE FROM traces WHERE ts < ?", (ts,)).rowcount
self._conn.commit()
return n
def records_of(self, person: str) -> List[Record]:
with self._lock:
rows = self._conn.execute(
@@ -207,6 +259,7 @@ class HistoryStore:
with self._lock:
n = self._conn.execute("DELETE FROM history WHERE author_id=? OR to_id=?", (person, person)).rowcount
self._conn.execute("DELETE FROM presence WHERE user_id=?", (person,))
self._conn.execute("DELETE FROM traces WHERE user_id=?", (person,))
self._conn.commit()
debug_log(f"history: forgot a person ({n} records)", "memory")
@@ -214,6 +267,7 @@ class HistoryStore:
with self._lock:
n = self._conn.execute("DELETE FROM history WHERE ts < ?", (ts,)).rowcount
self._conn.execute("DELETE FROM presence WHERE left_at IS NOT NULL AND left_at < ?", (ts,))
self._conn.execute("DELETE FROM traces WHERE ts < ?", (ts,))
self._conn.commit()
debug_log(f"history: retention removed {n} records", "memory")

View File

@@ -27,6 +27,7 @@ from .enrichment import (
digest_tool_result_for_query,
digest_loop_for_max_turns,
)
from . import turn_state as _turn_state
from .prompt_dump import dump_reply_turn, is_enabled as _prompt_dump_enabled, new_session_id
from .prompts import ModelSize, detect_model_size, get_system_prompts
from .compound_query import split_compound_query
@@ -61,6 +62,22 @@ if TYPE_CHECKING:
# ── Helpers ─────────────────────────────────────────────────────────────────
# How much of a tool result and of the model's thinking a trace keeps (history.spec.md, "Traces").
TRACE_RESULT_CHARS = 2000
TRACE_THINKING_CHARS = 8000
def _clip(text: str, limit: int) -> str:
return text if len(text) <= limit else text[:limit] + "…"
def _trace_tool(name: str, args, result) -> None:
"""One executed tool call in the turn's trace: name, arguments, outcome and the start of its result."""
text = result.reply_text or result.error_message or ""
_turn_state.trace("tool", name=name, args=args or {}, ok=bool(result.success),
result=_clip(str(text), TRACE_RESULT_CHARS))
def _indent_text(text: str, prefix: str = " ") -> str:
return f"\n{prefix}".join(text.splitlines())
@@ -1419,6 +1436,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
routed_tools = [t for t in routed_tools if t != "webSearch"]
debug_log("screen-share: site named — dropping webSearch so controlBrowser wins", "tools")
_turn_state.trace("tools", names=list(routed_tools))
_planner_schema = generate_tools_json_schema(routed_tools, mcp_tools)
_planner_tool_catalog = planner_tool_catalog(_planner_schema or [])
@@ -1438,6 +1456,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
debug_log(f"planner step failed (non-fatal): {_plan_exc}", "planning")
action_plan = []
if action_plan:
_turn_state.trace("plan", steps=list(action_plan))
_plan_preview = " | ".join(s[:50] for s in action_plan)
print(
f" 🗺️ Plan: {len(action_plan)} step(s) — {_plan_preview}",
@@ -2389,6 +2408,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
max_retries=1,
language=language,
)
_trace_tool(_name, _args, _plan_result)
if _plan_result.reply_text:
_plan_text = _maybe_digest_tool_result(
cfg=cfg,
@@ -2602,6 +2622,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
# Always show thinking if present, regardless of content
if thinking:
_turn_state.trace("think", text=_clip(str(thinking), TRACE_THINKING_CHARS))
debug_log(f" 💭 LLM thinking: '{thinking[:300]}{'...' if len(thinking) > 300 else ''}'", "planning")
# Extract tool call if present
@@ -2647,6 +2668,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
# Check if tool is not allowed - respond with tool error
if tool_name not in allowed_tools:
debug_log(f" ⚠️ tool not allowed: {tool_name}", "planning")
_turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="not_allowed")
print(f" ⚠️ Tool '{tool_name}' not in allow-list", flush=True)
# Use tool response instead of system message to maintain native tool calling compatibility
messages.append({
@@ -2694,6 +2716,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
if signature in recent_tool_signatures:
debug_log(f" ⚠️ Duplicate {tool_name} call - returning cached guidance", "planning")
_turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="duplicate")
if use_text_tools:
messages.append({"role": "user", "content": f"[Tool: {tool_name}] You already called this tool with these arguments. Use the results from the previous tool call to answer the user."})
else:
@@ -2713,6 +2736,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
)
if duplicate_tool_count >= 2:
debug_log(f" ⚠️ Too many {tool_name} calls ({duplicate_tool_count}) - returning guidance", "planning")
_turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="duplicate")
if use_text_tools:
messages.append({"role": "user", "content": f"[Tool: {tool_name}] You have already called this tool {duplicate_tool_count} times. Use the results from those calls to answer the user's question."})
else:
@@ -2731,6 +2755,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
max_retries=1,
language=language,
)
_trace_tool(tool_name, tool_args, result)
# Handle stop tool - end conversation without response
if result.reply_text == STOP_SIGNAL:

View File

@@ -31,6 +31,9 @@ notes : Optional[object]
screen : Optional[object]
v2: the app's screen backend for the asker (screen_view.spec.md). None means the screenView
tool is not offered.
trace : Optional[list]
v2: where the engine writes down how it made this answer (history.spec.md, "Traces"): the
tools it was given, the plan, the model's thinking and each tool call. None records nothing.
"""
from __future__ import annotations
@@ -50,6 +53,22 @@ def reset() -> None:
_state.schedule = None
_state.notes = None
_state.screen = None
_state.trace = None
def set_trace(steps: Optional[list]) -> None:
_state.trace = steps
def get_trace() -> Optional[list]:
return getattr(_state, "trace", None)
def trace(kind: str, **data) -> None:
"""Add one step to this turn's trace, if one is kept."""
steps = get_trace()
if steps is not None:
steps.append({"kind": kind, **data})
def set_screen(backend) -> None:

View File

@@ -19,7 +19,7 @@ from jarvis.memory.history import HistoryContext, Scope
class Brain(Protocol):
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
task: bool = False, notes: Optional[list] = None, notes_backend=None,
screen=None) -> Optional[str]:
screen=None, trace: Optional[list] = None) -> Optional[str]:
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
reply for different people may overlap). ``chat``: the chat backend for
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
@@ -27,7 +27,8 @@ class Brain(Protocol):
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
screenView tool (screen_view.spec.md)."""
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
this answer to (history.spec.md, "Traces")."""
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
"""Blocking: was ``text`` said to the assistant (called ``names``)? None when it could not tell.
@@ -296,7 +297,7 @@ class JarvisBrain:
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
task: bool = False, notes: Optional[list] = None, notes_backend=None,
screen=None) -> Optional[str]:
screen=None, trace: Optional[list] = None) -> Optional[str]:
from jarvis.memory.conversation import DialogueMemory
from jarvis.reply import turn_state
from jarvis.reply.engine import run_reply_engine
@@ -317,6 +318,7 @@ class JarvisBrain:
turn_state.set_schedule(None if task else schedule)
turn_state.set_notes(None if task else notes_backend)
turn_state.set_screen(None if task else screen)
turn_state.set_trace(trace)
try:
return run_reply_engine(self.db, cfg, None, text, memory,
language=self.language, history=history, legacy_memory=False,

View File

@@ -51,6 +51,8 @@ class _LlmPaused(Exception):
# The asker's standing instructions the app sends with a reply (personal_notes.spec.md).
MAX_NOTES = 20
MAX_NOTE_CHARS = 300
# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts.
TRACE_KEEP_DAYS = 30
class VoiceWorker:
@@ -116,6 +118,9 @@ class VoiceWorker:
n = self.history.close_open_presence(time.time())
if n:
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
n = self.history.purge_traces_older_than(time.time() - TRACE_KEEP_DAYS * 86400)
if n:
debug_log(f"removed {n} answer traces older than {TRACE_KEEP_DAYS} days", "voice_worker")
if self.transcriber is not None:
self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held)
self.scheduler.start()
@@ -603,6 +608,23 @@ class Session:
waiting[user] = waiting.get(user, 0) + 1
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes))
@staticmethod
def _trace_start(history, notes: list | None) -> list:
"""The first step of an answer's trace: the records and notes it was given."""
return [{"kind": "records", "recent": list(history.recent), "matches": list(history.matches),
"notes": len(notes or [])}]
def _trace_end(self, via: str, channel: str | None, user: str, question: str, answer: str,
steps: list, began: float) -> None:
"""Keep how this answer was made (history.spec.md, "Traces"); never stops the answer."""
if self.worker.history is None:
return
try:
self.worker.history.add_trace(via, channel, user, question, answer, steps,
ms=round((time.time() - began) * 1000), ts=time.time())
except Exception as e: # noqa: BLE001 - a trace is a view, not part of the answer
debug_log(f"trace not kept: {type(e).__name__}", "llm")
async def _task(self, rid: str, user: str, text: str, notes: list | None = None) -> None:
"""A scheduled task (schedule.spec.md, "Tasks"): answered in the chat lane with the read-only
tools, its own short-term memory and the person's history scope. It is not a turn of their
@@ -622,12 +644,14 @@ class Session:
from jarvis.memory.history import HistoryContext
history = HistoryContext()
try:
steps = self._trace_start(history, notes)
async with self._llm():
answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True,
notes=notes or [])
notes=notes or [], trace=steps)
except Exception as e: # the brain must never take the session down
debug_log(f"brain task {rid} failed: {type(e).__name__}", "llm")
answer = (answer or "").strip()
answer = (answer or "").strip()
self._trace_end("task", None, user, text, answer, steps, now)
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
def _on_llm(self, msg: dict) -> None:
@@ -834,13 +858,15 @@ class Session:
schedule = AppBridge(self, asyncio.get_running_loop(), "schedule", *origin)
notes_backend = AppBridge(self, asyncio.get_running_loop(), "note", *origin)
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
steps = self._trace_start(history, notes)
try:
async with self._llm(rid if speak else None):
w.answering += 1
try:
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
schedule=schedule, notes=notes or [],
notes_backend=notes_backend, screen=screen)
notes_backend=notes_backend, screen=screen,
trace=steps)
finally:
w.answering -= 1
if not w.answering and w.scheduler:
@@ -848,7 +874,8 @@ class Session:
except Exception as e: # the brain must never take the session down
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
answer = None
answer = (answer or "").strip()
answer = (answer or "").strip()
self._trace_end(via, channel if via == "voice" else chat_channel, user, text, answer, steps, now)
if not answer and rid in self.tts_requests:
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})

View File

@@ -0,0 +1,59 @@
"""The reply engine writes down how it made an answer (history.spec.md, "Traces").
With a trace list set on the turn state, a reply records the tools it was given, the plan, the
model's thinking and each tool call with its outcome. Without one nothing is recorded.
"""
from unittest.mock import Mock, patch
import pytest
from src.jarvis.memory.conversation import DialogueMemory
from src.jarvis.reply import turn_state
from src.jarvis.reply.engine import run_reply_engine
from tests.test_engine_tool_carryover import _mock_cfg
def _run(trace):
tool = Mock(reply_text="서울: 맑음, 12도", error_message=None, success=True)
replies = [
{"message": {"content": "", "thinking": "날씨를 찾아봐야겠다", "tool_calls": [{
"id": "c1", "type": "function",
"function": {"name": "webSearch", "arguments": {"query": "서울 날씨"}}}]}},
{"message": {"content": "서울은 맑고 12도입니다."}},
]
turn_state.reset()
turn_state.set_trace(trace)
try:
with patch("src.jarvis.reply.engine.plan_query", return_value=["Search the web", "Reply to the user."]), \
patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \
patch("src.jarvis.reply.engine.run_tool_with_retries", return_value=tool), \
patch("src.jarvis.reply.engine.extract_text_from_response",
side_effect=lambda r: r["message"]["content"]), \
patch("src.jarvis.reply.engine.chat_with_messages", side_effect=replies):
return run_reply_engine(db=Mock(), cfg=_mock_cfg(), tts=None, text="서울 날씨 어때",
dialogue_memory=DialogueMemory())
finally:
turn_state.reset()
@pytest.mark.unit
def test_a_reply_records_its_tools_plan_thinking_and_tool_calls():
trace = []
assert _run(trace)
kinds = [s["kind"] for s in trace]
assert kinds.index("tools") < kinds.index("plan") < kinds.index("think") < kinds.index("tool")
tools = next(s for s in trace if s["kind"] == "tools")
assert "webSearch" in tools["names"]
assert next(s for s in trace if s["kind"] == "plan")["steps"] == ["Search the web", "Reply to the user."]
assert next(s for s in trace if s["kind"] == "think")["text"] == "날씨를 찾아봐야겠다"
call = next(s for s in trace if s["kind"] == "tool")
assert (call["name"], call["args"], call["ok"], call["result"]) == \
("webSearch", {"query": "서울 날씨"}, True, "서울: 맑음, 12도")
@pytest.mark.unit
def test_without_a_trace_nothing_is_recorded():
assert _run(None)
assert turn_state.get_trace() is None

View File

@@ -238,3 +238,26 @@ def test_chat_tool_reads_and_searches_one_channels_messages(store):
assert sorted(texts(store.search_chat(["c1", "c2"], "회의"))) == ["회의 취소", "회의는 3시"]
assert store.search_chat([], "회의") == []
assert store.chat_channel_of("m3") == "c2" and store.chat_channel_of("nope") is None
def test_how_an_answer_was_made_is_kept_per_person_and_never_searched(store):
steps = [{"kind": "tools", "names": ["getWeather"]},
{"kind": "tool", "name": "getWeather", "args": {"location": "서울"}, "ok": True, "result": "맑음"}]
store.add_trace("voice", VC, "a", "서울 날씨 어때", "맑습니다", steps, ms=1200, ts=T0)
store.add_trace("chat", CHAT, "b", "안녕", "", [], ms=50, ts=T0 + 1)
[t] = store.traces_of("a")
assert (t.via, t.channel_id, t.question, t.answer, t.ms, t.steps) == ("voice", VC, "서울 날씨 어때", "맑습니다", 1200, steps)
# A trace is no record: the brain never reads it back as history.
assert store.search("a", "날씨", scope=Scope()) == []
assert store.recent("a", T0 + 2, scope=Scope()) == []
store.forget_person("a")
assert store.traces_of("a") == [] and len(store.traces_of("b")) == 1
def test_traces_have_their_own_retention_and_follow_the_history_retention(store):
store.add_trace("voice", VC, "a", "옛 질문", "옛 답", [], ms=1, ts=T0)
store.add_trace("voice", VC, "a", "새 질문", "새 답", [], ms=1, ts=T0 + 100)
store.purge_traces_older_than(T0 + 50)
assert [t.question for t in store.traces_of("a")] == ["새 질문"]
store.purge_older_than(T0 + 200)
assert store.traces_of("a") == []

View File

@@ -29,8 +29,10 @@ class FakeBrain:
self.calls = []
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
screen=None):
screen=None, trace=None):
self.calls.append((user, text, history))
if trace is not None:
trace.append({"kind": "tools", "names": ["getWeather"]})
self.chat, self.schedule, self.task = chat, schedule, task
self.notes, self.notes_backend, self.screen = notes, notes_backend, screen
return self.answer(user, text)
@@ -211,6 +213,37 @@ def test_a_failed_brain_answer_is_reported(store, answer):
assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True}
@pytest.mark.unit
def test_each_answer_keeps_how_it_was_made_with_the_records_it_was_given(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="what did I say", via="voice", speak=False,
notes=["반말로 해"])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False, channel="ch1")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1")
await send(ws, type="reply", id="t1", user="100", text="task", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "t1")
brain = FakeBrain(lambda u, t: None if t == "hi" else f"answer for {t}")
asyncio.run(with_worker(body, store=store, brain=brain))
voice, chat, task = store.traces_of("100")
assert (voice.via, voice.channel_id, voice.question, voice.answer) == \
("voice", "vc1", "what did I say", "answer for what did I say")
assert voice.ms >= 0
records = voice.steps[0]
assert records["kind"] == "records" and records["notes"] == 1
assert any("Alice: level 0.9" in x for x in records["recent"])
assert voice.steps[1:] == [{"kind": "tools", "names": ["getWeather"]}]
# A failed answer is kept too, with no answer; a task has no channel.
assert (chat.via, chat.channel_id, chat.answer) == ("chat", "ch1", "")
assert (task.via, task.channel_id, task.answer) == ("task", None, "answer for task")
@pytest.mark.unit
def test_without_a_brain_reply_is_an_error():
async def body(url):
@@ -300,6 +333,21 @@ def test_presence_left_open_by_a_previous_run_is_closed_at_start(store):
assert store.recent("old", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
@pytest.mark.unit
def test_traces_older_than_thirty_days_are_removed_at_start(store):
import time
now = time.time()
store.add_trace("voice", "vc1", "100", "old", "a", [], ms=1, ts=now - 31 * 86400)
store.add_trace("voice", "vc1", "100", "new", "a", [], ms=1, ts=now - 29 * 86400)
async def body(url):
async with connect(url) as ws:
await hello(ws)
asyncio.run(with_worker(body, store=store))
assert [t.question for t in store.traces_of("100")] == ["new"]
@pytest.mark.unit
def test_the_brain_prints_no_text_from_its_thread(tmp_path, monkeypatch, capfd):
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)