feat(brain): keep how each answer was made (records, tools, plan, thinking, tool calls)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@ registered chat channel messages, searched per asker. Contract:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sqlite3
|
||||
@@ -41,6 +42,19 @@ CREATE TABLE IF NOT EXISTS presence (
|
||||
left_at REAL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS presence_user ON presence(user_id, channel_id);
|
||||
CREATE TABLE IF NOT EXISTS traces (
|
||||
id INTEGER PRIMARY KEY,
|
||||
ts REAL NOT NULL, -- when the answer was ready
|
||||
via TEXT NOT NULL, -- voice | chat | task
|
||||
channel_id TEXT, -- where it was asked; none for a task
|
||||
user_id TEXT NOT NULL,
|
||||
question TEXT NOT NULL,
|
||||
answer TEXT NOT NULL, -- empty when the brain gave none
|
||||
ms INTEGER NOT NULL,
|
||||
steps TEXT NOT NULL -- JSON list (history.spec.md, "Traces")
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS traces_ts ON traces(ts);
|
||||
CREATE INDEX IF NOT EXISTS traces_user ON traces(user_id);
|
||||
"""
|
||||
|
||||
_WORD = re.compile(r"\w{2,}", re.UNICODE)
|
||||
@@ -70,6 +84,21 @@ class Record:
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Trace:
|
||||
"""How one answer was made: never read back into a prompt (``history.spec.md``, "Traces")."""
|
||||
|
||||
id: int
|
||||
ts: float
|
||||
via: str
|
||||
channel_id: Optional[str]
|
||||
user_id: str
|
||||
question: str
|
||||
answer: str
|
||||
ms: int
|
||||
steps: list
|
||||
|
||||
|
||||
@dataclass
|
||||
class HistoryContext:
|
||||
"""Lines for the prompt: the recent window and the search matches, kept apart for labelling."""
|
||||
@@ -196,6 +225,29 @@ class HistoryStore:
|
||||
|
||||
# -- maintenance --------------------------------------------------------
|
||||
|
||||
def add_trace(self, via: str, channel_id: Optional[str], user_id: str, question: str, answer: str,
|
||||
steps: list, ms: int, ts: float) -> int:
|
||||
with self._lock:
|
||||
cur = self._conn.execute(
|
||||
"INSERT INTO traces(ts, via, channel_id, user_id, question, answer, ms, steps) VALUES (?,?,?,?,?,?,?,?)",
|
||||
(ts, via, channel_id, user_id, question, answer or "", int(ms),
|
||||
json.dumps(steps, ensure_ascii=False, default=str)))
|
||||
self._conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
def traces_of(self, person: str) -> List[Trace]:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT id, ts, via, channel_id, user_id, question, answer, ms, steps FROM traces"
|
||||
" WHERE user_id=? ORDER BY ts, id", (person,)).fetchall()
|
||||
return [Trace(**{**dict(r), "steps": json.loads(r["steps"])}) for r in rows]
|
||||
|
||||
def purge_traces_older_than(self, ts: float) -> int:
|
||||
with self._lock:
|
||||
n = self._conn.execute("DELETE FROM traces WHERE ts < ?", (ts,)).rowcount
|
||||
self._conn.commit()
|
||||
return n
|
||||
|
||||
def records_of(self, person: str) -> List[Record]:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
@@ -207,6 +259,7 @@ class HistoryStore:
|
||||
with self._lock:
|
||||
n = self._conn.execute("DELETE FROM history WHERE author_id=? OR to_id=?", (person, person)).rowcount
|
||||
self._conn.execute("DELETE FROM presence WHERE user_id=?", (person,))
|
||||
self._conn.execute("DELETE FROM traces WHERE user_id=?", (person,))
|
||||
self._conn.commit()
|
||||
debug_log(f"history: forgot a person ({n} records)", "memory")
|
||||
|
||||
@@ -214,6 +267,7 @@ class HistoryStore:
|
||||
with self._lock:
|
||||
n = self._conn.execute("DELETE FROM history WHERE ts < ?", (ts,)).rowcount
|
||||
self._conn.execute("DELETE FROM presence WHERE left_at IS NOT NULL AND left_at < ?", (ts,))
|
||||
self._conn.execute("DELETE FROM traces WHERE ts < ?", (ts,))
|
||||
self._conn.commit()
|
||||
debug_log(f"history: retention removed {n} records", "memory")
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ from .enrichment import (
|
||||
digest_tool_result_for_query,
|
||||
digest_loop_for_max_turns,
|
||||
)
|
||||
from . import turn_state as _turn_state
|
||||
from .prompt_dump import dump_reply_turn, is_enabled as _prompt_dump_enabled, new_session_id
|
||||
from .prompts import ModelSize, detect_model_size, get_system_prompts
|
||||
from .compound_query import split_compound_query
|
||||
@@ -61,6 +62,22 @@ if TYPE_CHECKING:
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
# How much of a tool result and of the model's thinking a trace keeps (history.spec.md, "Traces").
|
||||
TRACE_RESULT_CHARS = 2000
|
||||
TRACE_THINKING_CHARS = 8000
|
||||
|
||||
|
||||
def _clip(text: str, limit: int) -> str:
|
||||
return text if len(text) <= limit else text[:limit] + "…"
|
||||
|
||||
|
||||
def _trace_tool(name: str, args, result) -> None:
|
||||
"""One executed tool call in the turn's trace: name, arguments, outcome and the start of its result."""
|
||||
text = result.reply_text or result.error_message or ""
|
||||
_turn_state.trace("tool", name=name, args=args or {}, ok=bool(result.success),
|
||||
result=_clip(str(text), TRACE_RESULT_CHARS))
|
||||
|
||||
|
||||
def _indent_text(text: str, prefix: str = " ") -> str:
|
||||
return f"\n{prefix}".join(text.splitlines())
|
||||
|
||||
@@ -1419,6 +1436,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
routed_tools = [t for t in routed_tools if t != "webSearch"]
|
||||
debug_log("screen-share: site named — dropping webSearch so controlBrowser wins", "tools")
|
||||
|
||||
_turn_state.trace("tools", names=list(routed_tools))
|
||||
_planner_schema = generate_tools_json_schema(routed_tools, mcp_tools)
|
||||
_planner_tool_catalog = planner_tool_catalog(_planner_schema or [])
|
||||
|
||||
@@ -1438,6 +1456,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
debug_log(f"planner step failed (non-fatal): {_plan_exc}", "planning")
|
||||
action_plan = []
|
||||
if action_plan:
|
||||
_turn_state.trace("plan", steps=list(action_plan))
|
||||
_plan_preview = " | ".join(s[:50] for s in action_plan)
|
||||
print(
|
||||
f" 🗺️ Plan: {len(action_plan)} step(s) — {_plan_preview}",
|
||||
@@ -2389,6 +2408,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
max_retries=1,
|
||||
language=language,
|
||||
)
|
||||
_trace_tool(_name, _args, _plan_result)
|
||||
if _plan_result.reply_text:
|
||||
_plan_text = _maybe_digest_tool_result(
|
||||
cfg=cfg,
|
||||
@@ -2602,6 +2622,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
|
||||
# Always show thinking if present, regardless of content
|
||||
if thinking:
|
||||
_turn_state.trace("think", text=_clip(str(thinking), TRACE_THINKING_CHARS))
|
||||
debug_log(f" 💭 LLM thinking: '{thinking[:300]}{'...' if len(thinking) > 300 else ''}'", "planning")
|
||||
|
||||
# Extract tool call if present
|
||||
@@ -2647,6 +2668,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
# Check if tool is not allowed - respond with tool error
|
||||
if tool_name not in allowed_tools:
|
||||
debug_log(f" ⚠️ tool not allowed: {tool_name}", "planning")
|
||||
_turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="not_allowed")
|
||||
print(f" ⚠️ Tool '{tool_name}' not in allow-list", flush=True)
|
||||
# Use tool response instead of system message to maintain native tool calling compatibility
|
||||
messages.append({
|
||||
@@ -2694,6 +2716,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
|
||||
if signature in recent_tool_signatures:
|
||||
debug_log(f" ⚠️ Duplicate {tool_name} call - returning cached guidance", "planning")
|
||||
_turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="duplicate")
|
||||
if use_text_tools:
|
||||
messages.append({"role": "user", "content": f"[Tool: {tool_name}] You already called this tool with these arguments. Use the results from the previous tool call to answer the user."})
|
||||
else:
|
||||
@@ -2713,6 +2736,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
)
|
||||
if duplicate_tool_count >= 2:
|
||||
debug_log(f" ⚠️ Too many {tool_name} calls ({duplicate_tool_count}) - returning guidance", "planning")
|
||||
_turn_state.trace("tool", name=tool_name, args=tool_args or {}, ok=False, skipped="duplicate")
|
||||
if use_text_tools:
|
||||
messages.append({"role": "user", "content": f"[Tool: {tool_name}] You have already called this tool {duplicate_tool_count} times. Use the results from those calls to answer the user's question."})
|
||||
else:
|
||||
@@ -2731,6 +2755,7 @@ def run_reply_engine(db: "Database", cfg, tts: Optional[Any],
|
||||
max_retries=1,
|
||||
language=language,
|
||||
)
|
||||
_trace_tool(tool_name, tool_args, result)
|
||||
|
||||
# Handle stop tool - end conversation without response
|
||||
if result.reply_text == STOP_SIGNAL:
|
||||
|
||||
@@ -31,6 +31,9 @@ notes : Optional[object]
|
||||
screen : Optional[object]
|
||||
v2: the app's screen backend for the asker (screen_view.spec.md). None means the screenView
|
||||
tool is not offered.
|
||||
trace : Optional[list]
|
||||
v2: where the engine writes down how it made this answer (history.spec.md, "Traces"): the
|
||||
tools it was given, the plan, the model's thinking and each tool call. None records nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -50,6 +53,22 @@ def reset() -> None:
|
||||
_state.schedule = None
|
||||
_state.notes = None
|
||||
_state.screen = None
|
||||
_state.trace = None
|
||||
|
||||
|
||||
def set_trace(steps: Optional[list]) -> None:
|
||||
_state.trace = steps
|
||||
|
||||
|
||||
def get_trace() -> Optional[list]:
|
||||
return getattr(_state, "trace", None)
|
||||
|
||||
|
||||
def trace(kind: str, **data) -> None:
|
||||
"""Add one step to this turn's trace, if one is kept."""
|
||||
steps = get_trace()
|
||||
if steps is not None:
|
||||
steps.append({"kind": kind, **data})
|
||||
|
||||
|
||||
def set_screen(backend) -> None:
|
||||
|
||||
@@ -19,7 +19,7 @@ from jarvis.memory.history import HistoryContext, Scope
|
||||
class Brain(Protocol):
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None) -> Optional[str]:
|
||||
screen=None, trace: Optional[list] = None) -> Optional[str]:
|
||||
"""Blocking; called on a worker thread, one call per person at a time (a voice and a chat
|
||||
reply for different people may overlap). ``chat``: the chat backend for
|
||||
the chatChannels tool (voice_worker/chat_bridge.py), or None without registered channels.
|
||||
@@ -27,7 +27,8 @@ class Brain(Protocol):
|
||||
``task``: a scheduled task runs, with the read-only tools only (``TASK_TOOLS``).
|
||||
``notes``: the asker's standing instructions, put in the prompt; ``notes_backend``: the
|
||||
backend for the personalNotes tool (personal_notes.spec.md); ``screen``: the backend for the
|
||||
screenView tool (screen_view.spec.md)."""
|
||||
screenView tool (screen_view.spec.md); ``trace``: a list the engine adds the steps of
|
||||
this answer to (history.spec.md, "Traces")."""
|
||||
|
||||
def addressed(self, user: str, text: str, names: list[str]) -> Optional[bool]:
|
||||
"""Blocking: was ``text`` said to the assistant (called ``names``)? None when it could not tell.
|
||||
@@ -296,7 +297,7 @@ class JarvisBrain:
|
||||
|
||||
def reply(self, user: str, text: str, history: HistoryContext, chat=None, schedule=None,
|
||||
task: bool = False, notes: Optional[list] = None, notes_backend=None,
|
||||
screen=None) -> Optional[str]:
|
||||
screen=None, trace: Optional[list] = None) -> Optional[str]:
|
||||
from jarvis.memory.conversation import DialogueMemory
|
||||
from jarvis.reply import turn_state
|
||||
from jarvis.reply.engine import run_reply_engine
|
||||
@@ -317,6 +318,7 @@ class JarvisBrain:
|
||||
turn_state.set_schedule(None if task else schedule)
|
||||
turn_state.set_notes(None if task else notes_backend)
|
||||
turn_state.set_screen(None if task else screen)
|
||||
turn_state.set_trace(trace)
|
||||
try:
|
||||
return run_reply_engine(self.db, cfg, None, text, memory,
|
||||
language=self.language, history=history, legacy_memory=False,
|
||||
|
||||
@@ -51,6 +51,8 @@ class _LlmPaused(Exception):
|
||||
# The asker's standing instructions the app sends with a reply (personal_notes.spec.md).
|
||||
MAX_NOTES = 20
|
||||
MAX_NOTE_CHARS = 300
|
||||
# How long answer traces stay (history.spec.md, "Traces"); removed when the worker starts.
|
||||
TRACE_KEEP_DAYS = 30
|
||||
|
||||
|
||||
class VoiceWorker:
|
||||
@@ -116,6 +118,9 @@ class VoiceWorker:
|
||||
n = self.history.close_open_presence(time.time())
|
||||
if n:
|
||||
debug_log(f"closed {n} presence intervals left open by a previous run", "voice_worker")
|
||||
n = self.history.purge_traces_older_than(time.time() - TRACE_KEEP_DAYS * 86400)
|
||||
if n:
|
||||
debug_log(f"removed {n} answer traces older than {TRACE_KEEP_DAYS} days", "voice_worker")
|
||||
if self.transcriber is not None:
|
||||
self.scheduler = SttScheduler(self.transcriber, partials_held=self._partials_held)
|
||||
self.scheduler.start()
|
||||
@@ -603,6 +608,23 @@ class Session:
|
||||
waiting[user] = waiting.get(user, 0) + 1
|
||||
asyncio.get_running_loop().create_task(self._reply(rid, user, text, via, speak, early, channel, notes))
|
||||
|
||||
@staticmethod
|
||||
def _trace_start(history, notes: list | None) -> list:
|
||||
"""The first step of an answer's trace: the records and notes it was given."""
|
||||
return [{"kind": "records", "recent": list(history.recent), "matches": list(history.matches),
|
||||
"notes": len(notes or [])}]
|
||||
|
||||
def _trace_end(self, via: str, channel: str | None, user: str, question: str, answer: str,
|
||||
steps: list, began: float) -> None:
|
||||
"""Keep how this answer was made (history.spec.md, "Traces"); never stops the answer."""
|
||||
if self.worker.history is None:
|
||||
return
|
||||
try:
|
||||
self.worker.history.add_trace(via, channel, user, question, answer, steps,
|
||||
ms=round((time.time() - began) * 1000), ts=time.time())
|
||||
except Exception as e: # noqa: BLE001 - a trace is a view, not part of the answer
|
||||
debug_log(f"trace not kept: {type(e).__name__}", "llm")
|
||||
|
||||
async def _task(self, rid: str, user: str, text: str, notes: list | None = None) -> None:
|
||||
"""A scheduled task (schedule.spec.md, "Tasks"): answered in the chat lane with the read-only
|
||||
tools, its own short-term memory and the person's history scope. It is not a turn of their
|
||||
@@ -622,12 +644,14 @@ class Session:
|
||||
from jarvis.memory.history import HistoryContext
|
||||
history = HistoryContext()
|
||||
try:
|
||||
steps = self._trace_start(history, notes)
|
||||
async with self._llm():
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, task=True,
|
||||
notes=notes or [])
|
||||
notes=notes or [], trace=steps)
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain task {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = (answer or "").strip()
|
||||
answer = (answer or "").strip()
|
||||
self._trace_end("task", None, user, text, answer, steps, now)
|
||||
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
||||
|
||||
def _on_llm(self, msg: dict) -> None:
|
||||
@@ -834,13 +858,15 @@ class Session:
|
||||
schedule = AppBridge(self, asyncio.get_running_loop(), "schedule", *origin)
|
||||
notes_backend = AppBridge(self, asyncio.get_running_loop(), "note", *origin)
|
||||
screen = AppBridge(self, asyncio.get_running_loop(), "screen", *origin)
|
||||
steps = self._trace_start(history, notes)
|
||||
try:
|
||||
async with self._llm(rid if speak else None):
|
||||
w.answering += 1
|
||||
try:
|
||||
answer = await asyncio.to_thread(w.brain.reply, user, text, history, chat=chat,
|
||||
schedule=schedule, notes=notes or [],
|
||||
notes_backend=notes_backend, screen=screen)
|
||||
notes_backend=notes_backend, screen=screen,
|
||||
trace=steps)
|
||||
finally:
|
||||
w.answering -= 1
|
||||
if not w.answering and w.scheduler:
|
||||
@@ -848,7 +874,8 @@ class Session:
|
||||
except Exception as e: # the brain must never take the session down
|
||||
debug_log(f"brain reply {rid} failed: {type(e).__name__}", "llm")
|
||||
answer = None
|
||||
answer = (answer or "").strip()
|
||||
answer = (answer or "").strip()
|
||||
self._trace_end(via, channel if via == "voice" else chat_channel, user, text, answer, steps, now)
|
||||
if not answer and rid in self.tts_requests:
|
||||
self._on_tts({"type": "tts", "id": rid, "text": "", "more": False}) # ends the notice's speech
|
||||
self.send({"type": "reply_text", "id": rid, "text": answer, "failed": not answer})
|
||||
|
||||
Reference in New Issue
Block a user