feat(brain): keep how each answer was made (records, tools, plan, thinking, tool calls)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-16 03:00:10 +09:00
parent 2a140d8490
commit a20ebe4eae
8 changed files with 265 additions and 8 deletions

View File

@@ -0,0 +1,59 @@
"""The reply engine writes down how it made an answer (history.spec.md, "Traces").
With a trace list set on the turn state, a reply records the tools it was given, the plan, the
model's thinking and each tool call with its outcome. Without one nothing is recorded.
"""
from unittest.mock import Mock, patch
import pytest
from src.jarvis.memory.conversation import DialogueMemory
from src.jarvis.reply import turn_state
from src.jarvis.reply.engine import run_reply_engine
from tests.test_engine_tool_carryover import _mock_cfg
def _run(trace):
tool = Mock(reply_text="서울: 맑음, 12도", error_message=None, success=True)
replies = [
{"message": {"content": "", "thinking": "날씨를 찾아봐야겠다", "tool_calls": [{
"id": "c1", "type": "function",
"function": {"name": "webSearch", "arguments": {"query": "서울 날씨"}}}]}},
{"message": {"content": "서울은 맑고 12도입니다."}},
]
turn_state.reset()
turn_state.set_trace(trace)
try:
with patch("src.jarvis.reply.engine.plan_query", return_value=["Search the web", "Reply to the user."]), \
patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \
patch("src.jarvis.reply.engine.run_tool_with_retries", return_value=tool), \
patch("src.jarvis.reply.engine.extract_text_from_response",
side_effect=lambda r: r["message"]["content"]), \
patch("src.jarvis.reply.engine.chat_with_messages", side_effect=replies):
return run_reply_engine(db=Mock(), cfg=_mock_cfg(), tts=None, text="서울 날씨 어때",
dialogue_memory=DialogueMemory())
finally:
turn_state.reset()
@pytest.mark.unit
def test_a_reply_records_its_tools_plan_thinking_and_tool_calls():
trace = []
assert _run(trace)
kinds = [s["kind"] for s in trace]
assert kinds.index("tools") < kinds.index("plan") < kinds.index("think") < kinds.index("tool")
tools = next(s for s in trace if s["kind"] == "tools")
assert "webSearch" in tools["names"]
assert next(s for s in trace if s["kind"] == "plan")["steps"] == ["Search the web", "Reply to the user."]
assert next(s for s in trace if s["kind"] == "think")["text"] == "날씨를 찾아봐야겠다"
call = next(s for s in trace if s["kind"] == "tool")
assert (call["name"], call["args"], call["ok"], call["result"]) == \
("webSearch", {"query": "서울 날씨"}, True, "서울: 맑음, 12도")
@pytest.mark.unit
def test_without_a_trace_nothing_is_recorded():
assert _run(None)
assert turn_state.get_trace() is None

View File

@@ -238,3 +238,26 @@ def test_chat_tool_reads_and_searches_one_channels_messages(store):
assert sorted(texts(store.search_chat(["c1", "c2"], "회의"))) == ["회의 취소", "회의는 3시"]
assert store.search_chat([], "회의") == []
assert store.chat_channel_of("m3") == "c2" and store.chat_channel_of("nope") is None
def test_how_an_answer_was_made_is_kept_per_person_and_never_searched(store):
steps = [{"kind": "tools", "names": ["getWeather"]},
{"kind": "tool", "name": "getWeather", "args": {"location": "서울"}, "ok": True, "result": "맑음"}]
store.add_trace("voice", VC, "a", "서울 날씨 어때", "맑습니다", steps, ms=1200, ts=T0)
store.add_trace("chat", CHAT, "b", "안녕", "", [], ms=50, ts=T0 + 1)
[t] = store.traces_of("a")
assert (t.via, t.channel_id, t.question, t.answer, t.ms, t.steps) == ("voice", VC, "서울 날씨 어때", "맑습니다", 1200, steps)
# A trace is no record: the brain never reads it back as history.
assert store.search("a", "날씨", scope=Scope()) == []
assert store.recent("a", T0 + 2, scope=Scope()) == []
store.forget_person("a")
assert store.traces_of("a") == [] and len(store.traces_of("b")) == 1
def test_traces_have_their_own_retention_and_follow_the_history_retention(store):
store.add_trace("voice", VC, "a", "옛 질문", "옛 답", [], ms=1, ts=T0)
store.add_trace("voice", VC, "a", "새 질문", "새 답", [], ms=1, ts=T0 + 100)
store.purge_traces_older_than(T0 + 50)
assert [t.question for t in store.traces_of("a")] == ["새 질문"]
store.purge_older_than(T0 + 200)
assert store.traces_of("a") == []

View File

@@ -29,8 +29,10 @@ class FakeBrain:
self.calls = []
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
screen=None):
screen=None, trace=None):
self.calls.append((user, text, history))
if trace is not None:
trace.append({"kind": "tools", "names": ["getWeather"]})
self.chat, self.schedule, self.task = chat, schedule, task
self.notes, self.notes_backend, self.screen = notes, notes_backend, screen
return self.answer(user, text)
@@ -211,6 +213,37 @@ def test_a_failed_brain_answer_is_reported(store, answer):
assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True}
@pytest.mark.unit
def test_each_answer_keeps_how_it_was_made_with_the_records_it_was_given(store):
async def body(url):
async with connect(url) as ws:
await hello(ws, stt={"partials": False})
await send(ws, type="voice_join", channel="vc1")
await send(ws, type="member_join", user="100", name="Alice")
await say(ws, "100", 0.9)
await send(ws, type="reply", id="r1", user="100", text="what did I say", via="voice", speak=False,
notes=["반말로 해"])
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False, channel="ch1")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1")
await send(ws, type="reply", id="t1", user="100", text="task", via="task")
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "t1")
brain = FakeBrain(lambda u, t: None if t == "hi" else f"answer for {t}")
asyncio.run(with_worker(body, store=store, brain=brain))
voice, chat, task = store.traces_of("100")
assert (voice.via, voice.channel_id, voice.question, voice.answer) == \
("voice", "vc1", "what did I say", "answer for what did I say")
assert voice.ms >= 0
records = voice.steps[0]
assert records["kind"] == "records" and records["notes"] == 1
assert any("Alice: level 0.9" in x for x in records["recent"])
assert voice.steps[1:] == [{"kind": "tools", "names": ["getWeather"]}]
# A failed answer is kept too, with no answer; a task has no channel.
assert (chat.via, chat.channel_id, chat.answer) == ("chat", "ch1", "")
assert (task.via, task.channel_id, task.answer) == ("task", None, "answer for task")
@pytest.mark.unit
def test_without_a_brain_reply_is_an_error():
async def body(url):
@@ -300,6 +333,21 @@ def test_presence_left_open_by_a_previous_run_is_closed_at_start(store):
assert store.recent("old", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
@pytest.mark.unit
def test_traces_older_than_thirty_days_are_removed_at_start(store):
import time
now = time.time()
store.add_trace("voice", "vc1", "100", "old", "a", [], ms=1, ts=now - 31 * 86400)
store.add_trace("voice", "vc1", "100", "new", "a", [], ms=1, ts=now - 29 * 86400)
async def body(url):
async with connect(url) as ws:
await hello(ws)
asyncio.run(with_worker(body, store=store))
assert [t.question for t in store.traces_of("100")] == ["new"]
@pytest.mark.unit
def test_the_brain_prints_no_text_from_its_thread(tmp_path, monkeypatch, capfd):
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)