feat(brain): keep how each answer was made (records, tools, plan, thinking, tool calls)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
59
tests/test_engine_trace.py
Normal file
59
tests/test_engine_trace.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""The reply engine writes down how it made an answer (history.spec.md, "Traces").
|
||||
|
||||
With a trace list set on the turn state, a reply records the tools it was given, the plan, the
|
||||
model's thinking and each tool call with its outcome. Without one nothing is recorded.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.jarvis.memory.conversation import DialogueMemory
|
||||
from src.jarvis.reply import turn_state
|
||||
from src.jarvis.reply.engine import run_reply_engine
|
||||
|
||||
from tests.test_engine_tool_carryover import _mock_cfg
|
||||
|
||||
|
||||
def _run(trace):
|
||||
tool = Mock(reply_text="서울: 맑음, 12도", error_message=None, success=True)
|
||||
replies = [
|
||||
{"message": {"content": "", "thinking": "날씨를 찾아봐야겠다", "tool_calls": [{
|
||||
"id": "c1", "type": "function",
|
||||
"function": {"name": "webSearch", "arguments": {"query": "서울 날씨"}}}]}},
|
||||
{"message": {"content": "서울은 맑고 12도입니다."}},
|
||||
]
|
||||
turn_state.reset()
|
||||
turn_state.set_trace(trace)
|
||||
try:
|
||||
with patch("src.jarvis.reply.engine.plan_query", return_value=["Search the web", "Reply to the user."]), \
|
||||
patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \
|
||||
patch("src.jarvis.reply.engine.run_tool_with_retries", return_value=tool), \
|
||||
patch("src.jarvis.reply.engine.extract_text_from_response",
|
||||
side_effect=lambda r: r["message"]["content"]), \
|
||||
patch("src.jarvis.reply.engine.chat_with_messages", side_effect=replies):
|
||||
return run_reply_engine(db=Mock(), cfg=_mock_cfg(), tts=None, text="서울 날씨 어때",
|
||||
dialogue_memory=DialogueMemory())
|
||||
finally:
|
||||
turn_state.reset()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_reply_records_its_tools_plan_thinking_and_tool_calls():
|
||||
trace = []
|
||||
assert _run(trace)
|
||||
kinds = [s["kind"] for s in trace]
|
||||
assert kinds.index("tools") < kinds.index("plan") < kinds.index("think") < kinds.index("tool")
|
||||
tools = next(s for s in trace if s["kind"] == "tools")
|
||||
assert "webSearch" in tools["names"]
|
||||
assert next(s for s in trace if s["kind"] == "plan")["steps"] == ["Search the web", "Reply to the user."]
|
||||
assert next(s for s in trace if s["kind"] == "think")["text"] == "날씨를 찾아봐야겠다"
|
||||
call = next(s for s in trace if s["kind"] == "tool")
|
||||
assert (call["name"], call["args"], call["ok"], call["result"]) == \
|
||||
("webSearch", {"query": "서울 날씨"}, True, "서울: 맑음, 12도")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_without_a_trace_nothing_is_recorded():
|
||||
assert _run(None)
|
||||
assert turn_state.get_trace() is None
|
||||
@@ -238,3 +238,26 @@ def test_chat_tool_reads_and_searches_one_channels_messages(store):
|
||||
assert sorted(texts(store.search_chat(["c1", "c2"], "회의"))) == ["회의 취소", "회의는 3시"]
|
||||
assert store.search_chat([], "회의") == []
|
||||
assert store.chat_channel_of("m3") == "c2" and store.chat_channel_of("nope") is None
|
||||
|
||||
|
||||
def test_how_an_answer_was_made_is_kept_per_person_and_never_searched(store):
|
||||
steps = [{"kind": "tools", "names": ["getWeather"]},
|
||||
{"kind": "tool", "name": "getWeather", "args": {"location": "서울"}, "ok": True, "result": "맑음"}]
|
||||
store.add_trace("voice", VC, "a", "서울 날씨 어때", "맑습니다", steps, ms=1200, ts=T0)
|
||||
store.add_trace("chat", CHAT, "b", "안녕", "", [], ms=50, ts=T0 + 1)
|
||||
[t] = store.traces_of("a")
|
||||
assert (t.via, t.channel_id, t.question, t.answer, t.ms, t.steps) == ("voice", VC, "서울 날씨 어때", "맑습니다", 1200, steps)
|
||||
# A trace is no record: the brain never reads it back as history.
|
||||
assert store.search("a", "날씨", scope=Scope()) == []
|
||||
assert store.recent("a", T0 + 2, scope=Scope()) == []
|
||||
store.forget_person("a")
|
||||
assert store.traces_of("a") == [] and len(store.traces_of("b")) == 1
|
||||
|
||||
|
||||
def test_traces_have_their_own_retention_and_follow_the_history_retention(store):
|
||||
store.add_trace("voice", VC, "a", "옛 질문", "옛 답", [], ms=1, ts=T0)
|
||||
store.add_trace("voice", VC, "a", "새 질문", "새 답", [], ms=1, ts=T0 + 100)
|
||||
store.purge_traces_older_than(T0 + 50)
|
||||
assert [t.question for t in store.traces_of("a")] == ["새 질문"]
|
||||
store.purge_older_than(T0 + 200)
|
||||
assert store.traces_of("a") == []
|
||||
|
||||
@@ -29,8 +29,10 @@ class FakeBrain:
|
||||
self.calls = []
|
||||
|
||||
def reply(self, user, text, history, chat=None, schedule=None, task=False, notes=None, notes_backend=None,
|
||||
screen=None):
|
||||
screen=None, trace=None):
|
||||
self.calls.append((user, text, history))
|
||||
if trace is not None:
|
||||
trace.append({"kind": "tools", "names": ["getWeather"]})
|
||||
self.chat, self.schedule, self.task = chat, schedule, task
|
||||
self.notes, self.notes_backend, self.screen = notes, notes_backend, screen
|
||||
return self.answer(user, text)
|
||||
@@ -211,6 +213,37 @@ def test_a_failed_brain_answer_is_reported(store, answer):
|
||||
assert seen[-1] == {"type": "reply_text", "id": "r1", "text": "", "failed": True}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_each_answer_keeps_how_it_was_made_with_the_records_it_was_given(store):
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws, stt={"partials": False})
|
||||
await send(ws, type="voice_join", channel="vc1")
|
||||
await send(ws, type="member_join", user="100", name="Alice")
|
||||
await say(ws, "100", 0.9)
|
||||
await send(ws, type="reply", id="r1", user="100", text="what did I say", via="voice", speak=False,
|
||||
notes=["반말로 해"])
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "r1")
|
||||
await send(ws, type="reply", id="c1", user="100", text="hi", via="chat", speak=False, channel="ch1")
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "c1")
|
||||
await send(ws, type="reply", id="t1", user="100", text="task", via="task")
|
||||
await recv_until(ws, lambda m: isinstance(m, dict) and m.get("id") == "t1")
|
||||
|
||||
brain = FakeBrain(lambda u, t: None if t == "hi" else f"answer for {t}")
|
||||
asyncio.run(with_worker(body, store=store, brain=brain))
|
||||
voice, chat, task = store.traces_of("100")
|
||||
assert (voice.via, voice.channel_id, voice.question, voice.answer) == \
|
||||
("voice", "vc1", "what did I say", "answer for what did I say")
|
||||
assert voice.ms >= 0
|
||||
records = voice.steps[0]
|
||||
assert records["kind"] == "records" and records["notes"] == 1
|
||||
assert any("Alice: level 0.9" in x for x in records["recent"])
|
||||
assert voice.steps[1:] == [{"kind": "tools", "names": ["getWeather"]}]
|
||||
# A failed answer is kept too, with no answer; a task has no channel.
|
||||
assert (chat.via, chat.channel_id, chat.answer) == ("chat", "ch1", "")
|
||||
assert (task.via, task.channel_id, task.answer) == ("task", None, "answer for task")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_without_a_brain_reply_is_an_error():
|
||||
async def body(url):
|
||||
@@ -300,6 +333,21 @@ def test_presence_left_open_by_a_previous_run_is_closed_at_start(store):
|
||||
assert store.recent("old", now=4_000_000_000, scope=Scope(), minutes=10 ** 9) == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_traces_older_than_thirty_days_are_removed_at_start(store):
|
||||
import time
|
||||
now = time.time()
|
||||
store.add_trace("voice", "vc1", "100", "old", "a", [], ms=1, ts=now - 31 * 86400)
|
||||
store.add_trace("voice", "vc1", "100", "new", "a", [], ms=1, ts=now - 29 * 86400)
|
||||
|
||||
async def body(url):
|
||||
async with connect(url) as ws:
|
||||
await hello(ws)
|
||||
|
||||
asyncio.run(with_worker(body, store=store))
|
||||
assert [t.question for t in store.traces_of("100")] == ["new"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_brain_prints_no_text_from_its_thread(tmp_path, monkeypatch, capfd):
|
||||
pytest.importorskip("mcp") # the real brain needs the jarvis dependencies (--brain)
|
||||
|
||||
Reference in New Issue
Block a user