feat(brain): keep how each answer was made (records, tools, plan, thinking, tool calls)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
59
tests/test_engine_trace.py
Normal file
59
tests/test_engine_trace.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""The reply engine writes down how it made an answer (history.spec.md, "Traces").
|
||||
|
||||
With a trace list set on the turn state, a reply records the tools it was given, the plan, the
|
||||
model's thinking and each tool call with its outcome. Without one nothing is recorded.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.jarvis.memory.conversation import DialogueMemory
|
||||
from src.jarvis.reply import turn_state
|
||||
from src.jarvis.reply.engine import run_reply_engine
|
||||
|
||||
from tests.test_engine_tool_carryover import _mock_cfg
|
||||
|
||||
|
||||
def _run(trace):
|
||||
tool = Mock(reply_text="서울: 맑음, 12도", error_message=None, success=True)
|
||||
replies = [
|
||||
{"message": {"content": "", "thinking": "날씨를 찾아봐야겠다", "tool_calls": [{
|
||||
"id": "c1", "type": "function",
|
||||
"function": {"name": "webSearch", "arguments": {"query": "서울 날씨"}}}]}},
|
||||
{"message": {"content": "서울은 맑고 12도입니다."}},
|
||||
]
|
||||
turn_state.reset()
|
||||
turn_state.set_trace(trace)
|
||||
try:
|
||||
with patch("src.jarvis.reply.engine.plan_query", return_value=["Search the web", "Reply to the user."]), \
|
||||
patch("src.jarvis.reply.engine.extract_search_params_for_memory", return_value={}), \
|
||||
patch("src.jarvis.reply.engine.run_tool_with_retries", return_value=tool), \
|
||||
patch("src.jarvis.reply.engine.extract_text_from_response",
|
||||
side_effect=lambda r: r["message"]["content"]), \
|
||||
patch("src.jarvis.reply.engine.chat_with_messages", side_effect=replies):
|
||||
return run_reply_engine(db=Mock(), cfg=_mock_cfg(), tts=None, text="서울 날씨 어때",
|
||||
dialogue_memory=DialogueMemory())
|
||||
finally:
|
||||
turn_state.reset()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_reply_records_its_tools_plan_thinking_and_tool_calls():
|
||||
trace = []
|
||||
assert _run(trace)
|
||||
kinds = [s["kind"] for s in trace]
|
||||
assert kinds.index("tools") < kinds.index("plan") < kinds.index("think") < kinds.index("tool")
|
||||
tools = next(s for s in trace if s["kind"] == "tools")
|
||||
assert "webSearch" in tools["names"]
|
||||
assert next(s for s in trace if s["kind"] == "plan")["steps"] == ["Search the web", "Reply to the user."]
|
||||
assert next(s for s in trace if s["kind"] == "think")["text"] == "날씨를 찾아봐야겠다"
|
||||
call = next(s for s in trace if s["kind"] == "tool")
|
||||
assert (call["name"], call["args"], call["ok"], call["result"]) == \
|
||||
("webSearch", {"query": "서울 날씨"}, True, "서울: 맑음, 12도")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_without_a_trace_nothing_is_recorded():
|
||||
assert _run(None)
|
||||
assert turn_state.get_trace() is None
|
||||
Reference in New Issue
Block a user