- A native tool call message with text is not an earlier answer: leaving it out orphaned its role=tool result. Only plain assistant answers are left out. - The guard is off once any tool run started in the reply, whatever it returned: an MCP call with an empty result was not in invoked_tools_history, so a second try could run it again. - HistoryContext.answers marks the lines that are Jarvis's answers to the asker (to_id is the asker); only those leave the history, not someone else saying the same words. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
173 lines
8.6 KiB
Python
173 lines
8.6 KiB
Python
"""The v2 done-by-user guard (reply.spec.md, "Earlier answers after the user did it (v2)").
|
|
|
|
Field turn (2026-11-18): Jarvis said "speed control is not supported", the user changed the speed
|
|
in the app and said "I adjusted your speed, so say something", and Jarvis refused again. Its own
|
|
earlier answer in the context outweighed what the user had just said.
|
|
|
|
Behaviour: when the user says they did something themselves and the answer still says something
|
|
cannot be done, the answer is made once more without Jarvis's earlier answers (the user's own words
|
|
stay). Everything else keeps the first answer: a repeat the user asked for, an answer that does
|
|
what was asked, a turn with no earlier answer, a turn that ran a tool, and a judgement that failed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
REFUSAL = "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다."
|
|
REWORDED = "직접 조절하셨다면 제가 그걸 확인하거나 제어할 수 있는 기능은 제공되지 않습니다."
|
|
QUESTION = "자비스 너 말하는 속도 좀 빠르게 할 수 있어?"
|
|
SPOKEN = "네, 지금 이 속도로 말씀드리고 있습니다. 들어 보시고 더 필요하면 말씀해 주세요."
|
|
DID_IT = "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고"
|
|
REFUSALS = {REFUSAL, REWORDED}
|
|
|
|
|
|
def _content(text: str) -> dict:
|
|
return {"message": {"role": "assistant", "content": text}}
|
|
|
|
|
|
def _run(mock_config, db, dialogue_memory, *, text, chat_replies, done="yes", earlier=True, tool_first=False,
|
|
tool_text="ok", others=()):
|
|
"""``done``: the judge's word for "the user says they did it"; a reply is judged a refusal when
|
|
it is one of REFUSALS. ``tool_text``: what the tool returns; ``others``: history lines of other
|
|
people, after Jarvis's answer."""
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import engine as engine_mod
|
|
from jarvis.tools.types import ToolExecutionResult
|
|
|
|
mock_config.ollama_chat_model = "gpt-oss:20b"
|
|
mock_config.evaluator_enabled = False
|
|
mock_config.stream_browser = False
|
|
if earlier:
|
|
dialogue_memory.add_message("user", QUESTION)
|
|
dialogue_memory.add_message("assistant", REFUSAL)
|
|
answer_line = f"[2026-11-18 18:27] 자비스: {REFUSAL}"
|
|
history = HistoryContext(recent=[f"[2026-11-18 18:27] 사금향: {QUESTION}", answer_line, *others]
|
|
if earlier else [], answers=[answer_line] if earlier else [])
|
|
seen: list[list[dict]] = []
|
|
replies = iter(chat_replies)
|
|
judged: list[str] = []
|
|
|
|
def fake_chat(*args, **kwargs):
|
|
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
|
|
return next(replies)
|
|
|
|
def fake_judge(base_url, model, system, user_content, **kwargs):
|
|
judged.append(user_content)
|
|
if user_content.startswith("<utterance>"):
|
|
return done
|
|
return "yes" if user_content.removeprefix("<reply>").removesuffix("</reply>") in REFUSALS else "no"
|
|
|
|
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
|
|
return ToolExecutionResult(success=True, reply_text=tool_text, error_message=None)
|
|
|
|
routed = ["getWeather", "stop"] if tool_first else ["stop"]
|
|
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
|
|
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
|
|
patch.object(engine_mod, "select_tools", return_value=routed), \
|
|
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]), \
|
|
patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge):
|
|
reply = engine_mod.run_reply_engine(
|
|
db=db, cfg=mock_config, tts=None, text=text, dialogue_memory=dialogue_memory,
|
|
history=history, legacy_memory=False, intent_judge=True)
|
|
return reply, seen, judged
|
|
|
|
|
|
def _said(messages: list[dict]) -> str:
|
|
return "\n".join(str(m.get("content") or "") for m in messages)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("draft", [REFUSAL, REWORDED], ids=["same words", "reworded"])
|
|
def test_a_refusal_after_the_user_did_it_is_answered_again_without_earlier_answers(
|
|
mock_config, db, dialogue_memory, draft):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(draft), _content(SPOKEN)])
|
|
assert reply == SPOKEN
|
|
assert DID_IT in judged[0]
|
|
assert len(seen) == 2
|
|
assert REFUSAL in _said(seen[0])
|
|
# The second try sees neither the earlier answer nor its history line; the user's question stays.
|
|
assert REFUSAL not in _said(seen[1])
|
|
assert QUESTION in _said(seen[1])
|
|
# The turn is kept once, with the new answer.
|
|
kept = [m["content"] for m in dialogue_memory.get_recent_messages()]
|
|
assert kept.count(DID_IT) == 1 and kept[-1] == SPOKEN
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_repeat_the_user_asked_for_is_kept(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text="방금 뭐라고 했어? 다시 말해 줘",
|
|
chat_replies=[_content(REFUSAL)], done="no")
|
|
assert reply == REFUSAL
|
|
assert len(judged) == 1 and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_answer_that_does_it_is_kept(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(SPOKEN)])
|
|
assert reply == SPOKEN
|
|
assert len(judged) == 2 and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_no_earlier_answer_is_not_judged(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
|
|
earlier=False)
|
|
assert reply == REFUSAL
|
|
assert judged == [] and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("done", [None, "maybe"])
|
|
def test_no_usable_judgement_keeps_the_answer(mock_config, db, dialogue_memory, done):
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
|
|
done=done)
|
|
assert reply == REFUSAL and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("tool_text", ["ok", "", None], ids=["result", "empty", "none"])
|
|
def test_a_turn_that_ran_a_tool_is_not_made_again(mock_config, db, dialogue_memory, tool_text):
|
|
"""A second try would run the tool (and anything it does) twice, whatever it returned."""
|
|
call = {"message": {"role": "assistant", "content": "", "tool_calls": [
|
|
{"id": "call_1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}}
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[call, _content(REFUSAL)], tool_first=True, tool_text=tool_text)
|
|
assert reply == REFUSAL
|
|
assert judged == [] and len(seen) == 2
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_earlier_tool_call_and_its_result_stay_together(mock_config, db, dialogue_memory):
|
|
"""A native tool call message with text is not an answer to leave out: its tool result would be
|
|
left without the call it answers."""
|
|
dialogue_memory.add_message("user", "서울 날씨 어때?")
|
|
dialogue_memory.record_tool_turn([
|
|
{"role": "assistant", "content": REFUSAL, "tool_calls": [
|
|
{"id": "call_0", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]},
|
|
{"role": "tool", "tool_call_id": "call_0", "content": "Seoul: 13C, clear sky."}])
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(REFUSAL), _content(SPOKEN)])
|
|
assert reply == SPOKEN and len(seen) == 2
|
|
second = seen[1]
|
|
calls = {tc["id"] for m in second if m.get("role") == "assistant" for tc in (m.get("tool_calls") or [])}
|
|
results = [m["tool_call_id"] for m in second if m.get("role") == "tool"]
|
|
assert results == ["call_0"] and "call_0" in calls
|
|
# Jarvis's plain earlier answer is still left out.
|
|
assert not [m for m in second if m.get("role") == "assistant" and not m.get("tool_calls")
|
|
and m.get("content") == REFUSAL]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_only_jarvis_lines_leave_the_history(mock_config, db, dialogue_memory):
|
|
"""Someone else saying the same words is their record, not Jarvis's answer."""
|
|
quoted = f"[2026-11-18 18:28] 민수: {REFUSAL}"
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(REFUSAL), _content(SPOKEN)], others=[quoted])
|
|
assert reply == SPOKEN
|
|
assert quoted in _said(seen[1])
|
|
assert "자비스: " + REFUSAL not in _said(seen[1])
|