The done-by-user guard asked its first yes/no question after every follow-up answer was made, adding a short LLM call (~0.4 s) before Jarvis spoke. The question depends only on the user's words, so the voice worker's prepare now asks it on the partial transcript after the tool pick (prepare_done_by_user, follow-up turns only) and the guard uses that judgement (prepared_done), asking only whether the answer refuses when the user said they did it. Without a prepared judgement, or when it failed, the guard asks for itself as before. The reply does not wait for the judgement, only the guard does. Measured on Qwen3.6 through the game PC's Strata, 12 pairs of short follow-up questions in alternating order: brain time median 1.06 s before, 0.69 s after. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
213 lines
10 KiB
Python
213 lines
10 KiB
Python
"""The v2 done-by-user guard (reply.spec.md, "Earlier answers after the user did it (v2)").
|
|
|
|
Field turn (2026-11-18): Jarvis said "speed control is not supported", the user changed the speed
|
|
in the app and said "I adjusted your speed, so say something", and Jarvis refused again. Its own
|
|
earlier answer in the context outweighed what the user had just said.
|
|
|
|
Behaviour: when the user says they did something themselves and the answer still says something
|
|
cannot be done, the answer is made once more without Jarvis's earlier answers (the user's own words
|
|
stay). Everything else keeps the first answer: a repeat the user asked for, an answer that does
|
|
what was asked, a turn with no earlier answer, a turn that ran a tool, and a judgement that failed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
REFUSAL = "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다."
|
|
REWORDED = "직접 조절하셨다면 제가 그걸 확인하거나 제어할 수 있는 기능은 제공되지 않습니다."
|
|
QUESTION = "자비스 너 말하는 속도 좀 빠르게 할 수 있어?"
|
|
SPOKEN = "네, 지금 이 속도로 말씀드리고 있습니다. 들어 보시고 더 필요하면 말씀해 주세요."
|
|
DID_IT = "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고"
|
|
REFUSALS = {REFUSAL, REWORDED}
|
|
|
|
|
|
def _content(text: str) -> dict:
|
|
return {"message": {"role": "assistant", "content": text}}
|
|
|
|
|
|
def _run(mock_config, db, dialogue_memory, *, text, chat_replies, done="yes", earlier=True, tool_first=False,
|
|
tool_text="ok", others=(), prepared_done=None):
|
|
"""``done``: the judge's word for "the user says they did it"; a reply is judged a refusal when
|
|
it is one of REFUSALS. ``tool_text``: what the tool returns; ``others``: history lines of other
|
|
people, after Jarvis's answer."""
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import engine as engine_mod
|
|
from jarvis.tools.types import ToolExecutionResult
|
|
|
|
mock_config.ollama_chat_model = "gpt-oss:20b"
|
|
mock_config.evaluator_enabled = False
|
|
mock_config.stream_browser = False
|
|
if earlier:
|
|
dialogue_memory.add_message("user", QUESTION)
|
|
dialogue_memory.add_message("assistant", REFUSAL)
|
|
answer_line = f"[2026-11-18 18:27] 자비스: {REFUSAL}"
|
|
history = HistoryContext(recent=[f"[2026-11-18 18:27] 사금향: {QUESTION}", answer_line, *others]
|
|
if earlier else [], answers=[answer_line] if earlier else [])
|
|
seen: list[list[dict]] = []
|
|
replies = iter(chat_replies)
|
|
judged: list[str] = []
|
|
|
|
def fake_chat(*args, **kwargs):
|
|
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
|
|
return next(replies)
|
|
|
|
def fake_judge(base_url, model, system, user_content, **kwargs):
|
|
judged.append(user_content)
|
|
if user_content.startswith("<utterance>"):
|
|
return done
|
|
return "yes" if user_content.removeprefix("<reply>").removesuffix("</reply>") in REFUSALS else "no"
|
|
|
|
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
|
|
return ToolExecutionResult(success=True, reply_text=tool_text, error_message=None)
|
|
|
|
routed = ["getWeather", "stop"] if tool_first else ["stop"]
|
|
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
|
|
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
|
|
patch.object(engine_mod, "select_tools", return_value=routed), \
|
|
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]), \
|
|
patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge):
|
|
reply = engine_mod.run_reply_engine(
|
|
db=db, cfg=mock_config, tts=None, text=text, dialogue_memory=dialogue_memory,
|
|
history=history, legacy_memory=False, intent_judge=True, prepared_done=prepared_done)
|
|
return reply, seen, judged
|
|
|
|
|
|
def _said(messages: list[dict]) -> str:
|
|
return "\n".join(str(m.get("content") or "") for m in messages)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("draft", [REFUSAL, REWORDED], ids=["same words", "reworded"])
|
|
def test_a_refusal_after_the_user_did_it_is_answered_again_without_earlier_answers(
|
|
mock_config, db, dialogue_memory, draft):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(draft), _content(SPOKEN)])
|
|
assert reply == SPOKEN
|
|
assert DID_IT in judged[0]
|
|
assert len(seen) == 2
|
|
assert REFUSAL in _said(seen[0])
|
|
# The second try sees neither the earlier answer nor its history line; the user's question stays.
|
|
assert REFUSAL not in _said(seen[1])
|
|
assert QUESTION in _said(seen[1])
|
|
# The turn is kept once, with the new answer.
|
|
kept = [m["content"] for m in dialogue_memory.get_recent_messages()]
|
|
assert kept.count(DID_IT) == 1 and kept[-1] == SPOKEN
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_repeat_the_user_asked_for_is_kept(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text="방금 뭐라고 했어? 다시 말해 줘",
|
|
chat_replies=[_content(REFUSAL)], done="no")
|
|
assert reply == REFUSAL
|
|
assert len(judged) == 1 and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_answer_that_does_it_is_kept(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(SPOKEN)])
|
|
assert reply == SPOKEN
|
|
assert len(judged) == 2 and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_no_earlier_answer_is_not_judged(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
|
|
earlier=False)
|
|
assert reply == REFUSAL
|
|
assert judged == [] and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("done", [None, "maybe"])
|
|
def test_no_usable_judgement_keeps_the_answer(mock_config, db, dialogue_memory, done):
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
|
|
done=done)
|
|
assert reply == REFUSAL and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("tool_text", ["ok", "", None], ids=["result", "empty", "none"])
|
|
def test_a_turn_that_ran_a_tool_is_not_made_again(mock_config, db, dialogue_memory, tool_text):
|
|
"""A second try would run the tool (and anything it does) twice, whatever it returned."""
|
|
call = {"message": {"role": "assistant", "content": "", "tool_calls": [
|
|
{"id": "call_1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}}
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[call, _content(REFUSAL)], tool_first=True, tool_text=tool_text)
|
|
assert reply == REFUSAL
|
|
assert judged == [] and len(seen) == 2
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_earlier_tool_call_and_its_result_stay_together(mock_config, db, dialogue_memory):
|
|
"""A native tool call message with text is not an answer to leave out: its tool result would be
|
|
left without the call it answers."""
|
|
dialogue_memory.add_message("user", "서울 날씨 어때?")
|
|
dialogue_memory.record_tool_turn([
|
|
{"role": "assistant", "content": REFUSAL, "tool_calls": [
|
|
{"id": "call_0", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]},
|
|
{"role": "tool", "tool_call_id": "call_0", "content": "Seoul: 13C, clear sky."}])
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(REFUSAL), _content(SPOKEN)])
|
|
assert reply == SPOKEN and len(seen) == 2
|
|
second = seen[1]
|
|
calls = {tc["id"] for m in second if m.get("role") == "assistant" for tc in (m.get("tool_calls") or [])}
|
|
results = [m["tool_call_id"] for m in second if m.get("role") == "tool"]
|
|
assert results == ["call_0"] and "call_0" in calls
|
|
# Jarvis's plain earlier answer is still left out.
|
|
assert not [m for m in second if m.get("role") == "assistant" and not m.get("tool_calls")
|
|
and m.get("content") == REFUSAL]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_only_jarvis_lines_leave_the_history(mock_config, db, dialogue_memory):
|
|
"""Someone else saying the same words is their record, not Jarvis's answer."""
|
|
quoted = f"[2026-11-18 18:28] 민수: {REFUSAL}"
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(REFUSAL), _content(SPOKEN)], others=[quoted])
|
|
assert reply == SPOKEN
|
|
assert quoted in _said(seen[1])
|
|
assert "자비스: " + REFUSAL not in _said(seen[1])
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("said, expected", [(True, SPOKEN), (False, REFUSAL)], ids=["did it", "did not"])
|
|
def test_a_judgement_made_ahead_is_not_asked_again(mock_config, db, dialogue_memory, said, expected):
|
|
"""The prepare stage judged the words while the user was still speaking (latency, reply.spec.md):
|
|
the guard uses that judgement and asks only whether the answer refuses."""
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(REFUSAL), _content(SPOKEN)], done="no" if said else "yes",
|
|
prepared_done=lambda: said)
|
|
assert reply == expected
|
|
assert not [j for j in judged if j.startswith("<utterance>")]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_no_judgement_made_ahead_asks_for_itself(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(REFUSAL), _content(SPOKEN)], prepared_done=lambda: None)
|
|
assert reply == SPOKEN
|
|
assert DID_IT in judged[0]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_said_done_ahead_judges_only_a_follow_up(mock_config, dialogue_memory):
|
|
"""No earlier answer: no guard would run, so nothing is asked ahead either."""
|
|
from jarvis.reply import engine as engine_mod
|
|
|
|
asked = []
|
|
|
|
def fake_judge(base_url, model, system, user_content, **kwargs):
|
|
asked.append(user_content)
|
|
return "yes"
|
|
|
|
with patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge):
|
|
assert engine_mod.prepare_done_by_user(mock_config, DID_IT, dialogue_memory) is None
|
|
assert asked == []
|
|
dialogue_memory.add_message("user", QUESTION)
|
|
dialogue_memory.add_message("assistant", REFUSAL)
|
|
assert engine_mod.prepare_done_by_user(mock_config, DID_IT, dialogue_memory) is True
|
|
assert len(asked) == 1 and DID_IT in asked[0]
|