"""The v2 done-by-user guard (reply.spec.md, "Earlier answers after the user did it (v2)"). Field turn (2026-11-18): Jarvis said "speed control is not supported", the user changed the speed in the app and said "I adjusted your speed, so say something", and Jarvis refused again. Its own earlier answer in the context outweighed what the user had just said. Behaviour: when the user says they did something themselves and the answer still says something cannot be done, the answer is made once more without Jarvis's earlier answers (the user's own words stay). Everything else keeps the first answer: a repeat the user asked for, an answer that does what was asked, a turn with no earlier answer, a turn that ran a tool, and a judgement that failed. """ from __future__ import annotations from unittest.mock import patch import pytest REFUSAL = "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다." REWORDED = "직접 조절하셨다면 제가 그걸 확인하거나 제어할 수 있는 기능은 제공되지 않습니다." QUESTION = "자비스 너 말하는 속도 좀 빠르게 할 수 있어?" SPOKEN = "네, 지금 이 속도로 말씀드리고 있습니다. 들어 보시고 더 필요하면 말씀해 주세요." DID_IT = "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고" REFUSALS = {REFUSAL, REWORDED} def _content(text: str) -> dict: return {"message": {"role": "assistant", "content": text}} def _run(mock_config, db, dialogue_memory, *, text, chat_replies, done="yes", earlier=True, tool_first=False, tool_text="ok", others=(), prepared_done=None): """``done``: the judge's word for "the user says they did it"; a reply is judged a refusal when it is one of REFUSALS. ``tool_text``: what the tool returns; ``others``: history lines of other people, after Jarvis's answer.""" from jarvis.memory.history import HistoryContext from jarvis.reply import engine as engine_mod from jarvis.tools.types import ToolExecutionResult mock_config.ollama_chat_model = "gpt-oss:20b" mock_config.evaluator_enabled = False mock_config.stream_browser = False if earlier: dialogue_memory.add_message("user", QUESTION) dialogue_memory.add_message("assistant", REFUSAL) answer_line = f"[2026-11-18 18:27] 자비스: {REFUSAL}" history = HistoryContext(recent=[f"[2026-11-18 18:27] 사금향: {QUESTION}", answer_line, *others] if earlier else [], answers=[answer_line] if earlier else []) seen: list[list[dict]] = [] replies = iter(chat_replies) judged: list[str] = [] def fake_chat(*args, **kwargs): seen.append([dict(m) for m in (kwargs.get("messages") or args[2])]) return next(replies) def fake_judge(base_url, model, system, user_content, **kwargs): judged.append(user_content) if user_content.startswith(""): return done return "yes" if user_content.removeprefix("").removesuffix("") in REFUSALS else "no" def fake_tool(db, cfg, tool_name, tool_args, **kwargs): return ToolExecutionResult(success=True, reply_text=tool_text, error_message=None) routed = ["getWeather", "stop"] if tool_first else ["stop"] with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \ patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \ patch.object(engine_mod, "select_tools", return_value=routed), \ patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]), \ patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge): reply = engine_mod.run_reply_engine( db=db, cfg=mock_config, tts=None, text=text, dialogue_memory=dialogue_memory, history=history, legacy_memory=False, intent_judge=True, prepared_done=prepared_done) return reply, seen, judged def _said(messages: list[dict]) -> str: return "\n".join(str(m.get("content") or "") for m in messages) @pytest.mark.unit @pytest.mark.parametrize("draft", [REFUSAL, REWORDED], ids=["same words", "reworded"]) def test_a_refusal_after_the_user_did_it_is_answered_again_without_earlier_answers( mock_config, db, dialogue_memory, draft): reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(draft), _content(SPOKEN)]) assert reply == SPOKEN assert DID_IT in judged[0] assert len(seen) == 2 assert REFUSAL in _said(seen[0]) # The second try sees neither the earlier answer nor its history line; the user's question stays. assert REFUSAL not in _said(seen[1]) assert QUESTION in _said(seen[1]) # The turn is kept once, with the new answer. kept = [m["content"] for m in dialogue_memory.get_recent_messages()] assert kept.count(DID_IT) == 1 and kept[-1] == SPOKEN @pytest.mark.unit def test_a_repeat_the_user_asked_for_is_kept(mock_config, db, dialogue_memory): reply, seen, judged = _run(mock_config, db, dialogue_memory, text="방금 뭐라고 했어? 다시 말해 줘", chat_replies=[_content(REFUSAL)], done="no") assert reply == REFUSAL assert len(judged) == 1 and len(seen) == 1 @pytest.mark.unit def test_an_answer_that_does_it_is_kept(mock_config, db, dialogue_memory): reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(SPOKEN)]) assert reply == SPOKEN assert len(judged) == 2 and len(seen) == 1 @pytest.mark.unit def test_no_earlier_answer_is_not_judged(mock_config, db, dialogue_memory): reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)], earlier=False) assert reply == REFUSAL assert judged == [] and len(seen) == 1 @pytest.mark.unit @pytest.mark.parametrize("done", [None, "maybe"]) def test_no_usable_judgement_keeps_the_answer(mock_config, db, dialogue_memory, done): reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)], done=done) assert reply == REFUSAL and len(seen) == 1 @pytest.mark.unit @pytest.mark.parametrize("tool_text", ["ok", "", None], ids=["result", "empty", "none"]) def test_a_turn_that_ran_a_tool_is_not_made_again(mock_config, db, dialogue_memory, tool_text): """A second try would run the tool (and anything it does) twice, whatever it returned.""" call = {"message": {"role": "assistant", "content": "", "tool_calls": [ {"id": "call_1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}} reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[call, _content(REFUSAL)], tool_first=True, tool_text=tool_text) assert reply == REFUSAL assert judged == [] and len(seen) == 2 @pytest.mark.unit def test_an_earlier_tool_call_and_its_result_stay_together(mock_config, db, dialogue_memory): """A native tool call message with text is not an answer to leave out: its tool result would be left without the call it answers.""" dialogue_memory.add_message("user", "서울 날씨 어때?") dialogue_memory.record_tool_turn([ {"role": "assistant", "content": REFUSAL, "tool_calls": [ {"id": "call_0", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}, {"role": "tool", "tool_call_id": "call_0", "content": "Seoul: 13C, clear sky."}]) reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL), _content(SPOKEN)]) assert reply == SPOKEN and len(seen) == 2 second = seen[1] calls = {tc["id"] for m in second if m.get("role") == "assistant" for tc in (m.get("tool_calls") or [])} results = [m["tool_call_id"] for m in second if m.get("role") == "tool"] assert results == ["call_0"] and "call_0" in calls # Jarvis's plain earlier answer is still left out. assert not [m for m in second if m.get("role") == "assistant" and not m.get("tool_calls") and m.get("content") == REFUSAL] @pytest.mark.unit def test_only_jarvis_lines_leave_the_history(mock_config, db, dialogue_memory): """Someone else saying the same words is their record, not Jarvis's answer.""" quoted = f"[2026-11-18 18:28] 민수: {REFUSAL}" reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL), _content(SPOKEN)], others=[quoted]) assert reply == SPOKEN assert quoted in _said(seen[1]) assert "자비스: " + REFUSAL not in _said(seen[1]) @pytest.mark.unit @pytest.mark.parametrize("said, expected", [(True, SPOKEN), (False, REFUSAL)], ids=["did it", "did not"]) def test_a_judgement_made_ahead_is_not_asked_again(mock_config, db, dialogue_memory, said, expected): """The prepare stage judged the words while the user was still speaking (latency, reply.spec.md): the guard uses that judgement and asks only whether the answer refuses.""" reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL), _content(SPOKEN)], done="no" if said else "yes", prepared_done=lambda: said) assert reply == expected assert not [j for j in judged if j.startswith("")] @pytest.mark.unit def test_no_judgement_made_ahead_asks_for_itself(mock_config, db, dialogue_memory): reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL), _content(SPOKEN)], prepared_done=lambda: None) assert reply == SPOKEN assert DID_IT in judged[0] @pytest.mark.unit def test_said_done_ahead_judges_only_a_follow_up(mock_config, dialogue_memory): """No earlier answer: no guard would run, so nothing is asked ahead either.""" from jarvis.reply import engine as engine_mod asked = [] def fake_judge(base_url, model, system, user_content, **kwargs): asked.append(user_content) return "yes" with patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge): assert engine_mod.prepare_done_by_user(mock_config, DID_IT, dialogue_memory) is None assert asked == [] dialogue_memory.add_message("user", QUESTION) dialogue_memory.add_message("assistant", REFUSAL) assert engine_mod.prepare_done_by_user(mock_config, DID_IT, dialogue_memory) is True assert len(asked) == 1 and DID_IT in asked[0]