Files
javis_bot/tests/test_engine_done_guard.py
javis-bot 9c1e82a890 fix(reply): answer again without earlier answers when the user did it and the answer refuses
Field turn 2026-11-18: Jarvis said speed control is not supported, the user changed
the speed in the app and said "I adjusted it, so say something", and Jarvis refused
again, often in other words. Prompt wording did not move it (five rules, two history
shapes tried).

v2 replies with an earlier Jarvis answer in the dialogue memory and no tool call now
ask the chat model two short yes/no questions: does the user say they did something
themselves, and only then, does the answer say something cannot be done. On two
yeses the answer is made once more with Jarvis's earlier answers (dialogue memory and
the history lines with the same text) left out of that reply's context.

Real model (qwen3.6 35B on Strata), field case with history and dialogue memory, 20
same-prompt pairs: refused 12/20 before, 1/20 after; "what did you say? say it
again" still said it again 20/20 before and after.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 20:42:15 +09:00

137 lines
6.4 KiB
Python

"""The v2 done-by-user guard (reply.spec.md, "Earlier answers after the user did it (v2)").
Field turn (2026-11-18): Jarvis said "speed control is not supported", the user changed the speed
in the app and said "I adjusted your speed, so say something", and Jarvis refused again. Its own
earlier answer in the context outweighed what the user had just said.
Behaviour: when the user says they did something themselves and the answer still says something
cannot be done, the answer is made once more without Jarvis's earlier answers (the user's own words
stay). Everything else keeps the first answer: a repeat the user asked for, an answer that does
what was asked, a turn with no earlier answer, a turn that ran a tool, and a judgement that failed.
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
REFUSAL = "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다."
REWORDED = "직접 조절하셨다면 제가 그걸 확인하거나 제어할 수 있는 기능은 제공되지 않습니다."
QUESTION = "자비스 너 말하는 속도 좀 빠르게 할 수 있어?"
SPOKEN = "네, 지금 이 속도로 말씀드리고 있습니다. 들어 보시고 더 필요하면 말씀해 주세요."
DID_IT = "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고"
REFUSALS = {REFUSAL, REWORDED}
def _content(text: str) -> dict:
return {"message": {"role": "assistant", "content": text}}
def _run(mock_config, db, dialogue_memory, *, text, chat_replies, done="yes", earlier=True, tool_first=False):
"""``done``: the judge's word for "the user says they did it"; a reply is judged a refusal when
it is one of REFUSALS."""
from jarvis.memory.history import HistoryContext
from jarvis.reply import engine as engine_mod
from jarvis.tools.types import ToolExecutionResult
mock_config.ollama_chat_model = "gpt-oss:20b"
mock_config.evaluator_enabled = False
mock_config.stream_browser = False
if earlier:
dialogue_memory.add_message("user", QUESTION)
dialogue_memory.add_message("assistant", REFUSAL)
history = HistoryContext(recent=[f"[2026-11-18 18:27] 사금향: {QUESTION}",
f"[2026-11-18 18:27] 자비스: {REFUSAL}"] if earlier else [])
seen: list[list[dict]] = []
replies = iter(chat_replies)
judged: list[str] = []
def fake_chat(*args, **kwargs):
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
return next(replies)
def fake_judge(base_url, model, system, user_content, **kwargs):
judged.append(user_content)
if user_content.startswith("<utterance>"):
return done
return "yes" if user_content.removeprefix("<reply>").removesuffix("</reply>") in REFUSALS else "no"
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
return ToolExecutionResult(success=True, reply_text="ok", error_message=None)
routed = ["getWeather", "stop"] if tool_first else ["stop"]
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
patch.object(engine_mod, "select_tools", return_value=routed), \
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]), \
patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge):
reply = engine_mod.run_reply_engine(
db=db, cfg=mock_config, tts=None, text=text, dialogue_memory=dialogue_memory,
history=history, legacy_memory=False, intent_judge=True)
return reply, seen, judged
def _said(messages: list[dict]) -> str:
return "\n".join(str(m.get("content") or "") for m in messages)
@pytest.mark.unit
@pytest.mark.parametrize("draft", [REFUSAL, REWORDED], ids=["same words", "reworded"])
def test_a_refusal_after_the_user_did_it_is_answered_again_without_earlier_answers(
mock_config, db, dialogue_memory, draft):
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
chat_replies=[_content(draft), _content(SPOKEN)])
assert reply == SPOKEN
assert DID_IT in judged[0]
assert len(seen) == 2
assert REFUSAL in _said(seen[0])
# The second try sees neither the earlier answer nor its history line; the user's question stays.
assert REFUSAL not in _said(seen[1])
assert QUESTION in _said(seen[1])
# The turn is kept once, with the new answer.
kept = [m["content"] for m in dialogue_memory.get_recent_messages()]
assert kept.count(DID_IT) == 1 and kept[-1] == SPOKEN
@pytest.mark.unit
def test_a_repeat_the_user_asked_for_is_kept(mock_config, db, dialogue_memory):
reply, seen, judged = _run(mock_config, db, dialogue_memory, text="방금 뭐라고 했어? 다시 말해 줘",
chat_replies=[_content(REFUSAL)], done="no")
assert reply == REFUSAL
assert len(judged) == 1 and len(seen) == 1
@pytest.mark.unit
def test_an_answer_that_does_it_is_kept(mock_config, db, dialogue_memory):
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(SPOKEN)])
assert reply == SPOKEN
assert len(judged) == 2 and len(seen) == 1
@pytest.mark.unit
def test_no_earlier_answer_is_not_judged(mock_config, db, dialogue_memory):
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
earlier=False)
assert reply == REFUSAL
assert judged == [] and len(seen) == 1
@pytest.mark.unit
@pytest.mark.parametrize("done", [None, "maybe"])
def test_no_usable_judgement_keeps_the_answer(mock_config, db, dialogue_memory, done):
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
done=done)
assert reply == REFUSAL and len(seen) == 1
@pytest.mark.unit
def test_a_turn_that_ran_a_tool_is_not_made_again(mock_config, db, dialogue_memory):
"""A second try would run the tool (and anything it does) twice."""
call = {"message": {"role": "assistant", "content": "", "tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}}
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
chat_replies=[call, _content(REFUSAL)], tool_first=True)
assert reply == REFUSAL
assert judged == [] and len(seen) == 2