Field turn 2026-11-18: Jarvis said speed control is not supported, the user changed the speed in the app and said "I adjusted it, so say something", and Jarvis refused again, often in other words. Prompt wording did not move it (five rules, two history shapes tried). v2 replies with an earlier Jarvis answer in the dialogue memory and no tool call now ask the chat model two short yes/no questions: does the user say they did something themselves, and only then, does the answer say something cannot be done. On two yeses the answer is made once more with Jarvis's earlier answers (dialogue memory and the history lines with the same text) left out of that reply's context. Real model (qwen3.6 35B on Strata), field case with history and dialogue memory, 20 same-prompt pairs: refused 12/20 before, 1/20 after; "what did you say? say it again" still said it again 20/20 before and after. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
137 lines
6.4 KiB
Python
137 lines
6.4 KiB
Python
"""The v2 done-by-user guard (reply.spec.md, "Earlier answers after the user did it (v2)").
|
|
|
|
Field turn (2026-11-18): Jarvis said "speed control is not supported", the user changed the speed
|
|
in the app and said "I adjusted your speed, so say something", and Jarvis refused again. Its own
|
|
earlier answer in the context outweighed what the user had just said.
|
|
|
|
Behaviour: when the user says they did something themselves and the answer still says something
|
|
cannot be done, the answer is made once more without Jarvis's earlier answers (the user's own words
|
|
stay). Everything else keeps the first answer: a repeat the user asked for, an answer that does
|
|
what was asked, a turn with no earlier answer, a turn that ran a tool, and a judgement that failed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
REFUSAL = "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다."
|
|
REWORDED = "직접 조절하셨다면 제가 그걸 확인하거나 제어할 수 있는 기능은 제공되지 않습니다."
|
|
QUESTION = "자비스 너 말하는 속도 좀 빠르게 할 수 있어?"
|
|
SPOKEN = "네, 지금 이 속도로 말씀드리고 있습니다. 들어 보시고 더 필요하면 말씀해 주세요."
|
|
DID_IT = "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고"
|
|
REFUSALS = {REFUSAL, REWORDED}
|
|
|
|
|
|
def _content(text: str) -> dict:
|
|
return {"message": {"role": "assistant", "content": text}}
|
|
|
|
|
|
def _run(mock_config, db, dialogue_memory, *, text, chat_replies, done="yes", earlier=True, tool_first=False):
|
|
"""``done``: the judge's word for "the user says they did it"; a reply is judged a refusal when
|
|
it is one of REFUSALS."""
|
|
from jarvis.memory.history import HistoryContext
|
|
from jarvis.reply import engine as engine_mod
|
|
from jarvis.tools.types import ToolExecutionResult
|
|
|
|
mock_config.ollama_chat_model = "gpt-oss:20b"
|
|
mock_config.evaluator_enabled = False
|
|
mock_config.stream_browser = False
|
|
if earlier:
|
|
dialogue_memory.add_message("user", QUESTION)
|
|
dialogue_memory.add_message("assistant", REFUSAL)
|
|
history = HistoryContext(recent=[f"[2026-11-18 18:27] 사금향: {QUESTION}",
|
|
f"[2026-11-18 18:27] 자비스: {REFUSAL}"] if earlier else [])
|
|
seen: list[list[dict]] = []
|
|
replies = iter(chat_replies)
|
|
judged: list[str] = []
|
|
|
|
def fake_chat(*args, **kwargs):
|
|
seen.append([dict(m) for m in (kwargs.get("messages") or args[2])])
|
|
return next(replies)
|
|
|
|
def fake_judge(base_url, model, system, user_content, **kwargs):
|
|
judged.append(user_content)
|
|
if user_content.startswith("<utterance>"):
|
|
return done
|
|
return "yes" if user_content.removeprefix("<reply>").removesuffix("</reply>") in REFUSALS else "no"
|
|
|
|
def fake_tool(db, cfg, tool_name, tool_args, **kwargs):
|
|
return ToolExecutionResult(success=True, reply_text="ok", error_message=None)
|
|
|
|
routed = ["getWeather", "stop"] if tool_first else ["stop"]
|
|
with patch.object(engine_mod, "run_tool_with_retries", side_effect=fake_tool), \
|
|
patch.object(engine_mod, "chat_with_messages", side_effect=fake_chat), \
|
|
patch.object(engine_mod, "select_tools", return_value=routed), \
|
|
patch.object(engine_mod, "plan_query", return_value=["Reply to the user."]), \
|
|
patch("jarvis.reply.done_guard.call_llm_direct", side_effect=fake_judge):
|
|
reply = engine_mod.run_reply_engine(
|
|
db=db, cfg=mock_config, tts=None, text=text, dialogue_memory=dialogue_memory,
|
|
history=history, legacy_memory=False, intent_judge=True)
|
|
return reply, seen, judged
|
|
|
|
|
|
def _said(messages: list[dict]) -> str:
|
|
return "\n".join(str(m.get("content") or "") for m in messages)
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("draft", [REFUSAL, REWORDED], ids=["same words", "reworded"])
|
|
def test_a_refusal_after_the_user_did_it_is_answered_again_without_earlier_answers(
|
|
mock_config, db, dialogue_memory, draft):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[_content(draft), _content(SPOKEN)])
|
|
assert reply == SPOKEN
|
|
assert DID_IT in judged[0]
|
|
assert len(seen) == 2
|
|
assert REFUSAL in _said(seen[0])
|
|
# The second try sees neither the earlier answer nor its history line; the user's question stays.
|
|
assert REFUSAL not in _said(seen[1])
|
|
assert QUESTION in _said(seen[1])
|
|
# The turn is kept once, with the new answer.
|
|
kept = [m["content"] for m in dialogue_memory.get_recent_messages()]
|
|
assert kept.count(DID_IT) == 1 and kept[-1] == SPOKEN
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_repeat_the_user_asked_for_is_kept(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text="방금 뭐라고 했어? 다시 말해 줘",
|
|
chat_replies=[_content(REFUSAL)], done="no")
|
|
assert reply == REFUSAL
|
|
assert len(judged) == 1 and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_an_answer_that_does_it_is_kept(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(SPOKEN)])
|
|
assert reply == SPOKEN
|
|
assert len(judged) == 2 and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_no_earlier_answer_is_not_judged(mock_config, db, dialogue_memory):
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
|
|
earlier=False)
|
|
assert reply == REFUSAL
|
|
assert judged == [] and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize("done", [None, "maybe"])
|
|
def test_no_usable_judgement_keeps_the_answer(mock_config, db, dialogue_memory, done):
|
|
reply, seen, _ = _run(mock_config, db, dialogue_memory, text=DID_IT, chat_replies=[_content(REFUSAL)],
|
|
done=done)
|
|
assert reply == REFUSAL and len(seen) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_a_turn_that_ran_a_tool_is_not_made_again(mock_config, db, dialogue_memory):
|
|
"""A second try would run the tool (and anything it does) twice."""
|
|
call = {"message": {"role": "assistant", "content": "", "tool_calls": [
|
|
{"id": "call_1", "type": "function", "function": {"name": "getWeather", "arguments": {}}}]}}
|
|
reply, seen, judged = _run(mock_config, db, dialogue_memory, text=DID_IT,
|
|
chat_replies=[call, _content(REFUSAL)], tool_first=True)
|
|
assert reply == REFUSAL
|
|
assert judged == [] and len(seen) == 2
|