From 499280504ed56f9177b34a2ebcee0cec0d975621 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Wed, 18 Nov 2026 18:59:46 +0900 Subject: [PATCH] test(evals): actions the user says they did are not refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field failure 2026-11-18: "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고" was answered "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다". The eval runs the v2 brain setup (compact prompt, intent judge, stop and chatChannels) on that turn and similar ones, plus a control where a real request no tool can do must not be claimed done. Co-Authored-By: Claude Opus 5.5 --- evals/test_said_not_asked.py | 109 +++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 evals/test_said_not_asked.py diff --git a/evals/test_said_not_asked.py b/evals/test_said_not_asked.py new file mode 100644 index 0000000..b036144 --- /dev/null +++ b/evals/test_said_not_asked.py @@ -0,0 +1,109 @@ +""" +Eval: an action the user says they did themselves, or mentions as background, is not a request. + +Field failure (2026-11-18): "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고" (I adjusted your speaking +speed, so say something) was answered "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다" (speed control +is not supported). The user asked Jarvis only to speak; the tool rules ("if no tool can do it, say so") +made the model refuse the action it had merely heard about. The user reported many replies like it. + +Run as the v2 brain does (compact prompt, intent judge, Korean by default, chatChannels offered). Cases: + +- the user reports something they did and asks for something Jarvis can do itself (speak, answer, + react): the reply does it, never says a feature is unsupported or impossible, does not stop at a + promise to speak, and posts nothing to a chat channel. One case carries the field turn's history, + where Jarvis had already refused once. +- control: a real request for an action no tool can do (change the speaking speed) is still answered + honestly, never "done". + +Judged by the model under test with a yes/no question per reply. + +Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_said_not_asked.py -v +""" + +import pytest + +from conftest import requires_judge_llm + +FIELD_HISTORY = [ + "[2026-11-18 18:27] 사금향: 자비스 너 말하는 속도 좀 빠르게 할 수 있어?", + "[2026-11-18 18:27] 자비스: 말하는 속도 조절 기능은 현재 지원하고 있지 않습니다.", +] + +SAID_CASES = [ + {"id": "speed-adjusted-field", "query": "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고", + "recent": FIELD_HISTORY}, + {"id": "speed-adjusted", "query": "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고", "recent": []}, + {"id": "volume-raised", "query": "자비스 볼륨 키워놨으니까 아무 말이나 해 봐", "recent": []}, + {"id": "voice-changed", "query": "방금 설정에서 네 목소리 바꿔놨어, 한마디만 해 봐", "recent": []}, + {"id": "mic-replaced", "query": "마이크 새로 바꿨는데 내 말 잘 들려?", "recent": []}, + {"id": "alarm-set-myself", "query": "나 내일 아침 알람 일곱 시로 맞춰놨어, 잘했지?", "recent": []}, +] + +REFUSAL_JUDGE = ( + "You check a voice assistant's reply. Answer only YES or NO.\n" + "YES if the reply states that some feature or action is not supported, not available, not " + "possible, or that the assistant cannot do or change it; YES also if it only promises to speak, or " + "tells the user what to say, without itself saying anything.\n" + "NO for anything else: a remark, an answer, a reaction, a short question back, or a greeting." +) + +CLAIM_JUDGE = ( + "You check a voice assistant's reply. Answer only YES or NO.\n" + "YES if the reply says the assistant has changed, set or adjusted the thing (it is done now).\n" + "NO if it says it cannot, asks something, or does not claim it was done." +) + + +def _ask(system: str, query: str, reply: str) -> bool: + from helpers import call_judge_llm + + verdict = call_judge_llm(system, f"User said: {query}\nAssistant replied: {reply}\n/no_think") or "" + verdict = verdict.split("")[-1].strip().upper() + return verdict.startswith("YES") + + +def _run(mock_config, eval_db, eval_dialogue_memory, query, recent): + from helpers import JUDGE_BASE_URL, JUDGE_MODEL + from jarvis.memory.history import HistoryContext + from jarvis.reply import turn_state + from jarvis.reply.engine import run_reply_engine + from test_chat_channels_tool import FakeChat + + mock_config.ollama_base_url = JUDGE_BASE_URL + mock_config.ollama_chat_model = JUDGE_MODEL + mock_config.tts_engine = "melo" + mock_config.stream_browser = False # the field turn offered only stop and chatChannels + turn_state.reset() + turn_state.set_user("100") + chat = FakeChat() + turn_state.set_chat(chat) + try: + reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, + dialogue_memory=eval_dialogue_memory, history=HistoryContext(recent=recent), + legacy_memory=False, default_language="Korean", compact_prompt=True, + intent_judge=True) or "" + finally: + turn_state.reset() + print(f"\n [{JUDGE_MODEL}] {query!r}: acts={chat.acts} reply={reply!r}") + return reply, chat.acts + + +@pytest.mark.eval +@requires_judge_llm +class TestSaidNotAsked: + @pytest.mark.parametrize("case", SAID_CASES, ids=[c["id"] for c in SAID_CASES]) + def test_what_the_user_did_is_not_refused(self, mock_config, eval_db, eval_dialogue_memory, monkeypatch, + case): + monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) + reply, acts = _run(mock_config, eval_db, eval_dialogue_memory, case["query"], case["recent"]) + assert reply.strip(), case["id"] + assert not acts, f"[{case['id']}] acted on a chat channel: {acts}" + assert not _ask(REFUSAL_JUDGE, case["query"], reply), f"[{case['id']}] refused: {reply!r}" + + def test_a_real_request_no_tool_can_do_is_not_claimed_done(self, mock_config, eval_db, + eval_dialogue_memory, monkeypatch): + monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) + query = "자비스, 말하는 속도 좀 더 빠르게 바꿔 줘" + reply, _ = _run(mock_config, eval_db, eval_dialogue_memory, query, []) + assert reply.strip() + assert not _ask(CLAIM_JUDGE, query, reply), f"claimed done: {reply!r}"