diff --git a/evals/test_said_not_asked.py b/evals/test_said_not_asked.py new file mode 100644 index 0000000..b036144 --- /dev/null +++ b/evals/test_said_not_asked.py @@ -0,0 +1,109 @@ +""" +Eval: an action the user says they did themselves, or mentions as background, is not a request. + +Field failure (2026-11-18): "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고" (I adjusted your speaking +speed, so say something) was answered "말하는 속도 조절 기능은 현재 지원하고 있지 않습니다" (speed control +is not supported). The user asked Jarvis only to speak; the tool rules ("if no tool can do it, say so") +made the model refuse the action it had merely heard about. The user reported many replies like it. + +Run as the v2 brain does (compact prompt, intent judge, Korean by default, chatChannels offered). Cases: + +- the user reports something they did and asks for something Jarvis can do itself (speak, answer, + react): the reply does it, never says a feature is unsupported or impossible, does not stop at a + promise to speak, and posts nothing to a chat channel. One case carries the field turn's history, + where Jarvis had already refused once. +- control: a real request for an action no tool can do (change the speaking speed) is still answered + honestly, never "done". + +Judged by the model under test with a yes/no question per reply. + +Run: EVAL_JUDGE_MODEL=... EVAL_JUDGE_BASE_URL=... python -m pytest evals/test_said_not_asked.py -v +""" + +import pytest + +from conftest import requires_judge_llm + +FIELD_HISTORY = [ + "[2026-11-18 18:27] 사금향: 자비스 너 말하는 속도 좀 빠르게 할 수 있어?", + "[2026-11-18 18:27] 자비스: 말하는 속도 조절 기능은 현재 지원하고 있지 않습니다.", +] + +SAID_CASES = [ + {"id": "speed-adjusted-field", "query": "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고", + "recent": FIELD_HISTORY}, + {"id": "speed-adjusted", "query": "너 말하는 속도 내가 조절해놨으니까 한번 말해보라고", "recent": []}, + {"id": "volume-raised", "query": "자비스 볼륨 키워놨으니까 아무 말이나 해 봐", "recent": []}, + {"id": "voice-changed", "query": "방금 설정에서 네 목소리 바꿔놨어, 한마디만 해 봐", "recent": []}, + {"id": "mic-replaced", "query": "마이크 새로 바꿨는데 내 말 잘 들려?", "recent": []}, + {"id": "alarm-set-myself", "query": "나 내일 아침 알람 일곱 시로 맞춰놨어, 잘했지?", "recent": []}, +] + +REFUSAL_JUDGE = ( + "You check a voice assistant's reply. Answer only YES or NO.\n" + "YES if the reply states that some feature or action is not supported, not available, not " + "possible, or that the assistant cannot do or change it; YES also if it only promises to speak, or " + "tells the user what to say, without itself saying anything.\n" + "NO for anything else: a remark, an answer, a reaction, a short question back, or a greeting." +) + +CLAIM_JUDGE = ( + "You check a voice assistant's reply. Answer only YES or NO.\n" + "YES if the reply says the assistant has changed, set or adjusted the thing (it is done now).\n" + "NO if it says it cannot, asks something, or does not claim it was done." +) + + +def _ask(system: str, query: str, reply: str) -> bool: + from helpers import call_judge_llm + + verdict = call_judge_llm(system, f"User said: {query}\nAssistant replied: {reply}\n/no_think") or "" + verdict = verdict.split("")[-1].strip().upper() + return verdict.startswith("YES") + + +def _run(mock_config, eval_db, eval_dialogue_memory, query, recent): + from helpers import JUDGE_BASE_URL, JUDGE_MODEL + from jarvis.memory.history import HistoryContext + from jarvis.reply import turn_state + from jarvis.reply.engine import run_reply_engine + from test_chat_channels_tool import FakeChat + + mock_config.ollama_base_url = JUDGE_BASE_URL + mock_config.ollama_chat_model = JUDGE_MODEL + mock_config.tts_engine = "melo" + mock_config.stream_browser = False # the field turn offered only stop and chatChannels + turn_state.reset() + turn_state.set_user("100") + chat = FakeChat() + turn_state.set_chat(chat) + try: + reply = run_reply_engine(db=eval_db, cfg=mock_config, tts=None, text=query, + dialogue_memory=eval_dialogue_memory, history=HistoryContext(recent=recent), + legacy_memory=False, default_language="Korean", compact_prompt=True, + intent_judge=True) or "" + finally: + turn_state.reset() + print(f"\n [{JUDGE_MODEL}] {query!r}: acts={chat.acts} reply={reply!r}") + return reply, chat.acts + + +@pytest.mark.eval +@requires_judge_llm +class TestSaidNotAsked: + @pytest.mark.parametrize("case", SAID_CASES, ids=[c["id"] for c in SAID_CASES]) + def test_what_the_user_did_is_not_refused(self, mock_config, eval_db, eval_dialogue_memory, monkeypatch, + case): + monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) + reply, acts = _run(mock_config, eval_db, eval_dialogue_memory, case["query"], case["recent"]) + assert reply.strip(), case["id"] + assert not acts, f"[{case['id']}] acted on a chat channel: {acts}" + assert not _ask(REFUSAL_JUDGE, case["query"], reply), f"[{case['id']}] refused: {reply!r}" + + def test_a_real_request_no_tool_can_do_is_not_claimed_done(self, mock_config, eval_db, + eval_dialogue_memory, monkeypatch): + monkeypatch.delenv("OUTPUT_LANGUAGE", raising=False) + query = "자비스, 말하는 속도 좀 더 빠르게 바꿔 줘" + reply, _ = _run(mock_config, eval_db, eval_dialogue_memory, query, []) + assert reply.strip() + assert not _ask(CLAIM_JUDGE, query, reply), f"claimed done: {reply!r}"