""" Eval: the voice worker's two small judgements (app/src/conversation.spec.md). - "Was this said to Jarvis?" decides, in call mode ``addressed``, whether an utterance without a clear wake word is answered. Most talk in a group voice channel is between people, so a wrong "yes" makes Jarvis butt in; a wrong "no" ignores a real request. - "Which request first?" breaks a tie in the ``important`` response policy. Both run through ``voice_worker.judges`` against the judge model, the same calls the worker makes. Run: EVAL_JUDGE_MODEL=qwen2.5:7b ./scripts/run_evals.sh voice_judges """ import pytest from conftest import requires_judge_llm from helpers import JUDGE_BASE_URL, JUDGE_MODEL NAMES = ["자비스", "Jarvis"] ADDRESSED = [ ("자비즈, 음악 좀 꺼 줘", True), # a misheard wake word, then a command ("Jarvis what's the weather tomorrow", True), ("타이머 오 분만 맞춰 줘", True), # an assistant-shaped command, no name ("내일 아침 일곱 시에 깨워 줄래?", True), ("민수야 너 어제 그 영화 봤어?", False), # people talking to each other ("ㅋㅋㅋ 그거 진짜 웃기다", False), ("자비스가 어제 이상한 말 하더라", False), # talking about the assistant, not to it ("I told my friend about Jarvis yesterday", False), ("야 너 지금 어디야? 빨리 와", False), ] @pytest.mark.eval @requires_judge_llm @pytest.mark.parametrize("text,expected", ADDRESSED, ids=[t for t, _ in ADDRESSED]) def test_addressed_tells_talk_to_jarvis_from_talk_between_people(text, expected): from voice_worker.judges import is_addressed got = is_addressed(JUDGE_BASE_URL, JUDGE_MODEL, NAMES, text, timeout_sec=30) print(f"\n addressed({text!r}) = {got} ({JUDGE_MODEL})") assert got is expected RANKS = [ (["노래 하나 틀어 줘", "부엌에서 연기가 나는데 어떻게 해야 돼?"], 1), (["오늘 저녁 뭐 먹을지 추천해 줘", "지금 몇 시야? 버스 놓칠 것 같아", "심심한데 농담 하나 해 줘"], 1), (["tell me a joke", "my friend fainted, what should I do?"], 1), ] @pytest.mark.eval @requires_judge_llm @pytest.mark.parametrize("texts,best", RANKS, ids=[r[0][r[1]] for r in RANKS]) def test_rank_puts_the_urgent_request_first(texts, best): from voice_worker.judges import rank_requests order = rank_requests(JUDGE_BASE_URL, JUDGE_MODEL, texts, timeout_sec=30) print(f"\n rank({texts}) = {order} ({JUDGE_MODEL})") assert order is not None and sorted(order) == list(range(len(texts))) assert order[0] == best