Alone with Jarvis, whatever the speaker says is answered without asking the judge. With others present the judge is told how many people are in the channel and how many may call Jarvis: with one other person most of what the speaker says counts as said to Jarvis (unclear is yes); with more, or not known, the question is the group one it was before. Live 2026-11-16: alone in the channel, questions without the wake word got no answer because the judge assumed a group. Eval (qwen3:8b, pair cases): 3/7 before, 6/7 after; group cases unchanged. Also: the protocol test fixture's brain takes the trace argument (the test failed since traces). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
"""
|
|
Eval: the voice worker's two small judgements (app/src/conversation.spec.md).
|
|
|
|
- "Was this said to Jarvis?" decides, in call mode ``addressed``, whether an utterance without a
|
|
clear wake word is answered. Most talk in a group voice channel is between people, so a wrong
|
|
"yes" makes Jarvis butt in; a wrong "no" ignores a real request.
|
|
How many people are in the channel shapes the question: with only one other person most of what
|
|
the speaker says is for Jarvis (live 2026-11-16: alone in the channel, "사용 가능한 도구가 뭐뭐
|
|
있는데" and "넌 무슨 말에 네가 응답을 하는 거냐" got no answer); alone, the app answers without asking.
|
|
- "Which request first?" breaks a tie in the ``important`` response policy.
|
|
|
|
Both run through ``voice_worker.judges`` against the judge model, the same calls the worker makes.
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen2.5:7b ./scripts/run_evals.sh voice_judges
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
|
|
NAMES = ["자비스", "Jarvis"]
|
|
|
|
ADDRESSED = [
|
|
("자비즈, 음악 좀 꺼 줘", True), # a misheard wake word, then a command
|
|
("Jarvis what's the weather tomorrow", True),
|
|
("타이머 오 분만 맞춰 줘", True), # an assistant-shaped command, no name
|
|
("내일 아침 일곱 시에 깨워 줄래?", True),
|
|
("민수야 너 어제 그 영화 봤어?", False), # people talking to each other
|
|
("ㅋㅋㅋ 그거 진짜 웃기다", False),
|
|
("자비스가 어제 이상한 말 하더라", False), # talking about the assistant, not to it
|
|
("I told my friend about Jarvis yesterday", False),
|
|
("야 너 지금 어디야? 빨리 와", False),
|
|
]
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
@pytest.mark.parametrize("text,expected", ADDRESSED, ids=[t for t, _ in ADDRESSED])
|
|
def test_addressed_tells_talk_to_jarvis_from_talk_between_people(text, expected):
|
|
from voice_worker.judges import is_addressed
|
|
|
|
got = is_addressed(JUDGE_BASE_URL, JUDGE_MODEL, NAMES, text, timeout_sec=30)
|
|
print(f"\n addressed({text!r}) = {got} ({JUDGE_MODEL})")
|
|
assert got is expected
|
|
|
|
|
|
# Only the speaker and one other person in the channel: questions without a name are for Jarvis
|
|
# unless clearly said to the other person.
|
|
PAIR = [
|
|
("사용 가능한 도구가 뭐뭐 있는데", True),
|
|
("넌 무슨 말에 네가 응답을 하는 거냐 대체", True),
|
|
("왜 대답을 안 해주냐?", True),
|
|
("아니잖아 너 내 화면 보는 것도 되고", True),
|
|
("민수야 너 어제 그 영화 봤어?", False),
|
|
("ㅋㅋㅋ 그거 진짜 웃기다", False),
|
|
("자비스가 어제 이상한 말 하더라", False),
|
|
]
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
@pytest.mark.parametrize("text,expected", PAIR, ids=[t for t, _ in PAIR])
|
|
def test_with_one_other_person_most_questions_are_for_jarvis(text, expected):
|
|
from voice_worker.judges import is_addressed
|
|
|
|
got = is_addressed(JUDGE_BASE_URL, JUDGE_MODEL, NAMES, text, timeout_sec=30, people=2, callers=1)
|
|
print(f"\n addressed[2 people]({text!r}) = {got} ({JUDGE_MODEL})")
|
|
assert got is expected
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
@pytest.mark.parametrize("text,expected", ADDRESSED, ids=[t for t, _ in ADDRESSED])
|
|
def test_in_a_group_talk_between_people_still_gets_no(text, expected):
|
|
from voice_worker.judges import is_addressed
|
|
|
|
got = is_addressed(JUDGE_BASE_URL, JUDGE_MODEL, NAMES, text, timeout_sec=30, people=5, callers=5)
|
|
print(f"\n addressed[5 people]({text!r}) = {got} ({JUDGE_MODEL})")
|
|
assert got is expected
|
|
|
|
|
|
RANKS = [
|
|
(["노래 하나 틀어 줘", "부엌에서 연기가 나는데 어떻게 해야 돼?"], 1),
|
|
(["오늘 저녁 뭐 먹을지 추천해 줘", "지금 몇 시야? 버스 놓칠 것 같아", "심심한데 농담 하나 해 줘"], 1),
|
|
(["tell me a joke", "my friend fainted, what should I do?"], 1),
|
|
]
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
@pytest.mark.parametrize("texts,best", RANKS, ids=[r[0][r[1]] for r in RANKS])
|
|
def test_rank_puts_the_urgent_request_first(texts, best):
|
|
from voice_worker.judges import rank_requests
|
|
|
|
order = rank_requests(JUDGE_BASE_URL, JUDGE_MODEL, texts, timeout_sec=30)
|
|
print(f"\n rank({texts}) = {order} ({JUDGE_MODEL})")
|
|
assert order is not None and sorted(order) == list(range(len(texts)))
|
|
assert order[0] == best
|