- conversation.rs: the loop between the core's events and the rest. Finals go through call detection and the response policy; answers are asked of the brain and spoken from the app's queue, an interruption cancels only that answer, a voice stop silences everything, and missed requests are kept without text (last 100). Tested with a fake voice side. - Core: answer(user, text) adopts a playback request the worker speaks under the same id, cancel(id) stops one request, ask_addressed and ask_rank pass the conversation's questions; new SpeakingStopped, Addressed and Ranked events. - Worker client: reply, addressed and rank messages and their results; a reply with no answer, refused or not sent ends its speech as failed. - Voice worker: addressed and rank messages on their own threads, beside a running reply; a spoken reply cancelled while the brain is answering is not spoken. judges.py holds the two short LLM judgements with the heard text fenced as data; eval evals/test_voice_judges.py. - Specs (conversation, discord core, voice worker), docs/llm_contexts.md and the roadmap updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
2.6 KiB
Python
62 lines
2.6 KiB
Python
"""
|
|
Eval: the voice worker's two small judgements (app/src/conversation.spec.md).
|
|
|
|
- "Was this said to Jarvis?" decides, in call mode ``addressed``, whether an utterance without a
|
|
clear wake word is answered. Most talk in a group voice channel is between people, so a wrong
|
|
"yes" makes Jarvis butt in; a wrong "no" ignores a real request.
|
|
- "Which request first?" breaks a tie in the ``important`` response policy.
|
|
|
|
Both run through ``voice_worker.judges`` against the judge model, the same calls the worker makes.
|
|
|
|
Run: EVAL_JUDGE_MODEL=qwen2.5:7b ./scripts/run_evals.sh voice_judges
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from conftest import requires_judge_llm
|
|
from helpers import JUDGE_BASE_URL, JUDGE_MODEL
|
|
|
|
NAMES = ["자비스", "Jarvis"]
|
|
|
|
ADDRESSED = [
|
|
("자비즈, 음악 좀 꺼 줘", True), # a misheard wake word, then a command
|
|
("Jarvis what's the weather tomorrow", True),
|
|
("타이머 오 분만 맞춰 줘", True), # an assistant-shaped command, no name
|
|
("내일 아침 일곱 시에 깨워 줄래?", True),
|
|
("민수야 너 어제 그 영화 봤어?", False), # people talking to each other
|
|
("ㅋㅋㅋ 그거 진짜 웃기다", False),
|
|
("자비스가 어제 이상한 말 하더라", False), # talking about the assistant, not to it
|
|
("I told my friend about Jarvis yesterday", False),
|
|
("야 너 지금 어디야? 빨리 와", False),
|
|
]
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
@pytest.mark.parametrize("text,expected", ADDRESSED, ids=[t for t, _ in ADDRESSED])
|
|
def test_addressed_tells_talk_to_jarvis_from_talk_between_people(text, expected):
|
|
from voice_worker.judges import is_addressed
|
|
|
|
got = is_addressed(JUDGE_BASE_URL, JUDGE_MODEL, NAMES, text, timeout_sec=30)
|
|
print(f"\n addressed({text!r}) = {got} ({JUDGE_MODEL})")
|
|
assert got is expected
|
|
|
|
|
|
RANKS = [
|
|
(["노래 하나 틀어 줘", "부엌에서 연기가 나는데 어떻게 해야 돼?"], 1),
|
|
(["오늘 저녁 뭐 먹을지 추천해 줘", "지금 몇 시야? 버스 놓칠 것 같아", "심심한데 농담 하나 해 줘"], 1),
|
|
(["tell me a joke", "my friend fainted, what should I do?"], 1),
|
|
]
|
|
|
|
|
|
@pytest.mark.eval
|
|
@requires_judge_llm
|
|
@pytest.mark.parametrize("texts,best", RANKS, ids=[r[0][r[1]] for r in RANKS])
|
|
def test_rank_puts_the_urgent_request_first(texts, best):
|
|
from voice_worker.judges import rank_requests
|
|
|
|
order = rank_requests(JUDGE_BASE_URL, JUDGE_MODEL, texts, timeout_sec=30)
|
|
print(f"\n rank({texts}) = {order} ({JUDGE_MODEL})")
|
|
assert order is not None and sorted(order) == list(range(len(texts)))
|
|
assert order[0] == best
|