Files
javis_bot/app/tests/worker_protocol.rs
javis-bot 72518ca50e feat(app): addressed calls follow who is in the voice channel
Alone with Jarvis, whatever the speaker says is answered without asking the judge. With others
present the judge is told how many people are in the channel and how many may call Jarvis: with
one other person most of what the speaker says counts as said to Jarvis (unclear is yes); with
more, or not known, the question is the group one it was before. Live 2026-11-16: alone in the
channel, questions without the wake word got no answer because the judge assumed a group.
Eval (qwen3:8b, pair cases): 3/7 before, 6/7 after; group cases unchanged.

Also: the protocol test fixture's brain takes the trace argument (the test failed since traces).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 15:43:10 +09:00

251 lines
11 KiB
Rust

//! The worker client and playback against the real Python voice worker server with fake engines, so
//! both sides of the protocol are checked together. Needs a Python with the worker's requirements:
//! JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol
use std::path::Path;
use std::time::Duration;
use jarvis_app::playback::{ClipSink, Playback, PlaybackEvent};
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions, WorkerEvent};
use jarvis_app::worker_process::free_port;
use serde_json::json;
use tokio::io::{AsyncBufReadExt, BufReader};
const TOKEN: &str = "protocol-test-token";
/// 20 ms of a 48 kHz stereo tone whose amplitude and pitch (x1000 Hz) are `level`.
fn frame(level: f64) -> Vec<i16> {
(0..960)
.flat_map(|i| {
let v = (level * (2.0 * std::f64::consts::PI * level * 1000.0 * i as f64 / 48000.0).sin() * 32767.0) as i16;
[v, v]
})
.collect()
}
#[derive(Default)]
struct Sink(Vec<usize>);
impl ClipSink for Sink {
fn play(&mut self, pcm: Vec<u8>) {
self.0.push(pcm.len());
}
fn stop(&mut self) {}
}
#[tokio::test]
async fn audio_and_tts_round_trip_through_the_python_server() {
let Ok(python) = std::env::var("JARVIS_VOICE_PYTHON") else {
eprintln!("skipped: set JARVIS_VOICE_PYTHON");
return;
};
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let port = free_port().unwrap();
let mut child = tokio::process::Command::new(python)
.arg(root.join("tests/fixtures/fake_engine_worker.py"))
.arg(port.to_string())
.env("PYTHONPATH", root.join("../src"))
.env("JARVIS_VOICE_TOKEN", TOKEN)
.stdout(std::process::Stdio::piped())
.kill_on_drop(true)
.spawn()
.unwrap();
let mut lines = BufReader::new(child.stdout.take().unwrap()).lines();
while let Some(l) = lines.next_line().await.unwrap() {
if l.starts_with("READY") {
break;
}
}
let mut o = WorkerClientOptions::new(format!("ws://127.0.0.1:{port}"), TOKEN);
o.stt = json!({"partials": false, "hotwords": ["자비스"]}).as_object().unwrap().clone();
let (c, mut rx) = WorkerClient::new(o);
c.start();
let ok = c.wait_ready(Duration::from_secs(5)).await.expect("ready");
assert!(ok.stt_ready && ok.tts_ready);
assert_eq!(ok.stt["hotwords"], json!(["자비스"]));
// Two Discord-format speakers; the second leaves mid-utterance and still gets its final.
let (a, b) = ("111111111111111111", "222222222222222222");
for _ in 0..30 {
c.send_audio(a, &frame(0.9));
c.send_audio(b, &frame(0.6));
}
c.speaking_end(a);
c.speaker_leave(b);
let mut finals = std::collections::HashMap::new();
tokio::time::timeout(Duration::from_secs(5), async {
while finals.len() < 2 {
if let Some(WorkerEvent::Final(f)) = rx.recv().await {
finals.insert(f.speaker.clone(), f);
}
}
})
.await
.expect("finals");
assert_eq!((finals[a].text.as_str(), finals[a].end_reason.as_str(), finals[a].utt), ("level 0.9", "speaking_end", 1));
assert_eq!((finals[b].text.as_str(), finals[b].end_reason.as_str()), ("level 0.6", "speaking_end"));
// TTS sentences play in order through Playback; 10 samples per character, stereo, 2 bytes each.
let mut p = Playback::new(c.clone(), Sink::default());
p.say("첫 번째 문장입니다. 두 번째 문장은 조금 더 깁니다.", false);
let end = tokio::time::timeout(Duration::from_secs(5), async {
loop {
match rx.recv().await.unwrap() {
WorkerEvent::TtsAudio { id, seq, pcm } => {
p.audio(&id, seq, pcm);
p.clip_ended();
}
WorkerEvent::TtsDone(d) => p.done(&d),
_ => {}
}
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
return e;
}
}
})
.await
.expect("speech end");
assert_eq!((end.played, end.cancelled, end.failed), (2, false, false));
// A spoken brain answer plays under the adopted id, like a TTS request.
let id = p.adopt();
assert!(c.reply(&id, 42, "자비스 안녕", "voice", true));
let end = tokio::time::timeout(Duration::from_secs(5), async {
loop {
match rx.recv().await.unwrap() {
WorkerEvent::TtsAudio { id, seq, pcm } => {
p.audio(&id, seq, pcm);
p.clip_ended();
}
WorkerEvent::TtsDone(d) => p.done(&d),
_ => {}
}
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
return e;
}
}
})
.await
.expect("answer end");
assert_eq!((end.id.as_str(), end.played, end.failed), (id.as_str(), 2, false));
// The two judgements come back under their ids.
assert!(c.addressed("q1", 42, "불 켜 줘", 2, 2));
assert!(c.rank("k1", &[("1".into(), 42, "a".into()), ("2".into(), 43, "b".into())]));
let (mut addressed, mut ranked) = (None, None);
tokio::time::timeout(Duration::from_secs(5), async {
while addressed.is_none() || ranked.is_none() {
match rx.recv().await.unwrap() {
WorkerEvent::Addressed { id, addressed: a } => addressed = Some((id, a)),
WorkerEvent::Ranked { id, order } => ranked = Some((id, order)),
_ => {}
}
}
})
.await
.expect("judgements");
assert_eq!(addressed, Some(("q1".to_string(), true)));
assert_eq!(ranked, Some(("k1".to_string(), vec!["2".to_string(), "1".to_string()])));
// The brain's schedule tool asks the app for the person and the channel they asked in; the
// app's store answers; a scheduled task comes back as text.
let data = std::env::temp_dir().join(format!("jarvis-proto-schedule-{}", std::process::id()));
let store = jarvis_app::schedule::Store::new(&data);
assert!(c.reply_in("c1", 42, "예약해 줘", "chat", false, Some(77), &[]));
let text = tokio::time::timeout(Duration::from_secs(5), async {
loop {
match rx.recv().await.unwrap() {
WorkerEvent::ScheduleAction(a) => {
assert_eq!((a.action.as_str(), a.user.as_str(), a.via.as_str(), a.channel.as_deref()),
("add", "42", "chat", Some("77")));
c.schedule_action_result(&a.id, jarvis_app::schedule::act(&store, &a, "가", 1000));
}
WorkerEvent::ReplyText { id, text, .. } if id == "c1" => return text,
_ => {}
}
}
})
.await
.expect("schedule round trip");
assert_eq!(text, "scheduled 1");
let kept = store.all().unwrap();
assert_eq!((kept[0].user, kept[0].channel, kept[0].repeat.as_str()), (42, Some(77), "daily"));
assert!(c.reply_in("t1", 42, "날씨", "task", false, None, &["섭씨로".to_string()]));
let task = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if let WorkerEvent::ReplyText { id, text, failed } = rx.recv().await.unwrap() {
if id == "t1" {
return (text, failed);
}
}
}
})
.await
.expect("task answer");
assert_eq!(task, ("task: 날씨 notes=섭씨로".to_string(), false));
// The personalNotes tool's actions come to the app's notes store for the asker.
let notes = jarvis_app::notes::Notes::new(&data);
assert!(c.reply("n1", 42, "지침 저장해 줘", "voice", false));
let noted = tokio::time::timeout(Duration::from_secs(5), async {
loop {
match rx.recv().await.unwrap() {
WorkerEvent::NoteAction(a) => {
assert_eq!((a.action.as_str(), a.user.as_str()), ("add", "42"));
c.note_action_result(&a.id, jarvis_app::notes::act(&notes, &a, "가", 1000));
}
WorkerEvent::ReplyText { id, text, .. } if id == "n1" => return text,
_ => {}
}
}
})
.await
.expect("note round trip");
assert_eq!(noted, "noted 1");
assert_eq!(notes.texts_of(42), vec!["반말로 해".to_string()]);
// The screenView tool's captures come to the app, which decides for the asker and sends the
// pictures back (a large message: a 4K monitor at full detail would be about 1 MB).
struct Grey;
impl jarvis_app::screen::Screen for Grey {
fn monitors(&self) -> Result<Vec<jarvis_app::screen::Monitor>, String> {
Ok(vec![jarvis_app::screen::Monitor { number: 2, rect: jarvis_app::screen::Rect { x: 0, y: 0, w: 3840, h: 2160 },
main: true, handle: 0 }])
}
fn windows(&self) -> Result<Vec<jarvis_app::screen::Window>, String> {
Ok(vec![])
}
fn capture_monitor(&self, m: &jarvis_app::screen::Monitor) -> Result<jarvis_app::screen::Image, String> {
let (w, h) = (m.rect.w as u32, m.rect.h as u32);
Ok(jarvis_app::screen::Image { w, h, bgra: (0..w * h * 4).map(|i| (i % 251) as u8 | 0x40).collect() })
}
fn capture_window(&self, _: &jarvis_app::screen::Window) -> Result<jarvis_app::screen::Image, String> {
Err("no windows".into())
}
}
let mut screen_settings = jarvis_app::settings::Settings::default();
screen_settings.pc_control.allowed_users = vec![42];
for (rid, user, want) in [("v1", 42u64, "saw "), ("v2", 7, "등록된 사람만")] {
assert!(c.reply(rid, user, "화면 봐 줘", "voice", false));
let seen = tokio::time::timeout(Duration::from_secs(10), async {
loop {
match rx.recv().await.unwrap() {
WorkerEvent::ScreenAction(a) => {
assert_eq!((a.action.as_str(), a.target.as_deref(), a.monitor, a.detail.as_deref()),
("capture", Some("monitor"), Some(2), Some("fast")));
c.screen_action_result(&a.id, jarvis_app::screen::act(&screen_settings, &Grey, &a));
}
WorkerEvent::ReplyText { id, text, .. } if id == rid => return text,
_ => {}
}
}
})
.await
.expect("screen round trip");
assert!(seen.contains(want), "{seen}");
if user == 42 {
assert!(seen.ends_with("모니터 2") && seen.split(' ').nth(1).unwrap().parse::<usize>().unwrap() > 1000, "{seen}");
}
}
let _ = std::fs::remove_dir_all(&data);
c.close();
let _ = child.kill().await;
}