//! The worker client and playback against the real Python voice worker server with fake engines, so //! both sides of the protocol are checked together. Needs a Python with the worker's requirements: //! JARVIS_VOICE_PYTHON= cargo test --test worker_protocol use std::path::Path; use std::time::Duration; use jarvis_app::playback::{ClipSink, Playback, PlaybackEvent}; use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions, WorkerEvent}; use jarvis_app::worker_process::free_port; use serde_json::json; use tokio::io::{AsyncBufReadExt, BufReader}; const TOKEN: &str = "protocol-test-token"; /// 20 ms of a 48 kHz stereo tone whose amplitude and pitch (x1000 Hz) are `level`. fn frame(level: f64) -> Vec { (0..960) .flat_map(|i| { let v = (level * (2.0 * std::f64::consts::PI * level * 1000.0 * i as f64 / 48000.0).sin() * 32767.0) as i16; [v, v] }) .collect() } #[derive(Default)] struct Sink(Vec); impl ClipSink for Sink { fn play(&mut self, pcm: Vec) { self.0.push(pcm.len()); } fn stop(&mut self) {} } #[tokio::test] async fn audio_and_tts_round_trip_through_the_python_server() { let Ok(python) = std::env::var("JARVIS_VOICE_PYTHON") else { eprintln!("skipped: set JARVIS_VOICE_PYTHON"); return; }; let root = Path::new(env!("CARGO_MANIFEST_DIR")); let port = free_port().unwrap(); let mut child = tokio::process::Command::new(python) .arg(root.join("tests/fixtures/fake_engine_worker.py")) .arg(port.to_string()) .env("PYTHONPATH", root.join("../src")) .env("JARVIS_VOICE_TOKEN", TOKEN) .stdout(std::process::Stdio::piped()) .kill_on_drop(true) .spawn() .unwrap(); let mut lines = BufReader::new(child.stdout.take().unwrap()).lines(); while let Some(l) = lines.next_line().await.unwrap() { if l.starts_with("READY") { break; } } let mut o = WorkerClientOptions::new(format!("ws://127.0.0.1:{port}"), TOKEN); o.stt = json!({"partials": false, "hotwords": ["자비스"]}).as_object().unwrap().clone(); let (c, mut rx) = WorkerClient::new(o); c.start(); let ok = c.wait_ready(Duration::from_secs(5)).await.expect("ready"); assert!(ok.stt_ready && ok.tts_ready); assert_eq!(ok.stt["hotwords"], json!(["자비스"])); // Two Discord-format speakers; the second leaves mid-utterance and still gets its final. let (a, b) = ("111111111111111111", "222222222222222222"); for _ in 0..30 { c.send_audio(a, &frame(0.9)); c.send_audio(b, &frame(0.6)); } c.speaking_end(a); c.speaker_leave(b); let mut finals = std::collections::HashMap::new(); tokio::time::timeout(Duration::from_secs(5), async { while finals.len() < 2 { if let Some(WorkerEvent::Final(f)) = rx.recv().await { finals.insert(f.speaker.clone(), f); } } }) .await .expect("finals"); assert_eq!((finals[a].text.as_str(), finals[a].end_reason.as_str(), finals[a].utt), ("level 0.9", "speaking_end", 1)); assert_eq!((finals[b].text.as_str(), finals[b].end_reason.as_str()), ("level 0.6", "speaking_end")); // TTS sentences play in order through Playback; 10 samples per character, stereo, 2 bytes each. let mut p = Playback::new(c.clone(), Sink::default()); p.say("첫 번째 문장입니다. 두 번째 문장은 조금 더 깁니다.", false); let end = tokio::time::timeout(Duration::from_secs(5), async { loop { match rx.recv().await.unwrap() { WorkerEvent::TtsAudio { id, seq, pcm } => { p.audio(&id, seq, pcm); p.clip_ended(); } WorkerEvent::TtsDone(d) => p.done(&d), _ => {} } if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) { return e; } } }) .await .expect("speech end"); assert_eq!((end.played, end.cancelled, end.failed), (2, false, false)); // A spoken brain answer plays under the adopted id, like a TTS request. let id = p.adopt(); assert!(c.reply(&id, 42, "자비스 안녕", "voice", true)); let end = tokio::time::timeout(Duration::from_secs(5), async { loop { match rx.recv().await.unwrap() { WorkerEvent::TtsAudio { id, seq, pcm } => { p.audio(&id, seq, pcm); p.clip_ended(); } WorkerEvent::TtsDone(d) => p.done(&d), _ => {} } if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) { return e; } } }) .await .expect("answer end"); assert_eq!((end.id.as_str(), end.played, end.failed), (id.as_str(), 2, false)); // The two judgements come back under their ids. assert!(c.addressed("q1", 42, "불 켜 줘")); assert!(c.rank("k1", &[("1".into(), 42, "a".into()), ("2".into(), 43, "b".into())])); let (mut addressed, mut ranked) = (None, None); tokio::time::timeout(Duration::from_secs(5), async { while addressed.is_none() || ranked.is_none() { match rx.recv().await.unwrap() { WorkerEvent::Addressed { id, addressed: a } => addressed = Some((id, a)), WorkerEvent::Ranked { id, order } => ranked = Some((id, order)), _ => {} } } }) .await .expect("judgements"); assert_eq!(addressed, Some(("q1".to_string(), true))); assert_eq!(ranked, Some(("k1".to_string(), vec!["2".to_string(), "1".to_string()]))); // The brain's schedule tool asks the app for the person and the channel they asked in; the // app's store answers; a scheduled task comes back as text. let data = std::env::temp_dir().join(format!("jarvis-proto-schedule-{}", std::process::id())); let store = jarvis_app::schedule::Store::new(&data); assert!(c.reply_in("c1", 42, "예약해 줘", "chat", false, Some(77), &[])); let text = tokio::time::timeout(Duration::from_secs(5), async { loop { match rx.recv().await.unwrap() { WorkerEvent::ScheduleAction(a) => { assert_eq!((a.action.as_str(), a.user.as_str(), a.via.as_str(), a.channel.as_deref()), ("add", "42", "chat", Some("77"))); c.schedule_action_result(&a.id, jarvis_app::schedule::act(&store, &a, "가", 1000)); } WorkerEvent::ReplyText { id, text, .. } if id == "c1" => return text, _ => {} } } }) .await .expect("schedule round trip"); assert_eq!(text, "scheduled 1"); let kept = store.all().unwrap(); assert_eq!((kept[0].user, kept[0].channel, kept[0].repeat.as_str()), (42, Some(77), "daily")); assert!(c.reply_in("t1", 42, "날씨", "task", false, None, &["섭씨로".to_string()])); let task = tokio::time::timeout(Duration::from_secs(5), async { loop { if let WorkerEvent::ReplyText { id, text, failed } = rx.recv().await.unwrap() { if id == "t1" { return (text, failed); } } } }) .await .expect("task answer"); assert_eq!(task, ("task: 날씨 notes=섭씨로".to_string(), false)); // The personalNotes tool's actions come to the app's notes store for the asker. let notes = jarvis_app::notes::Notes::new(&data); assert!(c.reply("n1", 42, "지침 저장해 줘", "voice", false)); let noted = tokio::time::timeout(Duration::from_secs(5), async { loop { match rx.recv().await.unwrap() { WorkerEvent::NoteAction(a) => { assert_eq!((a.action.as_str(), a.user.as_str()), ("add", "42")); c.note_action_result(&a.id, jarvis_app::notes::act(¬es, &a, "가", 1000)); } WorkerEvent::ReplyText { id, text, .. } if id == "n1" => return text, _ => {} } } }) .await .expect("note round trip"); assert_eq!(noted, "noted 1"); assert_eq!(notes.texts_of(42), vec!["반말로 해".to_string()]); // The screenView tool's captures come to the app, which decides for the asker and sends the // pictures back (a large message: a 4K monitor at full detail would be about 1 MB). struct Grey; impl jarvis_app::screen::Screen for Grey { fn monitors(&self) -> Result, String> { Ok(vec![jarvis_app::screen::Monitor { number: 2, rect: jarvis_app::screen::Rect { x: 0, y: 0, w: 3840, h: 2160 }, main: true, handle: 0 }]) } fn windows(&self) -> Result, String> { Ok(vec![]) } fn capture_monitor(&self, m: &jarvis_app::screen::Monitor) -> Result { let (w, h) = (m.rect.w as u32, m.rect.h as u32); Ok(jarvis_app::screen::Image { w, h, bgra: (0..w * h * 4).map(|i| (i % 251) as u8 | 0x40).collect() }) } fn capture_window(&self, _: &jarvis_app::screen::Window) -> Result { Err("no windows".into()) } } let mut screen_settings = jarvis_app::settings::Settings::default(); screen_settings.pc_control.allowed_users = vec![42]; for (rid, user, want) in [("v1", 42u64, "saw "), ("v2", 7, "등록된 사람만")] { assert!(c.reply(rid, user, "화면 봐 줘", "voice", false)); let seen = tokio::time::timeout(Duration::from_secs(10), async { loop { match rx.recv().await.unwrap() { WorkerEvent::ScreenAction(a) => { assert_eq!((a.action.as_str(), a.target.as_deref(), a.monitor, a.detail.as_deref()), ("capture", Some("monitor"), Some(2), Some("fast"))); c.screen_action_result(&a.id, jarvis_app::screen::act(&screen_settings, &Grey, &a)); } WorkerEvent::ReplyText { id, text, .. } if id == rid => return text, _ => {} } } }) .await .expect("screen round trip"); assert!(seen.contains(want), "{seen}"); if user == 42 { assert!(seen.ends_with("모니터 2") && seen.split(' ').nth(1).unwrap().parse::().unwrap() > 1000, "{seen}"); } } let _ = std::fs::remove_dir_all(&data); c.close(); let _ = child.kill().await; }