- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its title or program, or the list of monitors and windows; fast, normal or detailed pictures. - The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen viewing"), decides for the asker (only people registered for PC control), never picks, lists or shows excluded programs (pcControl.screenExclude, password managers by default), refuses minimised windows and black pictures, and records screen.view in the activity log. - Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only: not in the tool carryover, logs or prompt dumps. - The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a game), measured apart for the placement; otherwise screenView is a disabled tool. - Worker: screen_action / screen_action_result, 4 MiB messages. - Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
251 lines
11 KiB
Rust
251 lines
11 KiB
Rust
//! The worker client and playback against the real Python voice worker server with fake engines, so
|
|
//! both sides of the protocol are checked together. Needs a Python with the worker's requirements:
|
|
//! JARVIS_VOICE_PYTHON=<venv python> cargo test --test worker_protocol
|
|
|
|
use std::path::Path;
|
|
use std::time::Duration;
|
|
|
|
use jarvis_app::playback::{ClipSink, Playback, PlaybackEvent};
|
|
use jarvis_app::worker_client::{WorkerClient, WorkerClientOptions, WorkerEvent};
|
|
use jarvis_app::worker_process::free_port;
|
|
use serde_json::json;
|
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
|
|
|
const TOKEN: &str = "protocol-test-token";
|
|
|
|
/// 20 ms of a 48 kHz stereo tone whose amplitude and pitch (x1000 Hz) are `level`.
|
|
fn frame(level: f64) -> Vec<i16> {
|
|
(0..960)
|
|
.flat_map(|i| {
|
|
let v = (level * (2.0 * std::f64::consts::PI * level * 1000.0 * i as f64 / 48000.0).sin() * 32767.0) as i16;
|
|
[v, v]
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct Sink(Vec<usize>);
|
|
impl ClipSink for Sink {
|
|
fn play(&mut self, pcm: Vec<u8>) {
|
|
self.0.push(pcm.len());
|
|
}
|
|
fn stop(&mut self) {}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn audio_and_tts_round_trip_through_the_python_server() {
|
|
let Ok(python) = std::env::var("JARVIS_VOICE_PYTHON") else {
|
|
eprintln!("skipped: set JARVIS_VOICE_PYTHON");
|
|
return;
|
|
};
|
|
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
|
|
let port = free_port().unwrap();
|
|
let mut child = tokio::process::Command::new(python)
|
|
.arg(root.join("tests/fixtures/fake_engine_worker.py"))
|
|
.arg(port.to_string())
|
|
.env("PYTHONPATH", root.join("../src"))
|
|
.env("JARVIS_VOICE_TOKEN", TOKEN)
|
|
.stdout(std::process::Stdio::piped())
|
|
.kill_on_drop(true)
|
|
.spawn()
|
|
.unwrap();
|
|
let mut lines = BufReader::new(child.stdout.take().unwrap()).lines();
|
|
while let Some(l) = lines.next_line().await.unwrap() {
|
|
if l.starts_with("READY") {
|
|
break;
|
|
}
|
|
}
|
|
let mut o = WorkerClientOptions::new(format!("ws://127.0.0.1:{port}"), TOKEN);
|
|
o.stt = json!({"partials": false, "hotwords": ["자비스"]}).as_object().unwrap().clone();
|
|
let (c, mut rx) = WorkerClient::new(o);
|
|
c.start();
|
|
let ok = c.wait_ready(Duration::from_secs(5)).await.expect("ready");
|
|
assert!(ok.stt_ready && ok.tts_ready);
|
|
assert_eq!(ok.stt["hotwords"], json!(["자비스"]));
|
|
|
|
// Two Discord-format speakers; the second leaves mid-utterance and still gets its final.
|
|
let (a, b) = ("111111111111111111", "222222222222222222");
|
|
for _ in 0..30 {
|
|
c.send_audio(a, &frame(0.9));
|
|
c.send_audio(b, &frame(0.6));
|
|
}
|
|
c.speaking_end(a);
|
|
c.speaker_leave(b);
|
|
let mut finals = std::collections::HashMap::new();
|
|
tokio::time::timeout(Duration::from_secs(5), async {
|
|
while finals.len() < 2 {
|
|
if let Some(WorkerEvent::Final(f)) = rx.recv().await {
|
|
finals.insert(f.speaker.clone(), f);
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("finals");
|
|
assert_eq!((finals[a].text.as_str(), finals[a].end_reason.as_str(), finals[a].utt), ("level 0.9", "speaking_end", 1));
|
|
assert_eq!((finals[b].text.as_str(), finals[b].end_reason.as_str()), ("level 0.6", "speaking_end"));
|
|
|
|
// TTS sentences play in order through Playback; 10 samples per character, stereo, 2 bytes each.
|
|
let mut p = Playback::new(c.clone(), Sink::default());
|
|
p.say("첫 번째 문장입니다. 두 번째 문장은 조금 더 깁니다.", false);
|
|
let end = tokio::time::timeout(Duration::from_secs(5), async {
|
|
loop {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::TtsAudio { id, seq, pcm } => {
|
|
p.audio(&id, seq, pcm);
|
|
p.clip_ended();
|
|
}
|
|
WorkerEvent::TtsDone(d) => p.done(&d),
|
|
_ => {}
|
|
}
|
|
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
|
|
return e;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("speech end");
|
|
assert_eq!((end.played, end.cancelled, end.failed), (2, false, false));
|
|
|
|
// A spoken brain answer plays under the adopted id, like a TTS request.
|
|
let id = p.adopt();
|
|
assert!(c.reply(&id, 42, "자비스 안녕", "voice", true));
|
|
let end = tokio::time::timeout(Duration::from_secs(5), async {
|
|
loop {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::TtsAudio { id, seq, pcm } => {
|
|
p.audio(&id, seq, pcm);
|
|
p.clip_ended();
|
|
}
|
|
WorkerEvent::TtsDone(d) => p.done(&d),
|
|
_ => {}
|
|
}
|
|
if let Some(PlaybackEvent::End(e)) = p.take_events().into_iter().find(|e| matches!(e, PlaybackEvent::End(_))) {
|
|
return e;
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("answer end");
|
|
assert_eq!((end.id.as_str(), end.played, end.failed), (id.as_str(), 2, false));
|
|
|
|
// The two judgements come back under their ids.
|
|
assert!(c.addressed("q1", 42, "불 켜 줘"));
|
|
assert!(c.rank("k1", &[("1".into(), 42, "a".into()), ("2".into(), 43, "b".into())]));
|
|
let (mut addressed, mut ranked) = (None, None);
|
|
tokio::time::timeout(Duration::from_secs(5), async {
|
|
while addressed.is_none() || ranked.is_none() {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::Addressed { id, addressed: a } => addressed = Some((id, a)),
|
|
WorkerEvent::Ranked { id, order } => ranked = Some((id, order)),
|
|
_ => {}
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("judgements");
|
|
assert_eq!(addressed, Some(("q1".to_string(), true)));
|
|
assert_eq!(ranked, Some(("k1".to_string(), vec!["2".to_string(), "1".to_string()])));
|
|
|
|
// The brain's schedule tool asks the app for the person and the channel they asked in; the
|
|
// app's store answers; a scheduled task comes back as text.
|
|
let data = std::env::temp_dir().join(format!("jarvis-proto-schedule-{}", std::process::id()));
|
|
let store = jarvis_app::schedule::Store::new(&data);
|
|
assert!(c.reply_in("c1", 42, "예약해 줘", "chat", false, Some(77), &[]));
|
|
let text = tokio::time::timeout(Duration::from_secs(5), async {
|
|
loop {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::ScheduleAction(a) => {
|
|
assert_eq!((a.action.as_str(), a.user.as_str(), a.via.as_str(), a.channel.as_deref()),
|
|
("add", "42", "chat", Some("77")));
|
|
c.schedule_action_result(&a.id, jarvis_app::schedule::act(&store, &a, "가", 1000));
|
|
}
|
|
WorkerEvent::ReplyText { id, text, .. } if id == "c1" => return text,
|
|
_ => {}
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("schedule round trip");
|
|
assert_eq!(text, "scheduled 1");
|
|
let kept = store.all().unwrap();
|
|
assert_eq!((kept[0].user, kept[0].channel, kept[0].repeat.as_str()), (42, Some(77), "daily"));
|
|
assert!(c.reply_in("t1", 42, "날씨", "task", false, None, &["섭씨로".to_string()]));
|
|
let task = tokio::time::timeout(Duration::from_secs(5), async {
|
|
loop {
|
|
if let WorkerEvent::ReplyText { id, text, failed } = rx.recv().await.unwrap() {
|
|
if id == "t1" {
|
|
return (text, failed);
|
|
}
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("task answer");
|
|
assert_eq!(task, ("task: 날씨 notes=섭씨로".to_string(), false));
|
|
// The personalNotes tool's actions come to the app's notes store for the asker.
|
|
let notes = jarvis_app::notes::Notes::new(&data);
|
|
assert!(c.reply("n1", 42, "지침 저장해 줘", "voice", false));
|
|
let noted = tokio::time::timeout(Duration::from_secs(5), async {
|
|
loop {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::NoteAction(a) => {
|
|
assert_eq!((a.action.as_str(), a.user.as_str()), ("add", "42"));
|
|
c.note_action_result(&a.id, jarvis_app::notes::act(¬es, &a, "가", 1000));
|
|
}
|
|
WorkerEvent::ReplyText { id, text, .. } if id == "n1" => return text,
|
|
_ => {}
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("note round trip");
|
|
assert_eq!(noted, "noted 1");
|
|
assert_eq!(notes.texts_of(42), vec!["반말로 해".to_string()]);
|
|
// The screenView tool's captures come to the app, which decides for the asker and sends the
|
|
// pictures back (a large message: a 4K monitor at full detail would be about 1 MB).
|
|
struct Grey;
|
|
impl jarvis_app::screen::Screen for Grey {
|
|
fn monitors(&self) -> Result<Vec<jarvis_app::screen::Monitor>, String> {
|
|
Ok(vec![jarvis_app::screen::Monitor { number: 2, rect: jarvis_app::screen::Rect { x: 0, y: 0, w: 3840, h: 2160 },
|
|
main: true, handle: 0 }])
|
|
}
|
|
fn windows(&self) -> Result<Vec<jarvis_app::screen::Window>, String> {
|
|
Ok(vec![])
|
|
}
|
|
fn capture_monitor(&self, m: &jarvis_app::screen::Monitor) -> Result<jarvis_app::screen::Image, String> {
|
|
let (w, h) = (m.rect.w as u32, m.rect.h as u32);
|
|
Ok(jarvis_app::screen::Image { w, h, bgra: (0..w * h * 4).map(|i| (i % 251) as u8 | 0x40).collect() })
|
|
}
|
|
fn capture_window(&self, _: &jarvis_app::screen::Window) -> Result<jarvis_app::screen::Image, String> {
|
|
Err("no windows".into())
|
|
}
|
|
}
|
|
let mut screen_settings = jarvis_app::settings::Settings::default();
|
|
screen_settings.pc_control.allowed_users = vec![42];
|
|
for (rid, user, want) in [("v1", 42u64, "saw "), ("v2", 7, "등록된 사람만")] {
|
|
assert!(c.reply(rid, user, "화면 봐 줘", "voice", false));
|
|
let seen = tokio::time::timeout(Duration::from_secs(10), async {
|
|
loop {
|
|
match rx.recv().await.unwrap() {
|
|
WorkerEvent::ScreenAction(a) => {
|
|
assert_eq!((a.action.as_str(), a.target.as_deref(), a.monitor, a.detail.as_deref()),
|
|
("capture", Some("monitor"), Some(2), Some("fast")));
|
|
c.screen_action_result(&a.id, jarvis_app::screen::act(&screen_settings, &Grey, &a));
|
|
}
|
|
WorkerEvent::ReplyText { id, text, .. } if id == rid => return text,
|
|
_ => {}
|
|
}
|
|
}
|
|
})
|
|
.await
|
|
.expect("screen round trip");
|
|
assert!(seen.contains(want), "{seen}");
|
|
if user == 42 {
|
|
assert!(seen.ends_with("모니터 2") && seen.split(' ').nth(1).unwrap().parse::<usize>().unwrap() > 1000, "{seen}");
|
|
}
|
|
}
|
|
let _ = std::fs::remove_dir_all(&data);
|
|
c.close();
|
|
let _ = child.kill().await;
|
|
}
|