//! Screen viewing on a real Windows desktop (admin.spec.md, "Screen viewing"): the monitors and //! windows are found, a window, each monitor and the whole screen are captured as JPEG pictures of //! the expected size, and an excluded program is neither picked nor shown. Runs only on Windows //! with `JARVIS_TEST_SCREEN=1` (it opens Notepad); elsewhere it skips. use std::time::{Duration, Instant}; use base64::Engine as _; use jarvis_app::screen; use jarvis_app::settings::Settings; use jarvis_app::worker_client::ScreenAction; use serde_json::{json, Value}; fn action(v: Value) -> ScreenAction { let mut m = json!({"id": "s1", "user": "42"}); m.as_object_mut().unwrap().extend(v.as_object().unwrap().clone()); serde_json::from_value(m).unwrap() } /// The pictures of an answer, decoded: (width, height, RGB pixels). fn pictures(v: &Value) -> Vec<(u32, u32, Vec)> { v["images"].as_array().unwrap().iter().map(|i| { let bytes = base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap(); let mut d = jpeg_decoder::Decoder::new(&bytes[..]); let px = d.decode().unwrap(); let info = d.info().unwrap(); (info.width as u32, info.height as u32, px) }).collect() } /// The share of near-black pixels in `r` (screen coordinates) of a picture of `area`. fn black_share(pic: &(u32, u32, Vec), area: screen::Rect, r: screen::Rect) -> f64 { let (w, h, px) = pic; let (sx, sy) = (*w as f64 / area.w as f64, *h as f64 / area.h as f64); let (x0, x1) = (((r.x - area.x) as f64 * sx) as u32 + 2, ((r.x + r.w - area.x) as f64 * sx) as u32 - 2); let (y0, y1) = (((r.y - area.y) as f64 * sy) as u32 + 2, ((r.y + r.h - area.y) as f64 * sy) as u32 - 2); let (mut black, mut all) = (0, 0); for y in y0..y1.min(*h) { for x in x0..x1.min(*w) { let o = ((y * w + x) * 3) as usize; all += 1; if px[o] < 30 && px[o + 1] < 30 && px[o + 2] < 30 { black += 1; } } } black as f64 / all.max(1) as f64 } #[test] fn the_real_screen_is_seen_and_excluded_programs_are_not() { if std::env::var("JARVIS_TEST_SCREEN").is_err() { eprintln!("skipped: set JARVIS_TEST_SCREEN=1"); return; } let sys = screen::system_screen().expect("Windows has a screen"); let monitors = sys.monitors().unwrap(); println!("monitors: {:?}", monitors.iter().map(|m| (m.number, m.rect, m.main)).collect::>()); assert!(!monitors.is_empty()); // Closed however the test ends; no inherited output handles, so a waiting shell is not held. struct Closed(std::process::Child); impl Drop for Closed { fn drop(&mut self) { let _ = self.0.kill(); } } let _notepad = Closed(std::process::Command::new("notepad.exe").stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).spawn().unwrap()); let until = Instant::now() + Duration::from_secs(15); let win = loop { if let Some(w) = sys.windows().unwrap().into_iter().find(|w| w.program == "notepad.exe" && !w.secondary) { break w; } assert!(Instant::now() < until, "Notepad's window did not show"); std::thread::sleep(Duration::from_millis(200)); }; std::thread::sleep(Duration::from_secs(1)); println!("notepad: {:?} minimised={}", win.rect, win.minimised); let mut s = Settings::default(); s.pc_control.allowed_users = vec![42]; let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap(); println!("list: {} monitors, {} windows", list["item"]["monitors"].as_array().unwrap().len(), list["item"]["windows"].as_array().unwrap().len()); assert!(list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad")); let t = Instant::now(); let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap(); let pics = pictures(&v); println!("window: {} in {} ms, picture {}x{}", v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1); assert_eq!(v["item"]["what"], "창 notepad"); assert!(pics[0].0 > 100 && pics[0].1 > 100); for m in &monitors { let t = Instant::now(); let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "monitor", "monitor": m.number, "detail": "fast"}))).unwrap(); let pics = pictures(&v); println!("monitor {}: {} in {} ms, picture {}x{}", m.number, v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1); assert_eq!((v["item"]["width"].as_i64(), v["item"]["height"].as_i64()), (Some(m.rect.w as i64), Some(m.rect.h as i64))); assert!(pics[0].0.max(pics[0].1) <= screen::FAST_PX); } let area = screen::Rect { x: monitors.iter().map(|m| m.rect.x).min().unwrap(), y: monitors.iter().map(|m| m.rect.y).min().unwrap(), w: monitors.iter().map(|m| m.rect.x + m.rect.w).max().unwrap() - monitors.iter().map(|m| m.rect.x).min().unwrap(), h: monitors.iter().map(|m| m.rect.y + m.rect.h).max().unwrap() - monitors.iter().map(|m| m.rect.y).min().unwrap() }; let t = Instant::now(); let shown = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap(); let shown_pics = pictures(&shown); println!("screen: {} in {} ms, {} part(s)", shown["item"], t.elapsed().as_millis(), shown_pics.len()); let shown_black = black_share(&shown_pics[0], area, win.rect); println!("notepad area black when shown: {shown_black:.2}"); // Excluded: never picked or listed, and painted black on the screen. s.pc_control.screen_exclude = vec!["notepad".into()]; let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap_err(); println!("excluded window: {e}"); assert!(e.contains("찾지 못했습니다")); let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap(); assert!(!list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad")); let hidden = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap(); let hidden_black = black_share(&pictures(&hidden)[0], area, win.rect); println!("notepad area black when excluded: {hidden_black:.2}"); assert!(hidden_black > 0.99, "the excluded window is black"); assert!(shown_black < 0.5, "Notepad shows when not excluded"); // Someone not registered sees nothing. let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "list", "user": "7"}))).unwrap_err(); assert!(e.contains("등록된 사람만")); }