- screenView tool (screen_view.spec.md): whole screen, monitor by number, window by part of its title or program, or the list of monitors and windows; fast, normal or detailed pictures. - The app captures with Windows Graphics Capture (app/src/screen.rs, admin.spec.md "Screen viewing"), decides for the asker (only people registered for PC control), never picks, lists or shows excluded programs (pcControl.screenExclude, password managers by default), refuses minimised windows and black pictures, and records screen.view in the activity log. - Pictures ride on the tool message (ToolExecutionResult.images); llm.py sends them to OpenAI-compatible servers as image parts after the tool messages. They stay in that reply only: not in the tool carryover, logs or prompt dumps. - The LLM reads pictures only when screen viewing is available (--vision gpu normally, cpu in a game), measured apart for the placement; otherwise screenView is a disabled tool. - Worker: screen_action / screen_action_result, 4 MiB messages. - Eval evals/test_screen_view_tool.py; Windows check tests/screen_windows.rs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
130 lines
6.7 KiB
Rust
130 lines
6.7 KiB
Rust
//! Screen viewing on a real Windows desktop (admin.spec.md, "Screen viewing"): the monitors and
|
|
//! windows are found, a window, each monitor and the whole screen are captured as JPEG pictures of
|
|
//! the expected size, and an excluded program is neither picked nor shown. Runs only on Windows
|
|
//! with `JARVIS_TEST_SCREEN=1` (it opens Notepad); elsewhere it skips.
|
|
|
|
use std::time::{Duration, Instant};
|
|
|
|
use base64::Engine as _;
|
|
use jarvis_app::screen;
|
|
use jarvis_app::settings::Settings;
|
|
use jarvis_app::worker_client::ScreenAction;
|
|
use serde_json::{json, Value};
|
|
|
|
fn action(v: Value) -> ScreenAction {
|
|
let mut m = json!({"id": "s1", "user": "42"});
|
|
m.as_object_mut().unwrap().extend(v.as_object().unwrap().clone());
|
|
serde_json::from_value(m).unwrap()
|
|
}
|
|
|
|
/// The pictures of an answer, decoded: (width, height, RGB pixels).
|
|
fn pictures(v: &Value) -> Vec<(u32, u32, Vec<u8>)> {
|
|
v["images"].as_array().unwrap().iter().map(|i| {
|
|
let bytes = base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap();
|
|
let mut d = jpeg_decoder::Decoder::new(&bytes[..]);
|
|
let px = d.decode().unwrap();
|
|
let info = d.info().unwrap();
|
|
(info.width as u32, info.height as u32, px)
|
|
}).collect()
|
|
}
|
|
|
|
/// The share of near-black pixels in `r` (screen coordinates) of a picture of `area`.
|
|
fn black_share(pic: &(u32, u32, Vec<u8>), area: screen::Rect, r: screen::Rect) -> f64 {
|
|
let (w, h, px) = pic;
|
|
let (sx, sy) = (*w as f64 / area.w as f64, *h as f64 / area.h as f64);
|
|
let (x0, x1) = (((r.x - area.x) as f64 * sx) as u32 + 2, ((r.x + r.w - area.x) as f64 * sx) as u32 - 2);
|
|
let (y0, y1) = (((r.y - area.y) as f64 * sy) as u32 + 2, ((r.y + r.h - area.y) as f64 * sy) as u32 - 2);
|
|
let (mut black, mut all) = (0, 0);
|
|
for y in y0..y1.min(*h) {
|
|
for x in x0..x1.min(*w) {
|
|
let o = ((y * w + x) * 3) as usize;
|
|
all += 1;
|
|
if px[o] < 30 && px[o + 1] < 30 && px[o + 2] < 30 {
|
|
black += 1;
|
|
}
|
|
}
|
|
}
|
|
black as f64 / all.max(1) as f64
|
|
}
|
|
|
|
#[test]
|
|
fn the_real_screen_is_seen_and_excluded_programs_are_not() {
|
|
if std::env::var("JARVIS_TEST_SCREEN").is_err() {
|
|
eprintln!("skipped: set JARVIS_TEST_SCREEN=1");
|
|
return;
|
|
}
|
|
let sys = screen::system_screen().expect("Windows has a screen");
|
|
let monitors = sys.monitors().unwrap();
|
|
println!("monitors: {:?}", monitors.iter().map(|m| (m.number, m.rect, m.main)).collect::<Vec<_>>());
|
|
assert!(!monitors.is_empty());
|
|
// Closed however the test ends; no inherited output handles, so a waiting shell is not held.
|
|
struct Closed(std::process::Child);
|
|
impl Drop for Closed {
|
|
fn drop(&mut self) {
|
|
let _ = self.0.kill();
|
|
}
|
|
}
|
|
let _notepad = Closed(std::process::Command::new("notepad.exe").stdin(std::process::Stdio::null())
|
|
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).spawn().unwrap());
|
|
let until = Instant::now() + Duration::from_secs(15);
|
|
let win = loop {
|
|
if let Some(w) = sys.windows().unwrap().into_iter().find(|w| w.program == "notepad.exe" && !w.secondary) {
|
|
break w;
|
|
}
|
|
assert!(Instant::now() < until, "Notepad's window did not show");
|
|
std::thread::sleep(Duration::from_millis(200));
|
|
};
|
|
std::thread::sleep(Duration::from_secs(1));
|
|
println!("notepad: {:?} minimised={}", win.rect, win.minimised);
|
|
let mut s = Settings::default();
|
|
s.pc_control.allowed_users = vec![42];
|
|
|
|
let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap();
|
|
println!("list: {} monitors, {} windows", list["item"]["monitors"].as_array().unwrap().len(),
|
|
list["item"]["windows"].as_array().unwrap().len());
|
|
assert!(list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad"));
|
|
|
|
let t = Instant::now();
|
|
let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap();
|
|
let pics = pictures(&v);
|
|
println!("window: {} in {} ms, picture {}x{}", v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1);
|
|
assert_eq!(v["item"]["what"], "창 notepad");
|
|
assert!(pics[0].0 > 100 && pics[0].1 > 100);
|
|
|
|
for m in &monitors {
|
|
let t = Instant::now();
|
|
let v = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "monitor", "monitor": m.number, "detail": "fast"}))).unwrap();
|
|
let pics = pictures(&v);
|
|
println!("monitor {}: {} in {} ms, picture {}x{}", m.number, v["item"], t.elapsed().as_millis(), pics[0].0, pics[0].1);
|
|
assert_eq!((v["item"]["width"].as_i64(), v["item"]["height"].as_i64()), (Some(m.rect.w as i64), Some(m.rect.h as i64)));
|
|
assert!(pics[0].0.max(pics[0].1) <= screen::FAST_PX);
|
|
}
|
|
|
|
let area = screen::Rect { x: monitors.iter().map(|m| m.rect.x).min().unwrap(), y: monitors.iter().map(|m| m.rect.y).min().unwrap(),
|
|
w: monitors.iter().map(|m| m.rect.x + m.rect.w).max().unwrap() - monitors.iter().map(|m| m.rect.x).min().unwrap(),
|
|
h: monitors.iter().map(|m| m.rect.y + m.rect.h).max().unwrap() - monitors.iter().map(|m| m.rect.y).min().unwrap() };
|
|
let t = Instant::now();
|
|
let shown = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap();
|
|
let shown_pics = pictures(&shown);
|
|
println!("screen: {} in {} ms, {} part(s)", shown["item"], t.elapsed().as_millis(), shown_pics.len());
|
|
let shown_black = black_share(&shown_pics[0], area, win.rect);
|
|
println!("notepad area black when shown: {shown_black:.2}");
|
|
|
|
// Excluded: never picked or listed, and painted black on the screen.
|
|
s.pc_control.screen_exclude = vec!["notepad".into()];
|
|
let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "window", "window": "notepad"}))).unwrap_err();
|
|
println!("excluded window: {e}");
|
|
assert!(e.contains("찾지 못했습니다"));
|
|
let list = screen::act(&s, sys.as_ref(), &action(json!({"action": "list"}))).unwrap();
|
|
assert!(!list["item"]["windows"].as_array().unwrap().iter().any(|w| w["program"] == "notepad"));
|
|
let hidden = screen::act(&s, sys.as_ref(), &action(json!({"action": "capture", "target": "screen", "detail": "detailed"}))).unwrap();
|
|
let hidden_black = black_share(&pictures(&hidden)[0], area, win.rect);
|
|
println!("notepad area black when excluded: {hidden_black:.2}");
|
|
assert!(hidden_black > 0.99, "the excluded window is black");
|
|
assert!(shown_black < 0.5, "Notepad shows when not excluded");
|
|
|
|
// Someone not registered sees nothing.
|
|
let e = screen::act(&s, sys.as_ref(), &action(json!({"action": "list", "user": "7"}))).unwrap_err();
|
|
assert!(e.contains("등록된 사람만"));
|
|
}
|