//! Screen viewing for the brain's `screenView` tool (admin.spec.md, "Screen viewing"; //! `docs/v2/pc-features.md`, "화면 보기"): who may see, which monitor or window, the pictures it //! gets (scaled, excluded programs painted black, JPEG). The capture itself is Windows Graphics //! Capture (`win`); everything else is plain code the tests drive with a fake screen. use base64::Engine as _; use serde_json::{json, Value}; use crate::permissions::{decide, Action, Decision, Refusal}; use crate::settings::{Level, Settings}; use crate::worker_client::ScreenAction; /// Long side of a `fast` and a `normal` picture, and of each `detailed` part. pub const FAST_PX: u32 = 768; pub const NORMAL_PX: u32 = 1280; /// A picture whose long side is at most this stays one `detailed` part at full size. pub const ONE_PART_MAX_PX: u32 = 1600; pub const JPEG_QUALITY: u8 = 80; /// Pictures per answer (base64 grows them by a third; the worker takes 4 MiB messages). pub const MAX_BYTES: usize = 3 * 1024 * 1024; /// Windows in the list. pub const MAX_WINDOWS: usize = 30; /// Whether the brain gets the tool and the LLM reads pictures: Windows, the feature on, someone /// registered for PC control and viewing not turned off. pub fn available(s: &Settings) -> bool { cfg!(windows) && s.features.screen && !s.pc_control.allowed_users.is_empty() && s.pc_control.view != Level::Never } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Rect { pub x: i32, pub y: i32, pub w: i32, pub h: i32, } #[derive(Debug, Clone, PartialEq)] pub struct Monitor { /// As Windows numbers it (`\\.\DISPLAY`). pub number: u32, pub rect: Rect, pub main: bool, /// The opaque handle the capture takes. pub handle: isize, } /// A visible top-level window, front first. #[derive(Debug, Clone, PartialEq)] pub struct Window { pub handle: isize, /// Program file name, lower case (`chrome.exe`). pub program: String, pub title: String, pub rect: Rect, pub minimised: bool, /// Owned windows and tool windows (pop-ups, palettes): never listed or picked, but painted /// black when their program is excluded. pub secondary: bool, /// This app's own windows. pub own: bool, } /// A captured picture: top-down BGRA rows. #[derive(Debug, Clone, PartialEq)] pub struct Image { pub w: u32, pub h: u32, pub bgra: Vec, } impl Image { pub fn new(w: u32, h: u32) -> Self { Image { w, h, bgra: vec![0; (w * h * 4) as usize] } } } /// What the screen offers; Windows in `win`, a fake in the tests. pub trait Screen { /// The monitors without the bot browser's virtual one. fn monitors(&self) -> Result, String>; fn windows(&self) -> Result, String>; fn capture_monitor(&self, m: &Monitor) -> Result; fn capture_window(&self, w: &Window) -> Result; } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Detail { Fast, Normal, Detailed, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Target { Screen, Monitor(u32), Window(String), } /// The activity log's target: never a title. pub fn activity_target(t: &Target) -> String { match t { Target::Screen => "전체 화면".into(), Target::Monitor(n) => format!("모니터 {n}"), Target::Window(_) => "창".into(), } } /// The request in a `capture` action. pub fn target_of(a: &ScreenAction) -> Result<(Target, Detail), String> { let detail = match a.detail.as_deref() { Some("fast") => Detail::Fast, Some("detailed") => Detail::Detailed, _ => Detail::Normal, }; let target = match a.target.as_deref() { Some("monitor") => Target::Monitor(a.monitor.filter(|n| *n >= 1).ok_or("모니터 번호가 필요합니다")?), Some("window") => { let w = a.window.as_deref().map(str::trim).filter(|w| !w.is_empty()).ok_or("창 이름이 필요합니다")?; Target::Window(w.chars().take(200).collect()) } _ => Target::Screen, }; Ok((target, detail)) } /// Why the asker may not see the screen, or `None` when they may. pub fn refusal(s: &Settings, user: &str) -> Option { let Ok(id) = user.parse::() else { return Some(not_registered()) }; match decide(&s.pc_control, id, Action::View) { Decision::Allow => None, Decision::Refuse(Refusal::NotAllowed) => Some(not_registered()), // `ask` needs the next-message consent the PC control tool brings (M7, next step). Decision::Ask => Some("화면 보기가 '먼저 묻기'로 되어 있어 지금은 볼 수 없습니다(PC 제어 화면에서 '자동'으로 바꾸면 됩니다)".into()), Decision::Refuse(_) => Some("화면 보기가 꺼져 있습니다".into()), } } fn not_registered() -> String { "PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다".into() } /// Excluded: on the list, or a program that could not be told (its process refused the query), so /// no excluded program slips through unnamed. fn excluded(s: &Settings, program: &str) -> bool { program.is_empty() || crate::game::listed(program, &s.pc_control.screen_exclude) } /// Where excluded windows show now (minimised ones show nowhere). fn hidden_rects(s: &Settings, all: &[Window]) -> Vec { all.iter().filter(|w| !w.minimised && excluded(s, &w.program)).map(|w| w.rect).collect() } /// Captures of monitors and screens tried while excluded windows keep moving. const STILL_TRIES: usize = 3; /// The windows the list shows and a window request may pick: not secondary, not this app's, not /// excluded; front first. fn listable<'a>(s: &'a Settings, all: &'a [Window]) -> impl Iterator + 'a { all.iter().filter(move |w| !w.secondary && !w.own && !w.title.trim().is_empty() && !excluded(s, &w.program)) } /// The window nearest the front whose title or program name contains `text` (any case). pub fn pick_window<'a>(s: &'a Settings, all: &'a [Window], text: &str) -> Option<&'a Window> { let t = text.to_lowercase(); let stem = |p: &str| p.strip_suffix(".exe").unwrap_or(p).to_owned(); listable(s, all).find(|w| w.title.to_lowercase().contains(&t) || stem(&w.program).contains(&t)) } /// Whether nothing shows (protected video, exclusive full screen): every pixel near black. pub fn all_black(img: &Image) -> bool { img.bgra.chunks_exact(4).all(|p| p[0] <= 8 && p[1] <= 8 && p[2] <= 8) } /// Paint `rects` (screen coordinates) black on a picture of `area`; the picture may have another /// size than `area` (display scaling), so the rectangles are scaled to it. pub fn paint_black(img: &mut Image, area: Rect, rects: &[Rect]) { if area.w <= 0 || area.h <= 0 { return; } let (sx, sy) = (img.w as f64 / area.w as f64, img.h as f64 / area.h as f64); for r in rects { let x0 = (((r.x - area.x) as f64 * sx).floor().max(0.0) as u32).min(img.w); let y0 = (((r.y - area.y) as f64 * sy).floor().max(0.0) as u32).min(img.h); let x1 = (((r.x + r.w - area.x) as f64 * sx).ceil().max(0.0) as u32).min(img.w); let y1 = (((r.y + r.h - area.y) as f64 * sy).ceil().max(0.0) as u32).min(img.h); for y in y0..y1 { let row = (y * img.w) as usize * 4; img.bgra[row + x0 as usize * 4..row + x1 as usize * 4].fill(0); } } } /// `img` scaled down (area average) so its long side is at most `max`; never up. pub fn scale(img: &Image, max: u32) -> Image { let long = img.w.max(img.h); if long <= max || img.w == 0 || img.h == 0 { return img.clone(); } let (w, h) = (((img.w as u64 * max as u64) / long as u64).max(1) as u32, ((img.h as u64 * max as u64) / long as u64).max(1) as u32); let mut out = Image::new(w, h); let (fx, fy) = (img.w as f64 / w as f64, img.h as f64 / h as f64); for oy in 0..h { let (y0, y1) = ((oy as f64 * fy) as u32, (((oy + 1) as f64 * fy).ceil() as u32).min(img.h)); for ox in 0..w { let (x0, x1) = ((ox as f64 * fx) as u32, (((ox + 1) as f64 * fx).ceil() as u32).min(img.w)); let mut sum = [0u32; 3]; for y in y0..y1 { let row = (y * img.w) as usize * 4; for x in x0..x1 { let p = &img.bgra[row + x as usize * 4..]; sum[0] += p[0] as u32; sum[1] += p[1] as u32; sum[2] += p[2] as u32; } } let n = ((y1 - y0) * (x1 - x0)).max(1); let o = ((oy * w + ox) * 4) as usize; out.bgra[o..o + 4].copy_from_slice(&[(sum[0] / n) as u8, (sum[1] / n) as u8, (sum[2] / n) as u8, 255]); } } out } fn crop(img: &Image, x: u32, y: u32, w: u32, h: u32) -> Image { let mut out = Image::new(w, h); for row in 0..h { let src = (((y + row) * img.w + x) * 4) as usize; let dst = (row * w * 4) as usize; out.bgra[dst..dst + (w * 4) as usize].copy_from_slice(&img.bgra[src..src + (w * 4) as usize]); } out } /// The pictures of a request: one scaled picture, or for `detailed` four parts (left to right, /// top to bottom) of at most `NORMAL_PX` each. pub fn parts(img: &Image, detail: Detail) -> Vec { match detail { Detail::Fast => vec![scale(img, FAST_PX)], Detail::Normal => vec![scale(img, NORMAL_PX)], Detail::Detailed if img.w.max(img.h) <= ONE_PART_MAX_PX => vec![img.clone()], Detail::Detailed => { let (hw, hh) = (img.w / 2, img.h / 2); [(0, 0, hw, hh), (hw, 0, img.w - hw, hh), (0, hh, hw, img.h - hh), (hw, hh, img.w - hw, img.h - hh)] .into_iter().map(|(x, y, w, h)| scale(&crop(img, x, y, w, h), NORMAL_PX)).collect() } } } pub fn jpeg(img: &Image) -> Result, String> { let mut out = Vec::new(); jpeg_encoder::Encoder::new(&mut out, JPEG_QUALITY) .encode(&img.bgra, img.w as u16, img.h as u16, jpeg_encoder::ColorType::Bgra) .map_err(|e| format!("그림을 만들지 못했습니다: {e}"))?; Ok(out) } fn bounds(ms: &[Monitor]) -> Rect { let x0 = ms.iter().map(|m| m.rect.x).min().unwrap_or(0); let y0 = ms.iter().map(|m| m.rect.y).min().unwrap_or(0); let x1 = ms.iter().map(|m| m.rect.x + m.rect.w).max().unwrap_or(0); let y1 = ms.iter().map(|m| m.rect.y + m.rect.h).max().unwrap_or(0); Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 } } /// One picture of `ms` as they are placed; each monitor's capture is fitted to its place. fn whole(screen: &dyn Screen, ms: &[Monitor], excluded_rects: &[Rect]) -> Result<(Image, bool), String> { let b = bounds(ms); let mut out = Image::new(b.w.max(1) as u32, b.h.max(1) as u32); let mut black = true; for m in ms { let shot = screen.capture_monitor(m)?; black &= all_black(&shot); let mut fit = if shot.w as i32 == m.rect.w && shot.h as i32 == m.rect.h { shot } else { stretch(&shot, m.rect) }; paint_black(&mut fit, m.rect, excluded_rects); let (ox, oy) = ((m.rect.x - b.x) as u32, (m.rect.y - b.y) as u32); for row in 0..fit.h.min(out.h - oy) { let w = fit.w.min(out.w - ox) as usize * 4; let src = (row * fit.w * 4) as usize; let dst = (((oy + row) * out.w + ox) * 4) as usize; out.bgra[dst..dst + w].copy_from_slice(&fit.bgra[src..src + w]); } } Ok((out, black)) } /// `img` resized to exactly `r`'s size (nearest pixel): monitors whose capture differs from their /// placed size (display scaling) still fill their place. fn stretch(img: &Image, r: Rect) -> Image { let (w, h) = (r.w.max(1) as u32, r.h.max(1) as u32); let mut out = Image::new(w, h); for y in 0..h { let sy = (y as u64 * img.h as u64 / h as u64) as u32; for x in 0..w { let sx = (x as u64 * img.w as u64 / w as u64) as u32; let s = ((sy * img.w + sx) * 4) as usize; let d = ((y * w + x) * 4) as usize; out.bgra[d..d + 4].copy_from_slice(&img.bgra[s..s + 4]); } } out } /// Answer one `screen_action` for its asker: `{item, images}` or why not. pub fn act(s: &Settings, screen: &dyn Screen, a: &ScreenAction) -> Result { if let Some(why) = refusal(s, &a.user) { return Err(why); } match a.action.as_str() { "list" => { let monitors: Vec = screen.monitors()?.iter().map(|m| json!({"number": m.number, "width": m.rect.w, "height": m.rect.h, "main": m.main})).collect(); let all = screen.windows()?; let windows: Vec = listable(s, &all).take(MAX_WINDOWS).map(|w| json!({ "program": w.program.strip_suffix(".exe").unwrap_or(&w.program), "title": w.title, "minimised": w.minimised})).collect(); Ok(json!({"item": {"monitors": monitors, "windows": windows}})) } "capture" => { let (target, detail) = target_of(a)?; let (img, black, what) = match &target { Target::Window(text) => { let all = screen.windows()?; let w = pick_window(s, &all, text).ok_or("그런 창을 찾지 못했습니다")?; if w.minimised { return Err("그 창은 최소화되어 있어 볼 수 없습니다".into()); } let img = screen.capture_window(w)?; let black = all_black(&img); (img, black, format!("창 {}", w.program.strip_suffix(".exe").unwrap_or(&w.program))) } _ => { let ms = screen.monitors()?; let chosen: Vec = match target { Target::Monitor(n) => vec![ms.iter().find(|m| m.number == n).cloned() .ok_or_else(|| format!("{n}번 모니터가 없습니다 (있는 모니터: {})", ms.iter().map(|m| m.number.to_string()).collect::>().join(", ")))?], _ => ms, }; if chosen.is_empty() { return Err("모니터를 찾지 못했습니다".into()); } // Excluded windows are read before and after the capture; if one moved, was // brought back or opened meanwhile, the black areas may miss it: that capture // is thrown away. let mut tries = 0; let (img, black) = loop { let before = hidden_rects(s, &screen.windows()?); let shot = whole(screen, &chosen, &before)?; if hidden_rects(s, &screen.windows()?) == before { break shot; } tries += 1; if tries == STILL_TRIES { return Err("보면 안 되는 창이 움직이는 중이라 지금은 볼 수 없습니다".into()); } }; (img, black, activity_target(&target)) } }; if black { return Err("화면이 검게 나와 볼 수 없습니다 (보호된 영상이나 전체 화면 게임일 수 있습니다)".into()); } let pics = parts(&img, detail); let mut images = Vec::with_capacity(pics.len()); let mut total = 0; for p in &pics { let j = jpeg(p)?; total += j.len(); images.push(base64::engine::general_purpose::STANDARD.encode(j)); } if total > MAX_BYTES { return Err("그림이 너무 커서 보낼 수 없습니다 (화질을 낮춰 다시 보세요)".into()); } Ok(json!({"item": {"what": what, "width": img.w, "height": img.h, "parts": images.len()}, "images": images})) } other => Err(format!("모르는 화면 동작입니다: {}", other.chars().take(20).collect::())), } } /// The real screen (Windows), or none elsewhere. pub fn system_screen() -> Option> { #[cfg(windows)] return Some(Box::new(win::System)); #[cfg(not(windows))] None } #[cfg(windows)] mod win { use super::{Image, Monitor, Rect, Screen, Window}; use std::time::{Duration, Instant}; use windows::core::Interface; use windows::Graphics::Capture::{Direct3D11CaptureFramePool, GraphicsCaptureItem}; use windows::Graphics::DirectX::Direct3D11::IDirect3DDevice; use windows::Graphics::DirectX::DirectXPixelFormat; use windows::Win32::Foundation::{HMODULE, HWND}; use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}; use windows::Win32::Graphics::Direct3D11::*; use windows::Win32::Graphics::Dxgi::IDXGIDevice; use windows::Win32::Graphics::Gdi::HMONITOR; use windows::Win32::System::WinRT::Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess}; use windows::Win32::System::WinRT::Graphics::Capture::IGraphicsCaptureItemInterop; use windows::Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED}; use windows_sys::core::BOOL; use windows_sys::Win32::Foundation::{LPARAM, RECT}; use windows_sys::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED}; use windows_sys::Win32::Graphics::Gdi::{EnumDisplayMonitors, GetMonitorInfoW, HDC, MONITORINFOEXW}; use windows_sys::Win32::UI::WindowsAndMessaging::*; /// The first frame of a capture. const FRAME_TIMEOUT: Duration = Duration::from_secs(2); pub struct System; fn text(buf: &[u16]) -> String { String::from_utf16_lossy(&buf[..buf.iter().position(|&c| c == 0).unwrap_or(buf.len())]) } unsafe extern "system" fn add_monitor(h: windows_sys::Win32::Graphics::Gdi::HMONITOR, _: HDC, _: *mut RECT, lp: LPARAM) -> BOOL { // SAFETY: `lp` is the Vec `monitors` passed, alive for the whole enumeration. let out = unsafe { &mut *(lp as *mut Vec<(isize, String, Rect, bool)>) }; let mut mi: MONITORINFOEXW = unsafe { std::mem::zeroed() }; mi.monitorInfo.cbSize = std::mem::size_of::() as u32; // SAFETY: `mi` is a MONITORINFOEXW with its size set. if unsafe { GetMonitorInfoW(h, &mut mi as *mut MONITORINFOEXW as *mut _) } != 0 { let r = mi.monitorInfo.rcMonitor; out.push((h as isize, text(&mi.szDevice), Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top }, mi.monitorInfo.dwFlags & MONITORINFOF_PRIMARY != 0)); } 1 } unsafe extern "system" fn add_window(hwnd: windows_sys::Win32::Foundation::HWND, lp: LPARAM) -> BOOL { // SAFETY: `lp` is the Vec `windows` passed, alive for the whole enumeration. let out = unsafe { &mut *(lp as *mut Vec) }; out.push(hwnd); 1 } fn window(hwnd: windows_sys::Win32::Foundation::HWND, own: u32) -> Option { // SAFETY: plain queries on a window handle from EnumWindows; a stale one only fails them. unsafe { if IsWindowVisible(hwnd) == 0 { return None; } let mut cloaked: u32 = 0; DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED as u32, (&mut cloaked as *mut u32).cast(), 4); if cloaked != 0 { return None; } let mut buf = [0u16; 512]; let n = GetWindowTextW(hwnd, buf.as_mut_ptr(), buf.len() as i32).max(0) as usize; let mut pid = 0; GetWindowThreadProcessId(hwnd, &mut pid); let mut r: RECT = std::mem::zeroed(); GetWindowRect(hwnd, &mut r); let ex = GetWindowLongPtrW(hwnd, GWL_EXSTYLE) as u32; let program = crate::game::probe::image_path(pid).map(|p| crate::game::file_name(&p)).unwrap_or_default(); Some(Window { handle: hwnd as isize, program, title: String::from_utf16_lossy(&buf[..n]), rect: Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top }, minimised: IsIconic(hwnd) != 0, secondary: !GetWindow(hwnd, GW_OWNER).is_null() || ex & WS_EX_TOOLWINDOW != 0, own: pid == own }) } } fn device() -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext, IDirect3DDevice)> { let make = |kind: D3D_DRIVER_TYPE| -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext)> { let (mut d, mut c) = (None, None); // SAFETY: out pointers to locals; no adapter, no software module. unsafe { D3D11CreateDevice(None, kind, HMODULE::default(), D3D11_CREATE_DEVICE_BGRA_SUPPORT, None, D3D11_SDK_VERSION, Some(&mut d), None, Some(&mut c))? }; Ok((d.unwrap(), c.unwrap())) }; // Without a graphics card (a virtual machine) WARP still captures. let (d, c) = make(D3D_DRIVER_TYPE_HARDWARE).or_else(|_| make(D3D_DRIVER_TYPE_WARP))?; let dxgi: IDXGIDevice = d.cast()?; // SAFETY: a valid DXGI device. let winrt: IDirect3DDevice = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi)? }.cast()?; Ok((d, c, winrt)) } fn capture(item: GraphicsCaptureItem) -> Result { let e = |e: windows::core::Error| format!("화면을 캡처하지 못했습니다 ({:#x})", e.code().0); // SAFETY: a second call on a thread already initialised only reports it. let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; let (dev, ctx, winrt) = device().map_err(e)?; let size = item.Size().map_err(e)?; let pool = Direct3D11CaptureFramePool::CreateFreeThreaded(&winrt, DirectXPixelFormat::B8G8R8A8UIntNormalized, 1, size) .map_err(e)?; let session = pool.CreateCaptureSession(&item).map_err(e)?; let _ = session.SetIsCursorCaptureEnabled(false); // Windows 11 lets the app leave out the yellow border; older Windows refuses, which is fine. let _ = session.SetIsBorderRequired(false); session.StartCapture().map_err(e)?; let started = Instant::now(); let frame = loop { if let Ok(f) = pool.TryGetNextFrame() { break f; } if started.elapsed() > FRAME_TIMEOUT { let _ = session.Close(); let _ = pool.Close(); return Err("화면이 제때 오지 않았습니다".into()); } std::thread::sleep(Duration::from_millis(5)); }; let result = (|| -> windows::core::Result { let content = frame.ContentSize()?; let access: IDirect3DDxgiInterfaceAccess = frame.Surface()?.cast()?; // SAFETY: the frame's surface is a D3D11 texture. let tex: ID3D11Texture2D = unsafe { access.GetInterface()? }; let mut desc = D3D11_TEXTURE2D_DESC::default(); // SAFETY: `desc` is the out parameter. unsafe { tex.GetDesc(&mut desc) }; let (w, h) = ((content.Width.max(1) as u32).min(desc.Width), (content.Height.max(1) as u32).min(desc.Height)); desc.Usage = D3D11_USAGE_STAGING; desc.BindFlags = 0; desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; desc.MiscFlags = 0; let mut staging = None; // SAFETY: a staging copy of the frame's texture, read on this thread only. unsafe { dev.CreateTexture2D(&desc, None, Some(&mut staging))?; let staging = staging.unwrap(); ctx.CopyResource(&staging, &tex); let mut map = D3D11_MAPPED_SUBRESOURCE::default(); ctx.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut map))?; let mut img = Image::new(w, h); for y in 0..h as usize { let src = std::slice::from_raw_parts((map.pData as *const u8).add(y * map.RowPitch as usize), w as usize * 4); img.bgra[y * w as usize * 4..(y + 1) * w as usize * 4].copy_from_slice(src); } ctx.Unmap(&staging, 0); Ok(img) } })(); let _ = frame.Close(); let _ = session.Close(); let _ = pool.Close(); result.map_err(e) } fn interop() -> Result { windows::core::factory::() .map_err(|e| format!("화면 캡처를 쓸 수 없습니다 ({:#x})", e.code().0)) } impl Screen for System { fn monitors(&self) -> Result, String> { let mut found: Vec<(isize, String, Rect, bool)> = vec![]; // SAFETY: the callback only pushes into `found`, which outlives the call. unsafe { EnumDisplayMonitors(std::ptr::null_mut(), std::ptr::null(), Some(add_monitor), &mut found as *mut _ as LPARAM) }; let bot = crate::vdisplay::virtual_display_name(); Ok(found.into_iter().filter(|(_, name, _, _)| Some(name) != bot.as_ref()).map(|(h, name, rect, main)| Monitor { number: name.trim_start_matches(r"\\.\DISPLAY").parse().unwrap_or(0), rect, main, handle: h }).collect()) } fn windows(&self) -> Result, String> { let mut hwnds = vec![]; // SAFETY: the callback only pushes into `hwnds`, which outlives the call. unsafe { EnumWindows(Some(add_window), &mut hwnds as *mut _ as LPARAM) }; let own = std::process::id(); Ok(hwnds.into_iter().filter_map(|h| window(h, own)).collect()) } fn capture_monitor(&self, m: &Monitor) -> Result { // SAFETY: a monitor handle from EnumDisplayMonitors. let item = unsafe { interop()?.CreateForMonitor::(HMONITOR(m.handle as *mut _)) } .map_err(|e| format!("모니터를 캡처하지 못했습니다 ({:#x})", e.code().0))?; capture(item) } fn capture_window(&self, w: &Window) -> Result { // SAFETY: a window handle from EnumWindows; a closed window only fails the call. let item = unsafe { interop()?.CreateForWindow::(HWND(w.handle as *mut _)) } .map_err(|e| format!("창을 캡처하지 못했습니다 ({:#x})", e.code().0))?; capture(item) } } } #[cfg(test)] mod tests { use super::*; use std::cell::RefCell; const OWNER: &str = "123456789012345678"; fn settings() -> Settings { let mut s = Settings::default(); s.pc_control.allowed_users = vec![OWNER.parse().unwrap()]; s } fn solid(w: u32, h: u32, v: u8) -> Image { Image { w, h, bgra: [v, v, v, 255].repeat((w * h) as usize) } } struct Fake { monitors: Vec, windows: Vec, /// What the next reads of the windows give, one each, before `windows` again: a window /// that moves while the screen is captured. later: RefCell>>, shot: Image, captured: RefCell>, } impl Screen for Fake { fn monitors(&self) -> Result, String> { Ok(self.monitors.clone()) } fn windows(&self) -> Result, String> { let mut later = self.later.borrow_mut(); Ok(if later.is_empty() { self.windows.clone() } else { later.remove(0) }) } fn capture_monitor(&self, m: &Monitor) -> Result { self.captured.borrow_mut().push(format!("monitor {}", m.number)); Ok(solid(m.rect.w as u32, m.rect.h as u32, 200)) } fn capture_window(&self, w: &Window) -> Result { self.captured.borrow_mut().push(format!("window {}", w.program)); Ok(self.shot.clone()) } } fn win(program: &str, title: &str, rect: Rect) -> Window { Window { handle: 1, program: program.into(), title: title.into(), rect, minimised: false, secondary: false, own: false } } fn fake() -> Fake { Fake { monitors: vec![ Monitor { number: 1, rect: Rect { x: 0, y: 0, w: 400, h: 300 }, main: true, handle: 1 }, Monitor { number: 2, rect: Rect { x: 400, y: 0, w: 200, h: 100 }, main: false, handle: 2 }, ], windows: vec![ win("keepassxc.exe", "Passwords", Rect { x: 10, y: 10, w: 100, h: 50 }), win("chrome.exe", "메일 - Chrome", Rect { x: 0, y: 0, w: 300, h: 200 }), Window { secondary: true, ..win("chrome.exe", "pop-up", Rect { x: 0, y: 0, w: 10, h: 10 }) }, win("notepad.exe", "메모", Rect { x: 0, y: 0, w: 50, h: 50 }), Window { own: true, ..win("jarvis.exe", "Jarvis 관리", Rect { x: 0, y: 0, w: 50, h: 50 }) }, ], later: RefCell::new(vec![]), shot: solid(800, 600, 120), captured: RefCell::new(vec![]), } } fn action(action: &str, user: &str, extra: Value) -> ScreenAction { let mut v = json!({"id": "s1", "action": action, "user": user}); v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); serde_json::from_value(v).unwrap() } fn decode(v: &Value) -> Vec> { v["images"].as_array().unwrap().iter() .map(|i| base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap()).collect() } #[test] fn only_registered_people_see_the_screen_and_only_when_viewing_is_automatic() { let f = fake(); let mut s = settings(); for user in ["42", "not a number"] { let e = act(&s, &f, &action("capture", user, json!({}))).unwrap_err(); assert!(e.contains("등록된 사람만"), "{e}"); } s.pc_control.view = Level::Ask; assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("먼저 묻기")); s.pc_control.view = Level::Never; assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("꺼져")); assert!(f.captured.borrow().is_empty(), "nothing is captured for a refusal"); } #[test] fn the_tool_is_available_only_with_someone_registered_and_viewing_not_off() { let mut s = settings(); assert_eq!(available(&s), cfg!(windows)); s.features.screen = false; assert!(!available(&s)); let mut s = settings(); s.pc_control.view = Level::Never; assert!(!available(&s)); assert!(!available(&Settings::default()), "nobody is registered by default"); } #[test] fn the_whole_screen_is_one_picture_with_excluded_programs_painted_black() { let f = fake(); let s = settings(); let v = act(&s, &f, &action("capture", OWNER, json!({"target": "screen", "detail": "detailed"}))).unwrap(); assert_eq!(*f.captured.borrow(), vec!["monitor 1", "monitor 2"]); assert_eq!((v["item"]["width"].as_u64(), v["item"]["height"].as_u64()), (Some(600), Some(300))); assert_eq!((v["item"]["what"].as_str(), v["item"]["parts"].as_u64()), (Some("전체 화면"), Some(1))); assert!(decode(&v).iter().all(|j| j.starts_with(&[0xFF, 0xD8])), "JPEG pictures"); // The picture before encoding: the password manager's area is black, the rest is not; the // corner below monitor 2 that no monitor covers is black too. let hidden = vec![f.windows[0].rect]; let (img, black) = whole(&f, &f.monitors, &hidden).unwrap(); let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize]; assert!(!black); assert_eq!(at(50, 30), 0, "the excluded window is painted black"); assert!(at(200, 150) == 200 && at(500, 50) == 200, "the monitors show"); assert_eq!(at(500, 250), 0, "nothing outside the monitors"); } #[test] fn an_excluded_window_that_moves_or_comes_back_during_the_capture_never_shows() { let s = settings(); let moved = |x: i32, minimised: bool| { let mut w = fake().windows; w[0].rect.x = x; w[0].minimised = minimised; w }; // Moved between the reading before and after the capture: that capture is thrown away and // the next one, with the window still, is sent. let f = fake(); *f.later.borrow_mut() = vec![moved(10, false), moved(200, false)]; assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "screen"}))).is_ok()); assert_eq!(f.captured.borrow().len(), 4, "captured twice"); // Brought back from the task bar during the capture: the same. let f = fake(); *f.later.borrow_mut() = vec![moved(10, true), moved(10, false)]; assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 1}))).is_ok()); assert_eq!(f.captured.borrow().len(), 2, "captured twice"); // Moving all the time: nothing is sent. let f = fake(); *f.later.borrow_mut() = (0..20).map(|i| moved(i * 7, false)).collect(); let e = act(&s, &f, &action("capture", OWNER, json!({"target": "screen"}))).unwrap_err(); assert!(e.contains("움직"), "{e}"); } #[test] fn a_window_whose_program_cannot_be_told_is_treated_as_excluded() { let s = settings(); let mut f = fake(); f.windows = vec![win("", "Vault - unlocked", Rect { x: 10, y: 10, w: 100, h: 50 })]; assert!(pick_window(&s, &f.windows, "vault").is_none()); let v = act(&s, &f, &action("list", OWNER, json!({}))).unwrap(); assert_eq!(v["item"]["windows"], json!([])); let hidden = hidden_rects(&s, &f.windows); assert_eq!(hidden, vec![f.windows[0].rect], "painted black on the screen"); } #[test] fn excluded_areas_follow_a_monitor_captured_at_another_scale() { // A picture twice the placed size (display scaling): the black area doubles with it. let mut img = solid(800, 600, 200); paint_black(&mut img, Rect { x: 100, y: 0, w: 400, h: 300 }, &[Rect { x: 110, y: 10, w: 50, h: 20 }]); let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize]; assert_eq!((at(20, 20), at(119, 59)), (0, 0)); assert_eq!((at(121, 20), at(20, 61), at(19, 20)), (200, 200, 200)); } #[test] fn one_monitor_by_its_number_and_a_missing_one_names_those_there_are() { let f = fake(); let v = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 2}))).unwrap(); assert_eq!(*f.captured.borrow(), vec!["monitor 2"]); assert_eq!((v["item"]["what"].as_str(), v["item"]["width"].as_u64()), (Some("모니터 2"), Some(200))); let e = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 3}))).unwrap_err(); assert!(e.contains("3번") && e.contains("1, 2"), "{e}"); } #[test] fn a_window_is_picked_by_title_or_program_front_first_never_an_excluded_or_own_one() { let s = settings(); let f = fake(); let v = act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "CHROME"}))).unwrap(); assert_eq!(*f.captured.borrow(), vec!["window chrome.exe"]); assert_eq!(v["item"]["what"], "창 chrome"); assert_eq!(pick_window(&s, &f.windows, "메모").map(|w| w.program.as_str()), Some("notepad.exe")); for hidden in ["keepass", "passwords", "jarvis", "pop-up"] { assert!(pick_window(&s, &f.windows, hidden).is_none(), "{hidden}"); } let mut f = fake(); f.windows[3].minimised = true; assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "메모"}))).unwrap_err() .contains("최소화")); } #[test] fn a_black_picture_is_refused_and_pictures_are_scaled_to_the_detail() { let s = settings(); let mut f = fake(); f.shot = solid(800, 600, 0); assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "chrome"}))).unwrap_err() .contains("검게")); let big = solid(3840, 2160, 90); assert_eq!(parts(&big, Detail::Fast).iter().map(|p| (p.w, p.h)).collect::>(), vec![(768, 432)]); assert_eq!(parts(&big, Detail::Normal).iter().map(|p| (p.w, p.h)).collect::>(), vec![(1280, 720)]); let four = parts(&big, Detail::Detailed); assert_eq!(four.len(), 4); assert!(four.iter().all(|p| p.w.max(p.h) <= NORMAL_PX)); assert_eq!(parts(&solid(1600, 900, 90), Detail::Detailed).len(), 1); // Small pictures are never scaled up. assert_eq!(parts(&solid(300, 200, 90), Detail::Normal)[0].w, 300); } #[test] fn the_list_shows_monitors_and_windows_without_excluded_secondary_or_own_ones() { let v = act(&settings(), &fake(), &action("list", OWNER, json!({}))).unwrap(); assert_eq!(v["item"]["monitors"], json!([{"number": 1, "width": 400, "height": 300, "main": true}, {"number": 2, "width": 200, "height": 100, "main": false}])); let programs: Vec<&str> = v["item"]["windows"].as_array().unwrap().iter().map(|w| w["program"].as_str().unwrap()).collect(); assert_eq!(programs, vec!["chrome", "notepad"]); assert!(v.get("images").is_none()); } #[test] fn scaling_averages_the_area() { let mut img = solid(4, 2, 0); for y in 0..2 { for x in 0..2 { let o = ((y * 4 + x) * 4) as usize; img.bgra[o..o + 3].fill(200); } } let half = scale(&img, 2); assert_eq!((half.w, half.h), (2, 1)); assert_eq!(&half.bgra[..3], &[200, 200, 200]); assert_eq!(&half.bgra[4..7], &[0, 0, 0]); } }