Files
javis_bot/app/src/screen.rs
javis-bot a177d38ab2 fix(screen): an excluded window never shows, also when it moves during the capture or its program cannot be told
- Excluded windows are read before and after each monitor or whole-screen capture; when they
  differ, the capture is thrown away and taken again (at most 3 times, then refused).
- A window whose program cannot be told counts as excluded: not listed, not picked, painted black.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 17:11:19 +09:00

831 lines
38 KiB
Rust

//! Screen viewing for the brain's `screenView` tool (admin.spec.md, "Screen viewing";
//! `docs/v2/pc-features.md`, "화면 보기"): who may see, which monitor or window, the pictures it
//! gets (scaled, excluded programs painted black, JPEG). The capture itself is Windows Graphics
//! Capture (`win`); everything else is plain code the tests drive with a fake screen.
use base64::Engine as _;
use serde_json::{json, Value};
use crate::permissions::{decide, Action, Decision, Refusal};
use crate::settings::{Level, Settings};
use crate::worker_client::ScreenAction;
/// Long side of a `fast` and a `normal` picture, and of each `detailed` part.
pub const FAST_PX: u32 = 768;
pub const NORMAL_PX: u32 = 1280;
/// A picture whose long side is at most this stays one `detailed` part at full size.
pub const ONE_PART_MAX_PX: u32 = 1600;
pub const JPEG_QUALITY: u8 = 80;
/// Pictures per answer (base64 grows them by a third; the worker takes 4 MiB messages).
pub const MAX_BYTES: usize = 3 * 1024 * 1024;
/// Windows in the list.
pub const MAX_WINDOWS: usize = 30;
/// Whether the brain gets the tool and the LLM reads pictures: Windows, the feature on, someone
/// registered for PC control and viewing not turned off.
pub fn available(s: &Settings) -> bool {
cfg!(windows) && s.features.screen && !s.pc_control.allowed_users.is_empty() && s.pc_control.view != Level::Never
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Rect {
pub x: i32,
pub y: i32,
pub w: i32,
pub h: i32,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Monitor {
/// As Windows numbers it (`\\.\DISPLAY<n>`).
pub number: u32,
pub rect: Rect,
pub main: bool,
/// The opaque handle the capture takes.
pub handle: isize,
}
/// A visible top-level window, front first.
#[derive(Debug, Clone, PartialEq)]
pub struct Window {
pub handle: isize,
/// Program file name, lower case (`chrome.exe`).
pub program: String,
pub title: String,
pub rect: Rect,
pub minimised: bool,
/// Owned windows and tool windows (pop-ups, palettes): never listed or picked, but painted
/// black when their program is excluded.
pub secondary: bool,
/// This app's own windows.
pub own: bool,
}
/// A captured picture: top-down BGRA rows.
#[derive(Debug, Clone, PartialEq)]
pub struct Image {
pub w: u32,
pub h: u32,
pub bgra: Vec<u8>,
}
impl Image {
pub fn new(w: u32, h: u32) -> Self {
Image { w, h, bgra: vec![0; (w * h * 4) as usize] }
}
}
/// What the screen offers; Windows in `win`, a fake in the tests.
pub trait Screen {
/// The monitors without the bot browser's virtual one.
fn monitors(&self) -> Result<Vec<Monitor>, String>;
fn windows(&self) -> Result<Vec<Window>, String>;
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String>;
fn capture_window(&self, w: &Window) -> Result<Image, String>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Detail {
Fast,
Normal,
Detailed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Target {
Screen,
Monitor(u32),
Window(String),
}
/// The activity log's target: never a title.
pub fn activity_target(t: &Target) -> String {
match t {
Target::Screen => "전체 화면".into(),
Target::Monitor(n) => format!("모니터 {n}"),
Target::Window(_) => "창".into(),
}
}
/// The request in a `capture` action.
pub fn target_of(a: &ScreenAction) -> Result<(Target, Detail), String> {
let detail = match a.detail.as_deref() {
Some("fast") => Detail::Fast,
Some("detailed") => Detail::Detailed,
_ => Detail::Normal,
};
let target = match a.target.as_deref() {
Some("monitor") => Target::Monitor(a.monitor.filter(|n| *n >= 1).ok_or("모니터 번호가 필요합니다")?),
Some("window") => {
let w = a.window.as_deref().map(str::trim).filter(|w| !w.is_empty()).ok_or("창 이름이 필요합니다")?;
Target::Window(w.chars().take(200).collect())
}
_ => Target::Screen,
};
Ok((target, detail))
}
/// Why the asker may not see the screen, or `None` when they may.
pub fn refusal(s: &Settings, user: &str) -> Option<String> {
let Ok(id) = user.parse::<u64>() else { return Some(not_registered()) };
match decide(&s.pc_control, id, Action::View) {
Decision::Allow => None,
Decision::Refuse(Refusal::NotAllowed) => Some(not_registered()),
// `ask` needs the next-message consent the PC control tool brings (M7, next step).
Decision::Ask => Some("화면 보기가 '먼저 묻기'로 되어 있어 지금은 볼 수 없습니다(PC 제어 화면에서 '자동'으로 바꾸면 됩니다)".into()),
Decision::Refuse(_) => Some("화면 보기가 꺼져 있습니다".into()),
}
}
fn not_registered() -> String {
"PC 제어 화면에 등록된 사람만 화면을 볼 수 있습니다".into()
}
/// Excluded: on the list, or a program that could not be told (its process refused the query), so
/// no excluded program slips through unnamed.
fn excluded(s: &Settings, program: &str) -> bool {
program.is_empty() || crate::game::listed(program, &s.pc_control.screen_exclude)
}
/// Where excluded windows show now (minimised ones show nowhere).
fn hidden_rects(s: &Settings, all: &[Window]) -> Vec<Rect> {
all.iter().filter(|w| !w.minimised && excluded(s, &w.program)).map(|w| w.rect).collect()
}
/// Captures of monitors and screens tried while excluded windows keep moving.
const STILL_TRIES: usize = 3;
/// The windows the list shows and a window request may pick: not secondary, not this app's, not
/// excluded; front first.
fn listable<'a>(s: &'a Settings, all: &'a [Window]) -> impl Iterator<Item = &'a Window> + 'a {
all.iter().filter(move |w| !w.secondary && !w.own && !w.title.trim().is_empty() && !excluded(s, &w.program))
}
/// The window nearest the front whose title or program name contains `text` (any case).
pub fn pick_window<'a>(s: &'a Settings, all: &'a [Window], text: &str) -> Option<&'a Window> {
let t = text.to_lowercase();
let stem = |p: &str| p.strip_suffix(".exe").unwrap_or(p).to_owned();
listable(s, all).find(|w| w.title.to_lowercase().contains(&t) || stem(&w.program).contains(&t))
}
/// Whether nothing shows (protected video, exclusive full screen): every pixel near black.
pub fn all_black(img: &Image) -> bool {
img.bgra.chunks_exact(4).all(|p| p[0] <= 8 && p[1] <= 8 && p[2] <= 8)
}
/// Paint `rects` (screen coordinates) black on a picture of `area`; the picture may have another
/// size than `area` (display scaling), so the rectangles are scaled to it.
pub fn paint_black(img: &mut Image, area: Rect, rects: &[Rect]) {
if area.w <= 0 || area.h <= 0 {
return;
}
let (sx, sy) = (img.w as f64 / area.w as f64, img.h as f64 / area.h as f64);
for r in rects {
let x0 = (((r.x - area.x) as f64 * sx).floor().max(0.0) as u32).min(img.w);
let y0 = (((r.y - area.y) as f64 * sy).floor().max(0.0) as u32).min(img.h);
let x1 = (((r.x + r.w - area.x) as f64 * sx).ceil().max(0.0) as u32).min(img.w);
let y1 = (((r.y + r.h - area.y) as f64 * sy).ceil().max(0.0) as u32).min(img.h);
for y in y0..y1 {
let row = (y * img.w) as usize * 4;
img.bgra[row + x0 as usize * 4..row + x1 as usize * 4].fill(0);
}
}
}
/// `img` scaled down (area average) so its long side is at most `max`; never up.
pub fn scale(img: &Image, max: u32) -> Image {
let long = img.w.max(img.h);
if long <= max || img.w == 0 || img.h == 0 {
return img.clone();
}
let (w, h) = (((img.w as u64 * max as u64) / long as u64).max(1) as u32, ((img.h as u64 * max as u64) / long as u64).max(1) as u32);
let mut out = Image::new(w, h);
let (fx, fy) = (img.w as f64 / w as f64, img.h as f64 / h as f64);
for oy in 0..h {
let (y0, y1) = ((oy as f64 * fy) as u32, (((oy + 1) as f64 * fy).ceil() as u32).min(img.h));
for ox in 0..w {
let (x0, x1) = ((ox as f64 * fx) as u32, (((ox + 1) as f64 * fx).ceil() as u32).min(img.w));
let mut sum = [0u32; 3];
for y in y0..y1 {
let row = (y * img.w) as usize * 4;
for x in x0..x1 {
let p = &img.bgra[row + x as usize * 4..];
sum[0] += p[0] as u32;
sum[1] += p[1] as u32;
sum[2] += p[2] as u32;
}
}
let n = ((y1 - y0) * (x1 - x0)).max(1);
let o = ((oy * w + ox) * 4) as usize;
out.bgra[o..o + 4].copy_from_slice(&[(sum[0] / n) as u8, (sum[1] / n) as u8, (sum[2] / n) as u8, 255]);
}
}
out
}
fn crop(img: &Image, x: u32, y: u32, w: u32, h: u32) -> Image {
let mut out = Image::new(w, h);
for row in 0..h {
let src = (((y + row) * img.w + x) * 4) as usize;
let dst = (row * w * 4) as usize;
out.bgra[dst..dst + (w * 4) as usize].copy_from_slice(&img.bgra[src..src + (w * 4) as usize]);
}
out
}
/// The pictures of a request: one scaled picture, or for `detailed` four parts (left to right,
/// top to bottom) of at most `NORMAL_PX` each.
pub fn parts(img: &Image, detail: Detail) -> Vec<Image> {
match detail {
Detail::Fast => vec![scale(img, FAST_PX)],
Detail::Normal => vec![scale(img, NORMAL_PX)],
Detail::Detailed if img.w.max(img.h) <= ONE_PART_MAX_PX => vec![img.clone()],
Detail::Detailed => {
let (hw, hh) = (img.w / 2, img.h / 2);
[(0, 0, hw, hh), (hw, 0, img.w - hw, hh), (0, hh, hw, img.h - hh), (hw, hh, img.w - hw, img.h - hh)]
.into_iter().map(|(x, y, w, h)| scale(&crop(img, x, y, w, h), NORMAL_PX)).collect()
}
}
}
pub fn jpeg(img: &Image) -> Result<Vec<u8>, String> {
let mut out = Vec::new();
jpeg_encoder::Encoder::new(&mut out, JPEG_QUALITY)
.encode(&img.bgra, img.w as u16, img.h as u16, jpeg_encoder::ColorType::Bgra)
.map_err(|e| format!("그림을 만들지 못했습니다: {e}"))?;
Ok(out)
}
fn bounds(ms: &[Monitor]) -> Rect {
let x0 = ms.iter().map(|m| m.rect.x).min().unwrap_or(0);
let y0 = ms.iter().map(|m| m.rect.y).min().unwrap_or(0);
let x1 = ms.iter().map(|m| m.rect.x + m.rect.w).max().unwrap_or(0);
let y1 = ms.iter().map(|m| m.rect.y + m.rect.h).max().unwrap_or(0);
Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }
}
/// One picture of `ms` as they are placed; each monitor's capture is fitted to its place.
fn whole(screen: &dyn Screen, ms: &[Monitor], excluded_rects: &[Rect]) -> Result<(Image, bool), String> {
let b = bounds(ms);
let mut out = Image::new(b.w.max(1) as u32, b.h.max(1) as u32);
let mut black = true;
for m in ms {
let shot = screen.capture_monitor(m)?;
black &= all_black(&shot);
let mut fit = if shot.w as i32 == m.rect.w && shot.h as i32 == m.rect.h { shot } else { stretch(&shot, m.rect) };
paint_black(&mut fit, m.rect, excluded_rects);
let (ox, oy) = ((m.rect.x - b.x) as u32, (m.rect.y - b.y) as u32);
for row in 0..fit.h.min(out.h - oy) {
let w = fit.w.min(out.w - ox) as usize * 4;
let src = (row * fit.w * 4) as usize;
let dst = (((oy + row) * out.w + ox) * 4) as usize;
out.bgra[dst..dst + w].copy_from_slice(&fit.bgra[src..src + w]);
}
}
Ok((out, black))
}
/// `img` resized to exactly `r`'s size (nearest pixel): monitors whose capture differs from their
/// placed size (display scaling) still fill their place.
fn stretch(img: &Image, r: Rect) -> Image {
let (w, h) = (r.w.max(1) as u32, r.h.max(1) as u32);
let mut out = Image::new(w, h);
for y in 0..h {
let sy = (y as u64 * img.h as u64 / h as u64) as u32;
for x in 0..w {
let sx = (x as u64 * img.w as u64 / w as u64) as u32;
let s = ((sy * img.w + sx) * 4) as usize;
let d = ((y * w + x) * 4) as usize;
out.bgra[d..d + 4].copy_from_slice(&img.bgra[s..s + 4]);
}
}
out
}
/// Answer one `screen_action` for its asker: `{item, images}` or why not.
pub fn act(s: &Settings, screen: &dyn Screen, a: &ScreenAction) -> Result<Value, String> {
if let Some(why) = refusal(s, &a.user) {
return Err(why);
}
match a.action.as_str() {
"list" => {
let monitors: Vec<Value> = screen.monitors()?.iter().map(|m| json!({"number": m.number,
"width": m.rect.w, "height": m.rect.h, "main": m.main})).collect();
let all = screen.windows()?;
let windows: Vec<Value> = listable(s, &all).take(MAX_WINDOWS).map(|w| json!({
"program": w.program.strip_suffix(".exe").unwrap_or(&w.program), "title": w.title,
"minimised": w.minimised})).collect();
Ok(json!({"item": {"monitors": monitors, "windows": windows}}))
}
"capture" => {
let (target, detail) = target_of(a)?;
let (img, black, what) = match &target {
Target::Window(text) => {
let all = screen.windows()?;
let w = pick_window(s, &all, text).ok_or("그런 창을 찾지 못했습니다")?;
if w.minimised {
return Err("그 창은 최소화되어 있어 볼 수 없습니다".into());
}
let img = screen.capture_window(w)?;
let black = all_black(&img);
(img, black, format!("창 {}", w.program.strip_suffix(".exe").unwrap_or(&w.program)))
}
_ => {
let ms = screen.monitors()?;
let chosen: Vec<Monitor> = match target {
Target::Monitor(n) => vec![ms.iter().find(|m| m.number == n).cloned()
.ok_or_else(|| format!("{n}번 모니터가 없습니다 (있는 모니터: {})",
ms.iter().map(|m| m.number.to_string()).collect::<Vec<_>>().join(", ")))?],
_ => ms,
};
if chosen.is_empty() {
return Err("모니터를 찾지 못했습니다".into());
}
// Excluded windows are read before and after the capture; if one moved, was
// brought back or opened meanwhile, the black areas may miss it: that capture
// is thrown away.
let mut tries = 0;
let (img, black) = loop {
let before = hidden_rects(s, &screen.windows()?);
let shot = whole(screen, &chosen, &before)?;
if hidden_rects(s, &screen.windows()?) == before {
break shot;
}
tries += 1;
if tries == STILL_TRIES {
return Err("보면 안 되는 창이 움직이는 중이라 지금은 볼 수 없습니다".into());
}
};
(img, black, activity_target(&target))
}
};
if black {
return Err("화면이 검게 나와 볼 수 없습니다 (보호된 영상이나 전체 화면 게임일 수 있습니다)".into());
}
let pics = parts(&img, detail);
let mut images = Vec::with_capacity(pics.len());
let mut total = 0;
for p in &pics {
let j = jpeg(p)?;
total += j.len();
images.push(base64::engine::general_purpose::STANDARD.encode(j));
}
if total > MAX_BYTES {
return Err("그림이 너무 커서 보낼 수 없습니다 (화질을 낮춰 다시 보세요)".into());
}
Ok(json!({"item": {"what": what, "width": img.w, "height": img.h, "parts": images.len()}, "images": images}))
}
other => Err(format!("모르는 화면 동작입니다: {}", other.chars().take(20).collect::<String>())),
}
}
/// The real screen (Windows), or none elsewhere.
pub fn system_screen() -> Option<Box<dyn Screen + Send>> {
#[cfg(windows)]
return Some(Box::new(win::System));
#[cfg(not(windows))]
None
}
#[cfg(windows)]
mod win {
use super::{Image, Monitor, Rect, Screen, Window};
use std::time::{Duration, Instant};
use windows::core::Interface;
use windows::Graphics::Capture::{Direct3D11CaptureFramePool, GraphicsCaptureItem};
use windows::Graphics::DirectX::Direct3D11::IDirect3DDevice;
use windows::Graphics::DirectX::DirectXPixelFormat;
use windows::Win32::Foundation::{HMODULE, HWND};
use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP};
use windows::Win32::Graphics::Direct3D11::*;
use windows::Win32::Graphics::Dxgi::IDXGIDevice;
use windows::Win32::Graphics::Gdi::HMONITOR;
use windows::Win32::System::WinRT::Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess};
use windows::Win32::System::WinRT::Graphics::Capture::IGraphicsCaptureItemInterop;
use windows::Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED};
use windows_sys::core::BOOL;
use windows_sys::Win32::Foundation::{LPARAM, RECT};
use windows_sys::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED};
use windows_sys::Win32::Graphics::Gdi::{EnumDisplayMonitors, GetMonitorInfoW, HDC, MONITORINFOEXW};
use windows_sys::Win32::UI::WindowsAndMessaging::*;
/// The first frame of a capture.
const FRAME_TIMEOUT: Duration = Duration::from_secs(2);
pub struct System;
fn text(buf: &[u16]) -> String {
String::from_utf16_lossy(&buf[..buf.iter().position(|&c| c == 0).unwrap_or(buf.len())])
}
unsafe extern "system" fn add_monitor(h: windows_sys::Win32::Graphics::Gdi::HMONITOR, _: HDC, _: *mut RECT, lp: LPARAM) -> BOOL {
// SAFETY: `lp` is the Vec `monitors` passed, alive for the whole enumeration.
let out = unsafe { &mut *(lp as *mut Vec<(isize, String, Rect, bool)>) };
let mut mi: MONITORINFOEXW = unsafe { std::mem::zeroed() };
mi.monitorInfo.cbSize = std::mem::size_of::<MONITORINFOEXW>() as u32;
// SAFETY: `mi` is a MONITORINFOEXW with its size set.
if unsafe { GetMonitorInfoW(h, &mut mi as *mut MONITORINFOEXW as *mut _) } != 0 {
let r = mi.monitorInfo.rcMonitor;
out.push((h as isize, text(&mi.szDevice),
Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top },
mi.monitorInfo.dwFlags & MONITORINFOF_PRIMARY != 0));
}
1
}
unsafe extern "system" fn add_window(hwnd: windows_sys::Win32::Foundation::HWND, lp: LPARAM) -> BOOL {
// SAFETY: `lp` is the Vec `windows` passed, alive for the whole enumeration.
let out = unsafe { &mut *(lp as *mut Vec<windows_sys::Win32::Foundation::HWND>) };
out.push(hwnd);
1
}
fn window(hwnd: windows_sys::Win32::Foundation::HWND, own: u32) -> Option<Window> {
// SAFETY: plain queries on a window handle from EnumWindows; a stale one only fails them.
unsafe {
if IsWindowVisible(hwnd) == 0 {
return None;
}
let mut cloaked: u32 = 0;
DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED as u32, (&mut cloaked as *mut u32).cast(), 4);
if cloaked != 0 {
return None;
}
let mut buf = [0u16; 512];
let n = GetWindowTextW(hwnd, buf.as_mut_ptr(), buf.len() as i32).max(0) as usize;
let mut pid = 0;
GetWindowThreadProcessId(hwnd, &mut pid);
let mut r: RECT = std::mem::zeroed();
GetWindowRect(hwnd, &mut r);
let ex = GetWindowLongPtrW(hwnd, GWL_EXSTYLE) as u32;
let program = crate::game::probe::image_path(pid).map(|p| crate::game::file_name(&p)).unwrap_or_default();
Some(Window { handle: hwnd as isize, program, title: String::from_utf16_lossy(&buf[..n]),
rect: Rect { x: r.left, y: r.top, w: r.right - r.left, h: r.bottom - r.top },
minimised: IsIconic(hwnd) != 0,
secondary: !GetWindow(hwnd, GW_OWNER).is_null() || ex & WS_EX_TOOLWINDOW != 0,
own: pid == own })
}
}
fn device() -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext, IDirect3DDevice)> {
let make = |kind: D3D_DRIVER_TYPE| -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext)> {
let (mut d, mut c) = (None, None);
// SAFETY: out pointers to locals; no adapter, no software module.
unsafe { D3D11CreateDevice(None, kind, HMODULE::default(), D3D11_CREATE_DEVICE_BGRA_SUPPORT, None,
D3D11_SDK_VERSION, Some(&mut d), None, Some(&mut c))? };
Ok((d.unwrap(), c.unwrap()))
};
// Without a graphics card (a virtual machine) WARP still captures.
let (d, c) = make(D3D_DRIVER_TYPE_HARDWARE).or_else(|_| make(D3D_DRIVER_TYPE_WARP))?;
let dxgi: IDXGIDevice = d.cast()?;
// SAFETY: a valid DXGI device.
let winrt: IDirect3DDevice = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi)? }.cast()?;
Ok((d, c, winrt))
}
fn capture(item: GraphicsCaptureItem) -> Result<Image, String> {
let e = |e: windows::core::Error| format!("화면을 캡처하지 못했습니다 ({:#x})", e.code().0);
// SAFETY: a second call on a thread already initialised only reports it.
let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) };
let (dev, ctx, winrt) = device().map_err(e)?;
let size = item.Size().map_err(e)?;
let pool = Direct3D11CaptureFramePool::CreateFreeThreaded(&winrt, DirectXPixelFormat::B8G8R8A8UIntNormalized, 1, size)
.map_err(e)?;
let session = pool.CreateCaptureSession(&item).map_err(e)?;
let _ = session.SetIsCursorCaptureEnabled(false);
// Windows 11 lets the app leave out the yellow border; older Windows refuses, which is fine.
let _ = session.SetIsBorderRequired(false);
session.StartCapture().map_err(e)?;
let started = Instant::now();
let frame = loop {
if let Ok(f) = pool.TryGetNextFrame() {
break f;
}
if started.elapsed() > FRAME_TIMEOUT {
let _ = session.Close();
let _ = pool.Close();
return Err("화면이 제때 오지 않았습니다".into());
}
std::thread::sleep(Duration::from_millis(5));
};
let result = (|| -> windows::core::Result<Image> {
let content = frame.ContentSize()?;
let access: IDirect3DDxgiInterfaceAccess = frame.Surface()?.cast()?;
// SAFETY: the frame's surface is a D3D11 texture.
let tex: ID3D11Texture2D = unsafe { access.GetInterface()? };
let mut desc = D3D11_TEXTURE2D_DESC::default();
// SAFETY: `desc` is the out parameter.
unsafe { tex.GetDesc(&mut desc) };
let (w, h) = ((content.Width.max(1) as u32).min(desc.Width), (content.Height.max(1) as u32).min(desc.Height));
desc.Usage = D3D11_USAGE_STAGING;
desc.BindFlags = 0;
desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32;
desc.MiscFlags = 0;
let mut staging = None;
// SAFETY: a staging copy of the frame's texture, read on this thread only.
unsafe {
dev.CreateTexture2D(&desc, None, Some(&mut staging))?;
let staging = staging.unwrap();
ctx.CopyResource(&staging, &tex);
let mut map = D3D11_MAPPED_SUBRESOURCE::default();
ctx.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut map))?;
let mut img = Image::new(w, h);
for y in 0..h as usize {
let src = std::slice::from_raw_parts((map.pData as *const u8).add(y * map.RowPitch as usize), w as usize * 4);
img.bgra[y * w as usize * 4..(y + 1) * w as usize * 4].copy_from_slice(src);
}
ctx.Unmap(&staging, 0);
Ok(img)
}
})();
let _ = frame.Close();
let _ = session.Close();
let _ = pool.Close();
result.map_err(e)
}
fn interop() -> Result<IGraphicsCaptureItemInterop, String> {
windows::core::factory::<GraphicsCaptureItem, IGraphicsCaptureItemInterop>()
.map_err(|e| format!("화면 캡처를 쓸 수 없습니다 ({:#x})", e.code().0))
}
impl Screen for System {
fn monitors(&self) -> Result<Vec<Monitor>, String> {
let mut found: Vec<(isize, String, Rect, bool)> = vec![];
// SAFETY: the callback only pushes into `found`, which outlives the call.
unsafe { EnumDisplayMonitors(std::ptr::null_mut(), std::ptr::null(), Some(add_monitor), &mut found as *mut _ as LPARAM) };
let bot = crate::vdisplay::virtual_display_name();
Ok(found.into_iter().filter(|(_, name, _, _)| Some(name) != bot.as_ref()).map(|(h, name, rect, main)| Monitor {
number: name.trim_start_matches(r"\\.\DISPLAY").parse().unwrap_or(0), rect, main, handle: h }).collect())
}
fn windows(&self) -> Result<Vec<Window>, String> {
let mut hwnds = vec![];
// SAFETY: the callback only pushes into `hwnds`, which outlives the call.
unsafe { EnumWindows(Some(add_window), &mut hwnds as *mut _ as LPARAM) };
let own = std::process::id();
Ok(hwnds.into_iter().filter_map(|h| window(h, own)).collect())
}
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String> {
// SAFETY: a monitor handle from EnumDisplayMonitors.
let item = unsafe { interop()?.CreateForMonitor::<GraphicsCaptureItem>(HMONITOR(m.handle as *mut _)) }
.map_err(|e| format!("모니터를 캡처하지 못했습니다 ({:#x})", e.code().0))?;
capture(item)
}
fn capture_window(&self, w: &Window) -> Result<Image, String> {
// SAFETY: a window handle from EnumWindows; a closed window only fails the call.
let item = unsafe { interop()?.CreateForWindow::<GraphicsCaptureItem>(HWND(w.handle as *mut _)) }
.map_err(|e| format!("창을 캡처하지 못했습니다 ({:#x})", e.code().0))?;
capture(item)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::RefCell;
const OWNER: &str = "123456789012345678";
fn settings() -> Settings {
let mut s = Settings::default();
s.pc_control.allowed_users = vec![OWNER.parse().unwrap()];
s
}
fn solid(w: u32, h: u32, v: u8) -> Image {
Image { w, h, bgra: [v, v, v, 255].repeat((w * h) as usize) }
}
struct Fake {
monitors: Vec<Monitor>,
windows: Vec<Window>,
/// What the next reads of the windows give, one each, before `windows` again: a window
/// that moves while the screen is captured.
later: RefCell<Vec<Vec<Window>>>,
shot: Image,
captured: RefCell<Vec<String>>,
}
impl Screen for Fake {
fn monitors(&self) -> Result<Vec<Monitor>, String> {
Ok(self.monitors.clone())
}
fn windows(&self) -> Result<Vec<Window>, String> {
let mut later = self.later.borrow_mut();
Ok(if later.is_empty() { self.windows.clone() } else { later.remove(0) })
}
fn capture_monitor(&self, m: &Monitor) -> Result<Image, String> {
self.captured.borrow_mut().push(format!("monitor {}", m.number));
Ok(solid(m.rect.w as u32, m.rect.h as u32, 200))
}
fn capture_window(&self, w: &Window) -> Result<Image, String> {
self.captured.borrow_mut().push(format!("window {}", w.program));
Ok(self.shot.clone())
}
}
fn win(program: &str, title: &str, rect: Rect) -> Window {
Window { handle: 1, program: program.into(), title: title.into(), rect, minimised: false, secondary: false, own: false }
}
fn fake() -> Fake {
Fake {
monitors: vec![
Monitor { number: 1, rect: Rect { x: 0, y: 0, w: 400, h: 300 }, main: true, handle: 1 },
Monitor { number: 2, rect: Rect { x: 400, y: 0, w: 200, h: 100 }, main: false, handle: 2 },
],
windows: vec![
win("keepassxc.exe", "Passwords", Rect { x: 10, y: 10, w: 100, h: 50 }),
win("chrome.exe", "메일 - Chrome", Rect { x: 0, y: 0, w: 300, h: 200 }),
Window { secondary: true, ..win("chrome.exe", "pop-up", Rect { x: 0, y: 0, w: 10, h: 10 }) },
win("notepad.exe", "메모", Rect { x: 0, y: 0, w: 50, h: 50 }),
Window { own: true, ..win("jarvis.exe", "Jarvis 관리", Rect { x: 0, y: 0, w: 50, h: 50 }) },
],
later: RefCell::new(vec![]),
shot: solid(800, 600, 120),
captured: RefCell::new(vec![]),
}
}
fn action(action: &str, user: &str, extra: Value) -> ScreenAction {
let mut v = json!({"id": "s1", "action": action, "user": user});
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
serde_json::from_value(v).unwrap()
}
fn decode(v: &Value) -> Vec<Vec<u8>> {
v["images"].as_array().unwrap().iter()
.map(|i| base64::engine::general_purpose::STANDARD.decode(i.as_str().unwrap()).unwrap()).collect()
}
#[test]
fn only_registered_people_see_the_screen_and_only_when_viewing_is_automatic() {
let f = fake();
let mut s = settings();
for user in ["42", "not a number"] {
let e = act(&s, &f, &action("capture", user, json!({}))).unwrap_err();
assert!(e.contains("등록된 사람만"), "{e}");
}
s.pc_control.view = Level::Ask;
assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("먼저 묻기"));
s.pc_control.view = Level::Never;
assert!(act(&s, &f, &action("list", OWNER, json!({}))).unwrap_err().contains("꺼져"));
assert!(f.captured.borrow().is_empty(), "nothing is captured for a refusal");
}
#[test]
fn the_tool_is_available_only_with_someone_registered_and_viewing_not_off() {
let mut s = settings();
assert_eq!(available(&s), cfg!(windows));
s.features.screen = false;
assert!(!available(&s));
let mut s = settings();
s.pc_control.view = Level::Never;
assert!(!available(&s));
assert!(!available(&Settings::default()), "nobody is registered by default");
}
#[test]
fn the_whole_screen_is_one_picture_with_excluded_programs_painted_black() {
let f = fake();
let s = settings();
let v = act(&s, &f, &action("capture", OWNER, json!({"target": "screen", "detail": "detailed"}))).unwrap();
assert_eq!(*f.captured.borrow(), vec!["monitor 1", "monitor 2"]);
assert_eq!((v["item"]["width"].as_u64(), v["item"]["height"].as_u64()), (Some(600), Some(300)));
assert_eq!((v["item"]["what"].as_str(), v["item"]["parts"].as_u64()), (Some("전체 화면"), Some(1)));
assert!(decode(&v).iter().all(|j| j.starts_with(&[0xFF, 0xD8])), "JPEG pictures");
// The picture before encoding: the password manager's area is black, the rest is not; the
// corner below monitor 2 that no monitor covers is black too.
let hidden = vec![f.windows[0].rect];
let (img, black) = whole(&f, &f.monitors, &hidden).unwrap();
let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize];
assert!(!black);
assert_eq!(at(50, 30), 0, "the excluded window is painted black");
assert!(at(200, 150) == 200 && at(500, 50) == 200, "the monitors show");
assert_eq!(at(500, 250), 0, "nothing outside the monitors");
}
#[test]
fn an_excluded_window_that_moves_or_comes_back_during_the_capture_never_shows() {
let s = settings();
let moved = |x: i32, minimised: bool| {
let mut w = fake().windows;
w[0].rect.x = x;
w[0].minimised = minimised;
w
};
// Moved between the reading before and after the capture: that capture is thrown away and
// the next one, with the window still, is sent.
let f = fake();
*f.later.borrow_mut() = vec![moved(10, false), moved(200, false)];
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "screen"}))).is_ok());
assert_eq!(f.captured.borrow().len(), 4, "captured twice");
// Brought back from the task bar during the capture: the same.
let f = fake();
*f.later.borrow_mut() = vec![moved(10, true), moved(10, false)];
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 1}))).is_ok());
assert_eq!(f.captured.borrow().len(), 2, "captured twice");
// Moving all the time: nothing is sent.
let f = fake();
*f.later.borrow_mut() = (0..20).map(|i| moved(i * 7, false)).collect();
let e = act(&s, &f, &action("capture", OWNER, json!({"target": "screen"}))).unwrap_err();
assert!(e.contains("움직"), "{e}");
}
#[test]
fn a_window_whose_program_cannot_be_told_is_treated_as_excluded() {
let s = settings();
let mut f = fake();
f.windows = vec![win("", "Vault - unlocked", Rect { x: 10, y: 10, w: 100, h: 50 })];
assert!(pick_window(&s, &f.windows, "vault").is_none());
let v = act(&s, &f, &action("list", OWNER, json!({}))).unwrap();
assert_eq!(v["item"]["windows"], json!([]));
let hidden = hidden_rects(&s, &f.windows);
assert_eq!(hidden, vec![f.windows[0].rect], "painted black on the screen");
}
#[test]
fn excluded_areas_follow_a_monitor_captured_at_another_scale() {
// A picture twice the placed size (display scaling): the black area doubles with it.
let mut img = solid(800, 600, 200);
paint_black(&mut img, Rect { x: 100, y: 0, w: 400, h: 300 }, &[Rect { x: 110, y: 10, w: 50, h: 20 }]);
let at = |x: u32, y: u32| img.bgra[((y * img.w + x) * 4) as usize];
assert_eq!((at(20, 20), at(119, 59)), (0, 0));
assert_eq!((at(121, 20), at(20, 61), at(19, 20)), (200, 200, 200));
}
#[test]
fn one_monitor_by_its_number_and_a_missing_one_names_those_there_are() {
let f = fake();
let v = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 2}))).unwrap();
assert_eq!(*f.captured.borrow(), vec!["monitor 2"]);
assert_eq!((v["item"]["what"].as_str(), v["item"]["width"].as_u64()), (Some("모니터 2"), Some(200)));
let e = act(&settings(), &f, &action("capture", OWNER, json!({"target": "monitor", "monitor": 3}))).unwrap_err();
assert!(e.contains("3번") && e.contains("1, 2"), "{e}");
}
#[test]
fn a_window_is_picked_by_title_or_program_front_first_never_an_excluded_or_own_one() {
let s = settings();
let f = fake();
let v = act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "CHROME"}))).unwrap();
assert_eq!(*f.captured.borrow(), vec!["window chrome.exe"]);
assert_eq!(v["item"]["what"], "창 chrome");
assert_eq!(pick_window(&s, &f.windows, "메모").map(|w| w.program.as_str()), Some("notepad.exe"));
for hidden in ["keepass", "passwords", "jarvis", "pop-up"] {
assert!(pick_window(&s, &f.windows, hidden).is_none(), "{hidden}");
}
let mut f = fake();
f.windows[3].minimised = true;
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "메모"}))).unwrap_err()
.contains("최소화"));
}
#[test]
fn a_black_picture_is_refused_and_pictures_are_scaled_to_the_detail() {
let s = settings();
let mut f = fake();
f.shot = solid(800, 600, 0);
assert!(act(&s, &f, &action("capture", OWNER, json!({"target": "window", "window": "chrome"}))).unwrap_err()
.contains("검게"));
let big = solid(3840, 2160, 90);
assert_eq!(parts(&big, Detail::Fast).iter().map(|p| (p.w, p.h)).collect::<Vec<_>>(), vec![(768, 432)]);
assert_eq!(parts(&big, Detail::Normal).iter().map(|p| (p.w, p.h)).collect::<Vec<_>>(), vec![(1280, 720)]);
let four = parts(&big, Detail::Detailed);
assert_eq!(four.len(), 4);
assert!(four.iter().all(|p| p.w.max(p.h) <= NORMAL_PX));
assert_eq!(parts(&solid(1600, 900, 90), Detail::Detailed).len(), 1);
// Small pictures are never scaled up.
assert_eq!(parts(&solid(300, 200, 90), Detail::Normal)[0].w, 300);
}
#[test]
fn the_list_shows_monitors_and_windows_without_excluded_secondary_or_own_ones() {
let v = act(&settings(), &fake(), &action("list", OWNER, json!({}))).unwrap();
assert_eq!(v["item"]["monitors"], json!([{"number": 1, "width": 400, "height": 300, "main": true},
{"number": 2, "width": 200, "height": 100, "main": false}]));
let programs: Vec<&str> = v["item"]["windows"].as_array().unwrap().iter().map(|w| w["program"].as_str().unwrap()).collect();
assert_eq!(programs, vec!["chrome", "notepad"]);
assert!(v.get("images").is_none());
}
#[test]
fn scaling_averages_the_area() {
let mut img = solid(4, 2, 0);
for y in 0..2 {
for x in 0..2 {
let o = ((y * 4 + x) * 4) as usize;
img.bgra[o..o + 3].fill(200);
}
}
let half = scale(&img, 2);
assert_eq!((half.w, half.h), (2, 1));
assert_eq!(&half.bgra[..3], &[200, 200, 200]);
assert_eq!(&half.bgra[4..7], &[0, 0, 0]);
}
}