//! PC control for the brain's `pcControl` tool (admin.spec.md, "PC control"; `docs/v2/pc-features.md`, //! "키보드·마우스 제어와 안티치트"): who may, what is never done, the anti-cheat and emergency-stop //! blocks, and every action run through agent-desktop, one CLI process per command. The commands go //! through `Desktop`, so the tests drive everything with a fake; `Cli` is the real one. use std::collections::{HashSet, VecDeque}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Mutex; use std::time::{Duration, Instant}; use serde_json::{json, Value}; use crate::permissions::{decide, Action, Decision, Refusal}; use crate::settings::Settings; use crate::worker_client::PcAction; /// agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0), pinned as M0 ran it /// (`scripts/m0/m0lib/desktop.py`). pub const AGENT_DESKTOP_VERSION: &str = "1.2.0"; pub const AGENT_DESKTOP_URL: &str = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-pc-windows-msvc.tar.gz"; /// The release archive (`app/shell/package.sh` checks it). pub const AGENT_DESKTOP_ARCHIVE_SHA256: &str = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8"; /// `agent-desktop.exe` inside it, checked before every start of the services. pub const AGENT_DESKTOP_SHA256: &str = "d274667031322e50b9c2b1ab81c2ca46835d52cc932769a5c34bd4d481fd94c3"; pub const EXE_NAME: &str = "agent-desktop.exe"; /// One agent-desktop command at most. pub const COMMAND_LIMIT: Duration = Duration::from_secs(15); /// Elements in a reading (interactive ones first), and the characters kept of a name or value. pub const MAX_ELEMENTS: usize = 200; pub const MAX_CHARS: usize = 120; /// Characters of text put into a field. pub const MAX_TEXT: usize = 2000; pub const MAX_WINDOWS: usize = 30; pub const DEFAULT_STOP_HOTKEY: &str = "Ctrl+Shift+F12"; /// How long a reading may take (agent-desktop's budget; Chromium windows settle slowly). const SNAPSHOT_MS: &str = "8000"; /// How long `launch` waits for the program, and then this app for its window. const LAUNCH_MS: &str = "8000"; const WINDOW_WAIT: Duration = Duration::from_secs(5); /// Readings whose elements may still be acted on. const KEPT_READINGS: usize = 64; pub const NOT_REGISTERED: &str = "PC 제어 화면에 등록된 사람만 PC를 조작할 수 있습니다"; pub const TURNED_OFF: &str = "PC 제어 화면에서 이 동작이 꺼져 있습니다"; pub const PASSWORD: &str = "비밀번호 칸에는 입력하지 않습니다"; pub const ELEVATED: &str = "관리자 권한으로 실행된 창은 조작할 수 없습니다"; pub const STOPPED: &str = "비상 정지 중입니다. 단축키를 한 번 더 누르거나 PC 제어 화면에서 풀어 주세요"; pub const STALE: &str = "화면이 바뀌었습니다. 다시 읽어 주세요"; const NO_WINDOW: &str = "그 창을 찾지 못했습니다"; const NOT_FRONT: &str = "그 창을 앞으로 가져오지 못해 키를 보내지 않았습니다"; const NOT_IN_FRONT_NOW: &str = "게임 중이거나 전체 화면이라 그 창을 앞으로 가져오지 않았습니다. 키를 보내지 않았습니다"; const NOT_CLOSED: &str = "프로그램이 아직 닫히지 않았습니다(저장할지 묻는 창이 떠 있을 수 있습니다). 창을 다시 읽어 주세요"; const CLOSE_SEVERAL: &str = "그 프로그램이 여러 개 실행 중이라 이 창만 닫을 수 없습니다. 창의 닫기 단추를 눌러 주세요"; const RUNNING_SEVERAL: &str = "그 프로그램이 이미 여러 개 실행 중이라 새로 열지 못했습니다. 열린 창 중에서 골라 주세요"; /// Anti-cheats found by their processes (file names, lower case): the anti-cheat's name, and the /// game's when the process is the game itself. Vanguard's driver is always loaded, so it is found /// by Riot's game processes. Grown by updates. const ANTI_CHEATS: &[(&str, &str, Option<&str>)] = &[ ("valorant-win64-shipping.exe", "뱅가드", Some("VALORANT")), ("valorant.exe", "뱅가드", Some("VALORANT")), ("league of legends.exe", "뱅가드", Some("League of Legends")), ("easyanticheat.exe", "Easy Anti-Cheat", None), ("easyanticheat_eos.exe", "Easy Anti-Cheat", None), ("beservice.exe", "BattlEye", None), ("beservice_x64.exe", "BattlEye", None), ("faceit.exe", "FACEIT", None), ("faceitservice.exe", "FACEIT", None), ("gamemon.des", "nProtect GameGuard", None), ("gamemon64.des", "nProtect GameGuard", None), ("xcorona.xem", "XIGNCODE3", None), ("xcorona_x64.xem", "XIGNCODE3", None), ("xxd-0.xem", "XIGNCODE3", None), ]; /// Where agent-desktop is shipped: next to the app's program file. pub fn exe_path() -> Option { Some(std::env::current_exe().ok()?.parent()?.join(EXE_NAME)) } /// Whether `exe` is the pinned agent-desktop. pub fn exe_ok(exe: &Path) -> bool { crate::download::sha256_file(exe).is_ok_and(|h| h == AGENT_DESKTOP_SHA256) } /// Whether the brain gets the tool: Windows, the feature on, someone registered and agent-desktop /// present with the right hash (`installed`). pub fn available(s: &Settings, installed: bool) -> bool { cfg!(windows) && s.features.pc_control && !s.pc_control.allowed_users.is_empty() && installed } /// The agent-desktop the services use, when PC control is available (checked once per start). pub fn check(s: &Settings) -> Option { if !available(s, true) { return None; } match exe_path() { Some(exe) if exe_ok(&exe) => Some(exe), Some(exe) if exe.exists() => { crate::warn_log!("conversation", "Warning: agent-desktop has another hash than {AGENT_DESKTOP_VERSION}'s: PC control is off"); None } _ => { crate::warn_log!("conversation", "Warning: agent-desktop is not installed: PC control is off"); None } } } /// `Ctrl+Shift+F12` as the hotkey's modifiers (`MOD_ALT` 1, `MOD_CONTROL` 2, `MOD_SHIFT` 4, /// `MOD_WIN` 8) and virtual key (letters, digits, F1-F24); at least one modifier. pub fn parse_hotkey(s: &str) -> Option<(u32, u32)> { let parts: Vec = s.split('+').map(|p| p.trim().to_ascii_lowercase()).collect(); let (key, mods) = parts.split_last()?; let mut m = 0; for p in mods { m |= match p.as_str() { "alt" => 1, "ctrl" | "control" => 2, "shift" => 4, "win" => 8, _ => return None, }; } let vk = match key.as_bytes() { [c] if c.is_ascii_alphanumeric() => c.to_ascii_uppercase() as u32, [b'f', n @ ..] => match std::str::from_utf8(n).ok()?.parse::().ok()? { n @ 1..=24 => 0x6F + n, _ => return None, }, _ => return None, }; (m != 0).then_some((m, vk)) } // ---- emergency stop ----------------------------------------------------------------------------- /// The emergency stop: on until the hotkey is pressed again or the PC 제어 screen lifts it; an app /// restart lifts it (it lives in the process only). pub struct Stop { on: AtomicBool, /// How often it was turned on: a command started before the latest press is killed. presses: AtomicU64, } impl Default for Stop { fn default() -> Self { Self::new() } } impl Stop { pub const fn new() -> Self { Stop { on: AtomicBool::new(false), presses: AtomicU64::new(0) } } pub fn on(&self) -> bool { self.on.load(Ordering::SeqCst) } /// The hotkey: on, or off again. pub fn toggle(&self) { let now_on = !self.on(); if now_on { self.presses.fetch_add(1, Ordering::SeqCst); } self.on.store(now_on, Ordering::SeqCst); crate::info_log!("conversation", "PC control emergency stop {}", if now_on { "on" } else { "lifted" }); } /// The PC 제어 screen's 풀기. pub fn lift(&self) { if self.on.swap(false, Ordering::SeqCst) { crate::info_log!("conversation", "PC control emergency stop lifted"); } } fn presses(&self) -> u64 { self.presses.load(Ordering::SeqCst) } /// Whether a command started at `presses` is to be killed. fn pressed_since(&self, presses: u64) -> bool { self.on() && self.presses() != presses } } static EMERGENCY: Stop = Stop::new(); /// The app's emergency stop (the hotkey, the screen and the dashboard share it). pub fn emergency() -> &'static Stop { &EMERGENCY } // ---- agent-desktop ------------------------------------------------------------------------------ /// What PC control runs on: agent-desktop and two questions to Windows (`Cli`; a fake in tests). pub trait Desktop: Send + Sync { /// One agent-desktop command (never `--force`, never the raw mouse and key commands): its /// `data`, or its error code. A press of the emergency stop while it runs kills it /// (`APP_STOPPED`). fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result; /// Whether process `pid` runs as administrator; one that cannot be told counts as one. fn elevated(&self, pid: u64) -> bool; /// The running programs' file names, lower case. fn processes(&self) -> Vec; /// What `launch` is given for a program name: the name itself, or the program file a Start /// menu shortcut or Windows' App Paths name. fn program(&self, name: &str) -> String { name.to_owned() } /// The window in front now, as agent-desktop names it (`w-`). fn foreground(&self) -> Option { None } /// Whether the window in front fills its monitor without a frame (not the desktop), or a /// full-screen Direct3D program runs. fn fullscreen(&self) -> bool { false } /// Whether window `id` takes keys posted to it behind: it has one text control shown /// (`text_control`). fn takes_keys_behind(&self, _id: &str) -> bool { false } /// Post one key (`behind_key`) to window `id`'s text control, without bringing it forward. fn post_key(&self, _id: &str, _key: &str) -> Result<(), String> { Err("PLATFORM_NOT_SUPPORTED".into()) } } /// The real agent-desktop: one process per command, its APPDATA, LOCALAPPDATA and state folder in /// the data folder (`runtime/agent-desktop`). pub struct Cli { exe: PathBuf, home: PathBuf, limit: Duration, } impl Cli { pub fn new(exe: &Path, data: &Path) -> Self { let home = data.join("runtime").join("agent-desktop"); // agent-desktop refuses a state folder others may write to ("private file DACL grants an // untrusted principal": a data folder outside the user's profile inherits Authenticated // Users' Modify), so it is made afresh, private to this user; a run's readings are of no // use to the next. let _ = std::fs::remove_dir_all(&home); let _ = std::fs::create_dir_all(&home); #[cfg(windows)] if !win::make_private(&home) { crate::warn_log!("conversation", "Warning: agent-desktop's state folder could not be made private: PC control may fail"); } Cli { exe: exe.to_path_buf(), home, limit: COMMAND_LIMIT } } } /// An agent-desktop envelope `{ok, data, error{code}}`: the data, or the code (never the message, /// which can name windows). fn envelope(out: &[u8]) -> Result { let text = String::from_utf8_lossy(out); let text = text.trim(); let v: Value = serde_json::from_str(text).or_else(|_| serde_json::from_str(text.lines().last().unwrap_or(""))) .map_err(|_| "APP_NO_JSON".to_owned())?; match v["ok"].as_bool() { Some(true) => Ok(v["data"].clone()), Some(false) => Err(v["error"]["code"].as_str().unwrap_or("APP_UNKNOWN").to_owned()), None => Err("APP_NO_JSON".into()), } } impl Desktop for Cli { fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result { let mut cmd = std::process::Command::new(&self.exe); cmd.args(args); if args.first().map(String::as_str) == Some("launch") { // A started program gets the person's own folders, not agent-desktop's. for var in ["APPDATA", "LOCALAPPDATA"] { if let Some(v) = std::env::var_os(var) { cmd.arg("--env").arg(format!("{var}={}", v.to_string_lossy())); } } if let Some(home) = std::env::var_os("USERPROFILE").filter(|h| Path::new(h).is_absolute()) { cmd.arg("--cwd").arg(home); } } cmd.env("APPDATA", &self.home).env("LOCALAPPDATA", &self.home).env("AGENT_DESKTOP_HOME", &self.home) .stdin(if stdin.is_some() { std::process::Stdio::piped() } else { std::process::Stdio::null() }) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::null()); #[cfg(windows)] { use std::os::windows::process::CommandExt; cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW } let presses = stop.presses(); let mut child = cmd.spawn().map_err(|_| "APP_NOT_STARTED".to_owned())?; if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) { // Text goes in through stdin, never in the arguments; written aside so a full pipe // cannot hold this thread. let text = text.to_owned(); std::thread::spawn(move || { let _ = pipe.write_all(text.as_bytes()); }); } let mut out = child.stdout.take().expect("stdout is piped"); let reader = std::thread::spawn(move || { let mut b = Vec::new(); let _ = out.read_to_end(&mut b); b }); let started = Instant::now(); loop { match child.try_wait() { Ok(Some(_)) => break, Ok(None) => {} Err(_) => return Err("APP_NOT_STARTED".into()), } let killed = if stop.pressed_since(presses) { Some("APP_STOPPED") } else if started.elapsed() > self.limit { Some("APP_TIMEOUT") } else { None }; if let Some(code) = killed { let _ = child.kill(); let _ = child.wait(); return Err(code.into()); } std::thread::sleep(Duration::from_millis(10)); } envelope(&reader.join().unwrap_or_default()) } fn elevated(&self, pid: u64) -> bool { #[cfg(windows)] return win::elevated(pid); #[cfg(not(windows))] { let _ = pid; false } } fn processes(&self) -> Vec { #[cfg(windows)] return win::processes(); #[cfg(not(windows))] vec![] } fn program(&self, name: &str) -> String { #[cfg(windows)] return win::program(name).unwrap_or_else(|| name.to_owned()); #[cfg(not(windows))] name.to_owned() } #[cfg(windows)] fn foreground(&self) -> Option { win::foreground() } #[cfg(windows)] fn fullscreen(&self) -> bool { win::fullscreen() } #[cfg(windows)] fn takes_keys_behind(&self, id: &str) -> bool { win::text_control_of(id).is_some() } #[cfg(windows)] fn post_key(&self, id: &str, key: &str) -> Result<(), String> { let vk = behind_key(key).ok_or_else(|| "INVALID_ARGS".to_owned())?; let to = win::text_control_of(id).ok_or_else(|| "WINDOW_NOT_FOUND".to_owned())?; win::post_key(to, vk) } } // ---- the actions -------------------------------------------------------------------------------- /// A window as agent-desktop lists it. #[derive(Debug, Clone, PartialEq)] struct Win { id: String, /// Program file name, lower case (`notepad.exe`). program: String, title: String, pid: u64, focused: bool, minimised: bool, } impl Win { fn view(&self) -> Value { json!({"id": self.id, "program": stem(&self.program), "title": cut(&self.title)}) } } fn stem(program: &str) -> &str { program.strip_suffix(".exe").unwrap_or(program) } fn cut(s: &str) -> String { s.chars().take(MAX_CHARS).collect() } /// An activity record (admin.spec.md, "Activity"): the program's file name, never a title, /// element name or text. #[derive(Debug, Clone, PartialEq)] pub struct Record { pub kind: String, pub target: String, pub ok: bool, pub note: Option, } /// The answer to one `pc_action`, its result code for the log and what the activity records. #[derive(Debug, Clone, PartialEq)] pub struct Outcome { pub reply: Value, /// `ok`, `ask`, `refused` or agent-desktop's error code. pub code: String, pub activity: Option, } #[derive(Debug)] struct Fail { code: String, reason: String, } fn refused(reason: &str) -> Fail { Fail { code: "refused".into(), reason: reason.into() } } /// agent-desktop's error code as the reason the person hears. fn failed(code: &str, action: &str) -> Fail { let reason = match code { // Windows' foreground lock: focus-window could not bring the window forward, or press // could not confirm its focus; no key was sent. "ACTION_FAILED" | "STALE_REF" if action == "keys" => NOT_FRONT, // close-app gives up while the program still runs (a save question, say). "TIMEOUT" | "APP_TIMEOUT" if action == "close" => NOT_CLOSED, // close-app picks the program by name; with several running it cannot choose. "AMBIGUOUS_TARGET" if action == "close" => CLOSE_SEVERAL, // launch attaches to one running instance; with several it cannot choose. "AMBIGUOUS_TARGET" if action == "run" => RUNNING_SEVERAL, "STALE_REF" | "SNAPSHOT_NOT_FOUND" | "ELEMENT_NOT_FOUND" => STALE, "AMBIGUOUS_TARGET" => "같은 모양의 항목이 여럿이라 고를 수 없습니다. 다시 읽어 주세요", "APP_NOT_FOUND" if action == "run" => "그 프로그램을 찾지 못했습니다", "APP_NOT_FOUND" | "WINDOW_NOT_FOUND" => NO_WINDOW, "PERM_DENIED" => ELEVATED, "POLICY_DENIED" => "막혀 있는 동작이라 하지 않습니다(위험한 단축키 등)", "INVALID_ARGS" if action == "close" => "Windows가 지키는 프로그램이라 닫을 수 없습니다", "INVALID_ARGS" => "요청을 알아듣지 못했습니다. 이름이나 키를 확인해 주세요", "ACTION_NOT_SUPPORTED" => "그 항목은 이 동작을 받지 않습니다", "ACTION_FAILED" => "동작이 되지 않았습니다", "APP_UNRESPONSIVE" => "프로그램이 응답하지 않습니다", "TIMEOUT" | "APP_TIMEOUT" => "시간 안에 끝나지 않았습니다", "APP_STOPPED" => STOPPED, "PLATFORM_NOT_SUPPORTED" => "이 PC에서는 할 수 없는 동작입니다", _ => "PC 제어 도구가 실패했습니다", }; Fail { code: code.into(), reason: reason.into() } } enum Done { Item(Value), Ask(Value), } /// The permission kind of an action and its name in an `ask` answer. fn kind_of(action: &str) -> Option<(Action, &'static str)> { match action { "windows" | "read" => Some((Action::View, "view")), "click" | "close" => Some((Action::Click, "click")), "type" | "keys" => Some((Action::Type, "type")), "run" => Some((Action::Run, "run")), _ => None, } } const ACTING: &[&str] = &["click", "type", "keys", "run", "close"]; fn valid_ref(r: Option<&str>) -> Result<&str, Fail> { let r = r.map(str::trim).filter(|r| !r.is_empty()).ok_or_else(|| refused("누를 항목(ref)이 필요합니다"))?; let ok = r.strip_prefix('@').and_then(|r| r.split_once(":e")).is_some_and(|(snap, n)| { !snap.is_empty() && snap.len() <= 64 && snap.bytes().all(|b| b.is_ascii_alphanumeric()) && !n.is_empty() && n.len() <= 9 && n.bytes().all(|b| b.is_ascii_digit()) }); if ok { Ok(r) } else { Err(refused(STALE)) } } /// A key combination for `press`: names joined with `+` (`ctrl+s`, `enter`), nothing that could be /// read as an option. fn valid_keys(k: Option<&str>) -> Result { let k = k.map(|k| k.replace(' ', "")).filter(|k| !k.is_empty()).ok_or_else(|| refused("누를 키가 필요합니다"))?; let ok = k.len() <= 40 && k.split('+').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_alphanumeric())); if ok { Ok(k) } else { Err(refused("키를 알아듣지 못했습니다 (예: ctrl+s, enter)")) } } /// The virtual key of `keys` when it can be sent behind: one key without modifiers that types or /// moves, by agent-desktop's names (`a`-`z` and `0`-`9` as their keys on a US layout); else `None`. fn behind_key(keys: &str) -> Option { let k = keys.to_ascii_lowercase(); let vk = match k.as_str() { "return" | "enter" => 0x0D, "escape" | "esc" => 0x1B, "tab" => 0x09, "space" => 0x20, "delete" | "backspace" => 0x08, "forwarddelete" => 0x2E, "home" => 0x24, "end" => 0x23, "pageup" => 0x21, "pagedown" => 0x22, "left" => 0x25, "up" => 0x26, "right" => 0x27, "down" => 0x28, // F10 is Windows' menu key (a system key), so it goes in front. f if f.len() <= 3 && f.starts_with('f') => match f[1..].parse::() { Ok(n @ (1..=9 | 11 | 12)) => 0x6F + n, _ => return None, }, c if c.len() == 1 && c.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) => c.as_bytes()[0].to_ascii_uppercase() as u16, _ => return None, }; Some(vk) } /// Whether a control of this class takes keys posted behind: the text controls whose keys Windows /// gives to the control itself (classic Edit and RichEdit, the new Notepad's, Scintilla). pub fn text_control(class: &str) -> bool { ["Edit", "RichEdit20A", "RichEdit20W", "RICHEDIT50W", "RichEditD2DPT", "Scintilla"].contains(&class) } /// A program to start by name: no folders, nothing that could be read as an option. fn valid_program(p: Option<&str>) -> Result { let p = p.map(str::trim).filter(|p| !p.is_empty()).ok_or_else(|| refused("실행할 프로그램 이름이 필요합니다"))?; let ok = p.chars().count() <= 100 && !p.starts_with(['-', '.']) && !p.contains("..") && !p.chars().any(|c| c.is_control() || matches!(c, '\\' | '/' | ':' | '"' | '*' | '?' | '<' | '>' | '|')); if ok { Ok(p.to_owned()) } else { Err(refused("프로그램 이름으로만 실행할 수 있습니다 (예: notepad)")) } } /// A program name as a file name for the activity (`notepad` -> `notepad.exe`). fn program_file(name: &str) -> String { let n = name.to_lowercase(); if Path::new(&n).extension().is_some() { n } else { format!("{n}.exe") } } /// A reading's elements: interactive ones (with a ref) first, then the rest that have a name or a /// value; at most `MAX_ELEMENTS`, names and values cut to `MAX_CHARS`. fn elements(tree: &Value) -> Vec { fn walk<'a>(n: &'a Value, refs: &mut Vec<&'a Value>, rest: &mut Vec<&'a Value>) { for c in n["children"].as_array().into_iter().flatten() { if c["ref_id"].as_str().is_some_and(|r| !r.is_empty()) { refs.push(c); } else if c["name"].as_str().is_some_and(|s| !s.trim().is_empty()) || c["value"].as_str().is_some_and(|s| !s.trim().is_empty()) { rest.push(c); } walk(c, refs, rest); } } let (mut refs, mut rest) = (vec![], vec![]); walk(tree, &mut refs, &mut rest); refs.into_iter().chain(rest).take(MAX_ELEMENTS).map(|n| json!({ "ref": n["ref_id"].as_str().unwrap_or(""), "role": n["role"].as_str().unwrap_or(""), "name": cut(n["name"].as_str().unwrap_or("")), "value": cut(n["value"].as_str().unwrap_or("")), "states": n["states"].as_array().cloned().unwrap_or_default(), })).collect() } /// The refs of a reading's fields agent-desktop reports as secure (password fields), in the whole /// tree, not only the elements shown. fn secure_refs(tree: &Value) -> HashSet { fn walk(n: &Value, out: &mut HashSet) { for c in n["children"].as_array().into_iter().flatten() { if let Some(r) = c["ref_id"].as_str().filter(|_| c["states"].as_array().is_some_and(|s| s.iter().any(|x| x == "secure"))) { out.insert(r.to_owned()); } walk(c, out); } } let mut out = HashSet::new(); walk(tree, &mut out); out } fn args(a: &[&str]) -> Vec { a.iter().map(|s| s.to_string()).collect() } /// PC control for one run of the services. pub struct Pc { settings: Settings, desktop: Box, stop: &'static Stop, /// The game game detection sees now, for the anti-cheat sentence. game: Box Option + Send + Sync>, /// This app's process: its windows are never listed or acted on. own_pid: u64, /// The latest readings by snapshot id: the window, so an element names its window, and the /// fields read as secure (password fields; agent-desktop cannot address them again by `get`). readings: Mutex)>>, /// One action at a time. one: Mutex<()>, } impl Pc { pub fn new(settings: Settings, desktop: Box, stop: &'static Stop, game: Box Option + Send + Sync>) -> Self { Pc { settings, desktop, stop, game, own_pid: std::process::id() as u64, readings: Mutex::new(VecDeque::new()), one: Mutex::new(()) } } /// Answer one `pc_action` for its asker. pub fn act(&self, a: &PcAction) -> Outcome { let _one = self.one.lock().unwrap_or_else(|e| e.into_inner()); let mut program = None; let result = self.answer(a, &mut program); let acting = ACTING.contains(&a.action.as_str()); let (reply, code, note) = match result { Ok(Done::Item(item)) => (json!({"ok": true, "item": item}), "ok".to_owned(), None), Ok(Done::Ask(v)) => (v, "ask".to_owned(), None), Err(f) => (json!({"ok": false, "error": f.reason}), f.code, Some(f.reason)), }; let activity = (acting && code != "ask").then(|| Record { kind: format!("pc.{}", a.action), target: program.unwrap_or_else(|| "?".into()), ok: code == "ok", note }); Outcome { reply, code, activity } } fn answer(&self, a: &PcAction, program: &mut Option) -> Result { let (perm, kind) = kind_of(&a.action).ok_or_else(|| refused("모르는 PC 동작입니다"))?; let user = a.user.parse::().map_err(|_| refused(NOT_REGISTERED))?; let ask = match decide(&self.settings.pc_control, user, perm) { Decision::Allow => false, Decision::Ask => !a.consent, Decision::Refuse(Refusal::NotAllowed) => return Err(refused(NOT_REGISTERED)), Decision::Refuse(_) => return Err(refused(TURNED_OFF)), }; let asking = |window: Value| Done::Ask(json!({"ok": false, "ask": true, "kinds": [kind], "window": window})); match a.action.as_str() { "windows" => { if ask { return Ok(asking(json!({"id": "", "program": "", "title": ""}))); } let list: Vec = self.list()?.iter().take(MAX_WINDOWS).map(|w| { let mut v = w.view(); v["minimised"] = json!(w.minimised); v }).collect(); return Ok(Done::Item(json!({"windows": list}))); } "read" => { let w = self.window(a.window.as_deref())?; if ask { return Ok(asking(w.view())); } return self.read(&w).map(Done::Item); } _ => {} } // Acting: never during the emergency stop or while an anti-cheat game runs. if self.stop.on() { return Err(refused(STOPPED)); } if let Some(why) = self.anti_cheat() { return Err(refused(&why)); } if a.action == "run" { let name = valid_program(a.program.as_deref())?; *program = Some(program_file(&name)); if ask { return Ok(asking(json!({"id": "", "program": name, "title": ""}))); } return self.launch(&name).map(Done::Item); } let (w, element) = match a.action.as_str() { "click" | "type" => { let r = valid_ref(a.element.as_deref())?; (self.window_of(r)?, Some(r)) } _ => (self.window(a.window.as_deref())?, None), }; *program = Some(w.program.clone()); if self.desktop.elevated(w.pid) { return Err(refused(ELEVATED)); } let keys = if a.action == "keys" { Some(valid_keys(a.keys.as_deref())?) } else { None }; let text = a.text.as_deref().unwrap_or(""); if a.action == "type" { if text.chars().count() > MAX_TEXT { return Err(refused("글자가 너무 깁니다 (2,000자까지)")); } let read_secure = self.readings.lock().unwrap().iter().any(|(_, _, s)| s.contains(element.unwrap())); if read_secure { return Err(refused(PASSWORD)); } let states = self.cmd(&args(&["get", element.unwrap(), "--property", "states"]), None, "type")?; let secure = states["value"].as_array().is_none_or(|s| s.iter().any(|x| x == "secure")); if secure { return Err(refused(PASSWORD)); } } if ask { return Ok(asking(w.view())); } if self.stop.on() { return Err(refused(STOPPED)); } // Keys restore in their moment in front (restoring brings the window forward). if w.minimised && a.action != "keys" { self.cmd(&args(&["restore", "--window-id", &w.id]), None, &a.action)?; } match a.action.as_str() { "click" => { self.cmd(&args(&["click", element.unwrap()]), None, "click")?; } "type" => { self.cmd(&args(&["set-value", element.unwrap(), "--stdin"]), Some(text), "type")?; } "keys" => self.keys(&w, keys.as_deref().unwrap())?, _ => { self.cmd(&args(&["close-app", &w.program]), None, "close")?; return Ok(Done::Item(json!({"window": w.view(), "elements": []}))); } } match self.read(&w) { // The action closed its window (a dialog's button, Enter): done, nothing left to read. Err(f) if f.code == "WINDOW_NOT_FOUND" => Ok(Done::Item(json!({"window": w.view(), "elements": []}))), r => r.map(Done::Item), } } /// Press `keys` in window `w` one way, chosen first (admin.spec.md, "Keys"): behind it when it is /// one plain key and the window takes it so, else in front for a moment (a minimised window /// restored in it) and the window that was in front brought back. Never both ways: a key is /// never pressed twice. fn keys(&self, w: &Win, keys: &str) -> Result<(), Fail> { let front = self.desktop.foreground(); if front.as_deref() == Some(w.id.as_str()) { crate::debug_log!("conversation", "pc keys way=front-already"); return self.press(w, keys); } if !w.minimised && behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) { crate::debug_log!("conversation", "pc keys way=behind"); return self.desktop.post_key(&w.id, keys).map_err(|code| failed(&code, "keys")); } if (self.game)().is_some() || self.desktop.fullscreen() { crate::debug_log!("conversation", "pc keys way=none (game or full screen)"); return Err(refused(NOT_IN_FRONT_NOW)); } crate::debug_log!("conversation", "pc keys way=front"); let restored = if w.minimised { self.cmd(&args(&["restore", "--window-id", &w.id]), None, "keys").map(|_| ()) } else { Ok(()) }; let pressed = restored .and_then(|_| self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys")) .and_then(|_| self.press(w, keys)); if let Some(back) = front { if self.desktop.run(&args(&["focus-window", "--window-id", &back]), None, self.stop).is_err() { crate::warn_log!("conversation", "Warning: PC control could not bring back the window that was in front"); } } pressed.map(|_| ()) } /// `press` in window `w`, which is in front. agent-desktop cannot find some Store apps' process /// by the window (the new Calculator: APP_NOT_FOUND, nothing sent), so then the keys go to the /// window in front while it is still `w`. fn press(&self, w: &Win, keys: &str) -> Result<(), Fail> { match self.desktop.run(&args(&["press", keys, "--window-id", &w.id]), None, self.stop) { Err(code) if code == "APP_NOT_FOUND" && self.desktop.foreground().as_deref() == Some(w.id.as_str()) => { crate::debug_log!("conversation", "pc keys press=in-front (window's process not found)"); self.cmd(&args(&["press", keys]), None, "keys").map(|_| ()) } r => r.map(|_| ()).map_err(|code| failed(&code, "keys")), } } fn cmd(&self, args: &[String], stdin: Option<&str>, action: &str) -> Result { self.desktop.run(args, stdin, self.stop).map_err(|code| failed(&code, action)) } /// The windows that may be listed and acted on: not this app's, not an excluded program's /// (`pcControl.screenExclude`), with a title; in agent-desktop's order. fn list(&self) -> Result, Fail> { let data = self.cmd(&args(&["list-windows"]), None, "windows")?; let excluded = &self.settings.pc_control.screen_exclude; Ok(data.as_array().into_iter().flatten().filter_map(|w| { let program = crate::game::file_name(w["app_name"].as_str()?); let win = Win { id: w["id"].as_str()?.to_owned(), title: w["title"].as_str().unwrap_or("").to_owned(), pid: w["pid"].as_u64().unwrap_or(0), focused: w["is_focused"].as_bool().unwrap_or(false), minimised: w["minimized"].as_bool().unwrap_or(false), program }; let shown = w["visible"].as_bool() != Some(false) && !win.title.trim().is_empty() && !win.id.is_empty(); (shown && win.pid != self.own_pid && !win.program.is_empty() && !crate::game::listed(&win.program, excluded)).then_some(win) }).collect()) } /// A window by its id, else the one whose title or program contains `query` (the focused one /// first, then the first listed). fn window(&self, query: Option<&str>) -> Result { let q = query.map(str::trim).filter(|q| !q.is_empty()).ok_or_else(|| refused("창 이름이 필요합니다"))?; let list = self.list()?; if let Some(w) = list.iter().find(|w| w.id == q) { return Ok(w.clone()); } let q = q.to_lowercase(); let hits: Vec<&Win> = list.iter() .filter(|w| w.title.to_lowercase().contains(&q) || stem(&w.program).contains(&q)).collect(); hits.iter().find(|w| w.focused).or(hits.first()).map(|w| (*w).clone()).ok_or_else(|| refused(NO_WINDOW)) } /// The window an element was read from, as it is listed now. fn window_of(&self, element: &str) -> Result { let snap = element.trim_start_matches('@').split(':').next().unwrap_or(""); let id = self.readings.lock().unwrap().iter().find(|(s, _, _)| s == snap).map(|(_, w, _)| w.id.clone()) .ok_or_else(|| refused(STALE))?; self.list()?.into_iter().find(|w| w.id == id).ok_or_else(|| refused(STALE)) } /// Read a window afresh: `{window, elements}`. fn read(&self, w: &Win) -> Result { let data = self.cmd(&args(&["snapshot", "--window-id", &w.id, "--compact", "--timeout-ms", SNAPSHOT_MS]), None, "read")?; let mut w = w.clone(); if let Some(t) = data["window"]["title"].as_str() { w.title = t.to_owned(); } if let Some(snap) = data["snapshot_id"].as_str() { let mut r = self.readings.lock().unwrap(); r.push_back((snap.to_owned(), w.clone(), secure_refs(&data["tree"]))); while r.len() > KEPT_READINGS { r.pop_front(); } } Ok(json!({"window": w.view(), "elements": elements(&data["tree"])})) } /// Start a program (agent-desktop never asks for administrator rights) and read its window: /// the one `launch` names, else a new one of its process or name, else the only new one. fn launch(&self, name: &str) -> Result { let before: HashSet = self.list()?.into_iter().map(|w| w.id).collect(); let target = self.desktop.program(name); let launched = match self.desktop.run(&args(&["launch", &target, "--timeout", LAUNCH_MS]), None, self.stop) { Ok(d) => d, // A program that hands over to another process (the Windows calculator) ends at once. Err(code) if code == "APP_UNRESPONSIVE" => Value::Null, Err(code) => return Err(failed(&code, "run")), }; let (pid, id) = (launched["pid"].as_u64(), launched["window"]["id"].as_str().map(str::to_owned)); let lower = name.to_lowercase(); let started = Instant::now(); loop { let list = self.list()?; // A program already running is not started again: launch names its window. let named = list.iter().find(|w| Some(&w.id) == id.as_ref()); let new: Vec<&Win> = list.iter().filter(|w| !before.contains(&w.id)).collect(); let found = named .or_else(|| new.iter().copied().find(|w| pid.is_some_and(|p| p == w.pid))) .or_else(|| new.iter().copied().find(|w| stem(&w.program).contains(&lower) || w.title.to_lowercase().contains(&lower))) .or(if new.len() == 1 { new.first().copied() } else { None }); if let Some(w) = found { return self.read(w); } if self.stop.on() { return Err(refused(STOPPED)); } if started.elapsed() >= WINDOW_WAIT { return Err(Fail { code: "no_window".into(), reason: "프로그램을 시작했지만 창을 찾지 못했습니다".into() }); } std::thread::sleep(Duration::from_millis(300)); } } /// The anti-cheat sentence while a game with one runs (its processes, or a program the person /// listed), else `None`. fn anti_cheat(&self) -> Option { let running = self.desktop.processes(); let found = running.iter().find_map(|p| ANTI_CHEATS.iter().find(|(name, _, _)| name == p) .map(|(_, ac, game)| (*ac, game.map(str::to_owned).unwrap_or_else(|| crate::game::game_name(p, &[]))))) .or_else(|| running.iter().find(|p| crate::game::listed(p, &self.settings.pc_control.anti_cheat_games)) .map(|p| ("안티치트", crate::game::game_name(p, &[]))))?; let game = (self.game)().unwrap_or(found.1); Some(format!("{}가 적용된 게임({game})이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", found.0)) } } // ---- the emergency stop's hotkey ---------------------------------------------------------------- /// The emergency stop's global hotkey, on its own thread while the services run. pub struct Hotkey { #[cfg(windows)] thread: u32, #[cfg(windows)] join: Option>, } impl Hotkey { /// Register `spec` for `stop`; `None` elsewhere than Windows or when Windows refuses it /// (another program holds it: warned in the log). pub fn start(spec: &str, stop: &'static Stop) -> Option { let (mods, vk) = parse_hotkey(spec)?; #[cfg(windows)] return win::hotkey(mods, vk, stop).map(|(thread, join)| Hotkey { thread, join: Some(join) }); #[cfg(not(windows))] { let _ = (mods, vk, stop); None } } /// Unregister it and end its thread. pub fn end(mut self) { self.finish(); } fn finish(&mut self) { #[cfg(windows)] if let Some(j) = self.join.take() { win::end_hotkey(self.thread); let _ = j.join(); } } } impl Drop for Hotkey { fn drop(&mut self) { self.finish(); } } /// The program file a Start menu shortcut (`.lnk`) points at, from its link information (Unicode /// path when present, else an ASCII one); `None` for a shortcut with arguments (starting the file /// alone would do something else) or without a local path. pub fn shortcut_target(b: &[u8]) -> Option { let u32_at = |o: usize| b.get(o..o + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as usize); let u16_at = |o: usize| b.get(o..o + 2).map(|s| u16::from_le_bytes([s[0], s[1]]) as usize); if u32_at(0)? != 0x4C { return None; } let flags = u32_at(0x14)?; const HAS_ID_LIST: usize = 0x1; const HAS_LINK_INFO: usize = 0x2; const HAS_ARGUMENTS: usize = 0x20; if flags & HAS_LINK_INFO == 0 || flags & HAS_ARGUMENTS != 0 { return None; } let mut at = 0x4C; if flags & HAS_ID_LIST != 0 { at += 2 + u16_at(at)?; } let (header, info_flags) = (u32_at(at + 4)?, u32_at(at + 8)?); if info_flags & 0x1 == 0 { return None; } let path = if header >= 0x24 { let start = at + u32_at(at + 28)?; let units: Vec = b.get(start..)?.chunks_exact(2).map(|c| u16::from_le_bytes([c[0], c[1]])) .take_while(|&c| c != 0).collect(); String::from_utf16(&units).ok()? } else { let start = at + u32_at(at + 16)?; let bytes: Vec = b.get(start..)?.iter().copied().take_while(|&c| c != 0).collect(); if !bytes.is_ascii() { return None; } String::from_utf8(bytes).ok()? }; (!path.is_empty()).then_some(path) } #[cfg(windows)] mod win { use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE}; use windows_sys::Win32::Security::{GetTokenInformation, TokenElevation, TOKEN_ELEVATION, TOKEN_QUERY}; use windows_sys::Win32::System::Threading::{ GetCurrentThreadId, OpenProcess, OpenProcessToken, PROCESS_QUERY_LIMITED_INFORMATION, }; /// Whether process `pid`'s token is elevated; a process that cannot be asked counts as elevated. pub fn elevated(pid: u64) -> bool { // SAFETY: query-only handles, closed at once; the out structure is sized as passed. unsafe { let p = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid as u32); if p.is_null() { return true; } let mut token: HANDLE = std::ptr::null_mut(); let opened = OpenProcessToken(p, TOKEN_QUERY, &mut token) != 0; CloseHandle(p); if !opened { return true; } let mut e = TOKEN_ELEVATION { TokenIsElevated: 0 }; let mut len = 0u32; let ok = GetTokenInformation(token, TokenElevation, (&mut e as *mut TOKEN_ELEVATION).cast(), std::mem::size_of::() as u32, &mut len) != 0; CloseHandle(token); !ok || e.TokenIsElevated != 0 } } /// Make `dir` private to this user: a protected DACL with one entry, full control for the user, /// inherited by what is made inside. pub fn make_private(dir: &std::path::Path) -> bool { use std::os::windows::ffi::OsStrExt; use windows_sys::Win32::Security::{ AddAccessAllowedAceEx, GetLengthSid, InitializeAcl, InitializeSecurityDescriptor, SetFileSecurityW, SetSecurityDescriptorControl, SetSecurityDescriptorDacl, TokenUser, ACCESS_ALLOWED_ACE, ACL, ACL_REVISION, CONTAINER_INHERIT_ACE, DACL_SECURITY_INFORMATION, OBJECT_INHERIT_ACE, PROTECTED_DACL_SECURITY_INFORMATION, SECURITY_DESCRIPTOR, SE_DACL_PROTECTED, TOKEN_USER, }; use windows_sys::Win32::System::Threading::GetCurrentProcess; const FILE_ALL_ACCESS: u32 = 0x001F_01FF; // SAFETY: this process's token queried into a sized, aligned buffer and closed; the ACL and // security descriptor live in buffers that outlive the call that uses them. unsafe { let mut token: HANDLE = std::ptr::null_mut(); if OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token) == 0 { return false; } let mut user = [0u64; 64]; let mut len = 0u32; let ok = GetTokenInformation(token, TokenUser, user.as_mut_ptr().cast(), std::mem::size_of_val(&user) as u32, &mut len) != 0; CloseHandle(token); if !ok { return false; } let sid = (*(user.as_ptr() as *const TOKEN_USER)).User.Sid; let acl_len = std::mem::size_of::() + std::mem::size_of::() + GetLengthSid(sid) as usize; let mut acl = vec![0u32; acl_len.div_ceil(4)]; let acl = acl.as_mut_ptr() as *mut ACL; let mut sd: SECURITY_DESCRIPTOR = std::mem::zeroed(); let psd = (&mut sd as *mut SECURITY_DESCRIPTOR).cast(); let path: Vec = dir.as_os_str().encode_wide().chain([0]).collect(); InitializeAcl(acl, (acl_len.div_ceil(4) * 4) as u32, ACL_REVISION) != 0 && AddAccessAllowedAceEx(acl, ACL_REVISION, OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, FILE_ALL_ACCESS, sid) != 0 && InitializeSecurityDescriptor(psd, 1) != 0 && SetSecurityDescriptorDacl(psd, 1, acl, 0) != 0 && SetSecurityDescriptorControl(psd, SE_DACL_PROTECTED, SE_DACL_PROTECTED) != 0 && SetFileSecurityW(path.as_ptr(), DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, psd) != 0 } } /// Every running process's file name, lower case (also of processes this user may not open, /// such as anti-cheat services). pub fn processes() -> Vec { use windows_sys::Win32::System::Diagnostics::ToolHelp::{ CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS, }; let mut out = vec![]; // SAFETY: a process snapshot walked with a sized entry, then closed. unsafe { let snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); if snap == INVALID_HANDLE_VALUE { return out; } let mut e: PROCESSENTRY32W = std::mem::zeroed(); e.dwSize = std::mem::size_of::() as u32; let mut more = Process32FirstW(snap, &mut e) != 0; while more { let n = e.szExeFile.iter().position(|&c| c == 0).unwrap_or(e.szExeFile.len()); out.push(String::from_utf16_lossy(&e.szExeFile[..n]).to_lowercase()); more = Process32NextW(snap, &mut e) != 0; } CloseHandle(snap); } out } /// A program name as `launch` takes it: a name Windows' folders have stays as it is (agent-desktop /// finds it there), else the file Windows' App Paths name or a Start menu shortcut points at. pub fn program(name: &str) -> Option { let file = super::program_file(name); let windir = std::env::var_os("SystemRoot").or_else(|| std::env::var_os("windir"))?; let windir = std::path::Path::new(&windir); if windir.join("System32").join(&file).is_file() || windir.join(&file).is_file() { return None; } app_path(&file).or_else(|| start_menu(name)) } fn app_path(file: &str) -> Option { use windows_sys::Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, RRF_RT_REG_SZ}; let key: Vec = format!(r"Software\Microsoft\Windows\CurrentVersion\App Paths\{file}") .encode_utf16().chain([0]).collect(); for root in [HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE] { let mut buf = [0u16; 1024]; let mut len = (buf.len() * 2) as u32; // SAFETY: the default value read into a sized buffer. let r = unsafe { RegGetValueW(root, key.as_ptr(), std::ptr::null(), RRF_RT_REG_SZ, std::ptr::null_mut(), buf.as_mut_ptr().cast(), &mut len) }; if r == 0 { let n = buf.iter().position(|&c| c == 0).unwrap_or(buf.len()); let path = String::from_utf16_lossy(&buf[..n]).trim().trim_matches('"').to_owned(); if std::path::Path::new(&path).is_file() { return Some(path); } } } None } /// A Start menu shortcut named `name` (the person's, then everyone's), pointing at a program file. fn start_menu(name: &str) -> Option { let want = name.to_lowercase(); let roots = [("APPDATA", r"Microsoft\Windows\Start Menu\Programs"), ("ProgramData", r"Microsoft\Windows\Start Menu\Programs")]; for (var, sub) in roots { let Some(base) = std::env::var_os(var) else { continue }; let mut dirs = vec![std::path::Path::new(&base).join(sub)]; let mut seen = 0; while let Some(d) = dirs.pop() { for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { seen += 1; if seen > 5000 { break; } let p = e.path(); if p.is_dir() { dirs.push(p); } else if p.extension().is_some_and(|x| x.eq_ignore_ascii_case("lnk")) && p.file_stem().is_some_and(|s| s.to_string_lossy().to_lowercase() == want) { let target = std::fs::read(&p).ok().and_then(|b| super::shortcut_target(&b)); if let Some(t) = target.filter(|t| t.to_lowercase().ends_with(".exe") && std::path::Path::new(t).is_file()) { return Some(t); } } } } } None } pub fn hotkey(mods: u32, vk: u32, stop: &'static super::Stop) -> Option<(u32, std::thread::JoinHandle<()>)> { use windows_sys::Win32::UI::Input::KeyboardAndMouse::{RegisterHotKey, UnregisterHotKey, MOD_NOREPEAT}; use windows_sys::Win32::UI::WindowsAndMessaging::{GetMessageW, PeekMessageW, MSG, PM_NOREMOVE, WM_HOTKEY, WM_USER}; let (tx, rx) = std::sync::mpsc::channel(); let join = std::thread::spawn(move || { // SAFETY: a thread-bound hotkey and this thread's own message queue. unsafe { let mut msg: MSG = std::mem::zeroed(); // Make the queue before anyone posts to it. PeekMessageW(&mut msg, std::ptr::null_mut(), WM_USER, WM_USER, PM_NOREMOVE); let ok = RegisterHotKey(std::ptr::null_mut(), 1, mods | MOD_NOREPEAT, vk) != 0; let _ = tx.send((GetCurrentThreadId(), ok)); if !ok { return; } while GetMessageW(&mut msg, std::ptr::null_mut(), 0, 0) > 0 { if msg.message == WM_HOTKEY { stop.toggle(); } } UnregisterHotKey(std::ptr::null_mut(), 1); } }); match rx.recv() { Ok((thread, true)) => Some((thread, join)), _ => { let _ = join.join(); crate::warn_log!("conversation", "Warning: the PC control emergency stop hotkey could not be registered (another program may hold it)"); None } } } /// A window by agent-desktop's id (`w-`). fn handle(id: &str) -> Option { use windows_sys::Win32::UI::WindowsAndMessaging::IsWindow; let h = id.strip_prefix("w-")?.parse::().ok().filter(|h| *h > 0)? as windows_sys::Win32::Foundation::HWND; // SAFETY: a plain query on a handle value. (unsafe { IsWindow(h) } != 0).then_some(h) } fn class_of(h: windows_sys::Win32::Foundation::HWND) -> String { use windows_sys::Win32::UI::WindowsAndMessaging::GetClassNameW; let mut buf = [0u16; 256]; // SAFETY: the buffer is sized as passed. let n = unsafe { GetClassNameW(h, buf.as_mut_ptr(), buf.len() as i32) }; String::from_utf16_lossy(&buf[..n.max(0) as usize]) } pub fn foreground() -> Option { use windows_sys::Win32::UI::WindowsAndMessaging::GetForegroundWindow; // SAFETY: a plain query. let h = unsafe { GetForegroundWindow() }; (!h.is_null()).then(|| format!("w-{}", h as usize)) } /// The window in front covers its monitor without a title bar or sizing border (not the /// desktop), or Windows reports a full-screen Direct3D program. pub fn fullscreen() -> bool { use windows_sys::Win32::Foundation::RECT; use windows_sys::Win32::Graphics::Gdi::{GetMonitorInfoW, MonitorFromWindow, MONITORINFO, MONITOR_DEFAULTTONEAREST}; use windows_sys::Win32::UI::Shell::{SHQueryUserNotificationState, QUNS_RUNNING_D3D_FULL_SCREEN}; use windows_sys::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowLongPtrW, GetWindowRect, GWL_STYLE, WS_CAPTION, WS_THICKFRAME}; // SAFETY: plain queries on the window in front, into sized locals. unsafe { let mut state = 0; if SHQueryUserNotificationState(&mut state) == 0 && state == QUNS_RUNNING_D3D_FULL_SCREEN { return true; } let h = GetForegroundWindow(); if h.is_null() || matches!(class_of(h).as_str(), "Progman" | "WorkerW") { return false; } let mut r: RECT = std::mem::zeroed(); let mut mi = MONITORINFO { cbSize: std::mem::size_of::() as u32, ..Default::default() }; if GetWindowRect(h, &mut r) == 0 || GetMonitorInfoW(MonitorFromWindow(h, MONITOR_DEFAULTTONEAREST), &mut mi) == 0 { return false; } let m = mi.rcMonitor; let style = GetWindowLongPtrW(h, GWL_STYLE) as u32; r.left <= m.left && r.top <= m.top && r.right >= m.right && r.bottom >= m.bottom && style & WS_CAPTION != WS_CAPTION && style & WS_THICKFRAME == 0 } } /// Window `id`'s one text control shown (`text_control`), if it has exactly one. Windows does /// not tell which control of a window behind has the focus (a thread behind has none), so keys /// go behind only where there is no doubt where they land. pub fn text_control_of(id: &str) -> Option { use windows_sys::Win32::Foundation::{HWND, LPARAM}; use windows_sys::Win32::UI::WindowsAndMessaging::{EnumChildWindows, IsWindowVisible}; unsafe extern "system" fn each(child: HWND, found: LPARAM) -> i32 { // SAFETY: `found` is the vector below, alive for the enumeration. let found = unsafe { &mut *(found as *mut Vec) }; if unsafe { IsWindowVisible(child) } != 0 && super::text_control(&class_of(child)) { found.push(child); } 1 } let h = handle(id)?; let mut found: Vec = vec![]; // SAFETY: the callback only reads the children and writes the vector it is given. unsafe { EnumChildWindows(h, Some(each), &mut found as *mut Vec as LPARAM) }; (found.len() == 1).then(|| found[0]) } /// Key down and key up for `vk` posted to `to`; the program's message loop makes the character. pub fn post_key(to: windows_sys::Win32::Foundation::HWND, vk: u16) -> Result<(), String> { use windows_sys::Win32::UI::Input::KeyboardAndMouse::{MapVirtualKeyW, MAPVK_VK_TO_VSC}; use windows_sys::Win32::UI::WindowsAndMessaging::{PostMessageW, WM_KEYDOWN, WM_KEYUP}; // Arrows, Home, End, Page Up and Down, Delete: the extended keys. let extended = matches!(vk, 0x21..=0x28 | 0x2E); // SAFETY: plain message posts to a window handle. unsafe { let scan = MapVirtualKeyW(vk as u32, MAPVK_VK_TO_VSC) as isize; let down = 1 | (scan << 16) | if extended { 1 << 24 } else { 0 }; let up = down | (1 << 30) | (1 << 31); if PostMessageW(to, WM_KEYDOWN, vk as usize, down) == 0 || PostMessageW(to, WM_KEYUP, vk as usize, up) == 0 { return Err("ACTION_FAILED".into()); } } // Let the program take the keys before the window is read again. std::thread::sleep(std::time::Duration::from_millis(150)); Ok(()) } pub fn end_hotkey(thread: u32) { use windows_sys::Win32::UI::WindowsAndMessaging::{PostThreadMessageW, WM_QUIT}; // SAFETY: a quit message to the hotkey's own thread. unsafe { PostThreadMessageW(thread, WM_QUIT, 0, 0) }; } } #[cfg(test)] mod tests { use super::*; use crate::settings::Level; const OWNER: &str = "123456789012345678"; const TITLE: &str = "secret-plan.txt - Notepad"; const TYPED: &str = "typed words"; /// agent-desktop as a script: answers by command, remembers what it was asked. struct Fake { windows: Mutex>, processes: Vec, elevated: bool, secure: bool, /// Commands that fail with this code. fail: Option<(&'static str, &'static str)>, calls: Mutex, Option)>>, launched: Value, /// A password field in every reading, past the elements shown; `get` on it is refused as /// the real agent-desktop refuses it (a secure field's identity is not unique). secure_ref: Option<&'static str>, /// The window in front (`focus-window` moves it), a full-screen window in front, and /// whether windows take keys behind. front: Mutex>, fullscreen: bool, behind: bool, /// agent-desktop cannot find the window's process by its identity (the new Calculator, a /// Store app): `press --window-id` is APP_NOT_FOUND, nothing sent. unnamed: bool, } impl Fake { fn new() -> Self { Fake { windows: Mutex::new(VecDeque::from([json!([ {"id": "w-1", "title": TITLE, "app_name": "notepad.exe", "pid": 10, "is_focused": false, "accessible": true}, {"id": "w-2", "title": "Vault", "app_name": "KeePassXC.exe", "pid": 11, "is_focused": true, "accessible": true}, {"id": "w-3", "title": "Calc", "app_name": "CalculatorApp.exe", "pid": 12, "is_focused": false, "minimized": true}])])), processes: vec!["explorer.exe".into()], elevated: false, secure: false, fail: None, calls: Mutex::new(vec![]), launched: json!({"app": "notepad", "pid": 10}), secure_ref: None, front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false, unnamed: false} } fn front(&self) -> Option { self.front.lock().unwrap().clone() } /// The calls with their arguments, from the first named `from`. fn calls_from(&self, from: &str) -> Vec> { let calls = self.calls.lock().unwrap(); calls.iter().map(|(a, _)| a.clone()).skip_while(|a| a[0] != from).collect() } fn commands(&self) -> Vec { self.calls.lock().unwrap().iter().map(|(a, _)| a[0].clone()).collect() } } fn tree() -> Value { let mut kids: Vec = (0..150).map(|i| json!({"role": "statictext", "name": format!("label {i}")})).collect(); kids.extend((0..150).map(|i| json!({"role": "button", "name": format!("{i} {}", "x".repeat(300)), "ref_id": format!("@s1:e{i}"), "states": ["focused"]}))); json!({"role": "window", "name": TITLE, "children": [{"role": "group", "children": kids}]}) } impl Desktop for &'static Fake { fn run(&self, args: &[String], stdin: Option<&str>, _stop: &Stop) -> Result { self.calls.lock().unwrap().push((args.to_vec(), stdin.map(str::to_owned))); if let Some((cmd, code)) = self.fail { if args[0] == cmd { return Err(code.into()); } } if args[0] == "get" && self.secure_ref == Some(args[1].as_str()) { return Err("AMBIGUOUS_TARGET".into()); } if args[0] == "press" && self.unnamed && args.iter().any(|a| a == "--window-id") { return Err("APP_NOT_FOUND".into()); } Ok(match args[0].as_str() { "list-windows" => { let mut w = self.windows.lock().unwrap(); if w.len() > 1 { w.pop_front().unwrap() } else { w[0].clone() } } "snapshot" => { let mut t = tree(); if let Some(r) = self.secure_ref { t["children"][0]["children"].as_array_mut().unwrap() .push(json!({"role": "group", "children": [{"role": "textfield", "ref_id": r, "states": ["secure"]}]})); } json!({"snapshot_id": "s1", "window": {"id": args[2], "title": TITLE}, "tree": t}) } "get" => json!({"property": "states", "value": if self.secure { json!(["secure"]) } else { json!([]) }}), "launch" => self.launched.clone(), // restore activates the window (SW_RESTORE), as focus-window does. "focus-window" | "restore" => { *self.front.lock().unwrap() = Some(args[2].clone()); json!({"disposition": {"delivery": "delivered_unverified"}}) } _ => json!({"disposition": {"delivery": "delivered_unverified"}}), }) } fn elevated(&self, _pid: u64) -> bool { self.elevated } fn processes(&self) -> Vec { self.processes.clone() } fn foreground(&self) -> Option { self.front() } fn fullscreen(&self) -> bool { self.fullscreen } fn takes_keys_behind(&self, _id: &str) -> bool { self.behind } fn post_key(&self, id: &str, key: &str) -> Result<(), String> { self.calls.lock().unwrap().push((vec!["post".into(), id.into(), key.into()], None)); match self.fail { Some(("post", code)) => Err(code.into()), _ => Ok(()), } } } fn settings() -> Settings { let mut s = Settings::default(); s.pc_control.allowed_users = vec![OWNER.parse().unwrap()]; s } fn pc_with(s: Settings, fake: Fake) -> (Pc, &'static Fake, &'static Stop) { let fake: &'static Fake = Box::leak(Box::new(fake)); let stop: &'static Stop = Box::leak(Box::new(Stop::new())); (Pc::new(s, Box::new(fake), stop, Box::new(|| None)), fake, stop) } fn action(action: &str, user: &str, extra: Value) -> PcAction { let mut v = json!({"id": "p1", "action": action, "user": user, "via": "voice"}); v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); serde_json::from_value(v).unwrap() } fn read_first(pc: &Pc) { assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).code, "ok"); } #[test] fn people_not_registered_are_refused_before_anything_runs() { let (pc, fake, _) = pc_with(settings(), Fake::new()); for a in ["windows", "read", "click", "type", "keys", "run", "close"] { let o = pc.act(&action(a, "42", json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad"}))); assert_eq!(o.reply, json!({"ok": false, "error": NOT_REGISTERED}), "{a}"); } assert!(fake.commands().is_empty()); // Nobody is registered by default. let (pc, _, _) = pc_with(Settings::default(), Fake::new()); assert_eq!(pc.act(&action("windows", OWNER, json!({}))).reply["error"], NOT_REGISTERED); } #[test] fn a_kind_turned_off_is_refused_and_every_action_has_its_kind() { let mut s = settings(); s.pc_control.click = Level::Never; s.pc_control.type_text = Level::Never; s.pc_control.run = Level::Never; s.pc_control.view = Level::Never; let (pc, fake, _) = pc_with(s, Fake::new()); for a in ["windows", "read", "click", "close", "type", "keys", "run"] { let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad", "consent": true}))); assert_eq!(o.reply["error"], TURNED_OFF, "{a}"); } assert!(fake.commands().is_empty()); } #[test] fn asking_first_answers_ask_without_consent_and_acts_with_it() { let (pc, fake, _) = pc_with(settings(), Fake::new()); read_first(&pc); let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))); assert_eq!(o.reply, json!({"ok": false, "ask": true, "kinds": ["click"], "window": {"id": "w-1", "program": "notepad", "title": TITLE}})); assert_eq!(o.activity, None, "nothing was done yet"); assert!(!fake.commands().contains(&"click".to_owned())); let o = pc.act(&action("run", OWNER, json!({"program": "notepad"}))); assert_eq!(o.reply["kinds"], json!(["run"])); assert_eq!(o.reply["window"]["program"], "notepad"); assert!(!fake.commands().contains(&"launch".to_owned())); let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))); assert_eq!(o.reply["ok"], true); assert!(fake.commands().contains(&"click".to_owned())); // Done means seen: the window is read again after the click. assert_eq!(fake.commands().last().unwrap(), "snapshot"); assert_eq!(o.reply["item"]["window"]["id"], "w-1"); // At `auto` nothing is asked. let mut s = settings(); s.pc_control.type_text = Level::Auto; let (pc, fake, _) = pc_with(s, Fake::new()); read_first(&pc); let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))); assert_eq!(o.reply["ok"], true); let calls = fake.calls.lock().unwrap(); let set = calls.iter().find(|(a, _)| a[0] == "set-value").unwrap(); assert_eq!(set.1.as_deref(), Some(TYPED), "text goes through stdin"); assert!(!set.0.iter().any(|a| a.contains(TYPED)), "never in the arguments"); } #[test] fn an_anti_cheat_game_refuses_acting_but_not_reading() { let mut fake = Fake::new(); fake.processes = vec!["explorer.exe".into(), "valorant-win64-shipping.exe".into()]; let (pc, fake, _) = pc_with(settings(), fake); for a in ["click", "type", "keys", "run", "close"] { let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "keys": "enter", "program": "notepad", "consent": true}))); assert_eq!(o.reply["error"], "뱅가드가 적용된 게임(VALORANT)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", "{a}"); } assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot")); assert_eq!(pc.act(&action("windows", OWNER, json!({}))).code, "ok"); read_first(&pc); // A program the person listed, named by game detection when it sees the game. let mut s = settings(); s.pc_control.anti_cheat_games = vec!["MyShooter".into()]; let mut fake = Fake::new(); fake.processes = vec!["myshooter.exe".into()]; let fake: &'static Fake = Box::leak(Box::new(fake)); let pc = Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into()))); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter", "consent": true}))); assert_eq!(o.reply["error"], "안티치트가 적용된 게임(My Shooter)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다"); } #[test] fn the_emergency_stop_refuses_acting_until_lifted_and_kills_a_running_command() { let (pc, fake, stop) = pc_with(settings(), Fake::new()); read_first(&pc); stop.toggle(); let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))); assert_eq!(o.reply["error"], STOPPED); assert_eq!(o.activity.unwrap().ok, false); assert!(!fake.commands().contains(&"click".to_owned())); // Reading still works. read_first(&pc); stop.toggle(); assert_eq!(pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))).code, "ok"); stop.toggle(); stop.lift(); assert!(!stop.on()); // The real runner: a command running when the hotkey is pressed is killed at once. #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; let dir = crate::datadir::temp_dir("pcstop"); let exe = dir.join("agent-desktop"); std::fs::write(&exe, "#!/bin/sh\nsleep 5\necho '{\"ok\": true, \"data\": []}'\n").unwrap(); std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); let cli = Cli::new(&exe, &dir); let stop: &'static Stop = Box::leak(Box::new(Stop::new())); let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(200)); stop.toggle(); }); let started = Instant::now(); assert_eq!(cli.run(&args(&["click", "@s1:e1"]), None, stop), Err("APP_STOPPED".into())); assert!(started.elapsed() < Duration::from_secs(2)); t.join().unwrap(); // A command started while the stop is on (a reading) runs to its end. std::fs::write(&exe, "#!/bin/sh\necho \"{\\\"ok\\\": true, \\\"data\\\": \\\"$APPDATA\\\"}\"\n").unwrap(); let home = dir.join("runtime").join("agent-desktop"); assert_eq!(cli.run(&args(&["list-windows"]), None, stop), Ok(json!(home.to_string_lossy())), "agent-desktop's folders are in the data folder"); // The command limit. std::fs::write(&exe, "#!/bin/sh\nsleep 5\n").unwrap(); let cli = Cli { limit: Duration::from_millis(200), ..cli }; assert_eq!(cli.run(&args(&["list-windows"]), None, &Stop::new()), Err("APP_TIMEOUT".into())); } } #[test] fn a_password_field_and_an_administrator_window_are_refused_before_acting() { let mut s = settings(); s.pc_control.type_text = Level::Auto; let mut fake = Fake::new(); fake.secure = true; let (pc, fake, _) = pc_with(s.clone(), fake); read_first(&pc); let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))); assert_eq!(o.reply["error"], PASSWORD); assert!(!fake.commands().contains(&"set-value".to_owned())); // Even with consent. let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED, "consent": true}))); assert_eq!(o.reply["error"], PASSWORD); let mut fake = Fake::new(); fake.elevated = true; let (pc, fake, _) = pc_with(s, fake); read_first(&pc); for a in ["click", "type", "keys", "close"] { let o = pc.act(&action(a, OWNER, json!({"ref": "@s1:e3", "window": "notepad", "keys": "enter", "text": "x", "consent": true}))); assert_eq!(o.reply["error"], ELEVATED, "{a}"); } assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot")); } #[test] fn a_field_read_as_secure_is_refused_as_a_password_field() { let mut s = settings(); s.pc_control.type_text = Level::Auto; let (pc, fake, _) = pc_with(s, Fake { secure_ref: Some("@s1:e900"), ..Fake::new() }); read_first(&pc); let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e900", "text": TYPED, "consent": true}))); assert_eq!(o.reply, json!({"ok": false, "error": PASSWORD})); assert!(!fake.commands().contains(&"set-value".to_owned())); // Other fields of the same reading are typed into. assert_eq!(pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).code, "ok"); } #[test] fn a_reading_has_interactive_elements_first_at_most_200_and_cut_to_120_characters() { let (pc, _, _) = pc_with(settings(), Fake::new()); let o = pc.act(&action("read", OWNER, json!({"window": "notepad"}))); let els = o.reply["item"]["elements"].as_array().unwrap(); assert_eq!(els.len(), MAX_ELEMENTS); assert!(els[..150].iter().all(|e| e["ref"].as_str().unwrap().starts_with("@s1:e"))); assert!(els[150..].iter().all(|e| e["ref"] == "" && e["role"] == "statictext")); assert!(els.iter().all(|e| e["name"].as_str().unwrap().chars().count() <= MAX_CHARS)); assert_eq!(els[0]["name"].as_str().unwrap().chars().count(), MAX_CHARS); assert_eq!(els[0]["states"], json!(["focused"])); assert_eq!(o.reply["item"]["window"], json!({"id": "w-1", "program": "notepad", "title": TITLE})); // No window named: why. assert!(pc.act(&action("read", OWNER, json!({}))).reply["error"].is_string()); } #[test] fn a_stale_element_says_to_read_again() { let mut s = settings(); s.pc_control.click = Level::Auto; let mut fake = Fake::new(); fake.fail = Some(("click", "STALE_REF")); let (pc, _, _) = pc_with(s.clone(), fake); // An element of a reading this app does not know, or one in bad form. for r in ["@s9:e1", "e1", "--force"] { assert_eq!(pc.act(&action("click", OWNER, json!({"ref": r}))).reply["error"], STALE, "{r}"); } read_first(&pc); let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))); assert_eq!(o.reply, json!({"ok": false, "error": STALE})); assert_eq!(o.code, "STALE_REF"); } #[test] fn a_minimised_window_is_restored_before_acting_and_keys_bring_it_to_the_front() { let mut s = settings(); s.pc_control.type_text = Level::Auto; let (pc, fake, _) = pc_with(s, Fake::new()); let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+s"}))); assert_eq!(o.reply["ok"], true, "{o:?}"); assert_eq!(fake.commands(), ["list-windows", "restore", "focus-window", "press", "focus-window", "snapshot"]); let calls = fake.calls.lock().unwrap(); assert_eq!(calls[3].0, args(&["press", "ctrl+s", "--window-id", "w-3"])); assert!(calls.iter().all(|(a, _)| !a.iter().any(|x| x == "--force"))); drop(calls); for bad in ["--force", "ctrl+", "alt f4;rm"] { let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": bad}))); assert_eq!(o.code, "refused", "{bad}"); } } #[test] fn excluded_programs_and_this_apps_windows_are_never_listed_or_picked() { let (pc, _, _) = pc_with(settings(), Fake::new()); let o = pc.act(&action("windows", OWNER, json!({}))); assert_eq!(o.reply["item"]["windows"], json!([ {"id": "w-1", "program": "notepad", "title": TITLE, "minimised": false}, {"id": "w-3", "program": "calculatorapp", "title": "Calc", "minimised": true}])); assert!(pc.act(&action("read", OWNER, json!({"window": "Vault"}))).reply["error"].is_string()); } #[test] fn run_starts_a_program_by_name_and_reads_its_new_window() { let mut s = settings(); s.pc_control.run = Level::Auto; let fake = Fake::new(); let before = fake.windows.lock().unwrap()[0].clone(); let mut after = before.clone(); after.as_array_mut().unwrap().push(json!({"id": "w-9", "title": "Untitled - Notepad", "app_name": "notepad.exe", "pid": 99})); *fake.windows.lock().unwrap() = VecDeque::from([before, after]); let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 99}), ..fake }; let (pc, fake, _) = pc_with(s, fake); let o = pc.act(&action("run", OWNER, json!({"program": "notepad"}))); assert_eq!(o.reply["item"]["window"]["id"], "w-9", "{o:?}"); assert_eq!(fake.calls.lock().unwrap().iter().find(|(a, _)| a[0] == "launch").unwrap().0[1], "notepad"); assert_eq!(o.activity, Some(Record { kind: "pc.run".into(), target: "notepad.exe".into(), ok: true, note: None })); for bad in ["C:\\Windows\\cmd.exe", "-x", "..\\x"] { assert_eq!(pc.act(&action("run", OWNER, json!({"program": bad}))).code, "refused", "{bad}"); } } #[test] fn run_with_the_program_already_running_answers_the_window_launch_names() { let mut s = settings(); s.pc_control.run = Level::Auto; // agent-desktop attaches to a running instance and names its window; nothing new appears. let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 10, "window": {"id": "w-1", "title": TITLE}}), ..Fake::new() }; let (pc, _, _) = pc_with(s.clone(), fake); let started = Instant::now(); let o = pc.act(&action("run", OWNER, json!({"program": "notepad"}))); assert_eq!(o.reply["item"]["window"]["id"], "w-1", "{o:?}"); assert!(started.elapsed() < WINDOW_WAIT, "answered without waiting for a new window"); // Several running: launch cannot choose, and the answer says so (not an element's reason). let fake = Fake { fail: Some(("launch", "AMBIGUOUS_TARGET")), ..Fake::new() }; let (pc, _, _) = pc_with(s, fake); let o = pc.act(&action("run", OWNER, json!({"program": "notepad"}))); assert_eq!(o.reply, json!({"ok": false, "error": RUNNING_SEVERAL})); } #[test] fn keys_to_a_window_that_cannot_come_to_the_front_say_nothing_was_sent() { let mut s = settings(); s.pc_control.type_text = Level::Auto; // Windows' foreground lock: focus-window fails, or press cannot confirm the window's focus. for (cmd, code) in [("focus-window", "ACTION_FAILED"), ("press", "STALE_REF")] { let fake = Fake { fail: Some((cmd, code)), ..Fake::new() }; let (pc, fake, _) = pc_with(s.clone(), fake); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "ctrl+s"}))); assert_eq!(o.reply, json!({"ok": false, "error": NOT_FRONT}), "{cmd}"); assert!(cmd == "press" || !fake.commands().contains(&"press".to_owned()), "no key after a failed focus"); } } fn keys_auto(fake: Fake) -> (Pc, &'static Fake, &'static Stop) { let mut s = settings(); s.pc_control.type_text = Level::Auto; pc_with(s, fake) } #[test] fn one_key_without_modifiers_goes_behind_a_window_that_takes_it_and_nothing_moves() { for key in ["a", "7", "enter", "Tab", "space", "escape", "backspace", "forwarddelete", "left", "pagedown", "f5"] { let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() }); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key}))); assert_eq!(o.reply["ok"], true, "{key}: {o:?}"); assert_eq!(fake.commands(), ["list-windows", "post", "snapshot"], "{key}"); assert_eq!(fake.calls_from("post")[0][1], "w-1", "{key}"); assert_eq!(fake.front().as_deref(), Some("w-2"), "{key}: the window in front stays"); } } #[test] fn combinations_other_keys_and_windows_not_taking_keys_behind_go_in_front_and_back() { let in_front_and_back = |fake: &Fake, key: &str| { assert_eq!(fake.calls_from("focus-window"), [ args(&["focus-window", "--window-id", "w-1"]), args(&["press", key, "--window-id", "w-1"]), args(&["focus-window", "--window-id", "w-2"]), args(&["snapshot", "--window-id", "w-1", "--compact", "--timeout-ms", SNAPSHOT_MS])], "{key}"); assert!(!fake.commands().contains(&"post".to_owned()), "{key}: never both ways"); assert_eq!(fake.front().as_deref(), Some("w-2"), "{key}: the window in front is back"); }; for key in ["ctrl+s", "shift+tab", "alt+f", "cmd+d", "f10", "shift"] { let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() }); assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key}))).reply["ok"], true, "{key}"); in_front_and_back(fake, key); } // A window that does not take keys behind (a Store app, Chrome) gets even one letter in front. let (pc, fake, _) = keys_auto(Fake::new()); assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))).reply["ok"], true); in_front_and_back(fake, "a"); // A failed press still brings the window in front back, and is not tried behind. let (pc, fake, _) = keys_auto(Fake { fail: Some(("press", "STALE_REF")), ..Fake::new() }); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))); assert_eq!(o.reply, json!({"ok": false, "error": NOT_FRONT})); assert_eq!(fake.calls_from("focus-window").last().unwrap(), &args(&["focus-window", "--window-id", "w-2"])); assert_eq!(fake.front().as_deref(), Some("w-2")); } #[test] fn keys_to_a_minimised_window_bring_back_the_window_that_was_in_front_and_wait_for_no_game() { // Restoring brings the window forward, so it is part of the moment in front. let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() }); assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+a"}))).reply["ok"], true); assert_eq!(fake.calls_from("restore"), [ args(&["restore", "--window-id", "w-3"]), args(&["focus-window", "--window-id", "w-3"]), args(&["press", "ctrl+a", "--window-id", "w-3"]), args(&["focus-window", "--window-id", "w-2"]), args(&["snapshot", "--window-id", "w-3", "--compact", "--timeout-ms", SNAPSHOT_MS])]); assert_eq!(fake.front().as_deref(), Some("w-2"), "the window in front is back"); // During a game or full screen a minimised window is not even restored. let (pc, fake, _) = keys_auto(Fake { behind: true, fullscreen: true, ..Fake::new() }); let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "a"}))); assert_eq!(o.reply, json!({"ok": false, "error": NOT_IN_FRONT_NOW})); assert!(!fake.commands().contains(&"restore".to_owned()), "{:?}", fake.commands()); assert_eq!(fake.front().as_deref(), Some("w-2")); } #[test] fn a_store_app_agent_desktop_cannot_name_gets_its_keys_once_it_is_in_front() { // Brought in front, and already in front: the key is pressed once, in that window. for already in [false, true] { let fake = Fake { unnamed: true, ..Fake::new() }; if already { *fake.front.lock().unwrap() = Some("w-1".into()); } let (pc, fake, _) = keys_auto(fake); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "1"}))); assert_eq!(o.reply["ok"], true, "{o:?}"); let presses: Vec> = fake.calls_from("list-windows").into_iter().filter(|a| a[0] == "press").collect(); assert_eq!(presses.last().unwrap(), &args(&["press", "1"]), "{already}"); assert_eq!(fake.front().as_deref(), Some(if already { "w-1" } else { "w-2" }), "{already}: the window in front is back"); } } #[test] fn a_window_already_in_front_gets_its_keys_there_and_nothing_moves() { let fake = Fake::new(); *fake.front.lock().unwrap() = Some("w-1".into()); let (pc, fake, _) = keys_auto(fake); assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "ctrl+s"}))).reply["ok"], true); assert_eq!(fake.commands(), ["list-windows", "press", "snapshot"]); } #[test] fn keys_are_never_brought_in_front_during_a_game_or_full_screen_but_go_behind() { let game = |fake: Fake| { let mut s = settings(); s.pc_control.type_text = Level::Auto; let fake: &'static Fake = Box::leak(Box::new(fake)); (Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into()))), fake) }; let full = |fake: Fake| { let (pc, fake, _) = keys_auto(Fake { fullscreen: true, ..fake }); (pc, fake) }; for (i, make) in [&game as &dyn Fn(Fake) -> (Pc, &'static Fake), &full].into_iter().enumerate() { for (key, behind) in [("ctrl+s", true), ("a", false)] { let (pc, fake) = make(Fake { behind, ..Fake::new() }); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key}))); assert_eq!(o.reply, json!({"ok": false, "error": NOT_IN_FRONT_NOW}), "{i} {key}"); assert_eq!(fake.commands(), ["list-windows"], "{i} {key}: no key sent"); } let (pc, fake) = make(Fake { behind: true, ..Fake::new() }); assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))).reply["ok"], true, "{i}"); assert_eq!(fake.commands(), ["list-windows", "post", "snapshot"], "{i}"); } } #[test] fn a_key_not_taken_behind_is_not_pressed_again_in_front() { let (pc, fake, _) = keys_auto(Fake { behind: true, fail: Some(("post", "ACTION_FAILED")), ..Fake::new() }); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))); assert_eq!(o.reply["ok"], false); assert_eq!(fake.commands(), ["list-windows", "post"]); } #[test] fn only_text_controls_take_keys_behind() { for c in ["Edit", "RichEdit20W", "RICHEDIT50W", "RichEditD2DPT", "Scintilla"] { assert!(text_control(c), "{c}"); } for c in ["Notepad", "CalcFrame", "Button", "Windows.UI.Core.CoreWindow", "Chrome_RenderWidgetHostHWND", "Microsoft.UI.Content.DesktopChildSiteBridge", "MozillaWindowClass", "edit"] { assert!(!text_control(c), "{c}"); } } #[test] fn an_action_that_closes_its_window_is_done_with_nothing_left_to_read() { let mut s = settings(); s.pc_control.type_text = Level::Auto; // Enter on a dialog (or a click on its button) closes it: the fresh reading finds no window. let (pc, _, _) = pc_with(s, Fake { fail: Some(("snapshot", "WINDOW_NOT_FOUND")), ..Fake::new() }); let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter"}))); assert_eq!(o.reply, json!({"ok": true, "item": {"window": {"id": "w-1", "program": "notepad", "title": TITLE}, "elements": []}})); assert_eq!(o.code, "ok"); } #[test] fn a_program_that_does_not_close_in_time_is_said_to_be_still_open() { let mut s = settings(); s.pc_control.click = Level::Auto; // A save question keeps the program open; close-app gives up after its own deadline. let fake = Fake { fail: Some(("close-app", "TIMEOUT")), ..Fake::new() }; let (pc, _, _) = pc_with(s, fake); let o = pc.act(&action("close", OWNER, json!({"window": "notepad"}))); assert_eq!(o.reply, json!({"ok": false, "error": NOT_CLOSED})); // Several of it running: close-app picks by name and cannot choose. let mut s = settings(); s.pc_control.click = Level::Auto; let (pc, _, _) = pc_with(s, Fake { fail: Some(("close-app", "AMBIGUOUS_TARGET")), ..Fake::new() }); let o = pc.act(&action("close", OWNER, json!({"window": "notepad"}))); assert_eq!(o.reply, json!({"ok": false, "error": CLOSE_SEVERAL})); } #[test] fn the_activity_names_the_program_only() { let mut s = settings(); s.pc_control.type_text = Level::Auto; s.pc_control.click = Level::Auto; let (pc, _, stop) = pc_with(s, Fake::new()); read_first(&pc); let typed = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).activity.unwrap(); assert_eq!(typed, Record { kind: "pc.type".into(), target: "notepad.exe".into(), ok: true, note: None }); let closed = pc.act(&action("close", OWNER, json!({"window": "notepad"}))).activity.unwrap(); assert_eq!(closed, Record { kind: "pc.close".into(), target: "notepad.exe".into(), ok: true, note: None }); stop.toggle(); let refused = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))).activity.unwrap(); assert_eq!((refused.ok, refused.note.as_deref()), (false, Some(STOPPED))); // Readings are not activity; nothing recorded carries a title, an element or the text. assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).activity, None); assert_eq!(pc.act(&action("windows", OWNER, json!({}))).activity, None); let all = format!("{typed:?}{closed:?}{refused:?}"); for secret in [TITLE, "secret-plan", TYPED, "label", "@s1"] { assert!(!all.contains(secret), "{secret}"); } } #[test] fn the_tool_is_available_only_with_the_feature_someone_registered_and_agent_desktop() { let s = settings(); assert_eq!(available(&s, true), cfg!(windows)); assert!(!available(&s, false)); assert!(!available(&Settings::default(), true)); let mut off = settings(); off.features.pc_control = false; assert!(!available(&off, true)); // The pinned file is checked by its hash. let f = crate::datadir::temp_dir("pcexe").join(EXE_NAME); std::fs::write(&f, b"not agent-desktop").unwrap(); assert!(!exe_ok(&f)); assert!(!exe_ok(&f.with_file_name("missing.exe"))); } #[test] fn hotkeys_and_envelopes_are_read() { assert_eq!(parse_hotkey(DEFAULT_STOP_HOTKEY), Some((2 | 4, 0x7B))); assert_eq!(parse_hotkey("alt+win+q"), Some((1 | 8, b'Q' as u32))); for bad in ["F12", "Ctrl+", "Ctrl+F25", "Hyper+F1", ""] { assert_eq!(parse_hotkey(bad), None, "{bad}"); } assert_eq!(envelope(br#"{"ok": true, "data": [1]}"#), Ok(json!([1]))); assert_eq!(envelope(b"warning\n{\"ok\": false, \"error\": {\"code\": \"STALE_REF\", \"message\": \"x\"}}\n"), Err("STALE_REF".into())); assert_eq!(envelope(b"garbage"), Err("APP_NO_JSON".into())); } #[test] fn agent_desktops_state_folder_is_made_afresh_each_start() { // What a folder made under another ACL holds would keep that ACL, and readings of an // earlier run are of no use: the folder starts empty. let data = std::env::temp_dir().join(format!("jarvis-pc-{}-{}", std::process::id(), rand::random::())); let home = data.join("runtime").join("agent-desktop"); std::fs::create_dir_all(home.join("refmaps")).unwrap(); std::fs::write(home.join("refmaps").join("old.json"), "{}").unwrap(); let _cli = Cli::new(Path::new("agent-desktop.exe"), &data); assert!(home.is_dir()); assert_eq!(std::fs::read_dir(&home).unwrap().count(), 0); let _ = std::fs::remove_dir_all(&data); } #[test] fn the_package_and_m0_pin_the_same_agent_desktop() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); let package = std::fs::read_to_string(root.join("shell/package.sh")).unwrap(); for pin in [AGENT_DESKTOP_URL, AGENT_DESKTOP_ARCHIVE_SHA256, AGENT_DESKTOP_SHA256] { assert!(package.contains(pin), "{pin}"); } let m0 = std::fs::read_to_string(root.join("../scripts/m0/m0lib/desktop.py")).unwrap(); assert!(m0.contains(AGENT_DESKTOP_ARCHIVE_SHA256) && m0.contains(&format!("AD_VERSION = \"{AGENT_DESKTOP_VERSION}\""))); } #[test] fn a_start_menu_shortcut_gives_its_program_file() { // A shell link with link information and a Unicode local path (MS-SHLLINK). fn link(flags: u32, path: &str) -> Vec { let mut b = vec![0u8; 0x4C]; b[0] = 0x4C; b[0x14..0x18].copy_from_slice(&flags.to_le_bytes()); let wide: Vec = path.encode_utf16().chain([0]).flat_map(|c| c.to_le_bytes()).collect(); let header = 0x24u32; let mut info = vec![0u8; header as usize]; info[4..8].copy_from_slice(&header.to_le_bytes()); info[8..12].copy_from_slice(&1u32.to_le_bytes()); info[28..32].copy_from_slice(&header.to_le_bytes()); info.extend(&wide); let size = info.len() as u32; info[0..4].copy_from_slice(&size.to_le_bytes()); b.extend(info); b } let path = r"C:\Program Files\메모\memo.exe"; assert_eq!(shortcut_target(&link(0x2, path)).as_deref(), Some(path)); assert_eq!(shortcut_target(&link(0x2 | 0x20, path)), None, "a shortcut with arguments"); assert_eq!(shortcut_target(&link(0x0, path)), None); assert_eq!(shortcut_target(b"short"), None); } }