Files
javis_bot/app/src/pc.rs
javis-bot 5767ffa5ae fix(v2): pcControl keys reach a Store app agent-desktop cannot name by its window
On the game PC agent-desktop's press --window-id answers APP_NOT_FOUND for
the new Calculator ("not found with exact process identity", nothing sent),
though focus-window brings it in front. The keys now go to the window in
front while it is still that window. The Windows test checks Notepad's typed
text by element values only (a label such as Ctrl+Shift+X contains "hi").

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 10:55:19 +09:00

1947 lines
91 KiB
Rust

//! PC control for the brain's `pcControl` tool (admin.spec.md, "PC control"; `docs/v2/pc-features.md`,
//! "키보드·마우스 제어와 안티치트"): who may, what is never done, the anti-cheat and emergency-stop
//! blocks, and every action run through agent-desktop, one CLI process per command. The commands go
//! through `Desktop`, so the tests drive everything with a fake; `Cli` is the real one.
use std::collections::{HashSet, VecDeque};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Mutex;
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use crate::permissions::{decide, Action, Decision, Refusal};
use crate::settings::Settings;
use crate::worker_client::PcAction;
/// agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0), pinned as M0 ran it
/// (`scripts/m0/m0lib/desktop.py`).
pub const AGENT_DESKTOP_VERSION: &str = "1.2.0";
pub const AGENT_DESKTOP_URL: &str = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-pc-windows-msvc.tar.gz";
/// The release archive (`app/shell/package.sh` checks it).
pub const AGENT_DESKTOP_ARCHIVE_SHA256: &str = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8";
/// `agent-desktop.exe` inside it, checked before every start of the services.
pub const AGENT_DESKTOP_SHA256: &str = "d274667031322e50b9c2b1ab81c2ca46835d52cc932769a5c34bd4d481fd94c3";
pub const EXE_NAME: &str = "agent-desktop.exe";
/// One agent-desktop command at most.
pub const COMMAND_LIMIT: Duration = Duration::from_secs(15);
/// Elements in a reading (interactive ones first), and the characters kept of a name or value.
pub const MAX_ELEMENTS: usize = 200;
pub const MAX_CHARS: usize = 120;
/// Characters of text put into a field.
pub const MAX_TEXT: usize = 2000;
pub const MAX_WINDOWS: usize = 30;
pub const DEFAULT_STOP_HOTKEY: &str = "Ctrl+Shift+F12";
/// How long a reading may take (agent-desktop's budget; Chromium windows settle slowly).
const SNAPSHOT_MS: &str = "8000";
/// How long `launch` waits for the program, and then this app for its window.
const LAUNCH_MS: &str = "8000";
const WINDOW_WAIT: Duration = Duration::from_secs(5);
/// Readings whose elements may still be acted on.
const KEPT_READINGS: usize = 64;
pub const NOT_REGISTERED: &str = "PC 제어 화면에 등록된 사람만 PC를 조작할 수 있습니다";
pub const TURNED_OFF: &str = "PC 제어 화면에서 이 동작이 꺼져 있습니다";
pub const PASSWORD: &str = "비밀번호 칸에는 입력하지 않습니다";
pub const ELEVATED: &str = "관리자 권한으로 실행된 창은 조작할 수 없습니다";
pub const STOPPED: &str = "비상 정지 중입니다. 단축키를 한 번 더 누르거나 PC 제어 화면에서 풀어 주세요";
pub const STALE: &str = "화면이 바뀌었습니다. 다시 읽어 주세요";
const NO_WINDOW: &str = "그 창을 찾지 못했습니다";
const NOT_FRONT: &str = "그 창을 앞으로 가져오지 못해 키를 보내지 않았습니다";
const NOT_IN_FRONT_NOW: &str = "게임 중이거나 전체 화면이라 그 창을 앞으로 가져오지 않았습니다. 키를 보내지 않았습니다";
const NOT_CLOSED: &str = "프로그램이 아직 닫히지 않았습니다(저장할지 묻는 창이 떠 있을 수 있습니다). 창을 다시 읽어 주세요";
const CLOSE_SEVERAL: &str = "그 프로그램이 여러 개 실행 중이라 이 창만 닫을 수 없습니다. 창의 닫기 단추를 눌러 주세요";
const RUNNING_SEVERAL: &str = "그 프로그램이 이미 여러 개 실행 중이라 새로 열지 못했습니다. 열린 창 중에서 골라 주세요";
/// Anti-cheats found by their processes (file names, lower case): the anti-cheat's name, and the
/// game's when the process is the game itself. Vanguard's driver is always loaded, so it is found
/// by Riot's game processes. Grown by updates.
const ANTI_CHEATS: &[(&str, &str, Option<&str>)] = &[
("valorant-win64-shipping.exe", "뱅가드", Some("VALORANT")),
("valorant.exe", "뱅가드", Some("VALORANT")),
("league of legends.exe", "뱅가드", Some("League of Legends")),
("easyanticheat.exe", "Easy Anti-Cheat", None),
("easyanticheat_eos.exe", "Easy Anti-Cheat", None),
("beservice.exe", "BattlEye", None),
("beservice_x64.exe", "BattlEye", None),
("faceit.exe", "FACEIT", None),
("faceitservice.exe", "FACEIT", None),
("gamemon.des", "nProtect GameGuard", None),
("gamemon64.des", "nProtect GameGuard", None),
("xcorona.xem", "XIGNCODE3", None),
("xcorona_x64.xem", "XIGNCODE3", None),
("xxd-0.xem", "XIGNCODE3", None),
];
/// Where agent-desktop is shipped: next to the app's program file.
pub fn exe_path() -> Option<PathBuf> {
Some(std::env::current_exe().ok()?.parent()?.join(EXE_NAME))
}
/// Whether `exe` is the pinned agent-desktop.
pub fn exe_ok(exe: &Path) -> bool {
crate::download::sha256_file(exe).is_ok_and(|h| h == AGENT_DESKTOP_SHA256)
}
/// Whether the brain gets the tool: Windows, the feature on, someone registered and agent-desktop
/// present with the right hash (`installed`).
pub fn available(s: &Settings, installed: bool) -> bool {
cfg!(windows) && s.features.pc_control && !s.pc_control.allowed_users.is_empty() && installed
}
/// The agent-desktop the services use, when PC control is available (checked once per start).
pub fn check(s: &Settings) -> Option<PathBuf> {
if !available(s, true) {
return None;
}
match exe_path() {
Some(exe) if exe_ok(&exe) => Some(exe),
Some(exe) if exe.exists() => {
crate::warn_log!("conversation", "Warning: agent-desktop has another hash than {AGENT_DESKTOP_VERSION}'s: PC control is off");
None
}
_ => {
crate::warn_log!("conversation", "Warning: agent-desktop is not installed: PC control is off");
None
}
}
}
/// `Ctrl+Shift+F12` as the hotkey's modifiers (`MOD_ALT` 1, `MOD_CONTROL` 2, `MOD_SHIFT` 4,
/// `MOD_WIN` 8) and virtual key (letters, digits, F1-F24); at least one modifier.
pub fn parse_hotkey(s: &str) -> Option<(u32, u32)> {
let parts: Vec<String> = s.split('+').map(|p| p.trim().to_ascii_lowercase()).collect();
let (key, mods) = parts.split_last()?;
let mut m = 0;
for p in mods {
m |= match p.as_str() {
"alt" => 1,
"ctrl" | "control" => 2,
"shift" => 4,
"win" => 8,
_ => return None,
};
}
let vk = match key.as_bytes() {
[c] if c.is_ascii_alphanumeric() => c.to_ascii_uppercase() as u32,
[b'f', n @ ..] => match std::str::from_utf8(n).ok()?.parse::<u32>().ok()? {
n @ 1..=24 => 0x6F + n,
_ => return None,
},
_ => return None,
};
(m != 0).then_some((m, vk))
}
// ---- emergency stop -----------------------------------------------------------------------------
/// The emergency stop: on until the hotkey is pressed again or the PC 제어 screen lifts it; an app
/// restart lifts it (it lives in the process only).
pub struct Stop {
on: AtomicBool,
/// How often it was turned on: a command started before the latest press is killed.
presses: AtomicU64,
}
impl Default for Stop {
fn default() -> Self {
Self::new()
}
}
impl Stop {
pub const fn new() -> Self {
Stop { on: AtomicBool::new(false), presses: AtomicU64::new(0) }
}
pub fn on(&self) -> bool {
self.on.load(Ordering::SeqCst)
}
/// The hotkey: on, or off again.
pub fn toggle(&self) {
let now_on = !self.on();
if now_on {
self.presses.fetch_add(1, Ordering::SeqCst);
}
self.on.store(now_on, Ordering::SeqCst);
crate::info_log!("conversation", "PC control emergency stop {}", if now_on { "on" } else { "lifted" });
}
/// The PC 제어 screen's 풀기.
pub fn lift(&self) {
if self.on.swap(false, Ordering::SeqCst) {
crate::info_log!("conversation", "PC control emergency stop lifted");
}
}
fn presses(&self) -> u64 {
self.presses.load(Ordering::SeqCst)
}
/// Whether a command started at `presses` is to be killed.
fn pressed_since(&self, presses: u64) -> bool {
self.on() && self.presses() != presses
}
}
static EMERGENCY: Stop = Stop::new();
/// The app's emergency stop (the hotkey, the screen and the dashboard share it).
pub fn emergency() -> &'static Stop {
&EMERGENCY
}
// ---- agent-desktop ------------------------------------------------------------------------------
/// What PC control runs on: agent-desktop and two questions to Windows (`Cli`; a fake in tests).
pub trait Desktop: Send + Sync {
/// One agent-desktop command (never `--force`, never the raw mouse and key commands): its
/// `data`, or its error code. A press of the emergency stop while it runs kills it
/// (`APP_STOPPED`).
fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result<Value, String>;
/// Whether process `pid` runs as administrator; one that cannot be told counts as one.
fn elevated(&self, pid: u64) -> bool;
/// The running programs' file names, lower case.
fn processes(&self) -> Vec<String>;
/// What `launch` is given for a program name: the name itself, or the program file a Start
/// menu shortcut or Windows' App Paths name.
fn program(&self, name: &str) -> String {
name.to_owned()
}
/// The window in front now, as agent-desktop names it (`w-<handle>`).
fn foreground(&self) -> Option<String> {
None
}
/// Whether the window in front fills its monitor without a frame (not the desktop), or a
/// full-screen Direct3D program runs.
fn fullscreen(&self) -> bool {
false
}
/// Whether window `id` takes keys posted to it behind: it has one text control shown
/// (`text_control`).
fn takes_keys_behind(&self, _id: &str) -> bool {
false
}
/// Post one key (`behind_key`) to window `id`'s text control, without bringing it forward.
fn post_key(&self, _id: &str, _key: &str) -> Result<(), String> {
Err("PLATFORM_NOT_SUPPORTED".into())
}
}
/// The real agent-desktop: one process per command, its APPDATA, LOCALAPPDATA and state folder in
/// the data folder (`runtime/agent-desktop`).
pub struct Cli {
exe: PathBuf,
home: PathBuf,
limit: Duration,
}
impl Cli {
pub fn new(exe: &Path, data: &Path) -> Self {
let home = data.join("runtime").join("agent-desktop");
// agent-desktop refuses a state folder others may write to ("private file DACL grants an
// untrusted principal": a data folder outside the user's profile inherits Authenticated
// Users' Modify), so it is made afresh, private to this user; a run's readings are of no
// use to the next.
let _ = std::fs::remove_dir_all(&home);
let _ = std::fs::create_dir_all(&home);
#[cfg(windows)]
if !win::make_private(&home) {
crate::warn_log!("conversation", "Warning: agent-desktop's state folder could not be made private: PC control may fail");
}
Cli { exe: exe.to_path_buf(), home, limit: COMMAND_LIMIT }
}
}
/// An agent-desktop envelope `{ok, data, error{code}}`: the data, or the code (never the message,
/// which can name windows).
fn envelope(out: &[u8]) -> Result<Value, String> {
let text = String::from_utf8_lossy(out);
let text = text.trim();
let v: Value = serde_json::from_str(text).or_else(|_| serde_json::from_str(text.lines().last().unwrap_or("")))
.map_err(|_| "APP_NO_JSON".to_owned())?;
match v["ok"].as_bool() {
Some(true) => Ok(v["data"].clone()),
Some(false) => Err(v["error"]["code"].as_str().unwrap_or("APP_UNKNOWN").to_owned()),
None => Err("APP_NO_JSON".into()),
}
}
impl Desktop for Cli {
fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result<Value, String> {
let mut cmd = std::process::Command::new(&self.exe);
cmd.args(args);
if args.first().map(String::as_str) == Some("launch") {
// A started program gets the person's own folders, not agent-desktop's.
for var in ["APPDATA", "LOCALAPPDATA"] {
if let Some(v) = std::env::var_os(var) {
cmd.arg("--env").arg(format!("{var}={}", v.to_string_lossy()));
}
}
if let Some(home) = std::env::var_os("USERPROFILE").filter(|h| Path::new(h).is_absolute()) {
cmd.arg("--cwd").arg(home);
}
}
cmd.env("APPDATA", &self.home).env("LOCALAPPDATA", &self.home).env("AGENT_DESKTOP_HOME", &self.home)
.stdin(if stdin.is_some() { std::process::Stdio::piped() } else { std::process::Stdio::null() })
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null());
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
}
let presses = stop.presses();
let mut child = cmd.spawn().map_err(|_| "APP_NOT_STARTED".to_owned())?;
if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) {
// Text goes in through stdin, never in the arguments; written aside so a full pipe
// cannot hold this thread.
let text = text.to_owned();
std::thread::spawn(move || {
let _ = pipe.write_all(text.as_bytes());
});
}
let mut out = child.stdout.take().expect("stdout is piped");
let reader = std::thread::spawn(move || {
let mut b = Vec::new();
let _ = out.read_to_end(&mut b);
b
});
let started = Instant::now();
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => {}
Err(_) => return Err("APP_NOT_STARTED".into()),
}
let killed = if stop.pressed_since(presses) {
Some("APP_STOPPED")
} else if started.elapsed() > self.limit {
Some("APP_TIMEOUT")
} else {
None
};
if let Some(code) = killed {
let _ = child.kill();
let _ = child.wait();
return Err(code.into());
}
std::thread::sleep(Duration::from_millis(10));
}
envelope(&reader.join().unwrap_or_default())
}
fn elevated(&self, pid: u64) -> bool {
#[cfg(windows)]
return win::elevated(pid);
#[cfg(not(windows))]
{
let _ = pid;
false
}
}
fn processes(&self) -> Vec<String> {
#[cfg(windows)]
return win::processes();
#[cfg(not(windows))]
vec![]
}
fn program(&self, name: &str) -> String {
#[cfg(windows)]
return win::program(name).unwrap_or_else(|| name.to_owned());
#[cfg(not(windows))]
name.to_owned()
}
#[cfg(windows)]
fn foreground(&self) -> Option<String> {
win::foreground()
}
#[cfg(windows)]
fn fullscreen(&self) -> bool {
win::fullscreen()
}
#[cfg(windows)]
fn takes_keys_behind(&self, id: &str) -> bool {
win::text_control_of(id).is_some()
}
#[cfg(windows)]
fn post_key(&self, id: &str, key: &str) -> Result<(), String> {
let vk = behind_key(key).ok_or_else(|| "INVALID_ARGS".to_owned())?;
let to = win::text_control_of(id).ok_or_else(|| "WINDOW_NOT_FOUND".to_owned())?;
win::post_key(to, vk)
}
}
// ---- the actions --------------------------------------------------------------------------------
/// A window as agent-desktop lists it.
#[derive(Debug, Clone, PartialEq)]
struct Win {
id: String,
/// Program file name, lower case (`notepad.exe`).
program: String,
title: String,
pid: u64,
focused: bool,
minimised: bool,
}
impl Win {
fn view(&self) -> Value {
json!({"id": self.id, "program": stem(&self.program), "title": cut(&self.title)})
}
}
fn stem(program: &str) -> &str {
program.strip_suffix(".exe").unwrap_or(program)
}
fn cut(s: &str) -> String {
s.chars().take(MAX_CHARS).collect()
}
/// An activity record (admin.spec.md, "Activity"): the program's file name, never a title,
/// element name or text.
#[derive(Debug, Clone, PartialEq)]
pub struct Record {
pub kind: String,
pub target: String,
pub ok: bool,
pub note: Option<String>,
}
/// The answer to one `pc_action`, its result code for the log and what the activity records.
#[derive(Debug, Clone, PartialEq)]
pub struct Outcome {
pub reply: Value,
/// `ok`, `ask`, `refused` or agent-desktop's error code.
pub code: String,
pub activity: Option<Record>,
}
#[derive(Debug)]
struct Fail {
code: String,
reason: String,
}
fn refused(reason: &str) -> Fail {
Fail { code: "refused".into(), reason: reason.into() }
}
/// agent-desktop's error code as the reason the person hears.
fn failed(code: &str, action: &str) -> Fail {
let reason = match code {
// Windows' foreground lock: focus-window could not bring the window forward, or press
// could not confirm its focus; no key was sent.
"ACTION_FAILED" | "STALE_REF" if action == "keys" => NOT_FRONT,
// close-app gives up while the program still runs (a save question, say).
"TIMEOUT" | "APP_TIMEOUT" if action == "close" => NOT_CLOSED,
// close-app picks the program by name; with several running it cannot choose.
"AMBIGUOUS_TARGET" if action == "close" => CLOSE_SEVERAL,
// launch attaches to one running instance; with several it cannot choose.
"AMBIGUOUS_TARGET" if action == "run" => RUNNING_SEVERAL,
"STALE_REF" | "SNAPSHOT_NOT_FOUND" | "ELEMENT_NOT_FOUND" => STALE,
"AMBIGUOUS_TARGET" => "같은 모양의 항목이 여럿이라 고를 수 없습니다. 다시 읽어 주세요",
"APP_NOT_FOUND" if action == "run" => "그 프로그램을 찾지 못했습니다",
"APP_NOT_FOUND" | "WINDOW_NOT_FOUND" => NO_WINDOW,
"PERM_DENIED" => ELEVATED,
"POLICY_DENIED" => "막혀 있는 동작이라 하지 않습니다(위험한 단축키 등)",
"INVALID_ARGS" if action == "close" => "Windows가 지키는 프로그램이라 닫을 수 없습니다",
"INVALID_ARGS" => "요청을 알아듣지 못했습니다. 이름이나 키를 확인해 주세요",
"ACTION_NOT_SUPPORTED" => "그 항목은 이 동작을 받지 않습니다",
"ACTION_FAILED" => "동작이 되지 않았습니다",
"APP_UNRESPONSIVE" => "프로그램이 응답하지 않습니다",
"TIMEOUT" | "APP_TIMEOUT" => "시간 안에 끝나지 않았습니다",
"APP_STOPPED" => STOPPED,
"PLATFORM_NOT_SUPPORTED" => "이 PC에서는 할 수 없는 동작입니다",
_ => "PC 제어 도구가 실패했습니다",
};
Fail { code: code.into(), reason: reason.into() }
}
enum Done {
Item(Value),
Ask(Value),
}
/// The permission kind of an action and its name in an `ask` answer.
fn kind_of(action: &str) -> Option<(Action, &'static str)> {
match action {
"windows" | "read" => Some((Action::View, "view")),
"click" | "close" => Some((Action::Click, "click")),
"type" | "keys" => Some((Action::Type, "type")),
"run" => Some((Action::Run, "run")),
_ => None,
}
}
const ACTING: &[&str] = &["click", "type", "keys", "run", "close"];
fn valid_ref(r: Option<&str>) -> Result<&str, Fail> {
let r = r.map(str::trim).filter(|r| !r.is_empty()).ok_or_else(|| refused("누를 항목(ref)이 필요합니다"))?;
let ok = r.strip_prefix('@').and_then(|r| r.split_once(":e")).is_some_and(|(snap, n)| {
!snap.is_empty() && snap.len() <= 64 && snap.bytes().all(|b| b.is_ascii_alphanumeric())
&& !n.is_empty() && n.len() <= 9 && n.bytes().all(|b| b.is_ascii_digit())
});
if ok { Ok(r) } else { Err(refused(STALE)) }
}
/// A key combination for `press`: names joined with `+` (`ctrl+s`, `enter`), nothing that could be
/// read as an option.
fn valid_keys(k: Option<&str>) -> Result<String, Fail> {
let k = k.map(|k| k.replace(' ', "")).filter(|k| !k.is_empty()).ok_or_else(|| refused("누를 키가 필요합니다"))?;
let ok = k.len() <= 40 && k.split('+').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_alphanumeric()));
if ok { Ok(k) } else { Err(refused("키를 알아듣지 못했습니다 (예: ctrl+s, enter)")) }
}
/// The virtual key of `keys` when it can be sent behind: one key without modifiers that types or
/// moves, by agent-desktop's names (`a`-`z` and `0`-`9` as their keys on a US layout); else `None`.
fn behind_key(keys: &str) -> Option<u16> {
let k = keys.to_ascii_lowercase();
let vk = match k.as_str() {
"return" | "enter" => 0x0D,
"escape" | "esc" => 0x1B,
"tab" => 0x09,
"space" => 0x20,
"delete" | "backspace" => 0x08,
"forwarddelete" => 0x2E,
"home" => 0x24,
"end" => 0x23,
"pageup" => 0x21,
"pagedown" => 0x22,
"left" => 0x25,
"up" => 0x26,
"right" => 0x27,
"down" => 0x28,
// F10 is Windows' menu key (a system key), so it goes in front.
f if f.len() <= 3 && f.starts_with('f') => match f[1..].parse::<u16>() {
Ok(n @ (1..=9 | 11 | 12)) => 0x6F + n,
_ => return None,
},
c if c.len() == 1 && c.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) => c.as_bytes()[0].to_ascii_uppercase() as u16,
_ => return None,
};
Some(vk)
}
/// Whether a control of this class takes keys posted behind: the text controls whose keys Windows
/// gives to the control itself (classic Edit and RichEdit, the new Notepad's, Scintilla).
pub fn text_control(class: &str) -> bool {
["Edit", "RichEdit20A", "RichEdit20W", "RICHEDIT50W", "RichEditD2DPT", "Scintilla"].contains(&class)
}
/// A program to start by name: no folders, nothing that could be read as an option.
fn valid_program(p: Option<&str>) -> Result<String, Fail> {
let p = p.map(str::trim).filter(|p| !p.is_empty()).ok_or_else(|| refused("실행할 프로그램 이름이 필요합니다"))?;
let ok = p.chars().count() <= 100 && !p.starts_with(['-', '.']) && !p.contains("..")
&& !p.chars().any(|c| c.is_control() || matches!(c, '\\' | '/' | ':' | '"' | '*' | '?' | '<' | '>' | '|'));
if ok { Ok(p.to_owned()) } else { Err(refused("프로그램 이름으로만 실행할 수 있습니다 (예: notepad)")) }
}
/// A program name as a file name for the activity (`notepad` -> `notepad.exe`).
fn program_file(name: &str) -> String {
let n = name.to_lowercase();
if Path::new(&n).extension().is_some() { n } else { format!("{n}.exe") }
}
/// A reading's elements: interactive ones (with a ref) first, then the rest that have a name or a
/// value; at most `MAX_ELEMENTS`, names and values cut to `MAX_CHARS`.
fn elements(tree: &Value) -> Vec<Value> {
fn walk<'a>(n: &'a Value, refs: &mut Vec<&'a Value>, rest: &mut Vec<&'a Value>) {
for c in n["children"].as_array().into_iter().flatten() {
if c["ref_id"].as_str().is_some_and(|r| !r.is_empty()) {
refs.push(c);
} else if c["name"].as_str().is_some_and(|s| !s.trim().is_empty())
|| c["value"].as_str().is_some_and(|s| !s.trim().is_empty()) {
rest.push(c);
}
walk(c, refs, rest);
}
}
let (mut refs, mut rest) = (vec![], vec![]);
walk(tree, &mut refs, &mut rest);
refs.into_iter().chain(rest).take(MAX_ELEMENTS).map(|n| json!({
"ref": n["ref_id"].as_str().unwrap_or(""),
"role": n["role"].as_str().unwrap_or(""),
"name": cut(n["name"].as_str().unwrap_or("")),
"value": cut(n["value"].as_str().unwrap_or("")),
"states": n["states"].as_array().cloned().unwrap_or_default(),
})).collect()
}
/// The refs of a reading's fields agent-desktop reports as secure (password fields), in the whole
/// tree, not only the elements shown.
fn secure_refs(tree: &Value) -> HashSet<String> {
fn walk(n: &Value, out: &mut HashSet<String>) {
for c in n["children"].as_array().into_iter().flatten() {
if let Some(r) = c["ref_id"].as_str().filter(|_| c["states"].as_array().is_some_and(|s| s.iter().any(|x| x == "secure"))) {
out.insert(r.to_owned());
}
walk(c, out);
}
}
let mut out = HashSet::new();
walk(tree, &mut out);
out
}
fn args(a: &[&str]) -> Vec<String> {
a.iter().map(|s| s.to_string()).collect()
}
/// PC control for one run of the services.
pub struct Pc {
settings: Settings,
desktop: Box<dyn Desktop>,
stop: &'static Stop,
/// The game game detection sees now, for the anti-cheat sentence.
game: Box<dyn Fn() -> Option<String> + Send + Sync>,
/// This app's process: its windows are never listed or acted on.
own_pid: u64,
/// The latest readings by snapshot id: the window, so an element names its window, and the
/// fields read as secure (password fields; agent-desktop cannot address them again by `get`).
readings: Mutex<VecDeque<(String, Win, HashSet<String>)>>,
/// One action at a time.
one: Mutex<()>,
}
impl Pc {
pub fn new(settings: Settings, desktop: Box<dyn Desktop>, stop: &'static Stop,
game: Box<dyn Fn() -> Option<String> + Send + Sync>) -> Self {
Pc { settings, desktop, stop, game, own_pid: std::process::id() as u64, readings: Mutex::new(VecDeque::new()),
one: Mutex::new(()) }
}
/// Answer one `pc_action` for its asker.
pub fn act(&self, a: &PcAction) -> Outcome {
let _one = self.one.lock().unwrap_or_else(|e| e.into_inner());
let mut program = None;
let result = self.answer(a, &mut program);
let acting = ACTING.contains(&a.action.as_str());
let (reply, code, note) = match result {
Ok(Done::Item(item)) => (json!({"ok": true, "item": item}), "ok".to_owned(), None),
Ok(Done::Ask(v)) => (v, "ask".to_owned(), None),
Err(f) => (json!({"ok": false, "error": f.reason}), f.code, Some(f.reason)),
};
let activity = (acting && code != "ask").then(|| Record { kind: format!("pc.{}", a.action),
target: program.unwrap_or_else(|| "?".into()), ok: code == "ok", note });
Outcome { reply, code, activity }
}
fn answer(&self, a: &PcAction, program: &mut Option<String>) -> Result<Done, Fail> {
let (perm, kind) = kind_of(&a.action).ok_or_else(|| refused("모르는 PC 동작입니다"))?;
let user = a.user.parse::<u64>().map_err(|_| refused(NOT_REGISTERED))?;
let ask = match decide(&self.settings.pc_control, user, perm) {
Decision::Allow => false,
Decision::Ask => !a.consent,
Decision::Refuse(Refusal::NotAllowed) => return Err(refused(NOT_REGISTERED)),
Decision::Refuse(_) => return Err(refused(TURNED_OFF)),
};
let asking = |window: Value| Done::Ask(json!({"ok": false, "ask": true, "kinds": [kind], "window": window}));
match a.action.as_str() {
"windows" => {
if ask {
return Ok(asking(json!({"id": "", "program": "", "title": ""})));
}
let list: Vec<Value> = self.list()?.iter().take(MAX_WINDOWS).map(|w| {
let mut v = w.view();
v["minimised"] = json!(w.minimised);
v
}).collect();
return Ok(Done::Item(json!({"windows": list})));
}
"read" => {
let w = self.window(a.window.as_deref())?;
if ask {
return Ok(asking(w.view()));
}
return self.read(&w).map(Done::Item);
}
_ => {}
}
// Acting: never during the emergency stop or while an anti-cheat game runs.
if self.stop.on() {
return Err(refused(STOPPED));
}
if let Some(why) = self.anti_cheat() {
return Err(refused(&why));
}
if a.action == "run" {
let name = valid_program(a.program.as_deref())?;
*program = Some(program_file(&name));
if ask {
return Ok(asking(json!({"id": "", "program": name, "title": ""})));
}
return self.launch(&name).map(Done::Item);
}
let (w, element) = match a.action.as_str() {
"click" | "type" => {
let r = valid_ref(a.element.as_deref())?;
(self.window_of(r)?, Some(r))
}
_ => (self.window(a.window.as_deref())?, None),
};
*program = Some(w.program.clone());
if self.desktop.elevated(w.pid) {
return Err(refused(ELEVATED));
}
let keys = if a.action == "keys" { Some(valid_keys(a.keys.as_deref())?) } else { None };
let text = a.text.as_deref().unwrap_or("");
if a.action == "type" {
if text.chars().count() > MAX_TEXT {
return Err(refused("글자가 너무 깁니다 (2,000자까지)"));
}
let read_secure = self.readings.lock().unwrap().iter().any(|(_, _, s)| s.contains(element.unwrap()));
if read_secure {
return Err(refused(PASSWORD));
}
let states = self.cmd(&args(&["get", element.unwrap(), "--property", "states"]), None, "type")?;
let secure = states["value"].as_array().is_none_or(|s| s.iter().any(|x| x == "secure"));
if secure {
return Err(refused(PASSWORD));
}
}
if ask {
return Ok(asking(w.view()));
}
if self.stop.on() {
return Err(refused(STOPPED));
}
if w.minimised {
self.cmd(&args(&["restore", "--window-id", &w.id]), None, &a.action)?;
}
match a.action.as_str() {
"click" => {
self.cmd(&args(&["click", element.unwrap()]), None, "click")?;
}
"type" => {
self.cmd(&args(&["set-value", element.unwrap(), "--stdin"]), Some(text), "type")?;
}
"keys" => self.keys(&w, keys.as_deref().unwrap())?,
_ => {
self.cmd(&args(&["close-app", &w.program]), None, "close")?;
return Ok(Done::Item(json!({"window": w.view(), "elements": []})));
}
}
match self.read(&w) {
// The action closed its window (a dialog's button, Enter): done, nothing left to read.
Err(f) if f.code == "WINDOW_NOT_FOUND" => Ok(Done::Item(json!({"window": w.view(), "elements": []}))),
r => r.map(Done::Item),
}
}
/// Press `keys` in window `w` one way, chosen first (admin.spec.md, "Keys"): behind it when it is
/// one plain key and the window takes it so, else in front for a moment and the window that was
/// in front brought back. Never both ways: a key is never pressed twice.
fn keys(&self, w: &Win, keys: &str) -> Result<(), Fail> {
let front = self.desktop.foreground();
if front.as_deref() == Some(w.id.as_str()) {
crate::debug_log!("conversation", "pc keys way=front-already");
return self.press(w, keys);
}
if behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) {
crate::debug_log!("conversation", "pc keys way=behind");
return self.desktop.post_key(&w.id, keys).map_err(|code| failed(&code, "keys"));
}
if (self.game)().is_some() || self.desktop.fullscreen() {
crate::debug_log!("conversation", "pc keys way=none (game or full screen)");
return Err(refused(NOT_IN_FRONT_NOW));
}
crate::debug_log!("conversation", "pc keys way=front");
let pressed = self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys")
.and_then(|_| self.press(w, keys));
if let Some(back) = front {
if self.desktop.run(&args(&["focus-window", "--window-id", &back]), None, self.stop).is_err() {
crate::warn_log!("conversation", "Warning: PC control could not bring back the window that was in front");
}
}
pressed.map(|_| ())
}
/// `press` in window `w`, which is in front. agent-desktop cannot find some Store apps' process
/// by the window (the new Calculator: APP_NOT_FOUND, nothing sent), so then the keys go to the
/// window in front while it is still `w`.
fn press(&self, w: &Win, keys: &str) -> Result<(), Fail> {
match self.desktop.run(&args(&["press", keys, "--window-id", &w.id]), None, self.stop) {
Err(code) if code == "APP_NOT_FOUND" && self.desktop.foreground().as_deref() == Some(w.id.as_str()) => {
crate::debug_log!("conversation", "pc keys press=in-front (window's process not found)");
self.cmd(&args(&["press", keys]), None, "keys").map(|_| ())
}
r => r.map(|_| ()).map_err(|code| failed(&code, "keys")),
}
}
fn cmd(&self, args: &[String], stdin: Option<&str>, action: &str) -> Result<Value, Fail> {
self.desktop.run(args, stdin, self.stop).map_err(|code| failed(&code, action))
}
/// The windows that may be listed and acted on: not this app's, not an excluded program's
/// (`pcControl.screenExclude`), with a title; in agent-desktop's order.
fn list(&self) -> Result<Vec<Win>, Fail> {
let data = self.cmd(&args(&["list-windows"]), None, "windows")?;
let excluded = &self.settings.pc_control.screen_exclude;
Ok(data.as_array().into_iter().flatten().filter_map(|w| {
let program = crate::game::file_name(w["app_name"].as_str()?);
let win = Win { id: w["id"].as_str()?.to_owned(), title: w["title"].as_str().unwrap_or("").to_owned(),
pid: w["pid"].as_u64().unwrap_or(0), focused: w["is_focused"].as_bool().unwrap_or(false),
minimised: w["minimized"].as_bool().unwrap_or(false), program };
let shown = w["visible"].as_bool() != Some(false) && !win.title.trim().is_empty() && !win.id.is_empty();
(shown && win.pid != self.own_pid && !win.program.is_empty()
&& !crate::game::listed(&win.program, excluded)).then_some(win)
}).collect())
}
/// A window by its id, else the one whose title or program contains `query` (the focused one
/// first, then the first listed).
fn window(&self, query: Option<&str>) -> Result<Win, Fail> {
let q = query.map(str::trim).filter(|q| !q.is_empty()).ok_or_else(|| refused("창 이름이 필요합니다"))?;
let list = self.list()?;
if let Some(w) = list.iter().find(|w| w.id == q) {
return Ok(w.clone());
}
let q = q.to_lowercase();
let hits: Vec<&Win> = list.iter()
.filter(|w| w.title.to_lowercase().contains(&q) || stem(&w.program).contains(&q)).collect();
hits.iter().find(|w| w.focused).or(hits.first()).map(|w| (*w).clone()).ok_or_else(|| refused(NO_WINDOW))
}
/// The window an element was read from, as it is listed now.
fn window_of(&self, element: &str) -> Result<Win, Fail> {
let snap = element.trim_start_matches('@').split(':').next().unwrap_or("");
let id = self.readings.lock().unwrap().iter().find(|(s, _, _)| s == snap).map(|(_, w, _)| w.id.clone())
.ok_or_else(|| refused(STALE))?;
self.list()?.into_iter().find(|w| w.id == id).ok_or_else(|| refused(STALE))
}
/// Read a window afresh: `{window, elements}`.
fn read(&self, w: &Win) -> Result<Value, Fail> {
let data = self.cmd(&args(&["snapshot", "--window-id", &w.id, "--compact", "--timeout-ms", SNAPSHOT_MS]), None, "read")?;
let mut w = w.clone();
if let Some(t) = data["window"]["title"].as_str() {
w.title = t.to_owned();
}
if let Some(snap) = data["snapshot_id"].as_str() {
let mut r = self.readings.lock().unwrap();
r.push_back((snap.to_owned(), w.clone(), secure_refs(&data["tree"])));
while r.len() > KEPT_READINGS {
r.pop_front();
}
}
Ok(json!({"window": w.view(), "elements": elements(&data["tree"])}))
}
/// Start a program (agent-desktop never asks for administrator rights) and read its window:
/// the one `launch` names, else a new one of its process or name, else the only new one.
fn launch(&self, name: &str) -> Result<Value, Fail> {
let before: HashSet<String> = self.list()?.into_iter().map(|w| w.id).collect();
let target = self.desktop.program(name);
let launched = match self.desktop.run(&args(&["launch", &target, "--timeout", LAUNCH_MS]), None, self.stop) {
Ok(d) => d,
// A program that hands over to another process (the Windows calculator) ends at once.
Err(code) if code == "APP_UNRESPONSIVE" => Value::Null,
Err(code) => return Err(failed(&code, "run")),
};
let (pid, id) = (launched["pid"].as_u64(), launched["window"]["id"].as_str().map(str::to_owned));
let lower = name.to_lowercase();
let started = Instant::now();
loop {
let list = self.list()?;
// A program already running is not started again: launch names its window.
let named = list.iter().find(|w| Some(&w.id) == id.as_ref());
let new: Vec<&Win> = list.iter().filter(|w| !before.contains(&w.id)).collect();
let found = named
.or_else(|| new.iter().copied().find(|w| pid.is_some_and(|p| p == w.pid)))
.or_else(|| new.iter().copied().find(|w| stem(&w.program).contains(&lower) || w.title.to_lowercase().contains(&lower)))
.or(if new.len() == 1 { new.first().copied() } else { None });
if let Some(w) = found {
return self.read(w);
}
if self.stop.on() {
return Err(refused(STOPPED));
}
if started.elapsed() >= WINDOW_WAIT {
return Err(Fail { code: "no_window".into(), reason: "프로그램을 시작했지만 창을 찾지 못했습니다".into() });
}
std::thread::sleep(Duration::from_millis(300));
}
}
/// The anti-cheat sentence while a game with one runs (its processes, or a program the person
/// listed), else `None`.
fn anti_cheat(&self) -> Option<String> {
let running = self.desktop.processes();
let found = running.iter().find_map(|p| ANTI_CHEATS.iter().find(|(name, _, _)| name == p)
.map(|(_, ac, game)| (*ac, game.map(str::to_owned).unwrap_or_else(|| crate::game::game_name(p, &[])))))
.or_else(|| running.iter().find(|p| crate::game::listed(p, &self.settings.pc_control.anti_cheat_games))
.map(|p| ("안티치트", crate::game::game_name(p, &[]))))?;
let game = (self.game)().unwrap_or(found.1);
Some(format!("{}가 적용된 게임({game})이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", found.0))
}
}
// ---- the emergency stop's hotkey ----------------------------------------------------------------
/// The emergency stop's global hotkey, on its own thread while the services run.
pub struct Hotkey {
#[cfg(windows)]
thread: u32,
#[cfg(windows)]
join: Option<std::thread::JoinHandle<()>>,
}
impl Hotkey {
/// Register `spec` for `stop`; `None` elsewhere than Windows or when Windows refuses it
/// (another program holds it: warned in the log).
pub fn start(spec: &str, stop: &'static Stop) -> Option<Hotkey> {
let (mods, vk) = parse_hotkey(spec)?;
#[cfg(windows)]
return win::hotkey(mods, vk, stop).map(|(thread, join)| Hotkey { thread, join: Some(join) });
#[cfg(not(windows))]
{
let _ = (mods, vk, stop);
None
}
}
/// Unregister it and end its thread.
pub fn end(mut self) {
self.finish();
}
fn finish(&mut self) {
#[cfg(windows)]
if let Some(j) = self.join.take() {
win::end_hotkey(self.thread);
let _ = j.join();
}
}
}
impl Drop for Hotkey {
fn drop(&mut self) {
self.finish();
}
}
/// The program file a Start menu shortcut (`.lnk`) points at, from its link information (Unicode
/// path when present, else an ASCII one); `None` for a shortcut with arguments (starting the file
/// alone would do something else) or without a local path.
pub fn shortcut_target(b: &[u8]) -> Option<String> {
let u32_at = |o: usize| b.get(o..o + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as usize);
let u16_at = |o: usize| b.get(o..o + 2).map(|s| u16::from_le_bytes([s[0], s[1]]) as usize);
if u32_at(0)? != 0x4C {
return None;
}
let flags = u32_at(0x14)?;
const HAS_ID_LIST: usize = 0x1;
const HAS_LINK_INFO: usize = 0x2;
const HAS_ARGUMENTS: usize = 0x20;
if flags & HAS_LINK_INFO == 0 || flags & HAS_ARGUMENTS != 0 {
return None;
}
let mut at = 0x4C;
if flags & HAS_ID_LIST != 0 {
at += 2 + u16_at(at)?;
}
let (header, info_flags) = (u32_at(at + 4)?, u32_at(at + 8)?);
if info_flags & 0x1 == 0 {
return None;
}
let path = if header >= 0x24 {
let start = at + u32_at(at + 28)?;
let units: Vec<u16> = b.get(start..)?.chunks_exact(2).map(|c| u16::from_le_bytes([c[0], c[1]]))
.take_while(|&c| c != 0).collect();
String::from_utf16(&units).ok()?
} else {
let start = at + u32_at(at + 16)?;
let bytes: Vec<u8> = b.get(start..)?.iter().copied().take_while(|&c| c != 0).collect();
if !bytes.is_ascii() {
return None;
}
String::from_utf8(bytes).ok()?
};
(!path.is_empty()).then_some(path)
}
#[cfg(windows)]
mod win {
use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE};
use windows_sys::Win32::Security::{GetTokenInformation, TokenElevation, TOKEN_ELEVATION, TOKEN_QUERY};
use windows_sys::Win32::System::Threading::{
GetCurrentThreadId, OpenProcess, OpenProcessToken, PROCESS_QUERY_LIMITED_INFORMATION,
};
/// Whether process `pid`'s token is elevated; a process that cannot be asked counts as elevated.
pub fn elevated(pid: u64) -> bool {
// SAFETY: query-only handles, closed at once; the out structure is sized as passed.
unsafe {
let p = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid as u32);
if p.is_null() {
return true;
}
let mut token: HANDLE = std::ptr::null_mut();
let opened = OpenProcessToken(p, TOKEN_QUERY, &mut token) != 0;
CloseHandle(p);
if !opened {
return true;
}
let mut e = TOKEN_ELEVATION { TokenIsElevated: 0 };
let mut len = 0u32;
let ok = GetTokenInformation(token, TokenElevation, (&mut e as *mut TOKEN_ELEVATION).cast(),
std::mem::size_of::<TOKEN_ELEVATION>() as u32, &mut len) != 0;
CloseHandle(token);
!ok || e.TokenIsElevated != 0
}
}
/// Make `dir` private to this user: a protected DACL with one entry, full control for the user,
/// inherited by what is made inside.
pub fn make_private(dir: &std::path::Path) -> bool {
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Security::{
AddAccessAllowedAceEx, GetLengthSid, InitializeAcl, InitializeSecurityDescriptor, SetFileSecurityW,
SetSecurityDescriptorControl, SetSecurityDescriptorDacl, TokenUser, ACCESS_ALLOWED_ACE, ACL, ACL_REVISION,
CONTAINER_INHERIT_ACE, DACL_SECURITY_INFORMATION, OBJECT_INHERIT_ACE, PROTECTED_DACL_SECURITY_INFORMATION,
SECURITY_DESCRIPTOR, SE_DACL_PROTECTED, TOKEN_USER,
};
use windows_sys::Win32::System::Threading::GetCurrentProcess;
const FILE_ALL_ACCESS: u32 = 0x001F_01FF;
// SAFETY: this process's token queried into a sized, aligned buffer and closed; the ACL and
// security descriptor live in buffers that outlive the call that uses them.
unsafe {
let mut token: HANDLE = std::ptr::null_mut();
if OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token) == 0 {
return false;
}
let mut user = [0u64; 64];
let mut len = 0u32;
let ok = GetTokenInformation(token, TokenUser, user.as_mut_ptr().cast(), std::mem::size_of_val(&user) as u32, &mut len) != 0;
CloseHandle(token);
if !ok {
return false;
}
let sid = (*(user.as_ptr() as *const TOKEN_USER)).User.Sid;
let acl_len = std::mem::size_of::<ACL>() + std::mem::size_of::<ACCESS_ALLOWED_ACE>() + GetLengthSid(sid) as usize;
let mut acl = vec![0u32; acl_len.div_ceil(4)];
let acl = acl.as_mut_ptr() as *mut ACL;
let mut sd: SECURITY_DESCRIPTOR = std::mem::zeroed();
let psd = (&mut sd as *mut SECURITY_DESCRIPTOR).cast();
let path: Vec<u16> = dir.as_os_str().encode_wide().chain([0]).collect();
InitializeAcl(acl, (acl_len.div_ceil(4) * 4) as u32, ACL_REVISION) != 0
&& AddAccessAllowedAceEx(acl, ACL_REVISION, OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, FILE_ALL_ACCESS, sid) != 0
&& InitializeSecurityDescriptor(psd, 1) != 0
&& SetSecurityDescriptorDacl(psd, 1, acl, 0) != 0
&& SetSecurityDescriptorControl(psd, SE_DACL_PROTECTED, SE_DACL_PROTECTED) != 0
&& SetFileSecurityW(path.as_ptr(), DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, psd) != 0
}
}
/// Every running process's file name, lower case (also of processes this user may not open,
/// such as anti-cheat services).
pub fn processes() -> Vec<String> {
use windows_sys::Win32::System::Diagnostics::ToolHelp::{
CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS,
};
let mut out = vec![];
// SAFETY: a process snapshot walked with a sized entry, then closed.
unsafe {
let snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
if snap == INVALID_HANDLE_VALUE {
return out;
}
let mut e: PROCESSENTRY32W = std::mem::zeroed();
e.dwSize = std::mem::size_of::<PROCESSENTRY32W>() as u32;
let mut more = Process32FirstW(snap, &mut e) != 0;
while more {
let n = e.szExeFile.iter().position(|&c| c == 0).unwrap_or(e.szExeFile.len());
out.push(String::from_utf16_lossy(&e.szExeFile[..n]).to_lowercase());
more = Process32NextW(snap, &mut e) != 0;
}
CloseHandle(snap);
}
out
}
/// A program name as `launch` takes it: a name Windows' folders have stays as it is (agent-desktop
/// finds it there), else the file Windows' App Paths name or a Start menu shortcut points at.
pub fn program(name: &str) -> Option<String> {
let file = super::program_file(name);
let windir = std::env::var_os("SystemRoot").or_else(|| std::env::var_os("windir"))?;
let windir = std::path::Path::new(&windir);
if windir.join("System32").join(&file).is_file() || windir.join(&file).is_file() {
return None;
}
app_path(&file).or_else(|| start_menu(name))
}
fn app_path(file: &str) -> Option<String> {
use windows_sys::Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, RRF_RT_REG_SZ};
let key: Vec<u16> = format!(r"Software\Microsoft\Windows\CurrentVersion\App Paths\{file}")
.encode_utf16().chain([0]).collect();
for root in [HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE] {
let mut buf = [0u16; 1024];
let mut len = (buf.len() * 2) as u32;
// SAFETY: the default value read into a sized buffer.
let r = unsafe { RegGetValueW(root, key.as_ptr(), std::ptr::null(), RRF_RT_REG_SZ, std::ptr::null_mut(),
buf.as_mut_ptr().cast(), &mut len) };
if r == 0 {
let n = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
let path = String::from_utf16_lossy(&buf[..n]).trim().trim_matches('"').to_owned();
if std::path::Path::new(&path).is_file() {
return Some(path);
}
}
}
None
}
/// A Start menu shortcut named `name` (the person's, then everyone's), pointing at a program file.
fn start_menu(name: &str) -> Option<String> {
let want = name.to_lowercase();
let roots = [("APPDATA", r"Microsoft\Windows\Start Menu\Programs"), ("ProgramData", r"Microsoft\Windows\Start Menu\Programs")];
for (var, sub) in roots {
let Some(base) = std::env::var_os(var) else { continue };
let mut dirs = vec![std::path::Path::new(&base).join(sub)];
let mut seen = 0;
while let Some(d) = dirs.pop() {
for e in std::fs::read_dir(&d).into_iter().flatten().flatten() {
seen += 1;
if seen > 5000 {
break;
}
let p = e.path();
if p.is_dir() {
dirs.push(p);
} else if p.extension().is_some_and(|x| x.eq_ignore_ascii_case("lnk"))
&& p.file_stem().is_some_and(|s| s.to_string_lossy().to_lowercase() == want) {
let target = std::fs::read(&p).ok().and_then(|b| super::shortcut_target(&b));
if let Some(t) = target.filter(|t| t.to_lowercase().ends_with(".exe") && std::path::Path::new(t).is_file()) {
return Some(t);
}
}
}
}
}
None
}
pub fn hotkey(mods: u32, vk: u32, stop: &'static super::Stop) -> Option<(u32, std::thread::JoinHandle<()>)> {
use windows_sys::Win32::UI::Input::KeyboardAndMouse::{RegisterHotKey, UnregisterHotKey, MOD_NOREPEAT};
use windows_sys::Win32::UI::WindowsAndMessaging::{GetMessageW, PeekMessageW, MSG, PM_NOREMOVE, WM_HOTKEY, WM_USER};
let (tx, rx) = std::sync::mpsc::channel();
let join = std::thread::spawn(move || {
// SAFETY: a thread-bound hotkey and this thread's own message queue.
unsafe {
let mut msg: MSG = std::mem::zeroed();
// Make the queue before anyone posts to it.
PeekMessageW(&mut msg, std::ptr::null_mut(), WM_USER, WM_USER, PM_NOREMOVE);
let ok = RegisterHotKey(std::ptr::null_mut(), 1, mods | MOD_NOREPEAT, vk) != 0;
let _ = tx.send((GetCurrentThreadId(), ok));
if !ok {
return;
}
while GetMessageW(&mut msg, std::ptr::null_mut(), 0, 0) > 0 {
if msg.message == WM_HOTKEY {
stop.toggle();
}
}
UnregisterHotKey(std::ptr::null_mut(), 1);
}
});
match rx.recv() {
Ok((thread, true)) => Some((thread, join)),
_ => {
let _ = join.join();
crate::warn_log!("conversation", "Warning: the PC control emergency stop hotkey could not be registered (another program may hold it)");
None
}
}
}
/// A window by agent-desktop's id (`w-<handle>`).
fn handle(id: &str) -> Option<windows_sys::Win32::Foundation::HWND> {
use windows_sys::Win32::UI::WindowsAndMessaging::IsWindow;
let h = id.strip_prefix("w-")?.parse::<usize>().ok().filter(|h| *h > 0)? as windows_sys::Win32::Foundation::HWND;
// SAFETY: a plain query on a handle value.
(unsafe { IsWindow(h) } != 0).then_some(h)
}
fn class_of(h: windows_sys::Win32::Foundation::HWND) -> String {
use windows_sys::Win32::UI::WindowsAndMessaging::GetClassNameW;
let mut buf = [0u16; 256];
// SAFETY: the buffer is sized as passed.
let n = unsafe { GetClassNameW(h, buf.as_mut_ptr(), buf.len() as i32) };
String::from_utf16_lossy(&buf[..n.max(0) as usize])
}
pub fn foreground() -> Option<String> {
use windows_sys::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
// SAFETY: a plain query.
let h = unsafe { GetForegroundWindow() };
(!h.is_null()).then(|| format!("w-{}", h as usize))
}
/// The window in front covers its monitor without a title bar or sizing border (not the
/// desktop), or Windows reports a full-screen Direct3D program.
pub fn fullscreen() -> bool {
use windows_sys::Win32::Foundation::RECT;
use windows_sys::Win32::Graphics::Gdi::{GetMonitorInfoW, MonitorFromWindow, MONITORINFO, MONITOR_DEFAULTTONEAREST};
use windows_sys::Win32::UI::Shell::{SHQueryUserNotificationState, QUNS_RUNNING_D3D_FULL_SCREEN};
use windows_sys::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowLongPtrW, GetWindowRect, GWL_STYLE, WS_CAPTION, WS_THICKFRAME};
// SAFETY: plain queries on the window in front, into sized locals.
unsafe {
let mut state = 0;
if SHQueryUserNotificationState(&mut state) == 0 && state == QUNS_RUNNING_D3D_FULL_SCREEN {
return true;
}
let h = GetForegroundWindow();
if h.is_null() || matches!(class_of(h).as_str(), "Progman" | "WorkerW") {
return false;
}
let mut r: RECT = std::mem::zeroed();
let mut mi = MONITORINFO { cbSize: std::mem::size_of::<MONITORINFO>() as u32, ..Default::default() };
if GetWindowRect(h, &mut r) == 0 || GetMonitorInfoW(MonitorFromWindow(h, MONITOR_DEFAULTTONEAREST), &mut mi) == 0 {
return false;
}
let m = mi.rcMonitor;
let style = GetWindowLongPtrW(h, GWL_STYLE) as u32;
r.left <= m.left && r.top <= m.top && r.right >= m.right && r.bottom >= m.bottom
&& style & WS_CAPTION != WS_CAPTION && style & WS_THICKFRAME == 0
}
}
/// Window `id`'s one text control shown (`text_control`), if it has exactly one. Windows does
/// not tell which control of a window behind has the focus (a thread behind has none), so keys
/// go behind only where there is no doubt where they land.
pub fn text_control_of(id: &str) -> Option<windows_sys::Win32::Foundation::HWND> {
use windows_sys::Win32::Foundation::{HWND, LPARAM};
use windows_sys::Win32::UI::WindowsAndMessaging::{EnumChildWindows, IsWindowVisible};
unsafe extern "system" fn each(child: HWND, found: LPARAM) -> i32 {
// SAFETY: `found` is the vector below, alive for the enumeration.
let found = unsafe { &mut *(found as *mut Vec<HWND>) };
if unsafe { IsWindowVisible(child) } != 0 && super::text_control(&class_of(child)) {
found.push(child);
}
1
}
let h = handle(id)?;
let mut found: Vec<HWND> = vec![];
// SAFETY: the callback only reads the children and writes the vector it is given.
unsafe { EnumChildWindows(h, Some(each), &mut found as *mut Vec<HWND> as LPARAM) };
(found.len() == 1).then(|| found[0])
}
/// Key down and key up for `vk` posted to `to`; the program's message loop makes the character.
pub fn post_key(to: windows_sys::Win32::Foundation::HWND, vk: u16) -> Result<(), String> {
use windows_sys::Win32::UI::Input::KeyboardAndMouse::{MapVirtualKeyW, MAPVK_VK_TO_VSC};
use windows_sys::Win32::UI::WindowsAndMessaging::{PostMessageW, WM_KEYDOWN, WM_KEYUP};
// Arrows, Home, End, Page Up and Down, Delete: the extended keys.
let extended = matches!(vk, 0x21..=0x28 | 0x2E);
// SAFETY: plain message posts to a window handle.
unsafe {
let scan = MapVirtualKeyW(vk as u32, MAPVK_VK_TO_VSC) as isize;
let down = 1 | (scan << 16) | if extended { 1 << 24 } else { 0 };
let up = down | (1 << 30) | (1 << 31);
if PostMessageW(to, WM_KEYDOWN, vk as usize, down) == 0 || PostMessageW(to, WM_KEYUP, vk as usize, up) == 0 {
return Err("ACTION_FAILED".into());
}
}
// Let the program take the keys before the window is read again.
std::thread::sleep(std::time::Duration::from_millis(150));
Ok(())
}
pub fn end_hotkey(thread: u32) {
use windows_sys::Win32::UI::WindowsAndMessaging::{PostThreadMessageW, WM_QUIT};
// SAFETY: a quit message to the hotkey's own thread.
unsafe { PostThreadMessageW(thread, WM_QUIT, 0, 0) };
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::settings::Level;
const OWNER: &str = "123456789012345678";
const TITLE: &str = "secret-plan.txt - Notepad";
const TYPED: &str = "typed words";
/// agent-desktop as a script: answers by command, remembers what it was asked.
struct Fake {
windows: Mutex<VecDeque<Value>>,
processes: Vec<String>,
elevated: bool,
secure: bool,
/// Commands that fail with this code.
fail: Option<(&'static str, &'static str)>,
calls: Mutex<Vec<(Vec<String>, Option<String>)>>,
launched: Value,
/// A password field in every reading, past the elements shown; `get` on it is refused as
/// the real agent-desktop refuses it (a secure field's identity is not unique).
secure_ref: Option<&'static str>,
/// The window in front (`focus-window` moves it), a full-screen window in front, and
/// whether windows take keys behind.
front: Mutex<Option<String>>,
fullscreen: bool,
behind: bool,
/// agent-desktop cannot find the window's process by its identity (the new Calculator, a
/// Store app): `press --window-id` is APP_NOT_FOUND, nothing sent.
unnamed: bool,
}
impl Fake {
fn new() -> Self {
Fake { windows: Mutex::new(VecDeque::from([json!([
{"id": "w-1", "title": TITLE, "app_name": "notepad.exe", "pid": 10, "is_focused": false, "accessible": true},
{"id": "w-2", "title": "Vault", "app_name": "KeePassXC.exe", "pid": 11, "is_focused": true, "accessible": true},
{"id": "w-3", "title": "Calc", "app_name": "CalculatorApp.exe", "pid": 12, "is_focused": false, "minimized": true}])])),
processes: vec!["explorer.exe".into()], elevated: false, secure: false, fail: None,
calls: Mutex::new(vec![]), launched: json!({"app": "notepad", "pid": 10}), secure_ref: None,
front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false, unnamed: false}
}
fn front(&self) -> Option<String> {
self.front.lock().unwrap().clone()
}
/// The calls with their arguments, from the first named `from`.
fn calls_from(&self, from: &str) -> Vec<Vec<String>> {
let calls = self.calls.lock().unwrap();
calls.iter().map(|(a, _)| a.clone()).skip_while(|a| a[0] != from).collect()
}
fn commands(&self) -> Vec<String> {
self.calls.lock().unwrap().iter().map(|(a, _)| a[0].clone()).collect()
}
}
fn tree() -> Value {
let mut kids: Vec<Value> = (0..150).map(|i| json!({"role": "statictext", "name": format!("label {i}")})).collect();
kids.extend((0..150).map(|i| json!({"role": "button", "name": format!("{i} {}", "x".repeat(300)), "ref_id": format!("@s1:e{i}"),
"states": ["focused"]})));
json!({"role": "window", "name": TITLE, "children": [{"role": "group", "children": kids}]})
}
impl Desktop for &'static Fake {
fn run(&self, args: &[String], stdin: Option<&str>, _stop: &Stop) -> Result<Value, String> {
self.calls.lock().unwrap().push((args.to_vec(), stdin.map(str::to_owned)));
if let Some((cmd, code)) = self.fail {
if args[0] == cmd {
return Err(code.into());
}
}
if args[0] == "get" && self.secure_ref == Some(args[1].as_str()) {
return Err("AMBIGUOUS_TARGET".into());
}
if args[0] == "press" && self.unnamed && args.iter().any(|a| a == "--window-id") {
return Err("APP_NOT_FOUND".into());
}
Ok(match args[0].as_str() {
"list-windows" => {
let mut w = self.windows.lock().unwrap();
if w.len() > 1 { w.pop_front().unwrap() } else { w[0].clone() }
}
"snapshot" => {
let mut t = tree();
if let Some(r) = self.secure_ref {
t["children"][0]["children"].as_array_mut().unwrap()
.push(json!({"role": "group", "children": [{"role": "textfield", "ref_id": r, "states": ["secure"]}]}));
}
json!({"snapshot_id": "s1", "window": {"id": args[2], "title": TITLE}, "tree": t})
}
"get" => json!({"property": "states", "value": if self.secure { json!(["secure"]) } else { json!([]) }}),
"launch" => self.launched.clone(),
"focus-window" => {
*self.front.lock().unwrap() = Some(args[2].clone());
json!({"disposition": {"delivery": "delivered_unverified"}})
}
_ => json!({"disposition": {"delivery": "delivered_unverified"}}),
})
}
fn elevated(&self, _pid: u64) -> bool {
self.elevated
}
fn processes(&self) -> Vec<String> {
self.processes.clone()
}
fn foreground(&self) -> Option<String> {
self.front()
}
fn fullscreen(&self) -> bool {
self.fullscreen
}
fn takes_keys_behind(&self, _id: &str) -> bool {
self.behind
}
fn post_key(&self, id: &str, key: &str) -> Result<(), String> {
self.calls.lock().unwrap().push((vec!["post".into(), id.into(), key.into()], None));
match self.fail {
Some(("post", code)) => Err(code.into()),
_ => Ok(()),
}
}
}
fn settings() -> Settings {
let mut s = Settings::default();
s.pc_control.allowed_users = vec![OWNER.parse().unwrap()];
s
}
fn pc_with(s: Settings, fake: Fake) -> (Pc, &'static Fake, &'static Stop) {
let fake: &'static Fake = Box::leak(Box::new(fake));
let stop: &'static Stop = Box::leak(Box::new(Stop::new()));
(Pc::new(s, Box::new(fake), stop, Box::new(|| None)), fake, stop)
}
fn action(action: &str, user: &str, extra: Value) -> PcAction {
let mut v = json!({"id": "p1", "action": action, "user": user, "via": "voice"});
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
serde_json::from_value(v).unwrap()
}
fn read_first(pc: &Pc) {
assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).code, "ok");
}
#[test]
fn people_not_registered_are_refused_before_anything_runs() {
let (pc, fake, _) = pc_with(settings(), Fake::new());
for a in ["windows", "read", "click", "type", "keys", "run", "close"] {
let o = pc.act(&action(a, "42", json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_REGISTERED}), "{a}");
}
assert!(fake.commands().is_empty());
// Nobody is registered by default.
let (pc, _, _) = pc_with(Settings::default(), Fake::new());
assert_eq!(pc.act(&action("windows", OWNER, json!({}))).reply["error"], NOT_REGISTERED);
}
#[test]
fn a_kind_turned_off_is_refused_and_every_action_has_its_kind() {
let mut s = settings();
s.pc_control.click = Level::Never;
s.pc_control.type_text = Level::Never;
s.pc_control.run = Level::Never;
s.pc_control.view = Level::Never;
let (pc, fake, _) = pc_with(s, Fake::new());
for a in ["windows", "read", "click", "close", "type", "keys", "run"] {
let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad", "consent": true})));
assert_eq!(o.reply["error"], TURNED_OFF, "{a}");
}
assert!(fake.commands().is_empty());
}
#[test]
fn asking_first_answers_ask_without_consent_and_acts_with_it() {
let (pc, fake, _) = pc_with(settings(), Fake::new());
read_first(&pc);
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"})));
assert_eq!(o.reply, json!({"ok": false, "ask": true, "kinds": ["click"],
"window": {"id": "w-1", "program": "notepad", "title": TITLE}}));
assert_eq!(o.activity, None, "nothing was done yet");
assert!(!fake.commands().contains(&"click".to_owned()));
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply["kinds"], json!(["run"]));
assert_eq!(o.reply["window"]["program"], "notepad");
assert!(!fake.commands().contains(&"launch".to_owned()));
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true})));
assert_eq!(o.reply["ok"], true);
assert!(fake.commands().contains(&"click".to_owned()));
// Done means seen: the window is read again after the click.
assert_eq!(fake.commands().last().unwrap(), "snapshot");
assert_eq!(o.reply["item"]["window"]["id"], "w-1");
// At `auto` nothing is asked.
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let (pc, fake, _) = pc_with(s, Fake::new());
read_first(&pc);
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED})));
assert_eq!(o.reply["ok"], true);
let calls = fake.calls.lock().unwrap();
let set = calls.iter().find(|(a, _)| a[0] == "set-value").unwrap();
assert_eq!(set.1.as_deref(), Some(TYPED), "text goes through stdin");
assert!(!set.0.iter().any(|a| a.contains(TYPED)), "never in the arguments");
}
#[test]
fn an_anti_cheat_game_refuses_acting_but_not_reading() {
let mut fake = Fake::new();
fake.processes = vec!["explorer.exe".into(), "valorant-win64-shipping.exe".into()];
let (pc, fake, _) = pc_with(settings(), fake);
for a in ["click", "type", "keys", "run", "close"] {
let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "keys": "enter",
"program": "notepad", "consent": true})));
assert_eq!(o.reply["error"], "뱅가드가 적용된 게임(VALORANT)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", "{a}");
}
assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot"));
assert_eq!(pc.act(&action("windows", OWNER, json!({}))).code, "ok");
read_first(&pc);
// A program the person listed, named by game detection when it sees the game.
let mut s = settings();
s.pc_control.anti_cheat_games = vec!["MyShooter".into()];
let mut fake = Fake::new();
fake.processes = vec!["myshooter.exe".into()];
let fake: &'static Fake = Box::leak(Box::new(fake));
let pc = Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into())));
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter", "consent": true})));
assert_eq!(o.reply["error"], "안티치트가 적용된 게임(My Shooter)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다");
}
#[test]
fn the_emergency_stop_refuses_acting_until_lifted_and_kills_a_running_command() {
let (pc, fake, stop) = pc_with(settings(), Fake::new());
read_first(&pc);
stop.toggle();
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true})));
assert_eq!(o.reply["error"], STOPPED);
assert_eq!(o.activity.unwrap().ok, false);
assert!(!fake.commands().contains(&"click".to_owned()));
// Reading still works.
read_first(&pc);
stop.toggle();
assert_eq!(pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))).code, "ok");
stop.toggle();
stop.lift();
assert!(!stop.on());
// The real runner: a command running when the hotkey is pressed is killed at once.
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let dir = crate::datadir::temp_dir("pcstop");
let exe = dir.join("agent-desktop");
std::fs::write(&exe, "#!/bin/sh\nsleep 5\necho '{\"ok\": true, \"data\": []}'\n").unwrap();
std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();
let cli = Cli::new(&exe, &dir);
let stop: &'static Stop = Box::leak(Box::new(Stop::new()));
let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(200)); stop.toggle(); });
let started = Instant::now();
assert_eq!(cli.run(&args(&["click", "@s1:e1"]), None, stop), Err("APP_STOPPED".into()));
assert!(started.elapsed() < Duration::from_secs(2));
t.join().unwrap();
// A command started while the stop is on (a reading) runs to its end.
std::fs::write(&exe, "#!/bin/sh\necho \"{\\\"ok\\\": true, \\\"data\\\": \\\"$APPDATA\\\"}\"\n").unwrap();
let home = dir.join("runtime").join("agent-desktop");
assert_eq!(cli.run(&args(&["list-windows"]), None, stop), Ok(json!(home.to_string_lossy())),
"agent-desktop's folders are in the data folder");
// The command limit.
std::fs::write(&exe, "#!/bin/sh\nsleep 5\n").unwrap();
let cli = Cli { limit: Duration::from_millis(200), ..cli };
assert_eq!(cli.run(&args(&["list-windows"]), None, &Stop::new()), Err("APP_TIMEOUT".into()));
}
}
#[test]
fn a_password_field_and_an_administrator_window_are_refused_before_acting() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let mut fake = Fake::new();
fake.secure = true;
let (pc, fake, _) = pc_with(s.clone(), fake);
read_first(&pc);
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED})));
assert_eq!(o.reply["error"], PASSWORD);
assert!(!fake.commands().contains(&"set-value".to_owned()));
// Even with consent.
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED, "consent": true})));
assert_eq!(o.reply["error"], PASSWORD);
let mut fake = Fake::new();
fake.elevated = true;
let (pc, fake, _) = pc_with(s, fake);
read_first(&pc);
for a in ["click", "type", "keys", "close"] {
let o = pc.act(&action(a, OWNER, json!({"ref": "@s1:e3", "window": "notepad", "keys": "enter", "text": "x",
"consent": true})));
assert_eq!(o.reply["error"], ELEVATED, "{a}");
}
assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot"));
}
#[test]
fn a_field_read_as_secure_is_refused_as_a_password_field() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let (pc, fake, _) = pc_with(s, Fake { secure_ref: Some("@s1:e900"), ..Fake::new() });
read_first(&pc);
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e900", "text": TYPED, "consent": true})));
assert_eq!(o.reply, json!({"ok": false, "error": PASSWORD}));
assert!(!fake.commands().contains(&"set-value".to_owned()));
// Other fields of the same reading are typed into.
assert_eq!(pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).code, "ok");
}
#[test]
fn a_reading_has_interactive_elements_first_at_most_200_and_cut_to_120_characters() {
let (pc, _, _) = pc_with(settings(), Fake::new());
let o = pc.act(&action("read", OWNER, json!({"window": "notepad"})));
let els = o.reply["item"]["elements"].as_array().unwrap();
assert_eq!(els.len(), MAX_ELEMENTS);
assert!(els[..150].iter().all(|e| e["ref"].as_str().unwrap().starts_with("@s1:e")));
assert!(els[150..].iter().all(|e| e["ref"] == "" && e["role"] == "statictext"));
assert!(els.iter().all(|e| e["name"].as_str().unwrap().chars().count() <= MAX_CHARS));
assert_eq!(els[0]["name"].as_str().unwrap().chars().count(), MAX_CHARS);
assert_eq!(els[0]["states"], json!(["focused"]));
assert_eq!(o.reply["item"]["window"], json!({"id": "w-1", "program": "notepad", "title": TITLE}));
// No window named: why.
assert!(pc.act(&action("read", OWNER, json!({}))).reply["error"].is_string());
}
#[test]
fn a_stale_element_says_to_read_again() {
let mut s = settings();
s.pc_control.click = Level::Auto;
let mut fake = Fake::new();
fake.fail = Some(("click", "STALE_REF"));
let (pc, _, _) = pc_with(s.clone(), fake);
// An element of a reading this app does not know, or one in bad form.
for r in ["@s9:e1", "e1", "--force"] {
assert_eq!(pc.act(&action("click", OWNER, json!({"ref": r}))).reply["error"], STALE, "{r}");
}
read_first(&pc);
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"})));
assert_eq!(o.reply, json!({"ok": false, "error": STALE}));
assert_eq!(o.code, "STALE_REF");
}
#[test]
fn a_minimised_window_is_restored_before_acting_and_keys_bring_it_to_the_front() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let (pc, fake, _) = pc_with(s, Fake::new());
let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+s"})));
assert_eq!(o.reply["ok"], true, "{o:?}");
assert_eq!(fake.commands(), ["list-windows", "restore", "focus-window", "press", "focus-window", "snapshot"]);
let calls = fake.calls.lock().unwrap();
assert_eq!(calls[3].0, args(&["press", "ctrl+s", "--window-id", "w-3"]));
assert!(calls.iter().all(|(a, _)| !a.iter().any(|x| x == "--force")));
drop(calls);
for bad in ["--force", "ctrl+", "alt f4;rm"] {
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": bad})));
assert_eq!(o.code, "refused", "{bad}");
}
}
#[test]
fn excluded_programs_and_this_apps_windows_are_never_listed_or_picked() {
let (pc, _, _) = pc_with(settings(), Fake::new());
let o = pc.act(&action("windows", OWNER, json!({})));
assert_eq!(o.reply["item"]["windows"], json!([
{"id": "w-1", "program": "notepad", "title": TITLE, "minimised": false},
{"id": "w-3", "program": "calculatorapp", "title": "Calc", "minimised": true}]));
assert!(pc.act(&action("read", OWNER, json!({"window": "Vault"}))).reply["error"].is_string());
}
#[test]
fn run_starts_a_program_by_name_and_reads_its_new_window() {
let mut s = settings();
s.pc_control.run = Level::Auto;
let fake = Fake::new();
let before = fake.windows.lock().unwrap()[0].clone();
let mut after = before.clone();
after.as_array_mut().unwrap().push(json!({"id": "w-9", "title": "Untitled - Notepad", "app_name": "notepad.exe", "pid": 99}));
*fake.windows.lock().unwrap() = VecDeque::from([before, after]);
let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 99}), ..fake };
let (pc, fake, _) = pc_with(s, fake);
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply["item"]["window"]["id"], "w-9", "{o:?}");
assert_eq!(fake.calls.lock().unwrap().iter().find(|(a, _)| a[0] == "launch").unwrap().0[1], "notepad");
assert_eq!(o.activity, Some(Record { kind: "pc.run".into(), target: "notepad.exe".into(), ok: true, note: None }));
for bad in ["C:\\Windows\\cmd.exe", "-x", "..\\x"] {
assert_eq!(pc.act(&action("run", OWNER, json!({"program": bad}))).code, "refused", "{bad}");
}
}
#[test]
fn run_with_the_program_already_running_answers_the_window_launch_names() {
let mut s = settings();
s.pc_control.run = Level::Auto;
// agent-desktop attaches to a running instance and names its window; nothing new appears.
let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 10, "window": {"id": "w-1", "title": TITLE}}), ..Fake::new() };
let (pc, _, _) = pc_with(s.clone(), fake);
let started = Instant::now();
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply["item"]["window"]["id"], "w-1", "{o:?}");
assert!(started.elapsed() < WINDOW_WAIT, "answered without waiting for a new window");
// Several running: launch cannot choose, and the answer says so (not an element's reason).
let fake = Fake { fail: Some(("launch", "AMBIGUOUS_TARGET")), ..Fake::new() };
let (pc, _, _) = pc_with(s, fake);
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": RUNNING_SEVERAL}));
}
#[test]
fn keys_to_a_window_that_cannot_come_to_the_front_say_nothing_was_sent() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
// Windows' foreground lock: focus-window fails, or press cannot confirm the window's focus.
for (cmd, code) in [("focus-window", "ACTION_FAILED"), ("press", "STALE_REF")] {
let fake = Fake { fail: Some((cmd, code)), ..Fake::new() };
let (pc, fake, _) = pc_with(s.clone(), fake);
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "ctrl+s"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_FRONT}), "{cmd}");
assert!(cmd == "press" || !fake.commands().contains(&"press".to_owned()), "no key after a failed focus");
}
}
fn keys_auto(fake: Fake) -> (Pc, &'static Fake, &'static Stop) {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
pc_with(s, fake)
}
#[test]
fn one_key_without_modifiers_goes_behind_a_window_that_takes_it_and_nothing_moves() {
for key in ["a", "7", "enter", "Tab", "space", "escape", "backspace", "forwarddelete", "left", "pagedown", "f5"] {
let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key})));
assert_eq!(o.reply["ok"], true, "{key}: {o:?}");
assert_eq!(fake.commands(), ["list-windows", "post", "snapshot"], "{key}");
assert_eq!(fake.calls_from("post")[0][1], "w-1", "{key}");
assert_eq!(fake.front().as_deref(), Some("w-2"), "{key}: the window in front stays");
}
}
#[test]
fn combinations_other_keys_and_windows_not_taking_keys_behind_go_in_front_and_back() {
let in_front_and_back = |fake: &Fake, key: &str| {
assert_eq!(fake.calls_from("focus-window"), [
args(&["focus-window", "--window-id", "w-1"]),
args(&["press", key, "--window-id", "w-1"]),
args(&["focus-window", "--window-id", "w-2"]),
args(&["snapshot", "--window-id", "w-1", "--compact", "--timeout-ms", SNAPSHOT_MS])], "{key}");
assert!(!fake.commands().contains(&"post".to_owned()), "{key}: never both ways");
assert_eq!(fake.front().as_deref(), Some("w-2"), "{key}: the window in front is back");
};
for key in ["ctrl+s", "shift+tab", "alt+f", "cmd+d", "f10", "shift"] {
let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() });
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key}))).reply["ok"], true, "{key}");
in_front_and_back(fake, key);
}
// A window that does not take keys behind (a Store app, Chrome) gets even one letter in front.
let (pc, fake, _) = keys_auto(Fake::new());
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))).reply["ok"], true);
in_front_and_back(fake, "a");
// A failed press still brings the window in front back, and is not tried behind.
let (pc, fake, _) = keys_auto(Fake { fail: Some(("press", "STALE_REF")), ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_FRONT}));
assert_eq!(fake.calls_from("focus-window").last().unwrap(), &args(&["focus-window", "--window-id", "w-2"]));
assert_eq!(fake.front().as_deref(), Some("w-2"));
}
#[test]
fn a_store_app_agent_desktop_cannot_name_gets_its_keys_once_it_is_in_front() {
// Brought in front, and already in front: the key is pressed once, in that window.
for already in [false, true] {
let fake = Fake { unnamed: true, ..Fake::new() };
if already {
*fake.front.lock().unwrap() = Some("w-1".into());
}
let (pc, fake, _) = keys_auto(fake);
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "1"})));
assert_eq!(o.reply["ok"], true, "{o:?}");
let presses: Vec<Vec<String>> = fake.calls_from("list-windows").into_iter().filter(|a| a[0] == "press").collect();
assert_eq!(presses.last().unwrap(), &args(&["press", "1"]), "{already}");
assert_eq!(fake.front().as_deref(), Some(if already { "w-1" } else { "w-2" }), "{already}: the window in front is back");
}
}
#[test]
fn a_window_already_in_front_gets_its_keys_there_and_nothing_moves() {
let fake = Fake::new();
*fake.front.lock().unwrap() = Some("w-1".into());
let (pc, fake, _) = keys_auto(fake);
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "ctrl+s"}))).reply["ok"], true);
assert_eq!(fake.commands(), ["list-windows", "press", "snapshot"]);
}
#[test]
fn keys_are_never_brought_in_front_during_a_game_or_full_screen_but_go_behind() {
let game = |fake: Fake| {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let fake: &'static Fake = Box::leak(Box::new(fake));
(Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into()))), fake)
};
let full = |fake: Fake| {
let (pc, fake, _) = keys_auto(Fake { fullscreen: true, ..fake });
(pc, fake)
};
for (i, make) in [&game as &dyn Fn(Fake) -> (Pc, &'static Fake), &full].into_iter().enumerate() {
for (key, behind) in [("ctrl+s", true), ("a", false)] {
let (pc, fake) = make(Fake { behind, ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_IN_FRONT_NOW}), "{i} {key}");
assert_eq!(fake.commands(), ["list-windows"], "{i} {key}: no key sent");
}
let (pc, fake) = make(Fake { behind: true, ..Fake::new() });
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))).reply["ok"], true, "{i}");
assert_eq!(fake.commands(), ["list-windows", "post", "snapshot"], "{i}");
}
}
#[test]
fn a_key_not_taken_behind_is_not_pressed_again_in_front() {
let (pc, fake, _) = keys_auto(Fake { behind: true, fail: Some(("post", "ACTION_FAILED")), ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"})));
assert_eq!(o.reply["ok"], false);
assert_eq!(fake.commands(), ["list-windows", "post"]);
}
#[test]
fn only_text_controls_take_keys_behind() {
for c in ["Edit", "RichEdit20W", "RICHEDIT50W", "RichEditD2DPT", "Scintilla"] {
assert!(text_control(c), "{c}");
}
for c in ["Notepad", "CalcFrame", "Button", "Windows.UI.Core.CoreWindow", "Chrome_RenderWidgetHostHWND",
"Microsoft.UI.Content.DesktopChildSiteBridge", "MozillaWindowClass", "edit"] {
assert!(!text_control(c), "{c}");
}
}
#[test]
fn an_action_that_closes_its_window_is_done_with_nothing_left_to_read() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
// Enter on a dialog (or a click on its button) closes it: the fresh reading finds no window.
let (pc, _, _) = pc_with(s, Fake { fail: Some(("snapshot", "WINDOW_NOT_FOUND")), ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter"})));
assert_eq!(o.reply, json!({"ok": true, "item": {"window": {"id": "w-1", "program": "notepad", "title": TITLE}, "elements": []}}));
assert_eq!(o.code, "ok");
}
#[test]
fn a_program_that_does_not_close_in_time_is_said_to_be_still_open() {
let mut s = settings();
s.pc_control.click = Level::Auto;
// A save question keeps the program open; close-app gives up after its own deadline.
let fake = Fake { fail: Some(("close-app", "TIMEOUT")), ..Fake::new() };
let (pc, _, _) = pc_with(s, fake);
let o = pc.act(&action("close", OWNER, json!({"window": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_CLOSED}));
// Several of it running: close-app picks by name and cannot choose.
let mut s = settings();
s.pc_control.click = Level::Auto;
let (pc, _, _) = pc_with(s, Fake { fail: Some(("close-app", "AMBIGUOUS_TARGET")), ..Fake::new() });
let o = pc.act(&action("close", OWNER, json!({"window": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": CLOSE_SEVERAL}));
}
#[test]
fn the_activity_names_the_program_only() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
s.pc_control.click = Level::Auto;
let (pc, _, stop) = pc_with(s, Fake::new());
read_first(&pc);
let typed = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).activity.unwrap();
assert_eq!(typed, Record { kind: "pc.type".into(), target: "notepad.exe".into(), ok: true, note: None });
let closed = pc.act(&action("close", OWNER, json!({"window": "notepad"}))).activity.unwrap();
assert_eq!(closed, Record { kind: "pc.close".into(), target: "notepad.exe".into(), ok: true, note: None });
stop.toggle();
let refused = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))).activity.unwrap();
assert_eq!((refused.ok, refused.note.as_deref()), (false, Some(STOPPED)));
// Readings are not activity; nothing recorded carries a title, an element or the text.
assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).activity, None);
assert_eq!(pc.act(&action("windows", OWNER, json!({}))).activity, None);
let all = format!("{typed:?}{closed:?}{refused:?}");
for secret in [TITLE, "secret-plan", TYPED, "label", "@s1"] {
assert!(!all.contains(secret), "{secret}");
}
}
#[test]
fn the_tool_is_available_only_with_the_feature_someone_registered_and_agent_desktop() {
let s = settings();
assert_eq!(available(&s, true), cfg!(windows));
assert!(!available(&s, false));
assert!(!available(&Settings::default(), true));
let mut off = settings();
off.features.pc_control = false;
assert!(!available(&off, true));
// The pinned file is checked by its hash.
let f = crate::datadir::temp_dir("pcexe").join(EXE_NAME);
std::fs::write(&f, b"not agent-desktop").unwrap();
assert!(!exe_ok(&f));
assert!(!exe_ok(&f.with_file_name("missing.exe")));
}
#[test]
fn hotkeys_and_envelopes_are_read() {
assert_eq!(parse_hotkey(DEFAULT_STOP_HOTKEY), Some((2 | 4, 0x7B)));
assert_eq!(parse_hotkey("alt+win+q"), Some((1 | 8, b'Q' as u32)));
for bad in ["F12", "Ctrl+", "Ctrl+F25", "Hyper+F1", ""] {
assert_eq!(parse_hotkey(bad), None, "{bad}");
}
assert_eq!(envelope(br#"{"ok": true, "data": [1]}"#), Ok(json!([1])));
assert_eq!(envelope(b"warning\n{\"ok\": false, \"error\": {\"code\": \"STALE_REF\", \"message\": \"x\"}}\n"),
Err("STALE_REF".into()));
assert_eq!(envelope(b"garbage"), Err("APP_NO_JSON".into()));
}
#[test]
fn agent_desktops_state_folder_is_made_afresh_each_start() {
// What a folder made under another ACL holds would keep that ACL, and readings of an
// earlier run are of no use: the folder starts empty.
let data = std::env::temp_dir().join(format!("jarvis-pc-{}-{}", std::process::id(), rand::random::<u32>()));
let home = data.join("runtime").join("agent-desktop");
std::fs::create_dir_all(home.join("refmaps")).unwrap();
std::fs::write(home.join("refmaps").join("old.json"), "{}").unwrap();
let _cli = Cli::new(Path::new("agent-desktop.exe"), &data);
assert!(home.is_dir());
assert_eq!(std::fs::read_dir(&home).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&data);
}
#[test]
fn the_package_and_m0_pin_the_same_agent_desktop() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let package = std::fs::read_to_string(root.join("shell/package.sh")).unwrap();
for pin in [AGENT_DESKTOP_URL, AGENT_DESKTOP_ARCHIVE_SHA256, AGENT_DESKTOP_SHA256] {
assert!(package.contains(pin), "{pin}");
}
let m0 = std::fs::read_to_string(root.join("../scripts/m0/m0lib/desktop.py")).unwrap();
assert!(m0.contains(AGENT_DESKTOP_ARCHIVE_SHA256) && m0.contains(&format!("AD_VERSION = \"{AGENT_DESKTOP_VERSION}\"")));
}
#[test]
fn a_start_menu_shortcut_gives_its_program_file() {
// A shell link with link information and a Unicode local path (MS-SHLLINK).
fn link(flags: u32, path: &str) -> Vec<u8> {
let mut b = vec![0u8; 0x4C];
b[0] = 0x4C;
b[0x14..0x18].copy_from_slice(&flags.to_le_bytes());
let wide: Vec<u8> = path.encode_utf16().chain([0]).flat_map(|c| c.to_le_bytes()).collect();
let header = 0x24u32;
let mut info = vec![0u8; header as usize];
info[4..8].copy_from_slice(&header.to_le_bytes());
info[8..12].copy_from_slice(&1u32.to_le_bytes());
info[28..32].copy_from_slice(&header.to_le_bytes());
info.extend(&wide);
let size = info.len() as u32;
info[0..4].copy_from_slice(&size.to_le_bytes());
b.extend(info);
b
}
let path = r"C:\Program Files\메모\memo.exe";
assert_eq!(shortcut_target(&link(0x2, path)).as_deref(), Some(path));
assert_eq!(shortcut_target(&link(0x2 | 0x20, path)), None, "a shortcut with arguments");
assert_eq!(shortcut_target(&link(0x0, path)), None);
assert_eq!(shortcut_target(b"short"), None);
}
}