Files
javis_bot/app/src/pc.rs
javis-bot 20bd90f65e fix(v2): pcControl keys to a minimised window bring back the window that was in front
restore activates the window (SW_RESTORE), so restoring before keys made the
window already in front: the window that was in front before was not brought
back, and the game / full-screen check came after the window had moved. Keys
now note the window in front and check for a game first, then restore inside
the moment in front. A minimised window never takes keys behind.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 11:08:30 +09:00

1972 lines
93 KiB
Rust

//! PC control for the brain's `pcControl` tool (admin.spec.md, "PC control"; `docs/v2/pc-features.md`,
//! "키보드·마우스 제어와 안티치트"): who may, what is never done, the anti-cheat and emergency-stop
//! blocks, and every action run through agent-desktop, one CLI process per command. The commands go
//! through `Desktop`, so the tests drive everything with a fake; `Cli` is the real one.
use std::collections::{HashSet, VecDeque};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Mutex;
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use crate::permissions::{decide, Action, Decision, Refusal};
use crate::settings::Settings;
use crate::worker_client::PcAction;
/// agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0), pinned as M0 ran it
/// (`scripts/m0/m0lib/desktop.py`).
pub const AGENT_DESKTOP_VERSION: &str = "1.2.0";
pub const AGENT_DESKTOP_URL: &str = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-pc-windows-msvc.tar.gz";
/// The release archive (`app/shell/package.sh` checks it).
pub const AGENT_DESKTOP_ARCHIVE_SHA256: &str = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8";
/// `agent-desktop.exe` inside it, checked before every start of the services.
pub const AGENT_DESKTOP_SHA256: &str = "d274667031322e50b9c2b1ab81c2ca46835d52cc932769a5c34bd4d481fd94c3";
pub const EXE_NAME: &str = "agent-desktop.exe";
/// One agent-desktop command at most.
pub const COMMAND_LIMIT: Duration = Duration::from_secs(15);
/// Elements in a reading (interactive ones first), and the characters kept of a name or value.
pub const MAX_ELEMENTS: usize = 200;
pub const MAX_CHARS: usize = 120;
/// Characters of text put into a field.
pub const MAX_TEXT: usize = 2000;
pub const MAX_WINDOWS: usize = 30;
pub const DEFAULT_STOP_HOTKEY: &str = "Ctrl+Shift+F12";
/// How long a reading may take (agent-desktop's budget; Chromium windows settle slowly).
const SNAPSHOT_MS: &str = "8000";
/// How long `launch` waits for the program, and then this app for its window.
const LAUNCH_MS: &str = "8000";
const WINDOW_WAIT: Duration = Duration::from_secs(5);
/// Readings whose elements may still be acted on.
const KEPT_READINGS: usize = 64;
pub const NOT_REGISTERED: &str = "PC 제어 화면에 등록된 사람만 PC를 조작할 수 있습니다";
pub const TURNED_OFF: &str = "PC 제어 화면에서 이 동작이 꺼져 있습니다";
pub const PASSWORD: &str = "비밀번호 칸에는 입력하지 않습니다";
pub const ELEVATED: &str = "관리자 권한으로 실행된 창은 조작할 수 없습니다";
pub const STOPPED: &str = "비상 정지 중입니다. 단축키를 한 번 더 누르거나 PC 제어 화면에서 풀어 주세요";
pub const STALE: &str = "화면이 바뀌었습니다. 다시 읽어 주세요";
const NO_WINDOW: &str = "그 창을 찾지 못했습니다";
const NOT_FRONT: &str = "그 창을 앞으로 가져오지 못해 키를 보내지 않았습니다";
const NOT_IN_FRONT_NOW: &str = "게임 중이거나 전체 화면이라 그 창을 앞으로 가져오지 않았습니다. 키를 보내지 않았습니다";
const NOT_CLOSED: &str = "프로그램이 아직 닫히지 않았습니다(저장할지 묻는 창이 떠 있을 수 있습니다). 창을 다시 읽어 주세요";
const CLOSE_SEVERAL: &str = "그 프로그램이 여러 개 실행 중이라 이 창만 닫을 수 없습니다. 창의 닫기 단추를 눌러 주세요";
const RUNNING_SEVERAL: &str = "그 프로그램이 이미 여러 개 실행 중이라 새로 열지 못했습니다. 열린 창 중에서 골라 주세요";
/// Anti-cheats found by their processes (file names, lower case): the anti-cheat's name, and the
/// game's when the process is the game itself. Vanguard's driver is always loaded, so it is found
/// by Riot's game processes. Grown by updates.
const ANTI_CHEATS: &[(&str, &str, Option<&str>)] = &[
("valorant-win64-shipping.exe", "뱅가드", Some("VALORANT")),
("valorant.exe", "뱅가드", Some("VALORANT")),
("league of legends.exe", "뱅가드", Some("League of Legends")),
("easyanticheat.exe", "Easy Anti-Cheat", None),
("easyanticheat_eos.exe", "Easy Anti-Cheat", None),
("beservice.exe", "BattlEye", None),
("beservice_x64.exe", "BattlEye", None),
("faceit.exe", "FACEIT", None),
("faceitservice.exe", "FACEIT", None),
("gamemon.des", "nProtect GameGuard", None),
("gamemon64.des", "nProtect GameGuard", None),
("xcorona.xem", "XIGNCODE3", None),
("xcorona_x64.xem", "XIGNCODE3", None),
("xxd-0.xem", "XIGNCODE3", None),
];
/// Where agent-desktop is shipped: next to the app's program file.
pub fn exe_path() -> Option<PathBuf> {
Some(std::env::current_exe().ok()?.parent()?.join(EXE_NAME))
}
/// Whether `exe` is the pinned agent-desktop.
pub fn exe_ok(exe: &Path) -> bool {
crate::download::sha256_file(exe).is_ok_and(|h| h == AGENT_DESKTOP_SHA256)
}
/// Whether the brain gets the tool: Windows, the feature on, someone registered and agent-desktop
/// present with the right hash (`installed`).
pub fn available(s: &Settings, installed: bool) -> bool {
cfg!(windows) && s.features.pc_control && !s.pc_control.allowed_users.is_empty() && installed
}
/// The agent-desktop the services use, when PC control is available (checked once per start).
pub fn check(s: &Settings) -> Option<PathBuf> {
if !available(s, true) {
return None;
}
match exe_path() {
Some(exe) if exe_ok(&exe) => Some(exe),
Some(exe) if exe.exists() => {
crate::warn_log!("conversation", "Warning: agent-desktop has another hash than {AGENT_DESKTOP_VERSION}'s: PC control is off");
None
}
_ => {
crate::warn_log!("conversation", "Warning: agent-desktop is not installed: PC control is off");
None
}
}
}
/// `Ctrl+Shift+F12` as the hotkey's modifiers (`MOD_ALT` 1, `MOD_CONTROL` 2, `MOD_SHIFT` 4,
/// `MOD_WIN` 8) and virtual key (letters, digits, F1-F24); at least one modifier.
pub fn parse_hotkey(s: &str) -> Option<(u32, u32)> {
let parts: Vec<String> = s.split('+').map(|p| p.trim().to_ascii_lowercase()).collect();
let (key, mods) = parts.split_last()?;
let mut m = 0;
for p in mods {
m |= match p.as_str() {
"alt" => 1,
"ctrl" | "control" => 2,
"shift" => 4,
"win" => 8,
_ => return None,
};
}
let vk = match key.as_bytes() {
[c] if c.is_ascii_alphanumeric() => c.to_ascii_uppercase() as u32,
[b'f', n @ ..] => match std::str::from_utf8(n).ok()?.parse::<u32>().ok()? {
n @ 1..=24 => 0x6F + n,
_ => return None,
},
_ => return None,
};
(m != 0).then_some((m, vk))
}
// ---- emergency stop -----------------------------------------------------------------------------
/// The emergency stop: on until the hotkey is pressed again or the PC 제어 screen lifts it; an app
/// restart lifts it (it lives in the process only).
pub struct Stop {
on: AtomicBool,
/// How often it was turned on: a command started before the latest press is killed.
presses: AtomicU64,
}
impl Default for Stop {
fn default() -> Self {
Self::new()
}
}
impl Stop {
pub const fn new() -> Self {
Stop { on: AtomicBool::new(false), presses: AtomicU64::new(0) }
}
pub fn on(&self) -> bool {
self.on.load(Ordering::SeqCst)
}
/// The hotkey: on, or off again.
pub fn toggle(&self) {
let now_on = !self.on();
if now_on {
self.presses.fetch_add(1, Ordering::SeqCst);
}
self.on.store(now_on, Ordering::SeqCst);
crate::info_log!("conversation", "PC control emergency stop {}", if now_on { "on" } else { "lifted" });
}
/// The PC 제어 screen's 풀기.
pub fn lift(&self) {
if self.on.swap(false, Ordering::SeqCst) {
crate::info_log!("conversation", "PC control emergency stop lifted");
}
}
fn presses(&self) -> u64 {
self.presses.load(Ordering::SeqCst)
}
/// Whether a command started at `presses` is to be killed.
fn pressed_since(&self, presses: u64) -> bool {
self.on() && self.presses() != presses
}
}
static EMERGENCY: Stop = Stop::new();
/// The app's emergency stop (the hotkey, the screen and the dashboard share it).
pub fn emergency() -> &'static Stop {
&EMERGENCY
}
// ---- agent-desktop ------------------------------------------------------------------------------
/// What PC control runs on: agent-desktop and two questions to Windows (`Cli`; a fake in tests).
pub trait Desktop: Send + Sync {
/// One agent-desktop command (never `--force`, never the raw mouse and key commands): its
/// `data`, or its error code. A press of the emergency stop while it runs kills it
/// (`APP_STOPPED`).
fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result<Value, String>;
/// Whether process `pid` runs as administrator; one that cannot be told counts as one.
fn elevated(&self, pid: u64) -> bool;
/// The running programs' file names, lower case.
fn processes(&self) -> Vec<String>;
/// What `launch` is given for a program name: the name itself, or the program file a Start
/// menu shortcut or Windows' App Paths name.
fn program(&self, name: &str) -> String {
name.to_owned()
}
/// The window in front now, as agent-desktop names it (`w-<handle>`).
fn foreground(&self) -> Option<String> {
None
}
/// Whether the window in front fills its monitor without a frame (not the desktop), or a
/// full-screen Direct3D program runs.
fn fullscreen(&self) -> bool {
false
}
/// Whether window `id` takes keys posted to it behind: it has one text control shown
/// (`text_control`).
fn takes_keys_behind(&self, _id: &str) -> bool {
false
}
/// Post one key (`behind_key`) to window `id`'s text control, without bringing it forward.
fn post_key(&self, _id: &str, _key: &str) -> Result<(), String> {
Err("PLATFORM_NOT_SUPPORTED".into())
}
}
/// The real agent-desktop: one process per command, its APPDATA, LOCALAPPDATA and state folder in
/// the data folder (`runtime/agent-desktop`).
pub struct Cli {
exe: PathBuf,
home: PathBuf,
limit: Duration,
}
impl Cli {
pub fn new(exe: &Path, data: &Path) -> Self {
let home = data.join("runtime").join("agent-desktop");
// agent-desktop refuses a state folder others may write to ("private file DACL grants an
// untrusted principal": a data folder outside the user's profile inherits Authenticated
// Users' Modify), so it is made afresh, private to this user; a run's readings are of no
// use to the next.
let _ = std::fs::remove_dir_all(&home);
let _ = std::fs::create_dir_all(&home);
#[cfg(windows)]
if !win::make_private(&home) {
crate::warn_log!("conversation", "Warning: agent-desktop's state folder could not be made private: PC control may fail");
}
Cli { exe: exe.to_path_buf(), home, limit: COMMAND_LIMIT }
}
}
/// An agent-desktop envelope `{ok, data, error{code}}`: the data, or the code (never the message,
/// which can name windows).
fn envelope(out: &[u8]) -> Result<Value, String> {
let text = String::from_utf8_lossy(out);
let text = text.trim();
let v: Value = serde_json::from_str(text).or_else(|_| serde_json::from_str(text.lines().last().unwrap_or("")))
.map_err(|_| "APP_NO_JSON".to_owned())?;
match v["ok"].as_bool() {
Some(true) => Ok(v["data"].clone()),
Some(false) => Err(v["error"]["code"].as_str().unwrap_or("APP_UNKNOWN").to_owned()),
None => Err("APP_NO_JSON".into()),
}
}
impl Desktop for Cli {
fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result<Value, String> {
let mut cmd = std::process::Command::new(&self.exe);
cmd.args(args);
if args.first().map(String::as_str) == Some("launch") {
// A started program gets the person's own folders, not agent-desktop's.
for var in ["APPDATA", "LOCALAPPDATA"] {
if let Some(v) = std::env::var_os(var) {
cmd.arg("--env").arg(format!("{var}={}", v.to_string_lossy()));
}
}
if let Some(home) = std::env::var_os("USERPROFILE").filter(|h| Path::new(h).is_absolute()) {
cmd.arg("--cwd").arg(home);
}
}
cmd.env("APPDATA", &self.home).env("LOCALAPPDATA", &self.home).env("AGENT_DESKTOP_HOME", &self.home)
.stdin(if stdin.is_some() { std::process::Stdio::piped() } else { std::process::Stdio::null() })
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null());
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
}
let presses = stop.presses();
let mut child = cmd.spawn().map_err(|_| "APP_NOT_STARTED".to_owned())?;
if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) {
// Text goes in through stdin, never in the arguments; written aside so a full pipe
// cannot hold this thread.
let text = text.to_owned();
std::thread::spawn(move || {
let _ = pipe.write_all(text.as_bytes());
});
}
let mut out = child.stdout.take().expect("stdout is piped");
let reader = std::thread::spawn(move || {
let mut b = Vec::new();
let _ = out.read_to_end(&mut b);
b
});
let started = Instant::now();
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => {}
Err(_) => return Err("APP_NOT_STARTED".into()),
}
let killed = if stop.pressed_since(presses) {
Some("APP_STOPPED")
} else if started.elapsed() > self.limit {
Some("APP_TIMEOUT")
} else {
None
};
if let Some(code) = killed {
let _ = child.kill();
let _ = child.wait();
return Err(code.into());
}
std::thread::sleep(Duration::from_millis(10));
}
envelope(&reader.join().unwrap_or_default())
}
fn elevated(&self, pid: u64) -> bool {
#[cfg(windows)]
return win::elevated(pid);
#[cfg(not(windows))]
{
let _ = pid;
false
}
}
fn processes(&self) -> Vec<String> {
#[cfg(windows)]
return win::processes();
#[cfg(not(windows))]
vec![]
}
fn program(&self, name: &str) -> String {
#[cfg(windows)]
return win::program(name).unwrap_or_else(|| name.to_owned());
#[cfg(not(windows))]
name.to_owned()
}
#[cfg(windows)]
fn foreground(&self) -> Option<String> {
win::foreground()
}
#[cfg(windows)]
fn fullscreen(&self) -> bool {
win::fullscreen()
}
#[cfg(windows)]
fn takes_keys_behind(&self, id: &str) -> bool {
win::text_control_of(id).is_some()
}
#[cfg(windows)]
fn post_key(&self, id: &str, key: &str) -> Result<(), String> {
let vk = behind_key(key).ok_or_else(|| "INVALID_ARGS".to_owned())?;
let to = win::text_control_of(id).ok_or_else(|| "WINDOW_NOT_FOUND".to_owned())?;
win::post_key(to, vk)
}
}
// ---- the actions --------------------------------------------------------------------------------
/// A window as agent-desktop lists it.
#[derive(Debug, Clone, PartialEq)]
struct Win {
id: String,
/// Program file name, lower case (`notepad.exe`).
program: String,
title: String,
pid: u64,
focused: bool,
minimised: bool,
}
impl Win {
fn view(&self) -> Value {
json!({"id": self.id, "program": stem(&self.program), "title": cut(&self.title)})
}
}
fn stem(program: &str) -> &str {
program.strip_suffix(".exe").unwrap_or(program)
}
fn cut(s: &str) -> String {
s.chars().take(MAX_CHARS).collect()
}
/// An activity record (admin.spec.md, "Activity"): the program's file name, never a title,
/// element name or text.
#[derive(Debug, Clone, PartialEq)]
pub struct Record {
pub kind: String,
pub target: String,
pub ok: bool,
pub note: Option<String>,
}
/// The answer to one `pc_action`, its result code for the log and what the activity records.
#[derive(Debug, Clone, PartialEq)]
pub struct Outcome {
pub reply: Value,
/// `ok`, `ask`, `refused` or agent-desktop's error code.
pub code: String,
pub activity: Option<Record>,
}
#[derive(Debug)]
struct Fail {
code: String,
reason: String,
}
fn refused(reason: &str) -> Fail {
Fail { code: "refused".into(), reason: reason.into() }
}
/// agent-desktop's error code as the reason the person hears.
fn failed(code: &str, action: &str) -> Fail {
let reason = match code {
// Windows' foreground lock: focus-window could not bring the window forward, or press
// could not confirm its focus; no key was sent.
"ACTION_FAILED" | "STALE_REF" if action == "keys" => NOT_FRONT,
// close-app gives up while the program still runs (a save question, say).
"TIMEOUT" | "APP_TIMEOUT" if action == "close" => NOT_CLOSED,
// close-app picks the program by name; with several running it cannot choose.
"AMBIGUOUS_TARGET" if action == "close" => CLOSE_SEVERAL,
// launch attaches to one running instance; with several it cannot choose.
"AMBIGUOUS_TARGET" if action == "run" => RUNNING_SEVERAL,
"STALE_REF" | "SNAPSHOT_NOT_FOUND" | "ELEMENT_NOT_FOUND" => STALE,
"AMBIGUOUS_TARGET" => "같은 모양의 항목이 여럿이라 고를 수 없습니다. 다시 읽어 주세요",
"APP_NOT_FOUND" if action == "run" => "그 프로그램을 찾지 못했습니다",
"APP_NOT_FOUND" | "WINDOW_NOT_FOUND" => NO_WINDOW,
"PERM_DENIED" => ELEVATED,
"POLICY_DENIED" => "막혀 있는 동작이라 하지 않습니다(위험한 단축키 등)",
"INVALID_ARGS" if action == "close" => "Windows가 지키는 프로그램이라 닫을 수 없습니다",
"INVALID_ARGS" => "요청을 알아듣지 못했습니다. 이름이나 키를 확인해 주세요",
"ACTION_NOT_SUPPORTED" => "그 항목은 이 동작을 받지 않습니다",
"ACTION_FAILED" => "동작이 되지 않았습니다",
"APP_UNRESPONSIVE" => "프로그램이 응답하지 않습니다",
"TIMEOUT" | "APP_TIMEOUT" => "시간 안에 끝나지 않았습니다",
"APP_STOPPED" => STOPPED,
"PLATFORM_NOT_SUPPORTED" => "이 PC에서는 할 수 없는 동작입니다",
_ => "PC 제어 도구가 실패했습니다",
};
Fail { code: code.into(), reason: reason.into() }
}
enum Done {
Item(Value),
Ask(Value),
}
/// The permission kind of an action and its name in an `ask` answer.
fn kind_of(action: &str) -> Option<(Action, &'static str)> {
match action {
"windows" | "read" => Some((Action::View, "view")),
"click" | "close" => Some((Action::Click, "click")),
"type" | "keys" => Some((Action::Type, "type")),
"run" => Some((Action::Run, "run")),
_ => None,
}
}
const ACTING: &[&str] = &["click", "type", "keys", "run", "close"];
fn valid_ref(r: Option<&str>) -> Result<&str, Fail> {
let r = r.map(str::trim).filter(|r| !r.is_empty()).ok_or_else(|| refused("누를 항목(ref)이 필요합니다"))?;
let ok = r.strip_prefix('@').and_then(|r| r.split_once(":e")).is_some_and(|(snap, n)| {
!snap.is_empty() && snap.len() <= 64 && snap.bytes().all(|b| b.is_ascii_alphanumeric())
&& !n.is_empty() && n.len() <= 9 && n.bytes().all(|b| b.is_ascii_digit())
});
if ok { Ok(r) } else { Err(refused(STALE)) }
}
/// A key combination for `press`: names joined with `+` (`ctrl+s`, `enter`), nothing that could be
/// read as an option.
fn valid_keys(k: Option<&str>) -> Result<String, Fail> {
let k = k.map(|k| k.replace(' ', "")).filter(|k| !k.is_empty()).ok_or_else(|| refused("누를 키가 필요합니다"))?;
let ok = k.len() <= 40 && k.split('+').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_alphanumeric()));
if ok { Ok(k) } else { Err(refused("키를 알아듣지 못했습니다 (예: ctrl+s, enter)")) }
}
/// The virtual key of `keys` when it can be sent behind: one key without modifiers that types or
/// moves, by agent-desktop's names (`a`-`z` and `0`-`9` as their keys on a US layout); else `None`.
fn behind_key(keys: &str) -> Option<u16> {
let k = keys.to_ascii_lowercase();
let vk = match k.as_str() {
"return" | "enter" => 0x0D,
"escape" | "esc" => 0x1B,
"tab" => 0x09,
"space" => 0x20,
"delete" | "backspace" => 0x08,
"forwarddelete" => 0x2E,
"home" => 0x24,
"end" => 0x23,
"pageup" => 0x21,
"pagedown" => 0x22,
"left" => 0x25,
"up" => 0x26,
"right" => 0x27,
"down" => 0x28,
// F10 is Windows' menu key (a system key), so it goes in front.
f if f.len() <= 3 && f.starts_with('f') => match f[1..].parse::<u16>() {
Ok(n @ (1..=9 | 11 | 12)) => 0x6F + n,
_ => return None,
},
c if c.len() == 1 && c.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) => c.as_bytes()[0].to_ascii_uppercase() as u16,
_ => return None,
};
Some(vk)
}
/// Whether a control of this class takes keys posted behind: the text controls whose keys Windows
/// gives to the control itself (classic Edit and RichEdit, the new Notepad's, Scintilla).
pub fn text_control(class: &str) -> bool {
["Edit", "RichEdit20A", "RichEdit20W", "RICHEDIT50W", "RichEditD2DPT", "Scintilla"].contains(&class)
}
/// A program to start by name: no folders, nothing that could be read as an option.
fn valid_program(p: Option<&str>) -> Result<String, Fail> {
let p = p.map(str::trim).filter(|p| !p.is_empty()).ok_or_else(|| refused("실행할 프로그램 이름이 필요합니다"))?;
let ok = p.chars().count() <= 100 && !p.starts_with(['-', '.']) && !p.contains("..")
&& !p.chars().any(|c| c.is_control() || matches!(c, '\\' | '/' | ':' | '"' | '*' | '?' | '<' | '>' | '|'));
if ok { Ok(p.to_owned()) } else { Err(refused("프로그램 이름으로만 실행할 수 있습니다 (예: notepad)")) }
}
/// A program name as a file name for the activity (`notepad` -> `notepad.exe`).
fn program_file(name: &str) -> String {
let n = name.to_lowercase();
if Path::new(&n).extension().is_some() { n } else { format!("{n}.exe") }
}
/// A reading's elements: interactive ones (with a ref) first, then the rest that have a name or a
/// value; at most `MAX_ELEMENTS`, names and values cut to `MAX_CHARS`.
fn elements(tree: &Value) -> Vec<Value> {
fn walk<'a>(n: &'a Value, refs: &mut Vec<&'a Value>, rest: &mut Vec<&'a Value>) {
for c in n["children"].as_array().into_iter().flatten() {
if c["ref_id"].as_str().is_some_and(|r| !r.is_empty()) {
refs.push(c);
} else if c["name"].as_str().is_some_and(|s| !s.trim().is_empty())
|| c["value"].as_str().is_some_and(|s| !s.trim().is_empty()) {
rest.push(c);
}
walk(c, refs, rest);
}
}
let (mut refs, mut rest) = (vec![], vec![]);
walk(tree, &mut refs, &mut rest);
refs.into_iter().chain(rest).take(MAX_ELEMENTS).map(|n| json!({
"ref": n["ref_id"].as_str().unwrap_or(""),
"role": n["role"].as_str().unwrap_or(""),
"name": cut(n["name"].as_str().unwrap_or("")),
"value": cut(n["value"].as_str().unwrap_or("")),
"states": n["states"].as_array().cloned().unwrap_or_default(),
})).collect()
}
/// The refs of a reading's fields agent-desktop reports as secure (password fields), in the whole
/// tree, not only the elements shown.
fn secure_refs(tree: &Value) -> HashSet<String> {
fn walk(n: &Value, out: &mut HashSet<String>) {
for c in n["children"].as_array().into_iter().flatten() {
if let Some(r) = c["ref_id"].as_str().filter(|_| c["states"].as_array().is_some_and(|s| s.iter().any(|x| x == "secure"))) {
out.insert(r.to_owned());
}
walk(c, out);
}
}
let mut out = HashSet::new();
walk(tree, &mut out);
out
}
fn args(a: &[&str]) -> Vec<String> {
a.iter().map(|s| s.to_string()).collect()
}
/// PC control for one run of the services.
pub struct Pc {
settings: Settings,
desktop: Box<dyn Desktop>,
stop: &'static Stop,
/// The game game detection sees now, for the anti-cheat sentence.
game: Box<dyn Fn() -> Option<String> + Send + Sync>,
/// This app's process: its windows are never listed or acted on.
own_pid: u64,
/// The latest readings by snapshot id: the window, so an element names its window, and the
/// fields read as secure (password fields; agent-desktop cannot address them again by `get`).
readings: Mutex<VecDeque<(String, Win, HashSet<String>)>>,
/// One action at a time.
one: Mutex<()>,
}
impl Pc {
pub fn new(settings: Settings, desktop: Box<dyn Desktop>, stop: &'static Stop,
game: Box<dyn Fn() -> Option<String> + Send + Sync>) -> Self {
Pc { settings, desktop, stop, game, own_pid: std::process::id() as u64, readings: Mutex::new(VecDeque::new()),
one: Mutex::new(()) }
}
/// Answer one `pc_action` for its asker.
pub fn act(&self, a: &PcAction) -> Outcome {
let _one = self.one.lock().unwrap_or_else(|e| e.into_inner());
let mut program = None;
let result = self.answer(a, &mut program);
let acting = ACTING.contains(&a.action.as_str());
let (reply, code, note) = match result {
Ok(Done::Item(item)) => (json!({"ok": true, "item": item}), "ok".to_owned(), None),
Ok(Done::Ask(v)) => (v, "ask".to_owned(), None),
Err(f) => (json!({"ok": false, "error": f.reason}), f.code, Some(f.reason)),
};
let activity = (acting && code != "ask").then(|| Record { kind: format!("pc.{}", a.action),
target: program.unwrap_or_else(|| "?".into()), ok: code == "ok", note });
Outcome { reply, code, activity }
}
fn answer(&self, a: &PcAction, program: &mut Option<String>) -> Result<Done, Fail> {
let (perm, kind) = kind_of(&a.action).ok_or_else(|| refused("모르는 PC 동작입니다"))?;
let user = a.user.parse::<u64>().map_err(|_| refused(NOT_REGISTERED))?;
let ask = match decide(&self.settings.pc_control, user, perm) {
Decision::Allow => false,
Decision::Ask => !a.consent,
Decision::Refuse(Refusal::NotAllowed) => return Err(refused(NOT_REGISTERED)),
Decision::Refuse(_) => return Err(refused(TURNED_OFF)),
};
let asking = |window: Value| Done::Ask(json!({"ok": false, "ask": true, "kinds": [kind], "window": window}));
match a.action.as_str() {
"windows" => {
if ask {
return Ok(asking(json!({"id": "", "program": "", "title": ""})));
}
let list: Vec<Value> = self.list()?.iter().take(MAX_WINDOWS).map(|w| {
let mut v = w.view();
v["minimised"] = json!(w.minimised);
v
}).collect();
return Ok(Done::Item(json!({"windows": list})));
}
"read" => {
let w = self.window(a.window.as_deref())?;
if ask {
return Ok(asking(w.view()));
}
return self.read(&w).map(Done::Item);
}
_ => {}
}
// Acting: never during the emergency stop or while an anti-cheat game runs.
if self.stop.on() {
return Err(refused(STOPPED));
}
if let Some(why) = self.anti_cheat() {
return Err(refused(&why));
}
if a.action == "run" {
let name = valid_program(a.program.as_deref())?;
*program = Some(program_file(&name));
if ask {
return Ok(asking(json!({"id": "", "program": name, "title": ""})));
}
return self.launch(&name).map(Done::Item);
}
let (w, element) = match a.action.as_str() {
"click" | "type" => {
let r = valid_ref(a.element.as_deref())?;
(self.window_of(r)?, Some(r))
}
_ => (self.window(a.window.as_deref())?, None),
};
*program = Some(w.program.clone());
if self.desktop.elevated(w.pid) {
return Err(refused(ELEVATED));
}
let keys = if a.action == "keys" { Some(valid_keys(a.keys.as_deref())?) } else { None };
let text = a.text.as_deref().unwrap_or("");
if a.action == "type" {
if text.chars().count() > MAX_TEXT {
return Err(refused("글자가 너무 깁니다 (2,000자까지)"));
}
let read_secure = self.readings.lock().unwrap().iter().any(|(_, _, s)| s.contains(element.unwrap()));
if read_secure {
return Err(refused(PASSWORD));
}
let states = self.cmd(&args(&["get", element.unwrap(), "--property", "states"]), None, "type")?;
let secure = states["value"].as_array().is_none_or(|s| s.iter().any(|x| x == "secure"));
if secure {
return Err(refused(PASSWORD));
}
}
if ask {
return Ok(asking(w.view()));
}
if self.stop.on() {
return Err(refused(STOPPED));
}
// Keys restore in their moment in front (restoring brings the window forward).
if w.minimised && a.action != "keys" {
self.cmd(&args(&["restore", "--window-id", &w.id]), None, &a.action)?;
}
match a.action.as_str() {
"click" => {
self.cmd(&args(&["click", element.unwrap()]), None, "click")?;
}
"type" => {
self.cmd(&args(&["set-value", element.unwrap(), "--stdin"]), Some(text), "type")?;
}
"keys" => self.keys(&w, keys.as_deref().unwrap())?,
_ => {
self.cmd(&args(&["close-app", &w.program]), None, "close")?;
return Ok(Done::Item(json!({"window": w.view(), "elements": []})));
}
}
match self.read(&w) {
// The action closed its window (a dialog's button, Enter): done, nothing left to read.
Err(f) if f.code == "WINDOW_NOT_FOUND" => Ok(Done::Item(json!({"window": w.view(), "elements": []}))),
r => r.map(Done::Item),
}
}
/// Press `keys` in window `w` one way, chosen first (admin.spec.md, "Keys"): behind it when it is
/// one plain key and the window takes it so, else in front for a moment (a minimised window
/// restored in it) and the window that was in front brought back. Never both ways: a key is
/// never pressed twice.
fn keys(&self, w: &Win, keys: &str) -> Result<(), Fail> {
let front = self.desktop.foreground();
if front.as_deref() == Some(w.id.as_str()) {
crate::debug_log!("conversation", "pc keys way=front-already");
return self.press(w, keys);
}
if !w.minimised && behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) {
crate::debug_log!("conversation", "pc keys way=behind");
return self.desktop.post_key(&w.id, keys).map_err(|code| failed(&code, "keys"));
}
if (self.game)().is_some() || self.desktop.fullscreen() {
crate::debug_log!("conversation", "pc keys way=none (game or full screen)");
return Err(refused(NOT_IN_FRONT_NOW));
}
crate::debug_log!("conversation", "pc keys way=front");
let restored = if w.minimised { self.cmd(&args(&["restore", "--window-id", &w.id]), None, "keys").map(|_| ()) } else { Ok(()) };
let pressed = restored
.and_then(|_| self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys"))
.and_then(|_| self.press(w, keys));
if let Some(back) = front {
if self.desktop.run(&args(&["focus-window", "--window-id", &back]), None, self.stop).is_err() {
crate::warn_log!("conversation", "Warning: PC control could not bring back the window that was in front");
}
}
pressed.map(|_| ())
}
/// `press` in window `w`, which is in front. agent-desktop cannot find some Store apps' process
/// by the window (the new Calculator: APP_NOT_FOUND, nothing sent), so then the keys go to the
/// window in front while it is still `w`.
fn press(&self, w: &Win, keys: &str) -> Result<(), Fail> {
match self.desktop.run(&args(&["press", keys, "--window-id", &w.id]), None, self.stop) {
Err(code) if code == "APP_NOT_FOUND" && self.desktop.foreground().as_deref() == Some(w.id.as_str()) => {
crate::debug_log!("conversation", "pc keys press=in-front (window's process not found)");
self.cmd(&args(&["press", keys]), None, "keys").map(|_| ())
}
r => r.map(|_| ()).map_err(|code| failed(&code, "keys")),
}
}
fn cmd(&self, args: &[String], stdin: Option<&str>, action: &str) -> Result<Value, Fail> {
self.desktop.run(args, stdin, self.stop).map_err(|code| failed(&code, action))
}
/// The windows that may be listed and acted on: not this app's, not an excluded program's
/// (`pcControl.screenExclude`), with a title; in agent-desktop's order.
fn list(&self) -> Result<Vec<Win>, Fail> {
let data = self.cmd(&args(&["list-windows"]), None, "windows")?;
let excluded = &self.settings.pc_control.screen_exclude;
Ok(data.as_array().into_iter().flatten().filter_map(|w| {
let program = crate::game::file_name(w["app_name"].as_str()?);
let win = Win { id: w["id"].as_str()?.to_owned(), title: w["title"].as_str().unwrap_or("").to_owned(),
pid: w["pid"].as_u64().unwrap_or(0), focused: w["is_focused"].as_bool().unwrap_or(false),
minimised: w["minimized"].as_bool().unwrap_or(false), program };
let shown = w["visible"].as_bool() != Some(false) && !win.title.trim().is_empty() && !win.id.is_empty();
(shown && win.pid != self.own_pid && !win.program.is_empty()
&& !crate::game::listed(&win.program, excluded)).then_some(win)
}).collect())
}
/// A window by its id, else the one whose title or program contains `query` (the focused one
/// first, then the first listed).
fn window(&self, query: Option<&str>) -> Result<Win, Fail> {
let q = query.map(str::trim).filter(|q| !q.is_empty()).ok_or_else(|| refused("창 이름이 필요합니다"))?;
let list = self.list()?;
if let Some(w) = list.iter().find(|w| w.id == q) {
return Ok(w.clone());
}
let q = q.to_lowercase();
let hits: Vec<&Win> = list.iter()
.filter(|w| w.title.to_lowercase().contains(&q) || stem(&w.program).contains(&q)).collect();
hits.iter().find(|w| w.focused).or(hits.first()).map(|w| (*w).clone()).ok_or_else(|| refused(NO_WINDOW))
}
/// The window an element was read from, as it is listed now.
fn window_of(&self, element: &str) -> Result<Win, Fail> {
let snap = element.trim_start_matches('@').split(':').next().unwrap_or("");
let id = self.readings.lock().unwrap().iter().find(|(s, _, _)| s == snap).map(|(_, w, _)| w.id.clone())
.ok_or_else(|| refused(STALE))?;
self.list()?.into_iter().find(|w| w.id == id).ok_or_else(|| refused(STALE))
}
/// Read a window afresh: `{window, elements}`.
fn read(&self, w: &Win) -> Result<Value, Fail> {
let data = self.cmd(&args(&["snapshot", "--window-id", &w.id, "--compact", "--timeout-ms", SNAPSHOT_MS]), None, "read")?;
let mut w = w.clone();
if let Some(t) = data["window"]["title"].as_str() {
w.title = t.to_owned();
}
if let Some(snap) = data["snapshot_id"].as_str() {
let mut r = self.readings.lock().unwrap();
r.push_back((snap.to_owned(), w.clone(), secure_refs(&data["tree"])));
while r.len() > KEPT_READINGS {
r.pop_front();
}
}
Ok(json!({"window": w.view(), "elements": elements(&data["tree"])}))
}
/// Start a program (agent-desktop never asks for administrator rights) and read its window:
/// the one `launch` names, else a new one of its process or name, else the only new one.
fn launch(&self, name: &str) -> Result<Value, Fail> {
let before: HashSet<String> = self.list()?.into_iter().map(|w| w.id).collect();
let target = self.desktop.program(name);
let launched = match self.desktop.run(&args(&["launch", &target, "--timeout", LAUNCH_MS]), None, self.stop) {
Ok(d) => d,
// A program that hands over to another process (the Windows calculator) ends at once.
Err(code) if code == "APP_UNRESPONSIVE" => Value::Null,
Err(code) => return Err(failed(&code, "run")),
};
let (pid, id) = (launched["pid"].as_u64(), launched["window"]["id"].as_str().map(str::to_owned));
let lower = name.to_lowercase();
let started = Instant::now();
loop {
let list = self.list()?;
// A program already running is not started again: launch names its window.
let named = list.iter().find(|w| Some(&w.id) == id.as_ref());
let new: Vec<&Win> = list.iter().filter(|w| !before.contains(&w.id)).collect();
let found = named
.or_else(|| new.iter().copied().find(|w| pid.is_some_and(|p| p == w.pid)))
.or_else(|| new.iter().copied().find(|w| stem(&w.program).contains(&lower) || w.title.to_lowercase().contains(&lower)))
.or(if new.len() == 1 { new.first().copied() } else { None });
if let Some(w) = found {
return self.read(w);
}
if self.stop.on() {
return Err(refused(STOPPED));
}
if started.elapsed() >= WINDOW_WAIT {
return Err(Fail { code: "no_window".into(), reason: "프로그램을 시작했지만 창을 찾지 못했습니다".into() });
}
std::thread::sleep(Duration::from_millis(300));
}
}
/// The anti-cheat sentence while a game with one runs (its processes, or a program the person
/// listed), else `None`.
fn anti_cheat(&self) -> Option<String> {
let running = self.desktop.processes();
let found = running.iter().find_map(|p| ANTI_CHEATS.iter().find(|(name, _, _)| name == p)
.map(|(_, ac, game)| (*ac, game.map(str::to_owned).unwrap_or_else(|| crate::game::game_name(p, &[])))))
.or_else(|| running.iter().find(|p| crate::game::listed(p, &self.settings.pc_control.anti_cheat_games))
.map(|p| ("안티치트", crate::game::game_name(p, &[]))))?;
let game = (self.game)().unwrap_or(found.1);
Some(format!("{}가 적용된 게임({game})이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", found.0))
}
}
// ---- the emergency stop's hotkey ----------------------------------------------------------------
/// The emergency stop's global hotkey, on its own thread while the services run.
pub struct Hotkey {
#[cfg(windows)]
thread: u32,
#[cfg(windows)]
join: Option<std::thread::JoinHandle<()>>,
}
impl Hotkey {
/// Register `spec` for `stop`; `None` elsewhere than Windows or when Windows refuses it
/// (another program holds it: warned in the log).
pub fn start(spec: &str, stop: &'static Stop) -> Option<Hotkey> {
let (mods, vk) = parse_hotkey(spec)?;
#[cfg(windows)]
return win::hotkey(mods, vk, stop).map(|(thread, join)| Hotkey { thread, join: Some(join) });
#[cfg(not(windows))]
{
let _ = (mods, vk, stop);
None
}
}
/// Unregister it and end its thread.
pub fn end(mut self) {
self.finish();
}
fn finish(&mut self) {
#[cfg(windows)]
if let Some(j) = self.join.take() {
win::end_hotkey(self.thread);
let _ = j.join();
}
}
}
impl Drop for Hotkey {
fn drop(&mut self) {
self.finish();
}
}
/// The program file a Start menu shortcut (`.lnk`) points at, from its link information (Unicode
/// path when present, else an ASCII one); `None` for a shortcut with arguments (starting the file
/// alone would do something else) or without a local path.
pub fn shortcut_target(b: &[u8]) -> Option<String> {
let u32_at = |o: usize| b.get(o..o + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as usize);
let u16_at = |o: usize| b.get(o..o + 2).map(|s| u16::from_le_bytes([s[0], s[1]]) as usize);
if u32_at(0)? != 0x4C {
return None;
}
let flags = u32_at(0x14)?;
const HAS_ID_LIST: usize = 0x1;
const HAS_LINK_INFO: usize = 0x2;
const HAS_ARGUMENTS: usize = 0x20;
if flags & HAS_LINK_INFO == 0 || flags & HAS_ARGUMENTS != 0 {
return None;
}
let mut at = 0x4C;
if flags & HAS_ID_LIST != 0 {
at += 2 + u16_at(at)?;
}
let (header, info_flags) = (u32_at(at + 4)?, u32_at(at + 8)?);
if info_flags & 0x1 == 0 {
return None;
}
let path = if header >= 0x24 {
let start = at + u32_at(at + 28)?;
let units: Vec<u16> = b.get(start..)?.chunks_exact(2).map(|c| u16::from_le_bytes([c[0], c[1]]))
.take_while(|&c| c != 0).collect();
String::from_utf16(&units).ok()?
} else {
let start = at + u32_at(at + 16)?;
let bytes: Vec<u8> = b.get(start..)?.iter().copied().take_while(|&c| c != 0).collect();
if !bytes.is_ascii() {
return None;
}
String::from_utf8(bytes).ok()?
};
(!path.is_empty()).then_some(path)
}
#[cfg(windows)]
mod win {
use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE};
use windows_sys::Win32::Security::{GetTokenInformation, TokenElevation, TOKEN_ELEVATION, TOKEN_QUERY};
use windows_sys::Win32::System::Threading::{
GetCurrentThreadId, OpenProcess, OpenProcessToken, PROCESS_QUERY_LIMITED_INFORMATION,
};
/// Whether process `pid`'s token is elevated; a process that cannot be asked counts as elevated.
pub fn elevated(pid: u64) -> bool {
// SAFETY: query-only handles, closed at once; the out structure is sized as passed.
unsafe {
let p = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid as u32);
if p.is_null() {
return true;
}
let mut token: HANDLE = std::ptr::null_mut();
let opened = OpenProcessToken(p, TOKEN_QUERY, &mut token) != 0;
CloseHandle(p);
if !opened {
return true;
}
let mut e = TOKEN_ELEVATION { TokenIsElevated: 0 };
let mut len = 0u32;
let ok = GetTokenInformation(token, TokenElevation, (&mut e as *mut TOKEN_ELEVATION).cast(),
std::mem::size_of::<TOKEN_ELEVATION>() as u32, &mut len) != 0;
CloseHandle(token);
!ok || e.TokenIsElevated != 0
}
}
/// Make `dir` private to this user: a protected DACL with one entry, full control for the user,
/// inherited by what is made inside.
pub fn make_private(dir: &std::path::Path) -> bool {
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Security::{
AddAccessAllowedAceEx, GetLengthSid, InitializeAcl, InitializeSecurityDescriptor, SetFileSecurityW,
SetSecurityDescriptorControl, SetSecurityDescriptorDacl, TokenUser, ACCESS_ALLOWED_ACE, ACL, ACL_REVISION,
CONTAINER_INHERIT_ACE, DACL_SECURITY_INFORMATION, OBJECT_INHERIT_ACE, PROTECTED_DACL_SECURITY_INFORMATION,
SECURITY_DESCRIPTOR, SE_DACL_PROTECTED, TOKEN_USER,
};
use windows_sys::Win32::System::Threading::GetCurrentProcess;
const FILE_ALL_ACCESS: u32 = 0x001F_01FF;
// SAFETY: this process's token queried into a sized, aligned buffer and closed; the ACL and
// security descriptor live in buffers that outlive the call that uses them.
unsafe {
let mut token: HANDLE = std::ptr::null_mut();
if OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token) == 0 {
return false;
}
let mut user = [0u64; 64];
let mut len = 0u32;
let ok = GetTokenInformation(token, TokenUser, user.as_mut_ptr().cast(), std::mem::size_of_val(&user) as u32, &mut len) != 0;
CloseHandle(token);
if !ok {
return false;
}
let sid = (*(user.as_ptr() as *const TOKEN_USER)).User.Sid;
let acl_len = std::mem::size_of::<ACL>() + std::mem::size_of::<ACCESS_ALLOWED_ACE>() + GetLengthSid(sid) as usize;
let mut acl = vec![0u32; acl_len.div_ceil(4)];
let acl = acl.as_mut_ptr() as *mut ACL;
let mut sd: SECURITY_DESCRIPTOR = std::mem::zeroed();
let psd = (&mut sd as *mut SECURITY_DESCRIPTOR).cast();
let path: Vec<u16> = dir.as_os_str().encode_wide().chain([0]).collect();
InitializeAcl(acl, (acl_len.div_ceil(4) * 4) as u32, ACL_REVISION) != 0
&& AddAccessAllowedAceEx(acl, ACL_REVISION, OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, FILE_ALL_ACCESS, sid) != 0
&& InitializeSecurityDescriptor(psd, 1) != 0
&& SetSecurityDescriptorDacl(psd, 1, acl, 0) != 0
&& SetSecurityDescriptorControl(psd, SE_DACL_PROTECTED, SE_DACL_PROTECTED) != 0
&& SetFileSecurityW(path.as_ptr(), DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, psd) != 0
}
}
/// Every running process's file name, lower case (also of processes this user may not open,
/// such as anti-cheat services).
pub fn processes() -> Vec<String> {
use windows_sys::Win32::System::Diagnostics::ToolHelp::{
CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS,
};
let mut out = vec![];
// SAFETY: a process snapshot walked with a sized entry, then closed.
unsafe {
let snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
if snap == INVALID_HANDLE_VALUE {
return out;
}
let mut e: PROCESSENTRY32W = std::mem::zeroed();
e.dwSize = std::mem::size_of::<PROCESSENTRY32W>() as u32;
let mut more = Process32FirstW(snap, &mut e) != 0;
while more {
let n = e.szExeFile.iter().position(|&c| c == 0).unwrap_or(e.szExeFile.len());
out.push(String::from_utf16_lossy(&e.szExeFile[..n]).to_lowercase());
more = Process32NextW(snap, &mut e) != 0;
}
CloseHandle(snap);
}
out
}
/// A program name as `launch` takes it: a name Windows' folders have stays as it is (agent-desktop
/// finds it there), else the file Windows' App Paths name or a Start menu shortcut points at.
pub fn program(name: &str) -> Option<String> {
let file = super::program_file(name);
let windir = std::env::var_os("SystemRoot").or_else(|| std::env::var_os("windir"))?;
let windir = std::path::Path::new(&windir);
if windir.join("System32").join(&file).is_file() || windir.join(&file).is_file() {
return None;
}
app_path(&file).or_else(|| start_menu(name))
}
fn app_path(file: &str) -> Option<String> {
use windows_sys::Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, RRF_RT_REG_SZ};
let key: Vec<u16> = format!(r"Software\Microsoft\Windows\CurrentVersion\App Paths\{file}")
.encode_utf16().chain([0]).collect();
for root in [HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE] {
let mut buf = [0u16; 1024];
let mut len = (buf.len() * 2) as u32;
// SAFETY: the default value read into a sized buffer.
let r = unsafe { RegGetValueW(root, key.as_ptr(), std::ptr::null(), RRF_RT_REG_SZ, std::ptr::null_mut(),
buf.as_mut_ptr().cast(), &mut len) };
if r == 0 {
let n = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
let path = String::from_utf16_lossy(&buf[..n]).trim().trim_matches('"').to_owned();
if std::path::Path::new(&path).is_file() {
return Some(path);
}
}
}
None
}
/// A Start menu shortcut named `name` (the person's, then everyone's), pointing at a program file.
fn start_menu(name: &str) -> Option<String> {
let want = name.to_lowercase();
let roots = [("APPDATA", r"Microsoft\Windows\Start Menu\Programs"), ("ProgramData", r"Microsoft\Windows\Start Menu\Programs")];
for (var, sub) in roots {
let Some(base) = std::env::var_os(var) else { continue };
let mut dirs = vec![std::path::Path::new(&base).join(sub)];
let mut seen = 0;
while let Some(d) = dirs.pop() {
for e in std::fs::read_dir(&d).into_iter().flatten().flatten() {
seen += 1;
if seen > 5000 {
break;
}
let p = e.path();
if p.is_dir() {
dirs.push(p);
} else if p.extension().is_some_and(|x| x.eq_ignore_ascii_case("lnk"))
&& p.file_stem().is_some_and(|s| s.to_string_lossy().to_lowercase() == want) {
let target = std::fs::read(&p).ok().and_then(|b| super::shortcut_target(&b));
if let Some(t) = target.filter(|t| t.to_lowercase().ends_with(".exe") && std::path::Path::new(t).is_file()) {
return Some(t);
}
}
}
}
}
None
}
pub fn hotkey(mods: u32, vk: u32, stop: &'static super::Stop) -> Option<(u32, std::thread::JoinHandle<()>)> {
use windows_sys::Win32::UI::Input::KeyboardAndMouse::{RegisterHotKey, UnregisterHotKey, MOD_NOREPEAT};
use windows_sys::Win32::UI::WindowsAndMessaging::{GetMessageW, PeekMessageW, MSG, PM_NOREMOVE, WM_HOTKEY, WM_USER};
let (tx, rx) = std::sync::mpsc::channel();
let join = std::thread::spawn(move || {
// SAFETY: a thread-bound hotkey and this thread's own message queue.
unsafe {
let mut msg: MSG = std::mem::zeroed();
// Make the queue before anyone posts to it.
PeekMessageW(&mut msg, std::ptr::null_mut(), WM_USER, WM_USER, PM_NOREMOVE);
let ok = RegisterHotKey(std::ptr::null_mut(), 1, mods | MOD_NOREPEAT, vk) != 0;
let _ = tx.send((GetCurrentThreadId(), ok));
if !ok {
return;
}
while GetMessageW(&mut msg, std::ptr::null_mut(), 0, 0) > 0 {
if msg.message == WM_HOTKEY {
stop.toggle();
}
}
UnregisterHotKey(std::ptr::null_mut(), 1);
}
});
match rx.recv() {
Ok((thread, true)) => Some((thread, join)),
_ => {
let _ = join.join();
crate::warn_log!("conversation", "Warning: the PC control emergency stop hotkey could not be registered (another program may hold it)");
None
}
}
}
/// A window by agent-desktop's id (`w-<handle>`).
fn handle(id: &str) -> Option<windows_sys::Win32::Foundation::HWND> {
use windows_sys::Win32::UI::WindowsAndMessaging::IsWindow;
let h = id.strip_prefix("w-")?.parse::<usize>().ok().filter(|h| *h > 0)? as windows_sys::Win32::Foundation::HWND;
// SAFETY: a plain query on a handle value.
(unsafe { IsWindow(h) } != 0).then_some(h)
}
fn class_of(h: windows_sys::Win32::Foundation::HWND) -> String {
use windows_sys::Win32::UI::WindowsAndMessaging::GetClassNameW;
let mut buf = [0u16; 256];
// SAFETY: the buffer is sized as passed.
let n = unsafe { GetClassNameW(h, buf.as_mut_ptr(), buf.len() as i32) };
String::from_utf16_lossy(&buf[..n.max(0) as usize])
}
pub fn foreground() -> Option<String> {
use windows_sys::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
// SAFETY: a plain query.
let h = unsafe { GetForegroundWindow() };
(!h.is_null()).then(|| format!("w-{}", h as usize))
}
/// The window in front covers its monitor without a title bar or sizing border (not the
/// desktop), or Windows reports a full-screen Direct3D program.
pub fn fullscreen() -> bool {
use windows_sys::Win32::Foundation::RECT;
use windows_sys::Win32::Graphics::Gdi::{GetMonitorInfoW, MonitorFromWindow, MONITORINFO, MONITOR_DEFAULTTONEAREST};
use windows_sys::Win32::UI::Shell::{SHQueryUserNotificationState, QUNS_RUNNING_D3D_FULL_SCREEN};
use windows_sys::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowLongPtrW, GetWindowRect, GWL_STYLE, WS_CAPTION, WS_THICKFRAME};
// SAFETY: plain queries on the window in front, into sized locals.
unsafe {
let mut state = 0;
if SHQueryUserNotificationState(&mut state) == 0 && state == QUNS_RUNNING_D3D_FULL_SCREEN {
return true;
}
let h = GetForegroundWindow();
if h.is_null() || matches!(class_of(h).as_str(), "Progman" | "WorkerW") {
return false;
}
let mut r: RECT = std::mem::zeroed();
let mut mi = MONITORINFO { cbSize: std::mem::size_of::<MONITORINFO>() as u32, ..Default::default() };
if GetWindowRect(h, &mut r) == 0 || GetMonitorInfoW(MonitorFromWindow(h, MONITOR_DEFAULTTONEAREST), &mut mi) == 0 {
return false;
}
let m = mi.rcMonitor;
let style = GetWindowLongPtrW(h, GWL_STYLE) as u32;
r.left <= m.left && r.top <= m.top && r.right >= m.right && r.bottom >= m.bottom
&& style & WS_CAPTION != WS_CAPTION && style & WS_THICKFRAME == 0
}
}
/// Window `id`'s one text control shown (`text_control`), if it has exactly one. Windows does
/// not tell which control of a window behind has the focus (a thread behind has none), so keys
/// go behind only where there is no doubt where they land.
pub fn text_control_of(id: &str) -> Option<windows_sys::Win32::Foundation::HWND> {
use windows_sys::Win32::Foundation::{HWND, LPARAM};
use windows_sys::Win32::UI::WindowsAndMessaging::{EnumChildWindows, IsWindowVisible};
unsafe extern "system" fn each(child: HWND, found: LPARAM) -> i32 {
// SAFETY: `found` is the vector below, alive for the enumeration.
let found = unsafe { &mut *(found as *mut Vec<HWND>) };
if unsafe { IsWindowVisible(child) } != 0 && super::text_control(&class_of(child)) {
found.push(child);
}
1
}
let h = handle(id)?;
let mut found: Vec<HWND> = vec![];
// SAFETY: the callback only reads the children and writes the vector it is given.
unsafe { EnumChildWindows(h, Some(each), &mut found as *mut Vec<HWND> as LPARAM) };
(found.len() == 1).then(|| found[0])
}
/// Key down and key up for `vk` posted to `to`; the program's message loop makes the character.
pub fn post_key(to: windows_sys::Win32::Foundation::HWND, vk: u16) -> Result<(), String> {
use windows_sys::Win32::UI::Input::KeyboardAndMouse::{MapVirtualKeyW, MAPVK_VK_TO_VSC};
use windows_sys::Win32::UI::WindowsAndMessaging::{PostMessageW, WM_KEYDOWN, WM_KEYUP};
// Arrows, Home, End, Page Up and Down, Delete: the extended keys.
let extended = matches!(vk, 0x21..=0x28 | 0x2E);
// SAFETY: plain message posts to a window handle.
unsafe {
let scan = MapVirtualKeyW(vk as u32, MAPVK_VK_TO_VSC) as isize;
let down = 1 | (scan << 16) | if extended { 1 << 24 } else { 0 };
let up = down | (1 << 30) | (1 << 31);
if PostMessageW(to, WM_KEYDOWN, vk as usize, down) == 0 || PostMessageW(to, WM_KEYUP, vk as usize, up) == 0 {
return Err("ACTION_FAILED".into());
}
}
// Let the program take the keys before the window is read again.
std::thread::sleep(std::time::Duration::from_millis(150));
Ok(())
}
pub fn end_hotkey(thread: u32) {
use windows_sys::Win32::UI::WindowsAndMessaging::{PostThreadMessageW, WM_QUIT};
// SAFETY: a quit message to the hotkey's own thread.
unsafe { PostThreadMessageW(thread, WM_QUIT, 0, 0) };
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::settings::Level;
const OWNER: &str = "123456789012345678";
const TITLE: &str = "secret-plan.txt - Notepad";
const TYPED: &str = "typed words";
/// agent-desktop as a script: answers by command, remembers what it was asked.
struct Fake {
windows: Mutex<VecDeque<Value>>,
processes: Vec<String>,
elevated: bool,
secure: bool,
/// Commands that fail with this code.
fail: Option<(&'static str, &'static str)>,
calls: Mutex<Vec<(Vec<String>, Option<String>)>>,
launched: Value,
/// A password field in every reading, past the elements shown; `get` on it is refused as
/// the real agent-desktop refuses it (a secure field's identity is not unique).
secure_ref: Option<&'static str>,
/// The window in front (`focus-window` moves it), a full-screen window in front, and
/// whether windows take keys behind.
front: Mutex<Option<String>>,
fullscreen: bool,
behind: bool,
/// agent-desktop cannot find the window's process by its identity (the new Calculator, a
/// Store app): `press --window-id` is APP_NOT_FOUND, nothing sent.
unnamed: bool,
}
impl Fake {
fn new() -> Self {
Fake { windows: Mutex::new(VecDeque::from([json!([
{"id": "w-1", "title": TITLE, "app_name": "notepad.exe", "pid": 10, "is_focused": false, "accessible": true},
{"id": "w-2", "title": "Vault", "app_name": "KeePassXC.exe", "pid": 11, "is_focused": true, "accessible": true},
{"id": "w-3", "title": "Calc", "app_name": "CalculatorApp.exe", "pid": 12, "is_focused": false, "minimized": true}])])),
processes: vec!["explorer.exe".into()], elevated: false, secure: false, fail: None,
calls: Mutex::new(vec![]), launched: json!({"app": "notepad", "pid": 10}), secure_ref: None,
front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false, unnamed: false}
}
fn front(&self) -> Option<String> {
self.front.lock().unwrap().clone()
}
/// The calls with their arguments, from the first named `from`.
fn calls_from(&self, from: &str) -> Vec<Vec<String>> {
let calls = self.calls.lock().unwrap();
calls.iter().map(|(a, _)| a.clone()).skip_while(|a| a[0] != from).collect()
}
fn commands(&self) -> Vec<String> {
self.calls.lock().unwrap().iter().map(|(a, _)| a[0].clone()).collect()
}
}
fn tree() -> Value {
let mut kids: Vec<Value> = (0..150).map(|i| json!({"role": "statictext", "name": format!("label {i}")})).collect();
kids.extend((0..150).map(|i| json!({"role": "button", "name": format!("{i} {}", "x".repeat(300)), "ref_id": format!("@s1:e{i}"),
"states": ["focused"]})));
json!({"role": "window", "name": TITLE, "children": [{"role": "group", "children": kids}]})
}
impl Desktop for &'static Fake {
fn run(&self, args: &[String], stdin: Option<&str>, _stop: &Stop) -> Result<Value, String> {
self.calls.lock().unwrap().push((args.to_vec(), stdin.map(str::to_owned)));
if let Some((cmd, code)) = self.fail {
if args[0] == cmd {
return Err(code.into());
}
}
if args[0] == "get" && self.secure_ref == Some(args[1].as_str()) {
return Err("AMBIGUOUS_TARGET".into());
}
if args[0] == "press" && self.unnamed && args.iter().any(|a| a == "--window-id") {
return Err("APP_NOT_FOUND".into());
}
Ok(match args[0].as_str() {
"list-windows" => {
let mut w = self.windows.lock().unwrap();
if w.len() > 1 { w.pop_front().unwrap() } else { w[0].clone() }
}
"snapshot" => {
let mut t = tree();
if let Some(r) = self.secure_ref {
t["children"][0]["children"].as_array_mut().unwrap()
.push(json!({"role": "group", "children": [{"role": "textfield", "ref_id": r, "states": ["secure"]}]}));
}
json!({"snapshot_id": "s1", "window": {"id": args[2], "title": TITLE}, "tree": t})
}
"get" => json!({"property": "states", "value": if self.secure { json!(["secure"]) } else { json!([]) }}),
"launch" => self.launched.clone(),
// restore activates the window (SW_RESTORE), as focus-window does.
"focus-window" | "restore" => {
*self.front.lock().unwrap() = Some(args[2].clone());
json!({"disposition": {"delivery": "delivered_unverified"}})
}
_ => json!({"disposition": {"delivery": "delivered_unverified"}}),
})
}
fn elevated(&self, _pid: u64) -> bool {
self.elevated
}
fn processes(&self) -> Vec<String> {
self.processes.clone()
}
fn foreground(&self) -> Option<String> {
self.front()
}
fn fullscreen(&self) -> bool {
self.fullscreen
}
fn takes_keys_behind(&self, _id: &str) -> bool {
self.behind
}
fn post_key(&self, id: &str, key: &str) -> Result<(), String> {
self.calls.lock().unwrap().push((vec!["post".into(), id.into(), key.into()], None));
match self.fail {
Some(("post", code)) => Err(code.into()),
_ => Ok(()),
}
}
}
fn settings() -> Settings {
let mut s = Settings::default();
s.pc_control.allowed_users = vec![OWNER.parse().unwrap()];
s
}
fn pc_with(s: Settings, fake: Fake) -> (Pc, &'static Fake, &'static Stop) {
let fake: &'static Fake = Box::leak(Box::new(fake));
let stop: &'static Stop = Box::leak(Box::new(Stop::new()));
(Pc::new(s, Box::new(fake), stop, Box::new(|| None)), fake, stop)
}
fn action(action: &str, user: &str, extra: Value) -> PcAction {
let mut v = json!({"id": "p1", "action": action, "user": user, "via": "voice"});
v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone());
serde_json::from_value(v).unwrap()
}
fn read_first(pc: &Pc) {
assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).code, "ok");
}
#[test]
fn people_not_registered_are_refused_before_anything_runs() {
let (pc, fake, _) = pc_with(settings(), Fake::new());
for a in ["windows", "read", "click", "type", "keys", "run", "close"] {
let o = pc.act(&action(a, "42", json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_REGISTERED}), "{a}");
}
assert!(fake.commands().is_empty());
// Nobody is registered by default.
let (pc, _, _) = pc_with(Settings::default(), Fake::new());
assert_eq!(pc.act(&action("windows", OWNER, json!({}))).reply["error"], NOT_REGISTERED);
}
#[test]
fn a_kind_turned_off_is_refused_and_every_action_has_its_kind() {
let mut s = settings();
s.pc_control.click = Level::Never;
s.pc_control.type_text = Level::Never;
s.pc_control.run = Level::Never;
s.pc_control.view = Level::Never;
let (pc, fake, _) = pc_with(s, Fake::new());
for a in ["windows", "read", "click", "close", "type", "keys", "run"] {
let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad", "consent": true})));
assert_eq!(o.reply["error"], TURNED_OFF, "{a}");
}
assert!(fake.commands().is_empty());
}
#[test]
fn asking_first_answers_ask_without_consent_and_acts_with_it() {
let (pc, fake, _) = pc_with(settings(), Fake::new());
read_first(&pc);
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"})));
assert_eq!(o.reply, json!({"ok": false, "ask": true, "kinds": ["click"],
"window": {"id": "w-1", "program": "notepad", "title": TITLE}}));
assert_eq!(o.activity, None, "nothing was done yet");
assert!(!fake.commands().contains(&"click".to_owned()));
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply["kinds"], json!(["run"]));
assert_eq!(o.reply["window"]["program"], "notepad");
assert!(!fake.commands().contains(&"launch".to_owned()));
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true})));
assert_eq!(o.reply["ok"], true);
assert!(fake.commands().contains(&"click".to_owned()));
// Done means seen: the window is read again after the click.
assert_eq!(fake.commands().last().unwrap(), "snapshot");
assert_eq!(o.reply["item"]["window"]["id"], "w-1");
// At `auto` nothing is asked.
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let (pc, fake, _) = pc_with(s, Fake::new());
read_first(&pc);
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED})));
assert_eq!(o.reply["ok"], true);
let calls = fake.calls.lock().unwrap();
let set = calls.iter().find(|(a, _)| a[0] == "set-value").unwrap();
assert_eq!(set.1.as_deref(), Some(TYPED), "text goes through stdin");
assert!(!set.0.iter().any(|a| a.contains(TYPED)), "never in the arguments");
}
#[test]
fn an_anti_cheat_game_refuses_acting_but_not_reading() {
let mut fake = Fake::new();
fake.processes = vec!["explorer.exe".into(), "valorant-win64-shipping.exe".into()];
let (pc, fake, _) = pc_with(settings(), fake);
for a in ["click", "type", "keys", "run", "close"] {
let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "keys": "enter",
"program": "notepad", "consent": true})));
assert_eq!(o.reply["error"], "뱅가드가 적용된 게임(VALORANT)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", "{a}");
}
assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot"));
assert_eq!(pc.act(&action("windows", OWNER, json!({}))).code, "ok");
read_first(&pc);
// A program the person listed, named by game detection when it sees the game.
let mut s = settings();
s.pc_control.anti_cheat_games = vec!["MyShooter".into()];
let mut fake = Fake::new();
fake.processes = vec!["myshooter.exe".into()];
let fake: &'static Fake = Box::leak(Box::new(fake));
let pc = Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into())));
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter", "consent": true})));
assert_eq!(o.reply["error"], "안티치트가 적용된 게임(My Shooter)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다");
}
#[test]
fn the_emergency_stop_refuses_acting_until_lifted_and_kills_a_running_command() {
let (pc, fake, stop) = pc_with(settings(), Fake::new());
read_first(&pc);
stop.toggle();
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true})));
assert_eq!(o.reply["error"], STOPPED);
assert_eq!(o.activity.unwrap().ok, false);
assert!(!fake.commands().contains(&"click".to_owned()));
// Reading still works.
read_first(&pc);
stop.toggle();
assert_eq!(pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))).code, "ok");
stop.toggle();
stop.lift();
assert!(!stop.on());
// The real runner: a command running when the hotkey is pressed is killed at once.
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let dir = crate::datadir::temp_dir("pcstop");
let exe = dir.join("agent-desktop");
std::fs::write(&exe, "#!/bin/sh\nsleep 5\necho '{\"ok\": true, \"data\": []}'\n").unwrap();
std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();
let cli = Cli::new(&exe, &dir);
let stop: &'static Stop = Box::leak(Box::new(Stop::new()));
let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(200)); stop.toggle(); });
let started = Instant::now();
assert_eq!(cli.run(&args(&["click", "@s1:e1"]), None, stop), Err("APP_STOPPED".into()));
assert!(started.elapsed() < Duration::from_secs(2));
t.join().unwrap();
// A command started while the stop is on (a reading) runs to its end.
std::fs::write(&exe, "#!/bin/sh\necho \"{\\\"ok\\\": true, \\\"data\\\": \\\"$APPDATA\\\"}\"\n").unwrap();
let home = dir.join("runtime").join("agent-desktop");
assert_eq!(cli.run(&args(&["list-windows"]), None, stop), Ok(json!(home.to_string_lossy())),
"agent-desktop's folders are in the data folder");
// The command limit.
std::fs::write(&exe, "#!/bin/sh\nsleep 5\n").unwrap();
let cli = Cli { limit: Duration::from_millis(200), ..cli };
assert_eq!(cli.run(&args(&["list-windows"]), None, &Stop::new()), Err("APP_TIMEOUT".into()));
}
}
#[test]
fn a_password_field_and_an_administrator_window_are_refused_before_acting() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let mut fake = Fake::new();
fake.secure = true;
let (pc, fake, _) = pc_with(s.clone(), fake);
read_first(&pc);
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED})));
assert_eq!(o.reply["error"], PASSWORD);
assert!(!fake.commands().contains(&"set-value".to_owned()));
// Even with consent.
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED, "consent": true})));
assert_eq!(o.reply["error"], PASSWORD);
let mut fake = Fake::new();
fake.elevated = true;
let (pc, fake, _) = pc_with(s, fake);
read_first(&pc);
for a in ["click", "type", "keys", "close"] {
let o = pc.act(&action(a, OWNER, json!({"ref": "@s1:e3", "window": "notepad", "keys": "enter", "text": "x",
"consent": true})));
assert_eq!(o.reply["error"], ELEVATED, "{a}");
}
assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot"));
}
#[test]
fn a_field_read_as_secure_is_refused_as_a_password_field() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let (pc, fake, _) = pc_with(s, Fake { secure_ref: Some("@s1:e900"), ..Fake::new() });
read_first(&pc);
let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e900", "text": TYPED, "consent": true})));
assert_eq!(o.reply, json!({"ok": false, "error": PASSWORD}));
assert!(!fake.commands().contains(&"set-value".to_owned()));
// Other fields of the same reading are typed into.
assert_eq!(pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).code, "ok");
}
#[test]
fn a_reading_has_interactive_elements_first_at_most_200_and_cut_to_120_characters() {
let (pc, _, _) = pc_with(settings(), Fake::new());
let o = pc.act(&action("read", OWNER, json!({"window": "notepad"})));
let els = o.reply["item"]["elements"].as_array().unwrap();
assert_eq!(els.len(), MAX_ELEMENTS);
assert!(els[..150].iter().all(|e| e["ref"].as_str().unwrap().starts_with("@s1:e")));
assert!(els[150..].iter().all(|e| e["ref"] == "" && e["role"] == "statictext"));
assert!(els.iter().all(|e| e["name"].as_str().unwrap().chars().count() <= MAX_CHARS));
assert_eq!(els[0]["name"].as_str().unwrap().chars().count(), MAX_CHARS);
assert_eq!(els[0]["states"], json!(["focused"]));
assert_eq!(o.reply["item"]["window"], json!({"id": "w-1", "program": "notepad", "title": TITLE}));
// No window named: why.
assert!(pc.act(&action("read", OWNER, json!({}))).reply["error"].is_string());
}
#[test]
fn a_stale_element_says_to_read_again() {
let mut s = settings();
s.pc_control.click = Level::Auto;
let mut fake = Fake::new();
fake.fail = Some(("click", "STALE_REF"));
let (pc, _, _) = pc_with(s.clone(), fake);
// An element of a reading this app does not know, or one in bad form.
for r in ["@s9:e1", "e1", "--force"] {
assert_eq!(pc.act(&action("click", OWNER, json!({"ref": r}))).reply["error"], STALE, "{r}");
}
read_first(&pc);
let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"})));
assert_eq!(o.reply, json!({"ok": false, "error": STALE}));
assert_eq!(o.code, "STALE_REF");
}
#[test]
fn a_minimised_window_is_restored_before_acting_and_keys_bring_it_to_the_front() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let (pc, fake, _) = pc_with(s, Fake::new());
let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+s"})));
assert_eq!(o.reply["ok"], true, "{o:?}");
assert_eq!(fake.commands(), ["list-windows", "restore", "focus-window", "press", "focus-window", "snapshot"]);
let calls = fake.calls.lock().unwrap();
assert_eq!(calls[3].0, args(&["press", "ctrl+s", "--window-id", "w-3"]));
assert!(calls.iter().all(|(a, _)| !a.iter().any(|x| x == "--force")));
drop(calls);
for bad in ["--force", "ctrl+", "alt f4;rm"] {
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": bad})));
assert_eq!(o.code, "refused", "{bad}");
}
}
#[test]
fn excluded_programs_and_this_apps_windows_are_never_listed_or_picked() {
let (pc, _, _) = pc_with(settings(), Fake::new());
let o = pc.act(&action("windows", OWNER, json!({})));
assert_eq!(o.reply["item"]["windows"], json!([
{"id": "w-1", "program": "notepad", "title": TITLE, "minimised": false},
{"id": "w-3", "program": "calculatorapp", "title": "Calc", "minimised": true}]));
assert!(pc.act(&action("read", OWNER, json!({"window": "Vault"}))).reply["error"].is_string());
}
#[test]
fn run_starts_a_program_by_name_and_reads_its_new_window() {
let mut s = settings();
s.pc_control.run = Level::Auto;
let fake = Fake::new();
let before = fake.windows.lock().unwrap()[0].clone();
let mut after = before.clone();
after.as_array_mut().unwrap().push(json!({"id": "w-9", "title": "Untitled - Notepad", "app_name": "notepad.exe", "pid": 99}));
*fake.windows.lock().unwrap() = VecDeque::from([before, after]);
let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 99}), ..fake };
let (pc, fake, _) = pc_with(s, fake);
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply["item"]["window"]["id"], "w-9", "{o:?}");
assert_eq!(fake.calls.lock().unwrap().iter().find(|(a, _)| a[0] == "launch").unwrap().0[1], "notepad");
assert_eq!(o.activity, Some(Record { kind: "pc.run".into(), target: "notepad.exe".into(), ok: true, note: None }));
for bad in ["C:\\Windows\\cmd.exe", "-x", "..\\x"] {
assert_eq!(pc.act(&action("run", OWNER, json!({"program": bad}))).code, "refused", "{bad}");
}
}
#[test]
fn run_with_the_program_already_running_answers_the_window_launch_names() {
let mut s = settings();
s.pc_control.run = Level::Auto;
// agent-desktop attaches to a running instance and names its window; nothing new appears.
let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 10, "window": {"id": "w-1", "title": TITLE}}), ..Fake::new() };
let (pc, _, _) = pc_with(s.clone(), fake);
let started = Instant::now();
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply["item"]["window"]["id"], "w-1", "{o:?}");
assert!(started.elapsed() < WINDOW_WAIT, "answered without waiting for a new window");
// Several running: launch cannot choose, and the answer says so (not an element's reason).
let fake = Fake { fail: Some(("launch", "AMBIGUOUS_TARGET")), ..Fake::new() };
let (pc, _, _) = pc_with(s, fake);
let o = pc.act(&action("run", OWNER, json!({"program": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": RUNNING_SEVERAL}));
}
#[test]
fn keys_to_a_window_that_cannot_come_to_the_front_say_nothing_was_sent() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
// Windows' foreground lock: focus-window fails, or press cannot confirm the window's focus.
for (cmd, code) in [("focus-window", "ACTION_FAILED"), ("press", "STALE_REF")] {
let fake = Fake { fail: Some((cmd, code)), ..Fake::new() };
let (pc, fake, _) = pc_with(s.clone(), fake);
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "ctrl+s"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_FRONT}), "{cmd}");
assert!(cmd == "press" || !fake.commands().contains(&"press".to_owned()), "no key after a failed focus");
}
}
fn keys_auto(fake: Fake) -> (Pc, &'static Fake, &'static Stop) {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
pc_with(s, fake)
}
#[test]
fn one_key_without_modifiers_goes_behind_a_window_that_takes_it_and_nothing_moves() {
for key in ["a", "7", "enter", "Tab", "space", "escape", "backspace", "forwarddelete", "left", "pagedown", "f5"] {
let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key})));
assert_eq!(o.reply["ok"], true, "{key}: {o:?}");
assert_eq!(fake.commands(), ["list-windows", "post", "snapshot"], "{key}");
assert_eq!(fake.calls_from("post")[0][1], "w-1", "{key}");
assert_eq!(fake.front().as_deref(), Some("w-2"), "{key}: the window in front stays");
}
}
#[test]
fn combinations_other_keys_and_windows_not_taking_keys_behind_go_in_front_and_back() {
let in_front_and_back = |fake: &Fake, key: &str| {
assert_eq!(fake.calls_from("focus-window"), [
args(&["focus-window", "--window-id", "w-1"]),
args(&["press", key, "--window-id", "w-1"]),
args(&["focus-window", "--window-id", "w-2"]),
args(&["snapshot", "--window-id", "w-1", "--compact", "--timeout-ms", SNAPSHOT_MS])], "{key}");
assert!(!fake.commands().contains(&"post".to_owned()), "{key}: never both ways");
assert_eq!(fake.front().as_deref(), Some("w-2"), "{key}: the window in front is back");
};
for key in ["ctrl+s", "shift+tab", "alt+f", "cmd+d", "f10", "shift"] {
let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() });
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key}))).reply["ok"], true, "{key}");
in_front_and_back(fake, key);
}
// A window that does not take keys behind (a Store app, Chrome) gets even one letter in front.
let (pc, fake, _) = keys_auto(Fake::new());
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))).reply["ok"], true);
in_front_and_back(fake, "a");
// A failed press still brings the window in front back, and is not tried behind.
let (pc, fake, _) = keys_auto(Fake { fail: Some(("press", "STALE_REF")), ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_FRONT}));
assert_eq!(fake.calls_from("focus-window").last().unwrap(), &args(&["focus-window", "--window-id", "w-2"]));
assert_eq!(fake.front().as_deref(), Some("w-2"));
}
#[test]
fn keys_to_a_minimised_window_bring_back_the_window_that_was_in_front_and_wait_for_no_game() {
// Restoring brings the window forward, so it is part of the moment in front.
let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() });
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+a"}))).reply["ok"], true);
assert_eq!(fake.calls_from("restore"), [
args(&["restore", "--window-id", "w-3"]),
args(&["focus-window", "--window-id", "w-3"]),
args(&["press", "ctrl+a", "--window-id", "w-3"]),
args(&["focus-window", "--window-id", "w-2"]),
args(&["snapshot", "--window-id", "w-3", "--compact", "--timeout-ms", SNAPSHOT_MS])]);
assert_eq!(fake.front().as_deref(), Some("w-2"), "the window in front is back");
// During a game or full screen a minimised window is not even restored.
let (pc, fake, _) = keys_auto(Fake { behind: true, fullscreen: true, ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "a"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_IN_FRONT_NOW}));
assert!(!fake.commands().contains(&"restore".to_owned()), "{:?}", fake.commands());
assert_eq!(fake.front().as_deref(), Some("w-2"));
}
#[test]
fn a_store_app_agent_desktop_cannot_name_gets_its_keys_once_it_is_in_front() {
// Brought in front, and already in front: the key is pressed once, in that window.
for already in [false, true] {
let fake = Fake { unnamed: true, ..Fake::new() };
if already {
*fake.front.lock().unwrap() = Some("w-1".into());
}
let (pc, fake, _) = keys_auto(fake);
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "1"})));
assert_eq!(o.reply["ok"], true, "{o:?}");
let presses: Vec<Vec<String>> = fake.calls_from("list-windows").into_iter().filter(|a| a[0] == "press").collect();
assert_eq!(presses.last().unwrap(), &args(&["press", "1"]), "{already}");
assert_eq!(fake.front().as_deref(), Some(if already { "w-1" } else { "w-2" }), "{already}: the window in front is back");
}
}
#[test]
fn a_window_already_in_front_gets_its_keys_there_and_nothing_moves() {
let fake = Fake::new();
*fake.front.lock().unwrap() = Some("w-1".into());
let (pc, fake, _) = keys_auto(fake);
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "ctrl+s"}))).reply["ok"], true);
assert_eq!(fake.commands(), ["list-windows", "press", "snapshot"]);
}
#[test]
fn keys_are_never_brought_in_front_during_a_game_or_full_screen_but_go_behind() {
let game = |fake: Fake| {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
let fake: &'static Fake = Box::leak(Box::new(fake));
(Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into()))), fake)
};
let full = |fake: Fake| {
let (pc, fake, _) = keys_auto(Fake { fullscreen: true, ..fake });
(pc, fake)
};
for (i, make) in [&game as &dyn Fn(Fake) -> (Pc, &'static Fake), &full].into_iter().enumerate() {
for (key, behind) in [("ctrl+s", true), ("a", false)] {
let (pc, fake) = make(Fake { behind, ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": key})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_IN_FRONT_NOW}), "{i} {key}");
assert_eq!(fake.commands(), ["list-windows"], "{i} {key}: no key sent");
}
let (pc, fake) = make(Fake { behind: true, ..Fake::new() });
assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"}))).reply["ok"], true, "{i}");
assert_eq!(fake.commands(), ["list-windows", "post", "snapshot"], "{i}");
}
}
#[test]
fn a_key_not_taken_behind_is_not_pressed_again_in_front() {
let (pc, fake, _) = keys_auto(Fake { behind: true, fail: Some(("post", "ACTION_FAILED")), ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "a"})));
assert_eq!(o.reply["ok"], false);
assert_eq!(fake.commands(), ["list-windows", "post"]);
}
#[test]
fn only_text_controls_take_keys_behind() {
for c in ["Edit", "RichEdit20W", "RICHEDIT50W", "RichEditD2DPT", "Scintilla"] {
assert!(text_control(c), "{c}");
}
for c in ["Notepad", "CalcFrame", "Button", "Windows.UI.Core.CoreWindow", "Chrome_RenderWidgetHostHWND",
"Microsoft.UI.Content.DesktopChildSiteBridge", "MozillaWindowClass", "edit"] {
assert!(!text_control(c), "{c}");
}
}
#[test]
fn an_action_that_closes_its_window_is_done_with_nothing_left_to_read() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
// Enter on a dialog (or a click on its button) closes it: the fresh reading finds no window.
let (pc, _, _) = pc_with(s, Fake { fail: Some(("snapshot", "WINDOW_NOT_FOUND")), ..Fake::new() });
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter"})));
assert_eq!(o.reply, json!({"ok": true, "item": {"window": {"id": "w-1", "program": "notepad", "title": TITLE}, "elements": []}}));
assert_eq!(o.code, "ok");
}
#[test]
fn a_program_that_does_not_close_in_time_is_said_to_be_still_open() {
let mut s = settings();
s.pc_control.click = Level::Auto;
// A save question keeps the program open; close-app gives up after its own deadline.
let fake = Fake { fail: Some(("close-app", "TIMEOUT")), ..Fake::new() };
let (pc, _, _) = pc_with(s, fake);
let o = pc.act(&action("close", OWNER, json!({"window": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": NOT_CLOSED}));
// Several of it running: close-app picks by name and cannot choose.
let mut s = settings();
s.pc_control.click = Level::Auto;
let (pc, _, _) = pc_with(s, Fake { fail: Some(("close-app", "AMBIGUOUS_TARGET")), ..Fake::new() });
let o = pc.act(&action("close", OWNER, json!({"window": "notepad"})));
assert_eq!(o.reply, json!({"ok": false, "error": CLOSE_SEVERAL}));
}
#[test]
fn the_activity_names_the_program_only() {
let mut s = settings();
s.pc_control.type_text = Level::Auto;
s.pc_control.click = Level::Auto;
let (pc, _, stop) = pc_with(s, Fake::new());
read_first(&pc);
let typed = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).activity.unwrap();
assert_eq!(typed, Record { kind: "pc.type".into(), target: "notepad.exe".into(), ok: true, note: None });
let closed = pc.act(&action("close", OWNER, json!({"window": "notepad"}))).activity.unwrap();
assert_eq!(closed, Record { kind: "pc.close".into(), target: "notepad.exe".into(), ok: true, note: None });
stop.toggle();
let refused = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))).activity.unwrap();
assert_eq!((refused.ok, refused.note.as_deref()), (false, Some(STOPPED)));
// Readings are not activity; nothing recorded carries a title, an element or the text.
assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).activity, None);
assert_eq!(pc.act(&action("windows", OWNER, json!({}))).activity, None);
let all = format!("{typed:?}{closed:?}{refused:?}");
for secret in [TITLE, "secret-plan", TYPED, "label", "@s1"] {
assert!(!all.contains(secret), "{secret}");
}
}
#[test]
fn the_tool_is_available_only_with_the_feature_someone_registered_and_agent_desktop() {
let s = settings();
assert_eq!(available(&s, true), cfg!(windows));
assert!(!available(&s, false));
assert!(!available(&Settings::default(), true));
let mut off = settings();
off.features.pc_control = false;
assert!(!available(&off, true));
// The pinned file is checked by its hash.
let f = crate::datadir::temp_dir("pcexe").join(EXE_NAME);
std::fs::write(&f, b"not agent-desktop").unwrap();
assert!(!exe_ok(&f));
assert!(!exe_ok(&f.with_file_name("missing.exe")));
}
#[test]
fn hotkeys_and_envelopes_are_read() {
assert_eq!(parse_hotkey(DEFAULT_STOP_HOTKEY), Some((2 | 4, 0x7B)));
assert_eq!(parse_hotkey("alt+win+q"), Some((1 | 8, b'Q' as u32)));
for bad in ["F12", "Ctrl+", "Ctrl+F25", "Hyper+F1", ""] {
assert_eq!(parse_hotkey(bad), None, "{bad}");
}
assert_eq!(envelope(br#"{"ok": true, "data": [1]}"#), Ok(json!([1])));
assert_eq!(envelope(b"warning\n{\"ok\": false, \"error\": {\"code\": \"STALE_REF\", \"message\": \"x\"}}\n"),
Err("STALE_REF".into()));
assert_eq!(envelope(b"garbage"), Err("APP_NO_JSON".into()));
}
#[test]
fn agent_desktops_state_folder_is_made_afresh_each_start() {
// What a folder made under another ACL holds would keep that ACL, and readings of an
// earlier run are of no use: the folder starts empty.
let data = std::env::temp_dir().join(format!("jarvis-pc-{}-{}", std::process::id(), rand::random::<u32>()));
let home = data.join("runtime").join("agent-desktop");
std::fs::create_dir_all(home.join("refmaps")).unwrap();
std::fs::write(home.join("refmaps").join("old.json"), "{}").unwrap();
let _cli = Cli::new(Path::new("agent-desktop.exe"), &data);
assert!(home.is_dir());
assert_eq!(std::fs::read_dir(&home).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&data);
}
#[test]
fn the_package_and_m0_pin_the_same_agent_desktop() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let package = std::fs::read_to_string(root.join("shell/package.sh")).unwrap();
for pin in [AGENT_DESKTOP_URL, AGENT_DESKTOP_ARCHIVE_SHA256, AGENT_DESKTOP_SHA256] {
assert!(package.contains(pin), "{pin}");
}
let m0 = std::fs::read_to_string(root.join("../scripts/m0/m0lib/desktop.py")).unwrap();
assert!(m0.contains(AGENT_DESKTOP_ARCHIVE_SHA256) && m0.contains(&format!("AD_VERSION = \"{AGENT_DESKTOP_VERSION}\"")));
}
#[test]
fn a_start_menu_shortcut_gives_its_program_file() {
// A shell link with link information and a Unicode local path (MS-SHLLINK).
fn link(flags: u32, path: &str) -> Vec<u8> {
let mut b = vec![0u8; 0x4C];
b[0] = 0x4C;
b[0x14..0x18].copy_from_slice(&flags.to_le_bytes());
let wide: Vec<u8> = path.encode_utf16().chain([0]).flat_map(|c| c.to_le_bytes()).collect();
let header = 0x24u32;
let mut info = vec![0u8; header as usize];
info[4..8].copy_from_slice(&header.to_le_bytes());
info[8..12].copy_from_slice(&1u32.to_le_bytes());
info[28..32].copy_from_slice(&header.to_le_bytes());
info.extend(&wide);
let size = info.len() as u32;
info[0..4].copy_from_slice(&size.to_le_bytes());
b.extend(info);
b
}
let path = r"C:\Program Files\메모\memo.exe";
assert_eq!(shortcut_target(&link(0x2, path)).as_deref(), Some(path));
assert_eq!(shortcut_target(&link(0x2 | 0x20, path)), None, "a shortcut with arguments");
assert_eq!(shortcut_target(&link(0x0, path)), None);
assert_eq!(shortcut_target(b"short"), None);
}
}