From 39ac20682ece099d0a1092a73ef06381815d64ed Mon Sep 17 00:00:00 2001 From: javis-bot Date: Wed, 18 Nov 2026 22:42:48 +0900 Subject: [PATCH] feat(app): PC control through agent-desktop, with the emergency stop and the anti-cheat block app/src/pc.rs runs every pc_action from the worker (admin.spec.md, "PC control"): one agent-desktop 1.2.0 run per command with a 15 s limit and its state in the data folder, never --force or raw input, typed text only through stdin. Each action is decided by permissions::decide for its asker (not registered, never, ask without consent), then refused during the emergency stop, while an anti-cheat game runs (Vanguard, EAC, BattlEye, FACEIT, GameGuard, XIGNCODE3 and the person's list), on an administrator window and in a password field. Results carry the window and its elements read again (200 at most, cut to 120 chars), agent-desktop's error codes become Korean reasons, and each action is recorded as pc. with the program's file name only. The emergency stop is a global hotkey (pcControl.stopHotkey, default Ctrl+Shift+F12) on its own message loop; it kills the running command and blocks everything but reading until pressed again or lifted on the PC control screen. features.pcControl (on by default, nothing happens until someone is registered) and the pinned exe hash decide whether the brain gets the tool. The installer ships agent-desktop.exe from the pinned release archive, both hashes checked. Tested with a fake agent-desktop (342 lib tests pass) and cargo check for x86_64-pc-windows-gnu; not yet run against real agent-desktop. Co-Authored-By: Claude Opus 5.5 --- app/Cargo.toml | 2 +- app/shell/installer.nsi | 3 + app/shell/installer.sh | 2 +- app/shell/package.sh | 15 +- app/src/admin.spec.md | 69 +- app/src/admin_api.rs | 6 + app/src/control.rs | 1 + app/src/discord.rs | 28 + app/src/features.rs | 23 +- app/src/game.rs | 5 + app/src/lib.rs | 1 + app/src/pc.rs | 1391 ++++++++++++++++++++++++++++++++++++++ app/src/services.rs | 23 +- app/src/settings.rs | 56 +- app/src/settings_edit.rs | 7 +- app/src/strata.rs | 17 +- app/src/worker_client.rs | 39 ++ app/ui/app.js | 25 +- 18 files changed, 1658 insertions(+), 55 deletions(-) create mode 100644 app/src/pc.rs diff --git a/app/Cargo.toml b/app/Cargo.toml index 42e9beb..cbc63ff 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -50,7 +50,7 @@ codegen-units = 4 strip = "symbols" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp", "Win32_Graphics_Dwm"] } +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Globalization", "Win32_Security", "Win32_Security_Cryptography", "Win32_System_SystemInformation", "Win32_Devices_DeviceAndDriverInstallation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_UI_Input_KeyboardAndMouse", "Win32_System_Registry", "Win32_UI_Shell", "Win32_System_Threading", "Win32_System_Performance", "Win32_System_Diagnostics_ToolHelp", "Win32_Graphics_Dwm"] } windows = { version = "0.62", features = ["Foundation", "Graphics", "Graphics_Capture", "Graphics_DirectX", "Graphics_DirectX_Direct3D11", "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_System_WinRT", "Win32_System_WinRT_Direct3D11", "Win32_System_WinRT_Graphics_Capture"] } [target.'cfg(unix)'.dependencies] diff --git a/app/shell/installer.nsi b/app/shell/installer.nsi index 04e8c4d..1d0e103 100644 --- a/app/shell/installer.nsi +++ b/app/shell/installer.nsi @@ -226,6 +226,8 @@ Section "Jarvis (필수)" SecMain SetOutPath "$INSTDIR" File "${SRC}/jarvis.exe" File "${SRC}/WebView2Loader.dll" + ; PC control's agent-desktop (the app checks its hash before every start). + File "${SRC}/agent-desktop.exe" ; The shortcuts' icon, named by its content: Windows keeps an icon it has shown by the file's path, ; so a changed icon under the old path would stay the old one on the desktop and in search. Delete "$INSTDIR\jarvis-*.ico" @@ -334,6 +336,7 @@ Section "Uninstall" ; folder the person chose may hold other things; folders go only when empty. Delete "$INSTDIR\jarvis.exe" Delete "$INSTDIR\WebView2Loader.dll" + Delete "$INSTDIR\agent-desktop.exe" Delete "$INSTDIR\jarvis-*.ico" !include "${UNINSTALL_LIST}" Delete "$INSTDIR\uninstall.exe" diff --git a/app/shell/installer.sh b/app/shell/installer.sh index b3ae24e..e494a20 100755 --- a/app/shell/installer.sh +++ b/app/shell/installer.sh @@ -19,7 +19,7 @@ list="$target/uninstall-list.nsh" setup="$out/Jarvis-Setup-$version.exe" # The uninstall entry's size: what this setup installs, counted here. Counting the install folder on # the user's PC would also walk a data folder kept inside it (models, runtime: many thousand files). -size_kb=$(cd "$out/Jarvis" && du -sk --apparent-size -c jarvis.exe WebView2Loader.dll python-src display-driver | tail -1 | cut -f1) +size_kb=$(cd "$out/Jarvis" && du -sk --apparent-size -c jarvis.exe WebView2Loader.dll agent-desktop.exe python-src display-driver | tail -1 | cut -f1) # The shortcuts' icon file is named by its content, so a new icon gets a path Windows has not seen. icon_file="jarvis-$(sha256sum "$here/icons/icon.ico" | cut -c1-12).ico" makensis -V2 -DVERSION="$version" -DSRC="$out/Jarvis" -DOUT="$setup" -DICON="$here/icons/icon.ico" -DICON_FILE="$icon_file" \ diff --git a/app/shell/package.sh b/app/shell/package.sh index 0ee863a..22f6b7d 100755 --- a/app/shell/package.sh +++ b/app/shell/package.sh @@ -1,8 +1,9 @@ #!/bin/sh # Build the Windows management app from Linux and lay it out ready to zip (admin.spec.md, "Windows build"): # Jarvis/jarvis.exe, Jarvis/WebView2Loader.dll, Jarvis/python-src/{voice_worker,jarvis}, -# Jarvis/display-driver/ (Virtual Display Driver 25.7.23, MIT, SignPath-signed; "Virtual monitor") -# Needs the x86_64-pc-windows-gnu Rust target, mingw-w64, cmake (for Opus), curl and unzip. +# Jarvis/display-driver/ (Virtual Display Driver 25.7.23, MIT, SignPath-signed; "Virtual monitor"), +# Jarvis/agent-desktop.exe (agent-desktop 1.2.0, Apache-2.0; "PC control") +# Needs the x86_64-pc-windows-gnu Rust target, mingw-w64, cmake (for Opus), curl, unzip and tar. # app/shell/package.sh set -eu here=$(cd "$(dirname "$0")" && pwd) @@ -30,4 +31,14 @@ unzip -q "$vdd_zip" -d "$tmp" mkdir -p "$out/Jarvis/display-driver" cp "$tmp/VirtualDisplayDriver/MttVDD.inf" "$tmp/VirtualDisplayDriver/mttvdd.cat" "$tmp/VirtualDisplayDriver/MttVDD.dll" "$out/Jarvis/display-driver/" rm -rf "$tmp" +# PC control's agent-desktop, the archive and the program pinned by hash as in app/src/pc.rs (the +# app checks the program's hash again before every start of the services). +ad_url=https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-pc-windows-msvc.tar.gz +ad_sha=a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8 +ad_exe_sha=d274667031322e50b9c2b1ab81c2ca46835d52cc932769a5c34bd4d481fd94c3 +ad_tgz="$target/agent-desktop-1.2.0.tar.gz" +[ -f "$ad_tgz" ] || curl -fsSL -o "$ad_tgz" "$ad_url" +echo "$ad_sha $ad_tgz" | sha256sum -c - >/dev/null || { rm -f "$ad_tgz"; echo "agent-desktop hash mismatch" >&2; exit 1; } +tar -xzf "$ad_tgz" -C "$out/Jarvis" agent-desktop.exe +echo "$ad_exe_sha $out/Jarvis/agent-desktop.exe" | sha256sum -c - >/dev/null || { echo "agent-desktop.exe hash mismatch" >&2; exit 1; } echo "$out/Jarvis" diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 722b56e..8c4ff5d 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -66,9 +66,10 @@ manages the MCP servers it connects. "Virtual monitor"), off by default; `schedule` (reminders and scheduled tasks, below "Schedules", and the brain tool `schedule`), on by default; `notes` (personal notes, below "Personal notes", and the brain tool `personalNotes`), on by default; `screen` (화면 보기, below - "Screen viewing", and the brain tool `screenView`), on by default. The tools of a feature that - is off go into the brain's `disabled_tools`: they are neither offered nor run - (`voice_worker.spec.md`, "Tools"). PC control comes with its milestone (M7). + "Screen viewing", and the brain tool `screenView`), on by default; `pcControl` (PC 제어, below "PC + control", and the brain tool `pcControl`), on by default (like `screen`, nothing is done until + someone is in `pcControl.allowedUsers`). The tools of a feature that is off go into the brain's + `disabled_tools`: they are neither offered nor run (`voice_worker.spec.md`, "Tools"). - `mcpServers` (new list): each `{name, command, args, env, enabled}`. `name` is 1-32 letters, digits, `-` or `_` without `__` (the brain names a server's tools `__`), not listed twice; `command` is not empty; `args` a list of strings; `env` the names of the environment @@ -166,8 +167,12 @@ decision 2026-11-14). What Jarvis may do on this PC and for whom; the PC control agrees in their next message (checked by the PC control tool like the chat delete consent). - Never done, whatever the settings: payments, passwords and other secrets, deleting files, and anything that needs administrator rights. They have no setting. -- The PC 제어 screen edits the section and shows, read-only, what is never done and each registered - chat channel's send and delete switches (changed on the 채팅 채널 screen). +- `antiCheatGames` (program file names, ".exe" may be left out, at most 100, no folders; empty by + default) and `stopHotkey` (modifiers `Ctrl`, `Alt`, `Shift`, `Win` and one letter, digit or + F1-F24, at least one modifier; default `Ctrl+Shift+F12`): below "PC control". +- The PC 제어 screen edits the section and shows the emergency stop's state with 풀기 (below "PC + control"), and, read-only, what is never done and each registered chat channel's send and + delete switches (changed on the 채팅 채널 screen). ## Screen viewing (`app/src/screen.rs`) @@ -220,10 +225,25 @@ agent-desktop and decides who may. - **agent-desktop**: version 1.2.0 (Apache-2.0), pinned with its SHA-256, shipped in the installer next to `jarvis.exe` (`agent-desktop.exe`, about 4 MB) and checked before every start of the services (a wrong hash means not available). One CLI run per command (M0 `desktop`: 0.10 s, as - fast as its MCP server), its APPDATA and LOCALAPPDATA in the data folder (`runtime/agent-desktop`), - a 15 s limit per command. Actions are headless (no cursor, no focus), except `keys`, which brings - the window to the front first. `--force` (which lifts its block on Alt+F4, Win+R, Ctrl+Shift+Esc - and the like) and the raw mouse and key commands are never used. + fast as its MCP server), its APPDATA, LOCALAPPDATA and state folder (`AGENT_DESKTOP_HOME`, where + it keeps its readings) in the data folder (`runtime/agent-desktop`), a 15 s limit per command. A + program it starts gets the person's own APPDATA and LOCALAPPDATA back and starts in their user + folder. Actions are headless (no cursor, no focus), except `keys`, which brings the window to the + front first (`focus-window`, then `press`). `--force` (which lifts its block on Alt+F4, Win+R, + Ctrl+Shift+Esc and the like) and the raw mouse and key commands are never used; what the brain + sends is checked so nothing in it reads as an option, and typed text goes in through stdin + (`set-value --stdin`), never in the arguments. The commands: `list-windows`, `snapshot + --window-id` (`read`), `click`, `get --property states` then `set-value` (`type`), `focus-window` + then `press` (`keys`), `launch` (`run`), `close-app` (`close`: agent-desktop has no command for + one window, so the program closes gracefully with all its windows), `restore`. +- **Windows**: as `list-windows` gives them, without this app's own windows, windows without a + title and the programs of `pcControl.screenExclude` (never listed, read or acted on, as for + screen viewing); at most 30. `window` is a window id or part of a title or program name (the + focused match first, else the first listed). An element (`@:e`) acts on the window + it was read from; one from a reading this run did not make gets `STALE_REF`'s reason. `run` takes + a program name only (no folders): one in Windows' folders as it is, else the file Windows' App + Paths or a Start menu shortcut without arguments points at; the window is the one `launch` names, + else a new one of its process or name, else the only new one, waited for up to 5 s. - **Available** only on Windows with `features.pcControl` on, someone in `pcControl.allowedUsers` and agent-desktop present; otherwise `pcControl` is in the brain's `disabled_tools`. Settings apply on the next start. @@ -243,18 +263,26 @@ agent-desktop and decides who may. (VALORANT, League of Legends; the driver is always loaded), Easy Anti-Cheat, BattlEye, FACEIT, nProtect GameGuard and XIGNCODE3 by their own processes (list in `pc.rs`, grown by updates), and `pcControl.antiCheatGames` (program file names the person added on the PC 제어 screen; no switch - turns the check off). The game's name is the game detection's when it runs, else the program's. + turns the check off). The game's name is the game detection's when it runs, else the program's + (Riot's processes are named as their games). Processes are found by their file names in Windows' + process list, also those this user may not open. - **Emergency stop**: a global hotkey (`pcControl.stopHotkey`, default `Ctrl+Shift+F12`) stops at once: the running agent-desktop command is killed and every action but `windows` and `read` is refused ("비상 정지 중입니다. 단축키를 한 번 더 누르거나 PC 제어 화면에서 풀어 주세요") until the hotkey is pressed again or the PC 제어 screen lifts it. The dashboard shows "PC 제어: 비상 정지"; - an app restart lifts it. + an app restart lifts it. The hotkey is registered (`RegisterHotKey`, its own thread and message + loop) while the services run with PC control available; one Windows refuses (another program + holds it) is a warning in the log. A reading started while the stop is on runs to its end. - **Results**: `read` and every action answer with the window `{id, program, title}` and its interactive elements read afresh after the action (at most 200), so the brain says it was done only from what it sees. agent-desktop's error codes come back as Korean reasons (`STALE_REF`: "화면이 바뀌었습니다. 다시 읽어 주세요"; a minimised window is restored before acting). +- **Refused before asking**: the stop, the anti-cheat, an administrator window (a process whose + elevation cannot be read counts as one) and a password field are refused without asking. + `windows` and `read` at `ask` answer `ask: true` like the rest. - **Activity**: each action but `windows` and `read` is `pc.` with the program's file name - (never a title, element name or text), who asked and whether it worked ("Activity"). + (never a title, element name or text), who asked and whether it worked, with the refusal's reason + ("Activity"); an `ask` answer is not recorded (nothing was done). - Logs carry the action, the result code and the time taken; never a title, element name, value or typed text. @@ -309,7 +337,7 @@ records and "Traces"), so it works while the services are stopped. Nothing it re `docs/v2/proactive.md` (제안 5). What Jarvis did outside its answers, for the 로그 screen's 한 일 view: each chat channel action the brain's tool asked for (`chat.send`, `chat.edit`, `chat.react`, `chat.delete`, `chat.pin`, `chat.unpin`), each schedule delivery (`schedule.remind`, `schedule.task`) and each -news notification (`collect.notify`), each screen capture (`screen.view`, "Screen viewing"), done or failed. The other PC control actions join it with M7. Ordinary answers are not recorded. +news notification (`collect.notify`), each screen capture (`screen.view`, "Screen viewing") and each PC control action (`pc.click`, `pc.type`, `pc.keys`, `pc.run`, `pc.close`, "PC control"), done or failed. Ordinary answers are not recorded. - One JSON line per action in `activity/.jsonl` (the folder is made at the first one): `{at, kind, target, by, ok, note}`. `target` is the channel (and message) id or `예약 `; `by` @@ -772,6 +800,7 @@ back as Korean messages for the screen. The Tauri shell passes the screens' `inv | `settings_fields` / `settings_view` / `settings_apply` `{changes}` | the settings editor above; `restartNeeded` when the services run and something other than `voice.tts.speed` changed (the speed reaches the running worker at once, `discord_core.spec.md`); `sync` with the chat channels sent to Discord (below) | | `service_start` / `service_stop` / `service_restart` / `dashboard` | the controller and its snapshot; start and restart run in the background, stop also cancels a start | | `discord_channel_info` `{id}` / `channel_sync_status` | a channel's Discord name and topic / each synced chat channel's last result (above) | +| `pc_stop_status` / `pc_stop_lift` | `{stopped}`: PC control's emergency stop (above, "PC control") / lift it | | `notes_list` / `note_update` `{id, text}` / `note_delete` `{id}` | everyone's personal notes (above) / change one's text / delete one | | `history_query` `{said, answered, thought, via, person, channel, find, since, until, session, before, from, limit}` | the 대화 screen's records and answer traces (above, "Conversations") | | `history_stamp` | `{stamp}`: changes when what the 대화 screen shows may have changed (above, "Conversations") | @@ -920,7 +949,8 @@ both to a release with that tag. The repository's other releases (v1's `v1.x`) a `icons/icon.png` 512 px): voice bars in a ring on the accent colour's gradient; at 32 px and below only three thicker bars, so it stays readable in the taskbar and tray. - `app/shell/package.sh ` builds it on Linux (`x86_64-pc-windows-gnu`, mingw-w64) and lays - out `Jarvis/`: `jarvis.exe`, `WebView2Loader.dll`, `python-src/` (the `voice_worker` and + out `Jarvis/`: `jarvis.exe`, `WebView2Loader.dll`, `agent-desktop.exe` (PC control, from the + pinned release archive, both hashes checked), `python-src/` (the `voice_worker` and `jarvis` packages, no caches or specs) and `display-driver/` (the virtual monitor's driver). The worker's sources are `python-src` next to the executable when it is there, also over a `voiceWorker.cwd` the settings keep (another install's first-run setup may have written its own folder there, and an update must never run those older @@ -1049,7 +1079,16 @@ moves or is brought back during the capture never shown (taken again, or refused moving), a window of a program that cannot be told treated as excluded, windows by title or program front first (never excluded, secondary or the app's own), minimised and black pictures refused, the detail sizes and parts; the worker protocol test carries a 4K capture's pictures to a fake brain and -back. Performance stages (on every system, with a fake +back. PC control (on every system, with a fake agent-desktop): people not registered and kinds at +`never` refused before any command, `ask` answered without consent and done with it, the +anti-cheat refusing every action but reading (its own processes and the person's list, named by +game detection), the emergency stop refusing and lifted, and the real runner killing a running +command when it is pressed and at its time limit, a password field and an administrator window +refused before acting, readings capped at 200 elements (interactive first) and cut to 120 +characters, `STALE_REF`'s reason, a minimised window restored and `keys` brought to the front, +excluded and own windows never listed, `run` reading its new window, the activity naming the +program only, the hotkey and shortcut formats, and one pin of agent-desktop in `pc.rs`, +`package.sh` and M0. Performance stages (on every system, with a fake Strata and card): a game moves the LLM to the game placement (measured once at 16K and one request, 2 GiB on the game PC's numbers, half the 16 engine threads) and back after the wait, pause stops it, a card with no room for a game pauses, the normal stage counts a program opened since, the game diff --git a/app/src/admin_api.rs b/app/src/admin_api.rs index 6a5bce5..b8dccd9 100644 --- a/app/src/admin_api.rs +++ b/app/src/admin_api.rs @@ -410,6 +410,12 @@ impl AdminApi { v }).collect::>()})) } + // PC control's emergency stop (admin.spec.md, "PC control"): its state, and 풀기. + "pc_stop_status" => Ok(json!({"stopped": crate::pc::emergency().on()})), + "pc_stop_lift" => { + crate::pc::emergency().lift(); + Ok(json!({"stopped": false})) + } // Personal notes (admin.spec.md, "Personal notes"): everyone's, editable on the screen. "notes_list" => { let items = crate::notes::Notes::new(self.dir()?.path()).all()?; diff --git a/app/src/control.rs b/app/src/control.rs index be88e14..87331d7 100644 --- a/app/src/control.rs +++ b/app/src/control.rs @@ -303,6 +303,7 @@ impl Controller { view["llm"] = self.host.llm(); view["game"] = status.get("game").cloned().unwrap_or(Value::Null); view["performance"] = status.get("performance").cloned().unwrap_or(Value::Null); + view["pcStop"] = json!(crate::pc::emergency().on()); view } diff --git a/app/src/discord.rs b/app/src/discord.rs index 688fa45..ab03589 100644 --- a/app/src/discord.rs +++ b/app/src/discord.rs @@ -207,6 +207,8 @@ struct Inner { notes: OnceLock>, /// What Jarvis did outside its answers (admin.spec.md, "Activity"). activity: OnceLock>, + /// PC control, when it is available (admin.spec.md, "PC control"). + pc: OnceLock>, /// Speech requests whose end someone waits for (`say_to_end`), by request id. speech_waits: Mutex>>, /// The spoken transcription notice of the current voice session. @@ -334,6 +336,7 @@ impl DiscordCore { notice: Mutex::new(JoinNotice::default()), activity: OnceLock::new(), notes: OnceLock::new(), + pc: OnceLock::new(), }); tokio::spawn(player_task(Arc::downgrade(&inner), player_rx)); tokio::spawn(worker_task(Arc::downgrade(&inner), worker_events)); @@ -597,6 +600,11 @@ impl DiscordCore { let _ = self.inner.activity.set(activity); } + /// PC control for the brain's pcControl tool; without it its actions are refused. + pub fn set_pc(&self, pc: Arc) { + let _ = self.inner.pc.set(pc); + } + /// Run a scheduled task in the brain (read-only tools, not spoken) and wait for its answer. pub async fn run_task(&self, id: &str, user: u64, text: &str) -> Option { let (tx, rx) = tokio::sync::oneshot::channel(); @@ -1018,6 +1026,26 @@ async fn worker_task(core: std::sync::Weak, mut rx: mpsc::UnboundedReceiv inner.worker.screen_action_result(&a.id, result); }); } + WorkerEvent::PcAction(a) => { + let inner = inner.clone(); + tokio::spawn(async move { + let started = std::time::Instant::now(); + let pc = inner.pc.get().cloned(); + let req = a.clone(); + let out = tokio::task::spawn_blocking(move || pc.map(|pc| pc.act(&req))).await.ok().flatten(); + let out = out.unwrap_or_else(|| crate::pc::Outcome { + reply: serde_json::json!({"ok": false, "error": "이 PC에서는 PC 제어를 쓸 수 없습니다"}), + code: "unavailable".into(), activity: None }); + // The action, its result code and the time only: never a title, element or text. + debug_log!("conversation", "pc {} {} {} ms={}", a.id, a.action.chars().take(10).collect::(), + out.code, started.elapsed().as_millis()); + if let (Some(r), Some(activity)) = (&out.activity, inner.activity.get()) { + let by = a.user.parse().ok().map(|u| format!("{u} {}", name_of(&inner, u))); + activity.record(&r.kind, &r.target, by, r.ok, r.note.clone()); + } + inner.worker.pc_action_result(&a.id, out.reply); + }); + } WorkerEvent::ScheduleAction(a) => { let result = match inner.schedules.get() { Some(store) => { diff --git a/app/src/features.rs b/app/src/features.rs index a2f31e8..f1ea074 100644 --- a/app/src/features.rs +++ b/app/src/features.rs @@ -22,13 +22,17 @@ pub const FEATURE_TOOLS: &[(&str, &[&str])] = &[ ("collect", &["collectedNews"]), ]; -/// The built-in tools of the features that are off, and `screenView` unless screen viewing is -/// available (admin.spec.md, "Screen viewing"). -pub fn disabled_tools(s: &Settings) -> Vec<&'static str> { +/// The built-in tools of the features that are off, `screenView` unless screen viewing is +/// available (admin.spec.md, "Screen viewing") and `pcControl` unless PC control is (`pc`: checked +/// with agent-desktop's hash as the services start, admin.spec.md, "PC control"). +pub fn disabled_tools(s: &Settings, pc: bool) -> Vec<&'static str> { let mut off = feature_tools_off(&s.features); if !crate::screen::available(s) { off.push("screenView"); } + if !pc { + off.push("pcControl"); + } off } @@ -120,15 +124,18 @@ mod tests { let mut s = Settings::default(); s.pc_control.allowed_users = vec![1]; let screen = if cfg!(windows) { vec![] } else { vec!["screenView"] }; - assert_eq!(disabled_tools(&s), screen, "all on by default"); - s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true }; + assert_eq!(disabled_tools(&s, true), screen, "all on by default"); + s.features = FeatureSettings { weather: false, web_search: true, meals: false, collect: false, browser: false, browser_share: false, schedule: false, notes: false, screen: true, pc_control: true }; let mut want = vec!["getWeather", "logMeal", "fetchMeals", "deleteMeal", "botBrowser", "schedule", "personalNotes", "collectedNews"]; want.extend(&screen); - assert_eq!(disabled_tools(&s), want); + assert_eq!(disabled_tools(&s, true), want); // Screen viewing needs someone registered for PC control (and Windows). - assert!(disabled_tools(&Settings::default()).contains(&"screenView")); + assert!(disabled_tools(&Settings::default(), true).contains(&"screenView")); s.features.screen = false; - assert!(disabled_tools(&s).contains(&"screenView")); + assert!(disabled_tools(&s, true).contains(&"screenView")); + // PC control is offered only when it is available (`pc::available` and agent-desktop). + assert!(disabled_tools(&s, false).contains(&"pcControl")); + assert!(!disabled_tools(&s, true).contains(&"pcControl")); } #[test] diff --git a/app/src/game.rs b/app/src/game.rs index 72274eb..b6126d5 100644 --- a/app/src/game.rs +++ b/app/src/game.rs @@ -274,6 +274,11 @@ impl GameWatch { self.state.lock().unwrap().current().is_some() } + /// The game's name while game mode is on (PC control's anti-cheat sentence). + pub fn name(&self) -> Option { + self.state.lock().unwrap().current().map(|d| d.name.clone()) + } + /// The game for the dashboard: `{name, reason}` or null. pub fn view(&self) -> serde_json::Value { serde_json::json!(self.state.lock().unwrap().current()) diff --git a/app/src/lib.rs b/app/src/lib.rs index 2eb2ebb..7c2c608 100644 --- a/app/src/lib.rs +++ b/app/src/lib.rs @@ -25,6 +25,7 @@ pub mod install; pub mod logs; pub mod notes; pub mod playback; +pub mod pc; pub mod perf; pub mod permissions; pub mod placement; diff --git a/app/src/pc.rs b/app/src/pc.rs new file mode 100644 index 0000000..9a5ae6c --- /dev/null +++ b/app/src/pc.rs @@ -0,0 +1,1391 @@ +//! PC control for the brain's `pcControl` tool (admin.spec.md, "PC control"; `docs/v2/pc-features.md`, +//! "키보드·마우스 제어와 안티치트"): who may, what is never done, the anti-cheat and emergency-stop +//! blocks, and every action run through agent-desktop, one CLI process per command. The commands go +//! through `Desktop`, so the tests drive everything with a fake; `Cli` is the real one. + +use std::collections::{HashSet, VecDeque}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; + +use crate::permissions::{decide, Action, Decision, Refusal}; +use crate::settings::Settings; +use crate::worker_client::PcAction; + +/// agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0), pinned as M0 ran it +/// (`scripts/m0/m0lib/desktop.py`). +pub const AGENT_DESKTOP_VERSION: &str = "1.2.0"; +pub const AGENT_DESKTOP_URL: &str = "https://github.com/vercel-labs/agent-desktop/releases/download/v1.2.0/agent-desktop-v1.2.0-x86_64-pc-windows-msvc.tar.gz"; +/// The release archive (`app/shell/package.sh` checks it). +pub const AGENT_DESKTOP_ARCHIVE_SHA256: &str = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8"; +/// `agent-desktop.exe` inside it, checked before every start of the services. +pub const AGENT_DESKTOP_SHA256: &str = "d274667031322e50b9c2b1ab81c2ca46835d52cc932769a5c34bd4d481fd94c3"; +pub const EXE_NAME: &str = "agent-desktop.exe"; + +/// One agent-desktop command at most. +pub const COMMAND_LIMIT: Duration = Duration::from_secs(15); +/// Elements in a reading (interactive ones first), and the characters kept of a name or value. +pub const MAX_ELEMENTS: usize = 200; +pub const MAX_CHARS: usize = 120; +/// Characters of text put into a field. +pub const MAX_TEXT: usize = 2000; +pub const MAX_WINDOWS: usize = 30; +pub const DEFAULT_STOP_HOTKEY: &str = "Ctrl+Shift+F12"; + +/// How long a reading may take (agent-desktop's budget; Chromium windows settle slowly). +const SNAPSHOT_MS: &str = "8000"; +/// How long `launch` waits for the program, and then this app for its window. +const LAUNCH_MS: &str = "8000"; +const WINDOW_WAIT: Duration = Duration::from_secs(5); +/// Readings whose elements may still be acted on. +const KEPT_READINGS: usize = 64; + +pub const NOT_REGISTERED: &str = "PC 제어 화면에 등록된 사람만 PC를 조작할 수 있습니다"; +pub const TURNED_OFF: &str = "PC 제어 화면에서 이 동작이 꺼져 있습니다"; +pub const PASSWORD: &str = "비밀번호 칸에는 입력하지 않습니다"; +pub const ELEVATED: &str = "관리자 권한으로 실행된 창은 조작할 수 없습니다"; +pub const STOPPED: &str = "비상 정지 중입니다. 단축키를 한 번 더 누르거나 PC 제어 화면에서 풀어 주세요"; +pub const STALE: &str = "화면이 바뀌었습니다. 다시 읽어 주세요"; +const NO_WINDOW: &str = "그 창을 찾지 못했습니다"; + +/// Anti-cheats found by their processes (file names, lower case): the anti-cheat's name, and the +/// game's when the process is the game itself. Vanguard's driver is always loaded, so it is found +/// by Riot's game processes. Grown by updates. +const ANTI_CHEATS: &[(&str, &str, Option<&str>)] = &[ + ("valorant-win64-shipping.exe", "뱅가드", Some("VALORANT")), + ("valorant.exe", "뱅가드", Some("VALORANT")), + ("league of legends.exe", "뱅가드", Some("League of Legends")), + ("easyanticheat.exe", "Easy Anti-Cheat", None), + ("easyanticheat_eos.exe", "Easy Anti-Cheat", None), + ("beservice.exe", "BattlEye", None), + ("beservice_x64.exe", "BattlEye", None), + ("faceit.exe", "FACEIT", None), + ("faceitservice.exe", "FACEIT", None), + ("gamemon.des", "nProtect GameGuard", None), + ("gamemon64.des", "nProtect GameGuard", None), + ("xcorona.xem", "XIGNCODE3", None), + ("xcorona_x64.xem", "XIGNCODE3", None), + ("xxd-0.xem", "XIGNCODE3", None), +]; + +/// Where agent-desktop is shipped: next to the app's program file. +pub fn exe_path() -> Option { + Some(std::env::current_exe().ok()?.parent()?.join(EXE_NAME)) +} + +/// Whether `exe` is the pinned agent-desktop. +pub fn exe_ok(exe: &Path) -> bool { + crate::download::sha256_file(exe).is_ok_and(|h| h == AGENT_DESKTOP_SHA256) +} + +/// Whether the brain gets the tool: Windows, the feature on, someone registered and agent-desktop +/// present with the right hash (`installed`). +pub fn available(s: &Settings, installed: bool) -> bool { + cfg!(windows) && s.features.pc_control && !s.pc_control.allowed_users.is_empty() && installed +} + +/// The agent-desktop the services use, when PC control is available (checked once per start). +pub fn check(s: &Settings) -> Option { + if !available(s, true) { + return None; + } + match exe_path() { + Some(exe) if exe_ok(&exe) => Some(exe), + Some(exe) if exe.exists() => { + crate::warn_log!("conversation", "Warning: agent-desktop has another hash than {AGENT_DESKTOP_VERSION}'s: PC control is off"); + None + } + _ => { + crate::warn_log!("conversation", "Warning: agent-desktop is not installed: PC control is off"); + None + } + } +} + +/// `Ctrl+Shift+F12` as the hotkey's modifiers (`MOD_ALT` 1, `MOD_CONTROL` 2, `MOD_SHIFT` 4, +/// `MOD_WIN` 8) and virtual key (letters, digits, F1-F24); at least one modifier. +pub fn parse_hotkey(s: &str) -> Option<(u32, u32)> { + let parts: Vec = s.split('+').map(|p| p.trim().to_ascii_lowercase()).collect(); + let (key, mods) = parts.split_last()?; + let mut m = 0; + for p in mods { + m |= match p.as_str() { + "alt" => 1, + "ctrl" | "control" => 2, + "shift" => 4, + "win" => 8, + _ => return None, + }; + } + let vk = match key.as_bytes() { + [c] if c.is_ascii_alphanumeric() => c.to_ascii_uppercase() as u32, + [b'f', n @ ..] => match std::str::from_utf8(n).ok()?.parse::().ok()? { + n @ 1..=24 => 0x6F + n, + _ => return None, + }, + _ => return None, + }; + (m != 0).then_some((m, vk)) +} + +// ---- emergency stop ----------------------------------------------------------------------------- + +/// The emergency stop: on until the hotkey is pressed again or the PC 제어 screen lifts it; an app +/// restart lifts it (it lives in the process only). +pub struct Stop { + on: AtomicBool, + /// How often it was turned on: a command started before the latest press is killed. + presses: AtomicU64, +} + +impl Default for Stop { + fn default() -> Self { + Self::new() + } +} + +impl Stop { + pub const fn new() -> Self { + Stop { on: AtomicBool::new(false), presses: AtomicU64::new(0) } + } + + pub fn on(&self) -> bool { + self.on.load(Ordering::SeqCst) + } + + /// The hotkey: on, or off again. + pub fn toggle(&self) { + let now_on = !self.on(); + if now_on { + self.presses.fetch_add(1, Ordering::SeqCst); + } + self.on.store(now_on, Ordering::SeqCst); + crate::info_log!("conversation", "PC control emergency stop {}", if now_on { "on" } else { "lifted" }); + } + + /// The PC 제어 screen's 풀기. + pub fn lift(&self) { + if self.on.swap(false, Ordering::SeqCst) { + crate::info_log!("conversation", "PC control emergency stop lifted"); + } + } + + fn presses(&self) -> u64 { + self.presses.load(Ordering::SeqCst) + } + + /// Whether a command started at `presses` is to be killed. + fn pressed_since(&self, presses: u64) -> bool { + self.on() && self.presses() != presses + } +} + +static EMERGENCY: Stop = Stop::new(); + +/// The app's emergency stop (the hotkey, the screen and the dashboard share it). +pub fn emergency() -> &'static Stop { + &EMERGENCY +} + +// ---- agent-desktop ------------------------------------------------------------------------------ + +/// What PC control runs on: agent-desktop and two questions to Windows (`Cli`; a fake in tests). +pub trait Desktop: Send + Sync { + /// One agent-desktop command (never `--force`, never the raw mouse and key commands): its + /// `data`, or its error code. A press of the emergency stop while it runs kills it + /// (`APP_STOPPED`). + fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result; + /// Whether process `pid` runs as administrator; one that cannot be told counts as one. + fn elevated(&self, pid: u64) -> bool; + /// The running programs' file names, lower case. + fn processes(&self) -> Vec; + /// What `launch` is given for a program name: the name itself, or the program file a Start + /// menu shortcut or Windows' App Paths name. + fn program(&self, name: &str) -> String { + name.to_owned() + } +} + +/// The real agent-desktop: one process per command, its APPDATA, LOCALAPPDATA and state folder in +/// the data folder (`runtime/agent-desktop`). +pub struct Cli { + exe: PathBuf, + home: PathBuf, + limit: Duration, +} + +impl Cli { + pub fn new(exe: &Path, data: &Path) -> Self { + let home = data.join("runtime").join("agent-desktop"); + let _ = std::fs::create_dir_all(&home); + Cli { exe: exe.to_path_buf(), home, limit: COMMAND_LIMIT } + } +} + +/// An agent-desktop envelope `{ok, data, error{code}}`: the data, or the code (never the message, +/// which can name windows). +fn envelope(out: &[u8]) -> Result { + let text = String::from_utf8_lossy(out); + let text = text.trim(); + let v: Value = serde_json::from_str(text).or_else(|_| serde_json::from_str(text.lines().last().unwrap_or(""))) + .map_err(|_| "APP_NO_JSON".to_owned())?; + match v["ok"].as_bool() { + Some(true) => Ok(v["data"].clone()), + Some(false) => Err(v["error"]["code"].as_str().unwrap_or("APP_UNKNOWN").to_owned()), + None => Err("APP_NO_JSON".into()), + } +} + +impl Desktop for Cli { + fn run(&self, args: &[String], stdin: Option<&str>, stop: &Stop) -> Result { + let mut cmd = std::process::Command::new(&self.exe); + cmd.args(args); + if args.first().map(String::as_str) == Some("launch") { + // A started program gets the person's own folders, not agent-desktop's. + for var in ["APPDATA", "LOCALAPPDATA"] { + if let Some(v) = std::env::var_os(var) { + cmd.arg("--env").arg(format!("{var}={}", v.to_string_lossy())); + } + } + if let Some(home) = std::env::var_os("USERPROFILE").filter(|h| Path::new(h).is_absolute()) { + cmd.arg("--cwd").arg(home); + } + } + cmd.env("APPDATA", &self.home).env("LOCALAPPDATA", &self.home).env("AGENT_DESKTOP_HOME", &self.home) + .stdin(if stdin.is_some() { std::process::Stdio::piped() } else { std::process::Stdio::null() }) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW + } + let presses = stop.presses(); + let mut child = cmd.spawn().map_err(|_| "APP_NOT_STARTED".to_owned())?; + if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) { + // Text goes in through stdin, never in the arguments; written aside so a full pipe + // cannot hold this thread. + let text = text.to_owned(); + std::thread::spawn(move || { + let _ = pipe.write_all(text.as_bytes()); + }); + } + let mut out = child.stdout.take().expect("stdout is piped"); + let reader = std::thread::spawn(move || { + let mut b = Vec::new(); + let _ = out.read_to_end(&mut b); + b + }); + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) => {} + Err(_) => return Err("APP_NOT_STARTED".into()), + } + let killed = if stop.pressed_since(presses) { + Some("APP_STOPPED") + } else if started.elapsed() > self.limit { + Some("APP_TIMEOUT") + } else { + None + }; + if let Some(code) = killed { + let _ = child.kill(); + let _ = child.wait(); + return Err(code.into()); + } + std::thread::sleep(Duration::from_millis(10)); + } + envelope(&reader.join().unwrap_or_default()) + } + + fn elevated(&self, pid: u64) -> bool { + #[cfg(windows)] + return win::elevated(pid); + #[cfg(not(windows))] + { + let _ = pid; + false + } + } + + fn processes(&self) -> Vec { + #[cfg(windows)] + return win::processes(); + #[cfg(not(windows))] + vec![] + } + + fn program(&self, name: &str) -> String { + #[cfg(windows)] + return win::program(name).unwrap_or_else(|| name.to_owned()); + #[cfg(not(windows))] + name.to_owned() + } +} + +// ---- the actions -------------------------------------------------------------------------------- + +/// A window as agent-desktop lists it. +#[derive(Debug, Clone, PartialEq)] +struct Win { + id: String, + /// Program file name, lower case (`notepad.exe`). + program: String, + title: String, + pid: u64, + focused: bool, + minimised: bool, +} + +impl Win { + fn view(&self) -> Value { + json!({"id": self.id, "program": stem(&self.program), "title": cut(&self.title)}) + } +} + +fn stem(program: &str) -> &str { + program.strip_suffix(".exe").unwrap_or(program) +} + +fn cut(s: &str) -> String { + s.chars().take(MAX_CHARS).collect() +} + +/// An activity record (admin.spec.md, "Activity"): the program's file name, never a title, +/// element name or text. +#[derive(Debug, Clone, PartialEq)] +pub struct Record { + pub kind: String, + pub target: String, + pub ok: bool, + pub note: Option, +} + +/// The answer to one `pc_action`, its result code for the log and what the activity records. +#[derive(Debug, Clone, PartialEq)] +pub struct Outcome { + pub reply: Value, + /// `ok`, `ask`, `refused` or agent-desktop's error code. + pub code: String, + pub activity: Option, +} + +#[derive(Debug)] +struct Fail { + code: String, + reason: String, +} + +fn refused(reason: &str) -> Fail { + Fail { code: "refused".into(), reason: reason.into() } +} + +/// agent-desktop's error code as the reason the person hears. +fn failed(code: &str, action: &str) -> Fail { + let reason = match code { + "STALE_REF" | "SNAPSHOT_NOT_FOUND" | "ELEMENT_NOT_FOUND" => STALE, + "AMBIGUOUS_TARGET" => "같은 모양의 항목이 여럿이라 고를 수 없습니다. 다시 읽어 주세요", + "APP_NOT_FOUND" if action == "run" => "그 프로그램을 찾지 못했습니다", + "APP_NOT_FOUND" | "WINDOW_NOT_FOUND" => NO_WINDOW, + "PERM_DENIED" => ELEVATED, + "POLICY_DENIED" => "막혀 있는 동작이라 하지 않습니다(위험한 단축키 등)", + "INVALID_ARGS" if action == "close" => "Windows가 지키는 프로그램이라 닫을 수 없습니다", + "INVALID_ARGS" => "요청을 알아듣지 못했습니다. 이름이나 키를 확인해 주세요", + "ACTION_NOT_SUPPORTED" => "그 항목은 이 동작을 받지 않습니다", + "ACTION_FAILED" => "동작이 되지 않았습니다", + "APP_UNRESPONSIVE" => "프로그램이 응답하지 않습니다", + "TIMEOUT" | "APP_TIMEOUT" => "시간 안에 끝나지 않았습니다", + "APP_STOPPED" => STOPPED, + "PLATFORM_NOT_SUPPORTED" => "이 PC에서는 할 수 없는 동작입니다", + _ => "PC 제어 도구가 실패했습니다", + }; + Fail { code: code.into(), reason: reason.into() } +} + +enum Done { + Item(Value), + Ask(Value), +} + +/// The permission kind of an action and its name in an `ask` answer. +fn kind_of(action: &str) -> Option<(Action, &'static str)> { + match action { + "windows" | "read" => Some((Action::View, "view")), + "click" | "close" => Some((Action::Click, "click")), + "type" | "keys" => Some((Action::Type, "type")), + "run" => Some((Action::Run, "run")), + _ => None, + } +} + +const ACTING: &[&str] = &["click", "type", "keys", "run", "close"]; + +fn valid_ref(r: Option<&str>) -> Result<&str, Fail> { + let r = r.map(str::trim).filter(|r| !r.is_empty()).ok_or_else(|| refused("누를 항목(ref)이 필요합니다"))?; + let ok = r.strip_prefix('@').and_then(|r| r.split_once(":e")).is_some_and(|(snap, n)| { + !snap.is_empty() && snap.len() <= 64 && snap.bytes().all(|b| b.is_ascii_alphanumeric()) + && !n.is_empty() && n.len() <= 9 && n.bytes().all(|b| b.is_ascii_digit()) + }); + if ok { Ok(r) } else { Err(refused(STALE)) } +} + +/// A key combination for `press`: names joined with `+` (`ctrl+s`, `enter`), nothing that could be +/// read as an option. +fn valid_keys(k: Option<&str>) -> Result { + let k = k.map(|k| k.replace(' ', "")).filter(|k| !k.is_empty()).ok_or_else(|| refused("누를 키가 필요합니다"))?; + let ok = k.len() <= 40 && k.split('+').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_alphanumeric())); + if ok { Ok(k) } else { Err(refused("키를 알아듣지 못했습니다 (예: ctrl+s, enter)")) } +} + +/// A program to start by name: no folders, nothing that could be read as an option. +fn valid_program(p: Option<&str>) -> Result { + let p = p.map(str::trim).filter(|p| !p.is_empty()).ok_or_else(|| refused("실행할 프로그램 이름이 필요합니다"))?; + let ok = p.chars().count() <= 100 && !p.starts_with(['-', '.']) && !p.contains("..") + && !p.chars().any(|c| c.is_control() || matches!(c, '\\' | '/' | ':' | '"' | '*' | '?' | '<' | '>' | '|')); + if ok { Ok(p.to_owned()) } else { Err(refused("프로그램 이름으로만 실행할 수 있습니다 (예: notepad)")) } +} + +/// A program name as a file name for the activity (`notepad` -> `notepad.exe`). +fn program_file(name: &str) -> String { + let n = name.to_lowercase(); + if Path::new(&n).extension().is_some() { n } else { format!("{n}.exe") } +} + +/// A reading's elements: interactive ones (with a ref) first, then the rest that have a name or a +/// value; at most `MAX_ELEMENTS`, names and values cut to `MAX_CHARS`. +fn elements(tree: &Value) -> Vec { + fn walk<'a>(n: &'a Value, refs: &mut Vec<&'a Value>, rest: &mut Vec<&'a Value>) { + for c in n["children"].as_array().into_iter().flatten() { + if c["ref_id"].as_str().is_some_and(|r| !r.is_empty()) { + refs.push(c); + } else if c["name"].as_str().is_some_and(|s| !s.trim().is_empty()) + || c["value"].as_str().is_some_and(|s| !s.trim().is_empty()) { + rest.push(c); + } + walk(c, refs, rest); + } + } + let (mut refs, mut rest) = (vec![], vec![]); + walk(tree, &mut refs, &mut rest); + refs.into_iter().chain(rest).take(MAX_ELEMENTS).map(|n| json!({ + "ref": n["ref_id"].as_str().unwrap_or(""), + "role": n["role"].as_str().unwrap_or(""), + "name": cut(n["name"].as_str().unwrap_or("")), + "value": cut(n["value"].as_str().unwrap_or("")), + "states": n["states"].as_array().cloned().unwrap_or_default(), + })).collect() +} + +fn args(a: &[&str]) -> Vec { + a.iter().map(|s| s.to_string()).collect() +} + +/// PC control for one run of the services. +pub struct Pc { + settings: Settings, + desktop: Box, + stop: &'static Stop, + /// The game game detection sees now, for the anti-cheat sentence. + game: Box Option + Send + Sync>, + /// This app's process: its windows are never listed or acted on. + own_pid: u64, + /// The latest readings' windows by snapshot id, so an element names its window. + readings: Mutex>, + /// One action at a time. + one: Mutex<()>, +} + +impl Pc { + pub fn new(settings: Settings, desktop: Box, stop: &'static Stop, + game: Box Option + Send + Sync>) -> Self { + Pc { settings, desktop, stop, game, own_pid: std::process::id() as u64, readings: Mutex::new(VecDeque::new()), + one: Mutex::new(()) } + } + + /// Answer one `pc_action` for its asker. + pub fn act(&self, a: &PcAction) -> Outcome { + let _one = self.one.lock().unwrap_or_else(|e| e.into_inner()); + let mut program = None; + let result = self.answer(a, &mut program); + let acting = ACTING.contains(&a.action.as_str()); + let (reply, code, note) = match result { + Ok(Done::Item(item)) => (json!({"ok": true, "item": item}), "ok".to_owned(), None), + Ok(Done::Ask(v)) => (v, "ask".to_owned(), None), + Err(f) => (json!({"ok": false, "error": f.reason}), f.code, Some(f.reason)), + }; + let activity = (acting && code != "ask").then(|| Record { kind: format!("pc.{}", a.action), + target: program.unwrap_or_else(|| "?".into()), ok: code == "ok", note }); + Outcome { reply, code, activity } + } + + fn answer(&self, a: &PcAction, program: &mut Option) -> Result { + let (perm, kind) = kind_of(&a.action).ok_or_else(|| refused("모르는 PC 동작입니다"))?; + let user = a.user.parse::().map_err(|_| refused(NOT_REGISTERED))?; + let ask = match decide(&self.settings.pc_control, user, perm) { + Decision::Allow => false, + Decision::Ask => !a.consent, + Decision::Refuse(Refusal::NotAllowed) => return Err(refused(NOT_REGISTERED)), + Decision::Refuse(_) => return Err(refused(TURNED_OFF)), + }; + let asking = |window: Value| Done::Ask(json!({"ok": false, "ask": true, "kinds": [kind], "window": window})); + match a.action.as_str() { + "windows" => { + if ask { + return Ok(asking(json!({"id": "", "program": "", "title": ""}))); + } + let list: Vec = self.list()?.iter().take(MAX_WINDOWS).map(|w| { + let mut v = w.view(); + v["minimised"] = json!(w.minimised); + v + }).collect(); + return Ok(Done::Item(json!({"windows": list}))); + } + "read" => { + let w = self.window(a.window.as_deref())?; + if ask { + return Ok(asking(w.view())); + } + return self.read(&w).map(Done::Item); + } + _ => {} + } + // Acting: never during the emergency stop or while an anti-cheat game runs. + if self.stop.on() { + return Err(refused(STOPPED)); + } + if let Some(why) = self.anti_cheat() { + return Err(refused(&why)); + } + if a.action == "run" { + let name = valid_program(a.program.as_deref())?; + *program = Some(program_file(&name)); + if ask { + return Ok(asking(json!({"id": "", "program": name, "title": ""}))); + } + return self.launch(&name).map(Done::Item); + } + let (w, element) = match a.action.as_str() { + "click" | "type" => { + let r = valid_ref(a.element.as_deref())?; + (self.window_of(r)?, Some(r)) + } + _ => (self.window(a.window.as_deref())?, None), + }; + *program = Some(w.program.clone()); + if self.desktop.elevated(w.pid) { + return Err(refused(ELEVATED)); + } + let keys = if a.action == "keys" { Some(valid_keys(a.keys.as_deref())?) } else { None }; + let text = a.text.as_deref().unwrap_or(""); + if a.action == "type" { + if text.chars().count() > MAX_TEXT { + return Err(refused("글자가 너무 깁니다 (2,000자까지)")); + } + let states = self.cmd(&args(&["get", element.unwrap(), "--property", "states"]), None, "type")?; + let secure = states["value"].as_array().is_none_or(|s| s.iter().any(|x| x == "secure")); + if secure { + return Err(refused(PASSWORD)); + } + } + if ask { + return Ok(asking(w.view())); + } + if self.stop.on() { + return Err(refused(STOPPED)); + } + if w.minimised { + self.cmd(&args(&["restore", "--window-id", &w.id]), None, &a.action)?; + } + match a.action.as_str() { + "click" => { + self.cmd(&args(&["click", element.unwrap()]), None, "click")?; + } + "type" => { + self.cmd(&args(&["set-value", element.unwrap(), "--stdin"]), Some(text), "type")?; + } + "keys" => { + // Keys go to the window in front, so it is brought there first. + self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys")?; + self.cmd(&args(&["press", keys.as_deref().unwrap(), "--window-id", &w.id]), None, "keys")?; + } + _ => { + self.cmd(&args(&["close-app", &w.program]), None, "close")?; + return Ok(Done::Item(json!({"window": w.view(), "elements": []}))); + } + } + self.read(&w).map(Done::Item) + } + + fn cmd(&self, args: &[String], stdin: Option<&str>, action: &str) -> Result { + self.desktop.run(args, stdin, self.stop).map_err(|code| failed(&code, action)) + } + + /// The windows that may be listed and acted on: not this app's, not an excluded program's + /// (`pcControl.screenExclude`), with a title; in agent-desktop's order. + fn list(&self) -> Result, Fail> { + let data = self.cmd(&args(&["list-windows"]), None, "windows")?; + let excluded = &self.settings.pc_control.screen_exclude; + Ok(data.as_array().into_iter().flatten().filter_map(|w| { + let program = crate::game::file_name(w["app_name"].as_str()?); + let win = Win { id: w["id"].as_str()?.to_owned(), title: w["title"].as_str().unwrap_or("").to_owned(), + pid: w["pid"].as_u64().unwrap_or(0), focused: w["is_focused"].as_bool().unwrap_or(false), + minimised: w["minimized"].as_bool().unwrap_or(false), program }; + let shown = w["visible"].as_bool() != Some(false) && !win.title.trim().is_empty() && !win.id.is_empty(); + (shown && win.pid != self.own_pid && !win.program.is_empty() + && !crate::game::listed(&win.program, excluded)).then_some(win) + }).collect()) + } + + /// A window by its id, else the one whose title or program contains `query` (the focused one + /// first, then the first listed). + fn window(&self, query: Option<&str>) -> Result { + let q = query.map(str::trim).filter(|q| !q.is_empty()).ok_or_else(|| refused("창 이름이 필요합니다"))?; + let list = self.list()?; + if let Some(w) = list.iter().find(|w| w.id == q) { + return Ok(w.clone()); + } + let q = q.to_lowercase(); + let hits: Vec<&Win> = list.iter() + .filter(|w| w.title.to_lowercase().contains(&q) || stem(&w.program).contains(&q)).collect(); + hits.iter().find(|w| w.focused).or(hits.first()).map(|w| (*w).clone()).ok_or_else(|| refused(NO_WINDOW)) + } + + /// The window an element was read from, as it is listed now. + fn window_of(&self, element: &str) -> Result { + let snap = element.trim_start_matches('@').split(':').next().unwrap_or(""); + let id = self.readings.lock().unwrap().iter().find(|(s, _)| s == snap).map(|(_, w)| w.id.clone()) + .ok_or_else(|| refused(STALE))?; + self.list()?.into_iter().find(|w| w.id == id).ok_or_else(|| refused(STALE)) + } + + /// Read a window afresh: `{window, elements}`. + fn read(&self, w: &Win) -> Result { + let data = self.cmd(&args(&["snapshot", "--window-id", &w.id, "--compact", "--timeout-ms", SNAPSHOT_MS]), None, "read")?; + let mut w = w.clone(); + if let Some(t) = data["window"]["title"].as_str() { + w.title = t.to_owned(); + } + if let Some(snap) = data["snapshot_id"].as_str() { + let mut r = self.readings.lock().unwrap(); + r.push_back((snap.to_owned(), w.clone())); + while r.len() > KEPT_READINGS { + r.pop_front(); + } + } + Ok(json!({"window": w.view(), "elements": elements(&data["tree"])})) + } + + /// Start a program (agent-desktop never asks for administrator rights) and read its window: + /// the one `launch` names, else a new one of its process or name, else the only new one. + fn launch(&self, name: &str) -> Result { + let before: HashSet = self.list()?.into_iter().map(|w| w.id).collect(); + let target = self.desktop.program(name); + let launched = match self.desktop.run(&args(&["launch", &target, "--timeout", LAUNCH_MS]), None, self.stop) { + Ok(d) => d, + // A program that hands over to another process (the Windows calculator) ends at once. + Err(code) if code == "APP_UNRESPONSIVE" => Value::Null, + Err(code) => return Err(failed(&code, "run")), + }; + let (pid, id) = (launched["pid"].as_u64(), launched["window"]["id"].as_str().map(str::to_owned)); + let lower = name.to_lowercase(); + let started = Instant::now(); + loop { + let new: Vec = self.list()?.into_iter().filter(|w| !before.contains(&w.id)).collect(); + let found = new.iter().find(|w| Some(&w.id) == id.as_ref()) + .or_else(|| new.iter().find(|w| pid.is_some_and(|p| p == w.pid))) + .or_else(|| new.iter().find(|w| stem(&w.program).contains(&lower) || w.title.to_lowercase().contains(&lower))) + .or(if new.len() == 1 { new.first() } else { None }); + if let Some(w) = found { + return self.read(w); + } + if self.stop.on() { + return Err(refused(STOPPED)); + } + if started.elapsed() >= WINDOW_WAIT { + return Err(Fail { code: "no_window".into(), reason: "프로그램을 시작했지만 창을 찾지 못했습니다".into() }); + } + std::thread::sleep(Duration::from_millis(300)); + } + } + + /// The anti-cheat sentence while a game with one runs (its processes, or a program the person + /// listed), else `None`. + fn anti_cheat(&self) -> Option { + let running = self.desktop.processes(); + let found = running.iter().find_map(|p| ANTI_CHEATS.iter().find(|(name, _, _)| name == p) + .map(|(_, ac, game)| (*ac, game.map(str::to_owned).unwrap_or_else(|| crate::game::game_name(p, &[]))))) + .or_else(|| running.iter().find(|p| crate::game::listed(p, &self.settings.pc_control.anti_cheat_games)) + .map(|p| ("안티치트", crate::game::game_name(p, &[]))))?; + let game = (self.game)().unwrap_or(found.1); + Some(format!("{}가 적용된 게임({game})이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", found.0)) + } +} + +// ---- the emergency stop's hotkey ---------------------------------------------------------------- + +/// The emergency stop's global hotkey, on its own thread while the services run. +pub struct Hotkey { + #[cfg(windows)] + thread: u32, + #[cfg(windows)] + join: Option>, +} + +impl Hotkey { + /// Register `spec` for `stop`; `None` elsewhere than Windows or when Windows refuses it + /// (another program holds it: warned in the log). + pub fn start(spec: &str, stop: &'static Stop) -> Option { + let (mods, vk) = parse_hotkey(spec)?; + #[cfg(windows)] + return win::hotkey(mods, vk, stop).map(|(thread, join)| Hotkey { thread, join: Some(join) }); + #[cfg(not(windows))] + { + let _ = (mods, vk, stop); + None + } + } + + /// Unregister it and end its thread. + pub fn end(mut self) { + self.finish(); + } + + fn finish(&mut self) { + #[cfg(windows)] + if let Some(j) = self.join.take() { + win::end_hotkey(self.thread); + let _ = j.join(); + } + } +} + +impl Drop for Hotkey { + fn drop(&mut self) { + self.finish(); + } +} + +/// The program file a Start menu shortcut (`.lnk`) points at, from its link information (Unicode +/// path when present, else an ASCII one); `None` for a shortcut with arguments (starting the file +/// alone would do something else) or without a local path. +pub fn shortcut_target(b: &[u8]) -> Option { + let u32_at = |o: usize| b.get(o..o + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as usize); + let u16_at = |o: usize| b.get(o..o + 2).map(|s| u16::from_le_bytes([s[0], s[1]]) as usize); + if u32_at(0)? != 0x4C { + return None; + } + let flags = u32_at(0x14)?; + const HAS_ID_LIST: usize = 0x1; + const HAS_LINK_INFO: usize = 0x2; + const HAS_ARGUMENTS: usize = 0x20; + if flags & HAS_LINK_INFO == 0 || flags & HAS_ARGUMENTS != 0 { + return None; + } + let mut at = 0x4C; + if flags & HAS_ID_LIST != 0 { + at += 2 + u16_at(at)?; + } + let (header, info_flags) = (u32_at(at + 4)?, u32_at(at + 8)?); + if info_flags & 0x1 == 0 { + return None; + } + let path = if header >= 0x24 { + let start = at + u32_at(at + 28)?; + let units: Vec = b.get(start..)?.chunks_exact(2).map(|c| u16::from_le_bytes([c[0], c[1]])) + .take_while(|&c| c != 0).collect(); + String::from_utf16(&units).ok()? + } else { + let start = at + u32_at(at + 16)?; + let bytes: Vec = b.get(start..)?.iter().copied().take_while(|&c| c != 0).collect(); + if !bytes.is_ascii() { + return None; + } + String::from_utf8(bytes).ok()? + }; + (!path.is_empty()).then_some(path) +} + +#[cfg(windows)] +mod win { + use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Security::{GetTokenInformation, TokenElevation, TOKEN_ELEVATION, TOKEN_QUERY}; + use windows_sys::Win32::System::Threading::{ + GetCurrentThreadId, OpenProcess, OpenProcessToken, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + /// Whether process `pid`'s token is elevated; a process that cannot be asked counts as elevated. + pub fn elevated(pid: u64) -> bool { + // SAFETY: query-only handles, closed at once; the out structure is sized as passed. + unsafe { + let p = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid as u32); + if p.is_null() { + return true; + } + let mut token: HANDLE = std::ptr::null_mut(); + let opened = OpenProcessToken(p, TOKEN_QUERY, &mut token) != 0; + CloseHandle(p); + if !opened { + return true; + } + let mut e = TOKEN_ELEVATION { TokenIsElevated: 0 }; + let mut len = 0u32; + let ok = GetTokenInformation(token, TokenElevation, (&mut e as *mut TOKEN_ELEVATION).cast(), + std::mem::size_of::() as u32, &mut len) != 0; + CloseHandle(token); + !ok || e.TokenIsElevated != 0 + } + } + + /// Every running process's file name, lower case (also of processes this user may not open, + /// such as anti-cheat services). + pub fn processes() -> Vec { + use windows_sys::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS, + }; + let mut out = vec![]; + // SAFETY: a process snapshot walked with a sized entry, then closed. + unsafe { + let snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if snap == INVALID_HANDLE_VALUE { + return out; + } + let mut e: PROCESSENTRY32W = std::mem::zeroed(); + e.dwSize = std::mem::size_of::() as u32; + let mut more = Process32FirstW(snap, &mut e) != 0; + while more { + let n = e.szExeFile.iter().position(|&c| c == 0).unwrap_or(e.szExeFile.len()); + out.push(String::from_utf16_lossy(&e.szExeFile[..n]).to_lowercase()); + more = Process32NextW(snap, &mut e) != 0; + } + CloseHandle(snap); + } + out + } + + /// A program name as `launch` takes it: a name Windows' folders have stays as it is (agent-desktop + /// finds it there), else the file Windows' App Paths name or a Start menu shortcut points at. + pub fn program(name: &str) -> Option { + let file = super::program_file(name); + let windir = std::env::var_os("SystemRoot").or_else(|| std::env::var_os("windir"))?; + let windir = std::path::Path::new(&windir); + if windir.join("System32").join(&file).is_file() || windir.join(&file).is_file() { + return None; + } + app_path(&file).or_else(|| start_menu(name)) + } + + fn app_path(file: &str) -> Option { + use windows_sys::Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE, RRF_RT_REG_SZ}; + let key: Vec = format!(r"Software\Microsoft\Windows\CurrentVersion\App Paths\{file}") + .encode_utf16().chain([0]).collect(); + for root in [HKEY_CURRENT_USER, HKEY_LOCAL_MACHINE] { + let mut buf = [0u16; 1024]; + let mut len = (buf.len() * 2) as u32; + // SAFETY: the default value read into a sized buffer. + let r = unsafe { RegGetValueW(root, key.as_ptr(), std::ptr::null(), RRF_RT_REG_SZ, std::ptr::null_mut(), + buf.as_mut_ptr().cast(), &mut len) }; + if r == 0 { + let n = buf.iter().position(|&c| c == 0).unwrap_or(buf.len()); + let path = String::from_utf16_lossy(&buf[..n]).trim().trim_matches('"').to_owned(); + if std::path::Path::new(&path).is_file() { + return Some(path); + } + } + } + None + } + + /// A Start menu shortcut named `name` (the person's, then everyone's), pointing at a program file. + fn start_menu(name: &str) -> Option { + let want = name.to_lowercase(); + let roots = [("APPDATA", r"Microsoft\Windows\Start Menu\Programs"), ("ProgramData", r"Microsoft\Windows\Start Menu\Programs")]; + for (var, sub) in roots { + let Some(base) = std::env::var_os(var) else { continue }; + let mut dirs = vec![std::path::Path::new(&base).join(sub)]; + let mut seen = 0; + while let Some(d) = dirs.pop() { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + seen += 1; + if seen > 5000 { + break; + } + let p = e.path(); + if p.is_dir() { + dirs.push(p); + } else if p.extension().is_some_and(|x| x.eq_ignore_ascii_case("lnk")) + && p.file_stem().is_some_and(|s| s.to_string_lossy().to_lowercase() == want) { + let target = std::fs::read(&p).ok().and_then(|b| super::shortcut_target(&b)); + if let Some(t) = target.filter(|t| t.to_lowercase().ends_with(".exe") && std::path::Path::new(t).is_file()) { + return Some(t); + } + } + } + } + } + None + } + + pub fn hotkey(mods: u32, vk: u32, stop: &'static super::Stop) -> Option<(u32, std::thread::JoinHandle<()>)> { + use windows_sys::Win32::UI::Input::KeyboardAndMouse::{RegisterHotKey, UnregisterHotKey, MOD_NOREPEAT}; + use windows_sys::Win32::UI::WindowsAndMessaging::{GetMessageW, PeekMessageW, MSG, PM_NOREMOVE, WM_HOTKEY, WM_USER}; + let (tx, rx) = std::sync::mpsc::channel(); + let join = std::thread::spawn(move || { + // SAFETY: a thread-bound hotkey and this thread's own message queue. + unsafe { + let mut msg: MSG = std::mem::zeroed(); + // Make the queue before anyone posts to it. + PeekMessageW(&mut msg, std::ptr::null_mut(), WM_USER, WM_USER, PM_NOREMOVE); + let ok = RegisterHotKey(std::ptr::null_mut(), 1, mods | MOD_NOREPEAT, vk) != 0; + let _ = tx.send((GetCurrentThreadId(), ok)); + if !ok { + return; + } + while GetMessageW(&mut msg, std::ptr::null_mut(), 0, 0) > 0 { + if msg.message == WM_HOTKEY { + stop.toggle(); + } + } + UnregisterHotKey(std::ptr::null_mut(), 1); + } + }); + match rx.recv() { + Ok((thread, true)) => Some((thread, join)), + _ => { + let _ = join.join(); + crate::warn_log!("conversation", "Warning: the PC control emergency stop hotkey could not be registered (another program may hold it)"); + None + } + } + } + + pub fn end_hotkey(thread: u32) { + use windows_sys::Win32::UI::WindowsAndMessaging::{PostThreadMessageW, WM_QUIT}; + // SAFETY: a quit message to the hotkey's own thread. + unsafe { PostThreadMessageW(thread, WM_QUIT, 0, 0) }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::settings::Level; + + const OWNER: &str = "123456789012345678"; + const TITLE: &str = "secret-plan.txt - Notepad"; + const TYPED: &str = "typed words"; + + /// agent-desktop as a script: answers by command, remembers what it was asked. + struct Fake { + windows: Mutex>, + processes: Vec, + elevated: bool, + secure: bool, + /// Commands that fail with this code. + fail: Option<(&'static str, &'static str)>, + calls: Mutex, Option)>>, + launched: Value, + } + + impl Fake { + fn new() -> Self { + Fake { windows: Mutex::new(VecDeque::from([json!([ + {"id": "w-1", "title": TITLE, "app_name": "notepad.exe", "pid": 10, "is_focused": false, "accessible": true}, + {"id": "w-2", "title": "Vault", "app_name": "KeePassXC.exe", "pid": 11, "is_focused": true, "accessible": true}, + {"id": "w-3", "title": "Calc", "app_name": "CalculatorApp.exe", "pid": 12, "is_focused": false, "minimized": true}])])), + processes: vec!["explorer.exe".into()], elevated: false, secure: false, fail: None, + calls: Mutex::new(vec![]), launched: json!({"app": "notepad", "pid": 10})} + } + fn commands(&self) -> Vec { + self.calls.lock().unwrap().iter().map(|(a, _)| a[0].clone()).collect() + } + } + + fn tree() -> Value { + let mut kids: Vec = (0..150).map(|i| json!({"role": "statictext", "name": format!("label {i}")})).collect(); + kids.extend((0..150).map(|i| json!({"role": "button", "name": format!("{i} {}", "x".repeat(300)), "ref_id": format!("@s1:e{i}"), + "states": ["focused"]}))); + json!({"role": "window", "name": TITLE, "children": [{"role": "group", "children": kids}]}) + } + + impl Desktop for &'static Fake { + fn run(&self, args: &[String], stdin: Option<&str>, _stop: &Stop) -> Result { + self.calls.lock().unwrap().push((args.to_vec(), stdin.map(str::to_owned))); + if let Some((cmd, code)) = self.fail { + if args[0] == cmd { + return Err(code.into()); + } + } + Ok(match args[0].as_str() { + "list-windows" => { + let mut w = self.windows.lock().unwrap(); + if w.len() > 1 { w.pop_front().unwrap() } else { w[0].clone() } + } + "snapshot" => json!({"snapshot_id": "s1", "window": {"id": args[2], "title": TITLE}, "tree": tree()}), + "get" => json!({"property": "states", "value": if self.secure { json!(["secure"]) } else { json!([]) }}), + "launch" => self.launched.clone(), + _ => json!({"disposition": {"delivery": "delivered_unverified"}}), + }) + } + fn elevated(&self, _pid: u64) -> bool { + self.elevated + } + fn processes(&self) -> Vec { + self.processes.clone() + } + } + + fn settings() -> Settings { + let mut s = Settings::default(); + s.pc_control.allowed_users = vec![OWNER.parse().unwrap()]; + s + } + + fn pc_with(s: Settings, fake: Fake) -> (Pc, &'static Fake, &'static Stop) { + let fake: &'static Fake = Box::leak(Box::new(fake)); + let stop: &'static Stop = Box::leak(Box::new(Stop::new())); + (Pc::new(s, Box::new(fake), stop, Box::new(|| None)), fake, stop) + } + + fn action(action: &str, user: &str, extra: Value) -> PcAction { + let mut v = json!({"id": "p1", "action": action, "user": user, "via": "voice"}); + v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + serde_json::from_value(v).unwrap() + } + + fn read_first(pc: &Pc) { + assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).code, "ok"); + } + + #[test] + fn people_not_registered_are_refused_before_anything_runs() { + let (pc, fake, _) = pc_with(settings(), Fake::new()); + for a in ["windows", "read", "click", "type", "keys", "run", "close"] { + let o = pc.act(&action(a, "42", json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad"}))); + assert_eq!(o.reply, json!({"ok": false, "error": NOT_REGISTERED}), "{a}"); + } + assert!(fake.commands().is_empty()); + // Nobody is registered by default. + let (pc, _, _) = pc_with(Settings::default(), Fake::new()); + assert_eq!(pc.act(&action("windows", OWNER, json!({}))).reply["error"], NOT_REGISTERED); + } + + #[test] + fn a_kind_turned_off_is_refused_and_every_action_has_its_kind() { + let mut s = settings(); + s.pc_control.click = Level::Never; + s.pc_control.type_text = Level::Never; + s.pc_control.run = Level::Never; + s.pc_control.view = Level::Never; + let (pc, fake, _) = pc_with(s, Fake::new()); + for a in ["windows", "read", "click", "close", "type", "keys", "run"] { + let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "program": "notepad", "consent": true}))); + assert_eq!(o.reply["error"], TURNED_OFF, "{a}"); + } + assert!(fake.commands().is_empty()); + } + + #[test] + fn asking_first_answers_ask_without_consent_and_acts_with_it() { + let (pc, fake, _) = pc_with(settings(), Fake::new()); + read_first(&pc); + let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))); + assert_eq!(o.reply, json!({"ok": false, "ask": true, "kinds": ["click"], + "window": {"id": "w-1", "program": "notepad", "title": TITLE}})); + assert_eq!(o.activity, None, "nothing was done yet"); + assert!(!fake.commands().contains(&"click".to_owned())); + let o = pc.act(&action("run", OWNER, json!({"program": "notepad"}))); + assert_eq!(o.reply["kinds"], json!(["run"])); + assert_eq!(o.reply["window"]["program"], "notepad"); + assert!(!fake.commands().contains(&"launch".to_owned())); + + let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))); + assert_eq!(o.reply["ok"], true); + assert!(fake.commands().contains(&"click".to_owned())); + // Done means seen: the window is read again after the click. + assert_eq!(fake.commands().last().unwrap(), "snapshot"); + assert_eq!(o.reply["item"]["window"]["id"], "w-1"); + // At `auto` nothing is asked. + let mut s = settings(); + s.pc_control.type_text = Level::Auto; + let (pc, fake, _) = pc_with(s, Fake::new()); + read_first(&pc); + let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))); + assert_eq!(o.reply["ok"], true); + let calls = fake.calls.lock().unwrap(); + let set = calls.iter().find(|(a, _)| a[0] == "set-value").unwrap(); + assert_eq!(set.1.as_deref(), Some(TYPED), "text goes through stdin"); + assert!(!set.0.iter().any(|a| a.contains(TYPED)), "never in the arguments"); + } + + #[test] + fn an_anti_cheat_game_refuses_acting_but_not_reading() { + let mut fake = Fake::new(); + fake.processes = vec!["explorer.exe".into(), "valorant-win64-shipping.exe".into()]; + let (pc, fake, _) = pc_with(settings(), fake); + for a in ["click", "type", "keys", "run", "close"] { + let o = pc.act(&action(a, OWNER, json!({"window": "notepad", "ref": "@s1:e1", "keys": "enter", + "program": "notepad", "consent": true}))); + assert_eq!(o.reply["error"], "뱅가드가 적용된 게임(VALORANT)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다", "{a}"); + } + assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot")); + assert_eq!(pc.act(&action("windows", OWNER, json!({}))).code, "ok"); + read_first(&pc); + + // A program the person listed, named by game detection when it sees the game. + let mut s = settings(); + s.pc_control.anti_cheat_games = vec!["MyShooter".into()]; + let mut fake = Fake::new(); + fake.processes = vec!["myshooter.exe".into()]; + let fake: &'static Fake = Box::leak(Box::new(fake)); + let pc = Pc::new(s, Box::new(fake), Box::leak(Box::new(Stop::new())), Box::new(|| Some("My Shooter".into()))); + let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "enter", "consent": true}))); + assert_eq!(o.reply["error"], "안티치트가 적용된 게임(My Shooter)이 실행 중이어서 키보드/마우스 제어를 할 수 없습니다"); + } + + #[test] + fn the_emergency_stop_refuses_acting_until_lifted_and_kills_a_running_command() { + let (pc, fake, stop) = pc_with(settings(), Fake::new()); + read_first(&pc); + stop.toggle(); + let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))); + assert_eq!(o.reply["error"], STOPPED); + assert_eq!(o.activity.unwrap().ok, false); + assert!(!fake.commands().contains(&"click".to_owned())); + // Reading still works. + read_first(&pc); + stop.toggle(); + assert_eq!(pc.act(&action("click", OWNER, json!({"ref": "@s1:e3", "consent": true}))).code, "ok"); + stop.toggle(); + stop.lift(); + assert!(!stop.on()); + + // The real runner: a command running when the hotkey is pressed is killed at once. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let dir = crate::datadir::temp_dir("pcstop"); + let exe = dir.join("agent-desktop"); + std::fs::write(&exe, "#!/bin/sh\nsleep 5\necho '{\"ok\": true, \"data\": []}'\n").unwrap(); + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + let cli = Cli::new(&exe, &dir); + let stop: &'static Stop = Box::leak(Box::new(Stop::new())); + let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(200)); stop.toggle(); }); + let started = Instant::now(); + assert_eq!(cli.run(&args(&["click", "@s1:e1"]), None, stop), Err("APP_STOPPED".into())); + assert!(started.elapsed() < Duration::from_secs(2)); + t.join().unwrap(); + // A command started while the stop is on (a reading) runs to its end. + std::fs::write(&exe, "#!/bin/sh\necho \"{\\\"ok\\\": true, \\\"data\\\": \\\"$APPDATA\\\"}\"\n").unwrap(); + let home = dir.join("runtime").join("agent-desktop"); + assert_eq!(cli.run(&args(&["list-windows"]), None, stop), Ok(json!(home.to_string_lossy())), + "agent-desktop's folders are in the data folder"); + // The command limit. + std::fs::write(&exe, "#!/bin/sh\nsleep 5\n").unwrap(); + let cli = Cli { limit: Duration::from_millis(200), ..cli }; + assert_eq!(cli.run(&args(&["list-windows"]), None, &Stop::new()), Err("APP_TIMEOUT".into())); + } + } + + #[test] + fn a_password_field_and_an_administrator_window_are_refused_before_acting() { + let mut s = settings(); + s.pc_control.type_text = Level::Auto; + let mut fake = Fake::new(); + fake.secure = true; + let (pc, fake, _) = pc_with(s.clone(), fake); + read_first(&pc); + let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))); + assert_eq!(o.reply["error"], PASSWORD); + assert!(!fake.commands().contains(&"set-value".to_owned())); + // Even with consent. + let o = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED, "consent": true}))); + assert_eq!(o.reply["error"], PASSWORD); + + let mut fake = Fake::new(); + fake.elevated = true; + let (pc, fake, _) = pc_with(s, fake); + read_first(&pc); + for a in ["click", "type", "keys", "close"] { + let o = pc.act(&action(a, OWNER, json!({"ref": "@s1:e3", "window": "notepad", "keys": "enter", "text": "x", + "consent": true}))); + assert_eq!(o.reply["error"], ELEVATED, "{a}"); + } + assert!(fake.commands().iter().all(|c| c == "list-windows" || c == "snapshot")); + } + + #[test] + fn a_reading_has_interactive_elements_first_at_most_200_and_cut_to_120_characters() { + let (pc, _, _) = pc_with(settings(), Fake::new()); + let o = pc.act(&action("read", OWNER, json!({"window": "notepad"}))); + let els = o.reply["item"]["elements"].as_array().unwrap(); + assert_eq!(els.len(), MAX_ELEMENTS); + assert!(els[..150].iter().all(|e| e["ref"].as_str().unwrap().starts_with("@s1:e"))); + assert!(els[150..].iter().all(|e| e["ref"] == "" && e["role"] == "statictext")); + assert!(els.iter().all(|e| e["name"].as_str().unwrap().chars().count() <= MAX_CHARS)); + assert_eq!(els[0]["name"].as_str().unwrap().chars().count(), MAX_CHARS); + assert_eq!(els[0]["states"], json!(["focused"])); + assert_eq!(o.reply["item"]["window"], json!({"id": "w-1", "program": "notepad", "title": TITLE})); + // No window named: why. + assert!(pc.act(&action("read", OWNER, json!({}))).reply["error"].is_string()); + } + + #[test] + fn a_stale_element_says_to_read_again() { + let mut s = settings(); + s.pc_control.click = Level::Auto; + let mut fake = Fake::new(); + fake.fail = Some(("click", "STALE_REF")); + let (pc, _, _) = pc_with(s.clone(), fake); + // An element of a reading this app does not know, or one in bad form. + for r in ["@s9:e1", "e1", "--force"] { + assert_eq!(pc.act(&action("click", OWNER, json!({"ref": r}))).reply["error"], STALE, "{r}"); + } + read_first(&pc); + let o = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))); + assert_eq!(o.reply, json!({"ok": false, "error": STALE})); + assert_eq!(o.code, "STALE_REF"); + } + + #[test] + fn a_minimised_window_is_restored_before_acting_and_keys_bring_it_to_the_front() { + let mut s = settings(); + s.pc_control.type_text = Level::Auto; + let (pc, fake, _) = pc_with(s, Fake::new()); + let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+s"}))); + assert_eq!(o.reply["ok"], true, "{o:?}"); + assert_eq!(fake.commands(), ["list-windows", "restore", "focus-window", "press", "snapshot"]); + let calls = fake.calls.lock().unwrap(); + assert_eq!(calls[3].0, args(&["press", "ctrl+s", "--window-id", "w-3"])); + assert!(calls.iter().all(|(a, _)| !a.iter().any(|x| x == "--force"))); + drop(calls); + for bad in ["--force", "ctrl+", "alt f4;rm"] { + let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": bad}))); + assert_eq!(o.code, "refused", "{bad}"); + } + } + + #[test] + fn excluded_programs_and_this_apps_windows_are_never_listed_or_picked() { + let (pc, _, _) = pc_with(settings(), Fake::new()); + let o = pc.act(&action("windows", OWNER, json!({}))); + assert_eq!(o.reply["item"]["windows"], json!([ + {"id": "w-1", "program": "notepad", "title": TITLE, "minimised": false}, + {"id": "w-3", "program": "calculatorapp", "title": "Calc", "minimised": true}])); + assert!(pc.act(&action("read", OWNER, json!({"window": "Vault"}))).reply["error"].is_string()); + } + + #[test] + fn run_starts_a_program_by_name_and_reads_its_new_window() { + let mut s = settings(); + s.pc_control.run = Level::Auto; + let fake = Fake::new(); + let before = fake.windows.lock().unwrap()[0].clone(); + let mut after = before.clone(); + after.as_array_mut().unwrap().push(json!({"id": "w-9", "title": "Untitled - Notepad", "app_name": "notepad.exe", "pid": 99})); + *fake.windows.lock().unwrap() = VecDeque::from([before, after]); + let fake = Fake { launched: json!({"app": "notepad.exe", "pid": 99}), ..fake }; + let (pc, fake, _) = pc_with(s, fake); + let o = pc.act(&action("run", OWNER, json!({"program": "notepad"}))); + assert_eq!(o.reply["item"]["window"]["id"], "w-9", "{o:?}"); + assert_eq!(fake.calls.lock().unwrap().iter().find(|(a, _)| a[0] == "launch").unwrap().0[1], "notepad"); + assert_eq!(o.activity, Some(Record { kind: "pc.run".into(), target: "notepad.exe".into(), ok: true, note: None })); + for bad in ["C:\\Windows\\cmd.exe", "-x", "..\\x"] { + assert_eq!(pc.act(&action("run", OWNER, json!({"program": bad}))).code, "refused", "{bad}"); + } + } + + #[test] + fn the_activity_names_the_program_only() { + let mut s = settings(); + s.pc_control.type_text = Level::Auto; + s.pc_control.click = Level::Auto; + let (pc, _, stop) = pc_with(s, Fake::new()); + read_first(&pc); + let typed = pc.act(&action("type", OWNER, json!({"ref": "@s1:e3", "text": TYPED}))).activity.unwrap(); + assert_eq!(typed, Record { kind: "pc.type".into(), target: "notepad.exe".into(), ok: true, note: None }); + let closed = pc.act(&action("close", OWNER, json!({"window": "notepad"}))).activity.unwrap(); + assert_eq!(closed, Record { kind: "pc.close".into(), target: "notepad.exe".into(), ok: true, note: None }); + stop.toggle(); + let refused = pc.act(&action("click", OWNER, json!({"ref": "@s1:e3"}))).activity.unwrap(); + assert_eq!((refused.ok, refused.note.as_deref()), (false, Some(STOPPED))); + // Readings are not activity; nothing recorded carries a title, an element or the text. + assert_eq!(pc.act(&action("read", OWNER, json!({"window": "notepad"}))).activity, None); + assert_eq!(pc.act(&action("windows", OWNER, json!({}))).activity, None); + let all = format!("{typed:?}{closed:?}{refused:?}"); + for secret in [TITLE, "secret-plan", TYPED, "label", "@s1"] { + assert!(!all.contains(secret), "{secret}"); + } + } + + #[test] + fn the_tool_is_available_only_with_the_feature_someone_registered_and_agent_desktop() { + let s = settings(); + assert_eq!(available(&s, true), cfg!(windows)); + assert!(!available(&s, false)); + assert!(!available(&Settings::default(), true)); + let mut off = settings(); + off.features.pc_control = false; + assert!(!available(&off, true)); + // The pinned file is checked by its hash. + let f = crate::datadir::temp_dir("pcexe").join(EXE_NAME); + std::fs::write(&f, b"not agent-desktop").unwrap(); + assert!(!exe_ok(&f)); + assert!(!exe_ok(&f.with_file_name("missing.exe"))); + } + + #[test] + fn hotkeys_and_envelopes_are_read() { + assert_eq!(parse_hotkey(DEFAULT_STOP_HOTKEY), Some((2 | 4, 0x7B))); + assert_eq!(parse_hotkey("alt+win+q"), Some((1 | 8, b'Q' as u32))); + for bad in ["F12", "Ctrl+", "Ctrl+F25", "Hyper+F1", ""] { + assert_eq!(parse_hotkey(bad), None, "{bad}"); + } + assert_eq!(envelope(br#"{"ok": true, "data": [1]}"#), Ok(json!([1]))); + assert_eq!(envelope(b"warning\n{\"ok\": false, \"error\": {\"code\": \"STALE_REF\", \"message\": \"x\"}}\n"), + Err("STALE_REF".into())); + assert_eq!(envelope(b"garbage"), Err("APP_NO_JSON".into())); + } + + #[test] + fn the_package_and_m0_pin_the_same_agent_desktop() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let package = std::fs::read_to_string(root.join("shell/package.sh")).unwrap(); + for pin in [AGENT_DESKTOP_URL, AGENT_DESKTOP_ARCHIVE_SHA256, AGENT_DESKTOP_SHA256] { + assert!(package.contains(pin), "{pin}"); + } + let m0 = std::fs::read_to_string(root.join("../scripts/m0/m0lib/desktop.py")).unwrap(); + assert!(m0.contains(AGENT_DESKTOP_ARCHIVE_SHA256) && m0.contains(&format!("AD_VERSION = \"{AGENT_DESKTOP_VERSION}\""))); + } + + #[test] + fn a_start_menu_shortcut_gives_its_program_file() { + // A shell link with link information and a Unicode local path (MS-SHLLINK). + fn link(flags: u32, path: &str) -> Vec { + let mut b = vec![0u8; 0x4C]; + b[0] = 0x4C; + b[0x14..0x18].copy_from_slice(&flags.to_le_bytes()); + let wide: Vec = path.encode_utf16().chain([0]).flat_map(|c| c.to_le_bytes()).collect(); + let header = 0x24u32; + let mut info = vec![0u8; header as usize]; + info[4..8].copy_from_slice(&header.to_le_bytes()); + info[8..12].copy_from_slice(&1u32.to_le_bytes()); + info[28..32].copy_from_slice(&header.to_le_bytes()); + info.extend(&wide); + let size = info.len() as u32; + info[0..4].copy_from_slice(&size.to_le_bytes()); + b.extend(info); + b + } + let path = r"C:\Program Files\메모\memo.exe"; + assert_eq!(shortcut_target(&link(0x2, path)).as_deref(), Some(path)); + assert_eq!(shortcut_target(&link(0x2 | 0x20, path)), None, "a shortcut with arguments"); + assert_eq!(shortcut_target(&link(0x0, path)), None); + assert_eq!(shortcut_target(b"short"), None); + } +} diff --git a/app/src/services.rs b/app/src/services.rs index 5dddfcb..e3f7428 100644 --- a/app/src/services.rs +++ b/app/src/services.rs @@ -31,6 +31,8 @@ pub struct Services { pub schedules: std::sync::Mutex>>, /// Announces new collected items (admin.spec.md, "News notifications"). pub news: std::sync::Mutex>>, + /// PC control's emergency-stop hotkey while PC control is available (admin.spec.md, "PC control"). + pub pc_hotkey: std::sync::Mutex>, } /// The bot browser of a run (admin.spec.md, "Bot browser"). @@ -82,6 +84,9 @@ impl Services { n.abort(); let _ = n.await; } + if let Some(h) = self.pc_hotkey.lock().unwrap().take() { + h.end(); + } self.game.stop(); self.core.close().await; self.worker.close(); @@ -155,8 +160,10 @@ pub async fn start_services(data_dir: &Path, llm: Arc) -> Result f, Err(e) => return Err(after_llm_stop(&llm, format!("could not write the brain settings: {e}")).await.into()), }; @@ -214,7 +221,19 @@ pub async fn start_services(data_dir: &Path, llm: Arc) -> Result) -> Result, + /// Programs with an anti-cheat the person added (file names, ".exe" may be left out): while one + /// runs, nothing but reading is done (admin.spec.md, "PC control"). + pub anti_cheat_games: Vec, + /// The emergency stop's global hotkey (`Ctrl+Shift+F12`). + pub stop_hotkey: String, } /// An MCP server the brain starts (stdio). The values of `env` are in the secrets, never here. @@ -276,14 +283,15 @@ impl Default for Settings { response: PolicySettings::default(), memory: Map::new(), llm: LlmSettings { url: None, port: 8080, context: 32768 }, - features: FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true }, + features: FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true, pc_control: true }, mcp_servers: vec![], collect: CollectSettings { keep_days: 30, targets: vec![], notify_channel: None, digest_hour: 9 }, game: GameSettings { detect: true, games: vec![], not_games: vec![], other_room_gb: 1.0, reserve_gb: 8.0, revert_seconds: 30, pause_in_game: false }, schedule_channel: None, pc_control: PcControlSettings { allowed_users: vec![], view: Level::Auto, click: Level::Ask, type_text: Level::Ask, run: Level::Ask, - screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into() }, + screen_exclude: ["1Password", "Bitwarden", "KeePass", "KeePassXC"].map(String::from).into(), + anti_cheat_games: vec![], stop_hotkey: crate::pc::DEFAULT_STOP_HOTKEY.into() }, close_to_tray: false, location: Some(DEFAULT_LOCATION.into()), } @@ -627,6 +635,7 @@ pub fn parse_settings(raw: &Value) -> Result { schedule: get_bool(f, "schedule", "features", d.features.schedule)?, notes: get_bool(f, "notes", "features", d.features.notes)?, screen: get_bool(f, "screen", "features", d.features.screen)?, + pc_control: get_bool(f, "pcControl", "features", d.features.pc_control)?, } }, mcp_servers: parse_mcp_servers(raw)?, @@ -660,20 +669,27 @@ fn parse_pc_control(raw: &Map, d: &PcControlSettings) -> Result

err(format!("pcControl.{key} must be auto, ask or never")), } }; + let programs = |key: &str, default: &[String]| -> Result, SettingsError> { + let v: Vec = get_str_list(p, key, "pcControl", default)?.into_iter() + .map(|e| e.trim().to_owned()).filter(|e| !e.is_empty()).collect(); + if v.len() > 100 || v.iter().any(|e| e.chars().count() > 100 || e.contains(['\\', '/'])) { + return err(format!("pcControl.{key} must be at most 100 program names (no folders)")); + } + Ok(v) + }; + let stop_hotkey = get_str(p, "stopHotkey", "pcControl", &d.stop_hotkey)?.trim().to_owned(); + if crate::pc::parse_hotkey(&stop_hotkey).is_none() { + return err("pcControl.stopHotkey must be modifiers and a key, such as Ctrl+Shift+F12"); + } Ok(PcControlSettings { allowed_users: snowflake_list(p, "allowedUsers", "pcControl")?, view: level("view", d.view)?, click: level("click", d.click)?, type_text: level("type", d.type_text)?, run: level("run", d.run)?, - screen_exclude: { - let v: Vec = get_str_list(p, "screenExclude", "pcControl", &d.screen_exclude)?.into_iter() - .map(|e| e.trim().to_owned()).filter(|e| !e.is_empty()).collect(); - if v.len() > 100 || v.iter().any(|e| e.chars().count() > 100 || e.contains(['\\', '/'])) { - return err("pcControl.screenExclude must be at most 100 program names (no folders)"); - } - v - }, + screen_exclude: programs("screenExclude", &d.screen_exclude)?, + anti_cheat_games: programs("antiCheatGames", &d.anti_cheat_games)?, + stop_hotkey, }) } @@ -1054,13 +1070,13 @@ mod tests { #[test] fn features_and_mcp_servers_are_read_and_checked() { let s = parse_settings(&json!({})); - assert_eq!(s.unwrap().features, FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true }); + assert_eq!(s.unwrap().features, FeatureSettings { weather: true, web_search: true, meals: true, collect: true, browser: true, browser_share: false, schedule: true, notes: true, screen: true, pc_control: true }); let s = parse_settings(&json!({ - "features": {"weather": false, "meals": false, "browserShare": true, "schedule": false, "notes": false, "screen": false}, + "features": {"weather": false, "meals": false, "browserShare": true, "schedule": false, "notes": false, "screen": false, "pcControl": false}, "mcpServers": [{"name": "brave", "command": "npx", "args": ["-y", "brave"], "env": ["BRAVE_API_KEY", " "]}, {"name": "time-2", "command": "uvx", "enabled": false}] })).unwrap(); - assert_eq!(s.features, FeatureSettings { weather: false, web_search: true, meals: false, collect: true, browser: true, browser_share: true, schedule: false, notes: false, screen: false }); + assert_eq!(s.features, FeatureSettings { weather: false, web_search: true, meals: false, collect: true, browser: true, browser_share: true, schedule: false, notes: false, screen: false, pc_control: false }); assert_eq!(s.mcp_servers, vec![ McpServerSettings { name: "brave".into(), command: "npx".into(), args: vec!["-y".into(), "brave".into()], env: vec!["BRAVE_API_KEY".into()], enabled: true }, @@ -1071,13 +1087,23 @@ mod tests { assert!(pc.allowed_users.is_empty()); assert_eq!((pc.view, pc.click, pc.type_text, pc.run), (Level::Auto, Level::Ask, Level::Ask, Level::Ask)); assert!(pc.screen_exclude.contains(&"KeePassXC".to_owned()), "password managers are never seen by default"); + assert!(pc.anti_cheat_games.is_empty()); + assert_eq!(pc.stop_hotkey, "Ctrl+Shift+F12"); let pc = parse_settings(&json!({"pcControl": {"allowedUsers": ["123456789012345678"], "view": "never", - "click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""]}})).unwrap().pc_control; + "click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""], + "antiCheatGames": ["MyShooter"], "stopHotkey": "Ctrl+Shift+Nope"}})).unwrap_err(); + assert!(pc.0.contains("pcControl.stopHotkey"), "{pc}"); + let pc = parse_settings(&json!({"pcControl": {"allowedUsers": ["123456789012345678"], "view": "never", + "click": "auto", "type": "never", "run": "auto", "screenExclude": [" bank.exe ", ""], + "antiCheatGames": ["MyShooter"], "stopHotkey": " ctrl+alt+s "}})).unwrap().pc_control; assert_eq!(pc.allowed_users, vec![123456789012345678]); assert_eq!(pc.screen_exclude, vec!["bank.exe".to_owned()]); + assert_eq!(pc.anti_cheat_games, vec!["MyShooter".to_owned()]); + assert_eq!(pc.stop_hotkey, "ctrl+alt+s"); assert_eq!((pc.view, pc.click, pc.type_text, pc.run), (Level::Never, Level::Auto, Level::Never, Level::Auto)); for bad in [json!({"pcControl": {"click": "yes"}}), json!({"pcControl": {"allowedUsers": ["1"]}}), - json!({"pcControl": {"screenExclude": ["C:\\bank\\bank.exe"]}}), json!({"pcControl": {"screenExclude": "x"}})] { + json!({"pcControl": {"screenExclude": ["C:\\bank\\bank.exe"]}}), json!({"pcControl": {"screenExclude": "x"}}), + json!({"pcControl": {"antiCheatGames": ["D:\\games\\x.exe"]}}), json!({"pcControl": {"stopHotkey": "F12"}})] { assert!(parse_settings(&bad).unwrap_err().0.contains("pcControl."), "{bad}"); } // A schedule channel must be a registered one that Jarvis may send in. diff --git a/app/src/settings_edit.rs b/app/src/settings_edit.rs index 9e090d4..b858ba3 100644 --- a/app/src/settings_edit.rs +++ b/app/src/settings_edit.rs @@ -141,6 +141,7 @@ pub fn fields() -> Vec { f("features.schedule", Features, Bool, json!(true), "예약 알림과 정기 작업"), f("features.notes", Features, Bool, json!(true), "사람별 지침"), f("features.screen", Features, Bool, json!(true), "화면 보기(Windows, PC 제어에 등록한 사람만)"), + f("features.pcControl", Features, Bool, json!(true), "PC 제어(Windows, PC 제어에 등록한 사람만, 다음 시작부터)"), f("mcpServers", Features, McpServers, json!([]), "MCP 서버"), // Background collection (admin.spec.md, "Background collection"): no targets by default. f("collect.targets", Collect, CollectTargets, json!([]), "수집 대상"), @@ -157,7 +158,11 @@ pub fn fields() -> Vec { f("pcControl.type", PcControl, Choice { values: LEVELS }, json!("ask"), "글자·키 입력"), f("pcControl.run", PcControl, Choice { values: LEVELS }, json!("ask"), "프로그램 실행"), f("pcControl.screenExclude", PcControl, TextList { max_items: 100, nullable: false }, - json!(["1Password", "Bitwarden", "KeePass", "KeePassXC"]), "화면 보기에서 뺄 프로그램(파일 이름, 은행·비밀번호 관리자 등)"), + json!(["1Password", "Bitwarden", "KeePass", "KeePassXC"]), "화면 보기와 PC 제어에서 뺄 프로그램(파일 이름, 은행·비밀번호 관리자 등)"), + f("pcControl.antiCheatGames", PcControl, TextList { max_items: 100, nullable: false }, json!([]), + "안티치트가 있는 게임(파일 이름, 실행 중이면 키보드·마우스 제어를 하지 않음)"), + f("pcControl.stopHotkey", PcControl, Text { max: 40, non_empty: true }, json!(crate::pc::DEFAULT_STOP_HOTKEY), + "비상 정지 단축키(예: Ctrl+Shift+F12, 다음 시작부터)"), // Game detection (admin.spec.md, "Game detection"). f("game.detect", Game, Bool, json!(true), "게임 감지"), f("game.games", Game, TextList { max_items: 100, nullable: false }, json!([]), "게임으로 볼 프로그램(파일 이름, 예: MyGame.exe)"), diff --git a/app/src/strata.rs b/app/src/strata.rs index 6050c97..608b497 100644 --- a/app/src/strata.rs +++ b/app/src/strata.rs @@ -236,17 +236,17 @@ async fn warm_up(base: &str) -> Result<(), String> { } /// The brain's settings (`config/brain.json`, the worker's `JARVIS_CONFIG_PATH`): the LLM, the -/// memory database inside the data folder, the tools of the features that are off and the enabled -/// MCP servers (without their environment values, `features.rs`). The app owns this file and writes -/// it on every start. -pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings) +/// memory database inside the data folder, the tools of the features that are off (`pc`: whether PC +/// control is available) and the enabled MCP servers (without their environment values, +/// `features.rs`). The app owns this file and writes it on every start. +pub fn write_brain_config(data: &Path, base_url: &str, model: &str, settings: &crate::settings::Settings, pc: bool) -> Result { let file = data.join("config").join("brain.json"); let cfg = json!({ "ollama_base_url": base_url, "ollama_chat_model": model, "db_path": data.join("memory").join("jarvis.db").to_string_lossy(), - "disabled_tools": crate::features::disabled_tools(settings), + "disabled_tools": crate::features::disabled_tools(settings, pc), "mcps": crate::features::brain_mcps(&settings.mcp_servers), "location_name": settings.location, "_config_version": 1, @@ -722,16 +722,17 @@ pub(crate) mod tests { let l = layout("llm2"); let s = parse_settings(&json!({"features": {"weather": false}, "mcpServers": [{"name": "brave", "command": "npx", "env": ["BRAVE_API_KEY"]}]})).unwrap(); - let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s).unwrap(); + let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &s, false).unwrap(); let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap(); assert_eq!(v["ollama_base_url"], "http://127.0.0.1:8080/v1"); assert_eq!(v["ollama_chat_model"], "m"); assert!(Path::new(v["db_path"].as_str().unwrap()).starts_with(&l.data)); - assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s))); + assert_eq!(v["disabled_tools"], json!(crate::features::disabled_tools(&s, false))); + assert!(v["disabled_tools"].as_array().unwrap().contains(&json!("pcControl"))); assert_eq!(v["mcps"], crate::features::brain_mcps(&s.mcp_servers)); assert_eq!(v["location_name"], crate::settings::DEFAULT_LOCATION, "the people are in Korea by default"); let none = parse_settings(&json!({"brain": {"location": ""}})).unwrap(); - let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none).unwrap(); + let f = write_brain_config(&l.data, "http://127.0.0.1:8080/v1", "m", &none, false).unwrap(); let v: Value = serde_json::from_str(&std::fs::read_to_string(f).unwrap()).unwrap(); assert_eq!(v["location_name"], Value::Null); } diff --git a/app/src/worker_client.rs b/app/src/worker_client.rs index 9730a6d..cb9d40a 100644 --- a/app/src/worker_client.rs +++ b/app/src/worker_client.rs @@ -184,6 +184,35 @@ pub struct ScreenAction { pub detail: Option, } +/// The brain's pcControl tool asks the app for the person asking (voice_worker.spec.md, "PC control"). +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct PcAction { + pub id: String, + /// `windows`, `read`, `click`, `type`, `keys`, `run` or `close`. + pub action: String, + pub user: String, + /// `voice` or `chat`: where they asked. + #[serde(default)] + pub via: Option, + #[serde(default)] + pub channel: Option, + /// A window id from `windows`, or part of a program name or title. + #[serde(default)] + pub window: Option, + /// An element from the latest reading (`@:e`). + #[serde(default, rename = "ref")] + pub element: Option, + #[serde(default)] + pub text: Option, + #[serde(default)] + pub keys: Option, + #[serde(default)] + pub program: Option, + /// The person agreed in their next message (the tool checked it). + #[serde(default)] + pub consent: bool, +} + #[derive(Debug, Clone, PartialEq)] pub enum WorkerEvent { Ready(HelloOk), @@ -197,6 +226,7 @@ pub enum WorkerEvent { ScheduleAction(ScheduleAction), NoteAction(NoteAction), ScreenAction(ScreenAction), + PcAction(PcAction), /// The brain's answer to `reply` (empty and `failed` when it gave none). ReplyText { id: String, text: String, failed: bool }, /// Was the utterance said to Jarvis? (a failed judgement is a no) @@ -404,6 +434,14 @@ impl WorkerClient { self.app_action_result("screen_action_result", id, result); } + /// The outcome of a `pc_action`: `{ok, item}`, `{ok: false, error}` or `{ok: false, ask, kinds, + /// window}`, as `pc.rs` made it. + pub fn pc_action_result(&self, id: &str, mut reply: Value) { + reply["type"] = json!("pc_action_result"); + reply["id"] = json!(id); + self.send_json(reply); + } + fn app_action_result(&self, kind: &str, id: &str, result: Result) { let msg = match result { Ok(Value::Object(mut m)) => { @@ -718,6 +756,7 @@ fn on_text(inner: &Inner, t: &str) { "schedule_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScheduleAction), "note_action" => serde_json::from_value(v).ok().map(WorkerEvent::NoteAction), "screen_action" => serde_json::from_value(v).ok().map(WorkerEvent::ScreenAction), + "pc_action" => serde_json::from_value(v).ok().map(WorkerEvent::PcAction), "llm_idle" => { if let Some(id) = v["id"].as_u64() { inner.llm_idle.send_if_modified(|got| if id > *got { *got = id; true } else { false }); diff --git a/app/ui/app.js b/app/ui/app.js index 5d58d8c..75879df 100644 --- a/app/ui/app.js +++ b/app/ui/app.js @@ -340,6 +340,26 @@ async function scheduleList() { h("div", { class: "field" }, h("button", { class: "btn", onclick: load }, "새로 고침")), list); } +/** PC control's emergency stop (admin.spec.md, "PC control"): its state and 풀기. */ +async function pcStopCard() { + const state = h("div"); + const load = async () => { + try { + const { stopped } = await api("pc_stop_status"); + state.replaceChildren(stopped + ? h("div", {}, h("span", { class: "badge bad" }, "비상 정지 중"), " ", h("button", { class: "btn", onclick: async () => { + try { await api("pc_stop_lift"); toast("비상 정지를 풀었습니다"); load(); } catch (e) { fail(e); } + } }, "풀기")) + : h("div", { class: "muted" }, "비상 정지 중이 아닙니다")); + } catch (e) { fail(e); } + }; + load(); + return h("div", { class: "card" }, h("h2", {}, "비상 정지"), + h("div", { class: "hint" }, "비상 정지 단축키를 누르면 하던 조작을 바로 멈추고, 다시 누르거나 여기서 풀 때까지 화면 보기와 창 목록 말고는 하지 않습니다. " + + "앱을 다시 시작해도 풀립니다. 안티치트가 있는 게임이 실행 중일 때도 키보드·마우스 제어는 하지 않습니다."), + h("div", { class: "field" }, h("button", { class: "btn", onclick: load }, "새로 고침")), state); +} + /** What is never done, and the chat channels' send and delete switches, beside the PC control levels. */ async function permissionsCard() { const view = await api("settings_view"); @@ -802,7 +822,8 @@ function dashboardBody(d) { h("span", { class: "k" }, "게임 모드"), h("span", { class: d.game ? "" : "muted" }, d.game ? `켜짐 (${d.game.name}, ${GAME_REASON[d.game.reason.kind]?.(d.game.reason) ?? d.game.reason.kind})` : "꺼짐"), h("span", { class: "k" }, "성능 단계"), h("span", { class: d.performance && d.performance.stage !== "normal" ? "" : "muted" }, - perfText(d.performance))))), + perfText(d.performance)), + d.pcStop ? [h("span", { class: "k" }, "PC 제어"), h("span", {}, h("span", { class: "badge bad" }, "비상 정지"))] : null))), h("div", { class: "grid" }, h("div", { class: "card" }, h("h2", {}, "실시간 자막"), h("div", { class: "captions" }, (d.captions || []).length ? d.captions.map((c) => h("div", { class: "cap" + (c.final ? "" : " partial") }, h("b", {}, c.name), c.text)) @@ -1018,7 +1039,7 @@ async function show(name) { const view = await settingsScreen(name, titles[name], extra); if (name === "collect") view.append(await collectResults()); if (name === "schedule") view.append(await scheduleList()); - if (name === "pcControl") view.append(await permissionsCard()); + if (name === "pcControl") view.append(await pcStopCard(), await permissionsCard()); if (my !== gen) return; screen.replaceChildren(view); if (update) ticker = setInterval(update.tick, 1000);