From 20bd90f65e09b3790926ef7190517a0ed104be3c Mon Sep 17 00:00:00 2001 From: javis-bot Date: Thu, 19 Nov 2026 11:08:30 +0900 Subject: [PATCH] fix(v2): pcControl keys to a minimised window bring back the window that was in front restore activates the window (SW_RESTORE), so restoring before keys made the window already in front: the window that was in front before was not brought back, and the game / full-screen check came after the window had moved. Keys now note the window in front and check for a game first, then restore inside the moment in front. A minimised window never takes keys behind. Co-Authored-By: Claude Opus 5.5 --- app/src/admin.spec.md | 7 +++++-- app/src/pc.rs | 37 ++++++++++++++++++++++++++++++------ app/tests/pc_keys_windows.rs | 25 +++++++++++++++++++++++- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 9c74492..f679d2e 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -254,7 +254,9 @@ agent-desktop and decides who may. has none; win-ci VM 2026-11-19), so keys go behind only where there is no doubt where they land. Store and WinUI apps (the new Calculator), Chromium, Electron and Firefox windows have none. - **In front for a moment**: a combination, any other key, and every key for a window that does - not take keys behind. The window is brought to the front (`focus-window`, then `press`) and the + not take keys behind or is minimised. A minimised window is restored inside the moment + (restoring brings it forward, so the window in front is noted before and brought back after; + the window itself stays restored). The window is brought to the front (`focus-window`, then `press`) and the window that was in front before is brought back right after (`focus-window` on it, also when `press` failed; one that cannot be brought back is left, with a warning in the log). For a window whose process agent-desktop cannot find from the window (the new Calculator, a Store @@ -315,7 +317,8 @@ agent-desktop and decides who may. interactive elements read afresh after the action (at most 200), so the brain says it was done only from what it sees; an action that closes its window (a dialog's button, Enter) answers the window with no elements. agent-desktop's error codes come back as Korean reasons (`STALE_REF`: "화면이 - 바뀌었습니다. 다시 읽어 주세요"; a minimised window is restored before acting). + 바뀌었습니다. 다시 읽어 주세요"; a minimised window is restored before acting; for `keys` inside + the moment in front, "Keys"). - **Refused before asking**: the stop, the anti-cheat, an administrator window (a process whose elevation cannot be read counts as one) and a password field are refused without asking. `windows` and `read` at `ask` answer `ask: true` like the rest. diff --git a/app/src/pc.rs b/app/src/pc.rs index 4817f8a..f160d2b 100644 --- a/app/src/pc.rs +++ b/app/src/pc.rs @@ -719,7 +719,8 @@ impl Pc { if self.stop.on() { return Err(refused(STOPPED)); } - if w.minimised { + // Keys restore in their moment in front (restoring brings the window forward). + if w.minimised && a.action != "keys" { self.cmd(&args(&["restore", "--window-id", &w.id]), None, &a.action)?; } match a.action.as_str() { @@ -743,15 +744,16 @@ impl Pc { } /// Press `keys` in window `w` one way, chosen first (admin.spec.md, "Keys"): behind it when it is - /// one plain key and the window takes it so, else in front for a moment and the window that was - /// in front brought back. Never both ways: a key is never pressed twice. + /// one plain key and the window takes it so, else in front for a moment (a minimised window + /// restored in it) and the window that was in front brought back. Never both ways: a key is + /// never pressed twice. fn keys(&self, w: &Win, keys: &str) -> Result<(), Fail> { let front = self.desktop.foreground(); if front.as_deref() == Some(w.id.as_str()) { crate::debug_log!("conversation", "pc keys way=front-already"); return self.press(w, keys); } - if behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) { + if !w.minimised && behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) { crate::debug_log!("conversation", "pc keys way=behind"); return self.desktop.post_key(&w.id, keys).map_err(|code| failed(&code, "keys")); } @@ -760,7 +762,9 @@ impl Pc { return Err(refused(NOT_IN_FRONT_NOW)); } crate::debug_log!("conversation", "pc keys way=front"); - let pressed = self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys") + let restored = if w.minimised { self.cmd(&args(&["restore", "--window-id", &w.id]), None, "keys").map(|_| ()) } else { Ok(()) }; + let pressed = restored + .and_then(|_| self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys")) .and_then(|_| self.press(w, keys)); if let Some(back) = front { if self.desktop.run(&args(&["focus-window", "--window-id", &back]), None, self.stop).is_err() { @@ -1361,7 +1365,8 @@ mod tests { } "get" => json!({"property": "states", "value": if self.secure { json!(["secure"]) } else { json!([]) }}), "launch" => self.launched.clone(), - "focus-window" => { + // restore activates the window (SW_RESTORE), as focus-window does. + "focus-window" | "restore" => { *self.front.lock().unwrap() = Some(args[2].clone()); json!({"disposition": {"delivery": "delivered_unverified"}}) } @@ -1743,6 +1748,26 @@ mod tests { assert_eq!(fake.front().as_deref(), Some("w-2")); } + #[test] + fn keys_to_a_minimised_window_bring_back_the_window_that_was_in_front_and_wait_for_no_game() { + // Restoring brings the window forward, so it is part of the moment in front. + let (pc, fake, _) = keys_auto(Fake { behind: true, ..Fake::new() }); + assert_eq!(pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "ctrl+a"}))).reply["ok"], true); + assert_eq!(fake.calls_from("restore"), [ + args(&["restore", "--window-id", "w-3"]), + args(&["focus-window", "--window-id", "w-3"]), + args(&["press", "ctrl+a", "--window-id", "w-3"]), + args(&["focus-window", "--window-id", "w-2"]), + args(&["snapshot", "--window-id", "w-3", "--compact", "--timeout-ms", SNAPSHOT_MS])]); + assert_eq!(fake.front().as_deref(), Some("w-2"), "the window in front is back"); + // During a game or full screen a minimised window is not even restored. + let (pc, fake, _) = keys_auto(Fake { behind: true, fullscreen: true, ..Fake::new() }); + let o = pc.act(&action("keys", OWNER, json!({"window": "calc", "keys": "a"}))); + assert_eq!(o.reply, json!({"ok": false, "error": NOT_IN_FRONT_NOW})); + assert!(!fake.commands().contains(&"restore".to_owned()), "{:?}", fake.commands()); + assert_eq!(fake.front().as_deref(), Some("w-2")); + } + #[test] fn a_store_app_agent_desktop_cannot_name_gets_its_keys_once_it_is_in_front() { // Brought in front, and already in front: the key is pressed once, in that window. diff --git a/app/tests/pc_keys_windows.rs b/app/tests/pc_keys_windows.rs index 6ffb405..da0d6ba 100644 --- a/app/tests/pc_keys_windows.rs +++ b/app/tests/pc_keys_windows.rs @@ -52,6 +52,16 @@ impl Drop for Closed { } } +/// Minimise window `id` ("w-") as the person would. +#[cfg(windows)] +fn minimise(id: &str) { + let h: usize = id.trim_start_matches("w-").parse().unwrap(); + // SAFETY: a plain window call on a handle from list-windows. + unsafe { windows_sys::Win32::UI::WindowsAndMessaging::ShowWindow(h as _, windows_sys::Win32::UI::WindowsAndMessaging::SW_MINIMIZE) }; +} +#[cfg(not(windows))] +fn minimise(_id: &str) {} + fn wait_window(pc: &Pc, f: impl Fn(&Value) -> bool) -> String { let until = Instant::now() + Duration::from_secs(15); loop { @@ -117,7 +127,20 @@ fn keys_go_behind_or_in_front_for_a_moment_and_the_window_in_front_stays() { assert_eq!(r["ok"], true, "x: {r}"); let after = values(&act(&pc, "read", json!({"window": notepad}))); println!("notepad after ctrl+a, x: {after:?}"); - assert!(after.contains('x') && !after.contains("hi"), "ctrl+a selected the text and x replaced it"); + assert!(after.split(" | ").any(|v| v == "x") && !after.contains("hi"), "ctrl+a selected the text and x replaced it"); + + // A minimised window: restored in the moment in front, and the window in front is back. + minimise(¬epad); + std::thread::sleep(Duration::from_millis(500)); + let _ = probe.run(&["focus-window".into(), "--window-id".into(), front.clone()], None, stop); + let r = act(&pc, "keys", json!({"window": notepad, "keys": "ctrl+a"})); + assert_eq!(r["ok"], true, "minimised ctrl+a: {r}"); + assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "minimised ctrl+a: the window in front is back"); + let r = act(&pc, "keys", json!({"window": notepad, "keys": "y"})); + assert_eq!(r["ok"], true, "y: {r}"); + let after = values(&act(&pc, "read", json!({"window": notepad}))); + println!("notepad after minimised, ctrl+a, y: {after:?}"); + assert!(after.split(" | ").any(|v| v == "y") && !after.split(" | ").any(|v| v == "x"), "ctrl+a reached the restored Notepad"); // The Calculator: in front for a moment when it does not take keys behind (the Store app). let calc = act(&pc, "run", json!({"program": "calc"}));