diff --git a/app/src/admin.spec.md b/app/src/admin.spec.md index 81c51bb..9c74492 100644 --- a/app/src/admin.spec.md +++ b/app/src/admin.spec.md @@ -256,7 +256,11 @@ agent-desktop and decides who may. - **In front for a moment**: a combination, any other key, and every key for a window that does not take keys behind. The window is brought to the front (`focus-window`, then `press`) and the window that was in front before is brought back right after (`focus-window` on it, also when - `press` failed; one that cannot be brought back is left, with a warning in the log). A key the + `press` failed; one that cannot be brought back is left, with a warning in the log). For a + window whose process agent-desktop cannot find from the window (the new Calculator, a Store + app: `press --window-id` is `APP_NOT_FOUND` and sends nothing; game PC 2026-11-19) the keys + are pressed without naming the window (nothing was sent before), into the window in front, + only while that is still the window (checked just before). A key the person types in that moment may go to the window. When Windows does not let the window come forward (its foreground lock), no key is sent ("그 창을 앞으로 가져오지 못해 키를 보내지 않았습니다"). A window already in front gets the keys there and nothing else moves. diff --git a/app/src/pc.rs b/app/src/pc.rs index bebffc2..4817f8a 100644 --- a/app/src/pc.rs +++ b/app/src/pc.rs @@ -749,8 +749,7 @@ impl Pc { let front = self.desktop.foreground(); if front.as_deref() == Some(w.id.as_str()) { crate::debug_log!("conversation", "pc keys way=front-already"); - self.cmd(&args(&["press", keys, "--window-id", &w.id]), None, "keys")?; - return Ok(()); + return self.press(w, keys); } if behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) { crate::debug_log!("conversation", "pc keys way=behind"); @@ -762,7 +761,7 @@ impl Pc { } crate::debug_log!("conversation", "pc keys way=front"); let pressed = self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys") - .and_then(|_| self.cmd(&args(&["press", keys, "--window-id", &w.id]), None, "keys")); + .and_then(|_| self.press(w, keys)); if let Some(back) = front { if self.desktop.run(&args(&["focus-window", "--window-id", &back]), None, self.stop).is_err() { crate::warn_log!("conversation", "Warning: PC control could not bring back the window that was in front"); @@ -771,6 +770,19 @@ impl Pc { pressed.map(|_| ()) } + /// `press` in window `w`, which is in front. agent-desktop cannot find some Store apps' process + /// by the window (the new Calculator: APP_NOT_FOUND, nothing sent), so then the keys go to the + /// window in front while it is still `w`. + fn press(&self, w: &Win, keys: &str) -> Result<(), Fail> { + match self.desktop.run(&args(&["press", keys, "--window-id", &w.id]), None, self.stop) { + Err(code) if code == "APP_NOT_FOUND" && self.desktop.foreground().as_deref() == Some(w.id.as_str()) => { + crate::debug_log!("conversation", "pc keys press=in-front (window's process not found)"); + self.cmd(&args(&["press", keys]), None, "keys").map(|_| ()) + } + r => r.map(|_| ()).map_err(|code| failed(&code, "keys")), + } + } + fn cmd(&self, args: &[String], stdin: Option<&str>, action: &str) -> Result { self.desktop.run(args, stdin, self.stop).map_err(|code| failed(&code, action)) } @@ -1285,6 +1297,9 @@ mod tests { front: Mutex>, fullscreen: bool, behind: bool, + /// agent-desktop cannot find the window's process by its identity (the new Calculator, a + /// Store app): `press --window-id` is APP_NOT_FOUND, nothing sent. + unnamed: bool, } impl Fake { @@ -1295,7 +1310,7 @@ mod tests { {"id": "w-3", "title": "Calc", "app_name": "CalculatorApp.exe", "pid": 12, "is_focused": false, "minimized": true}])])), processes: vec!["explorer.exe".into()], elevated: false, secure: false, fail: None, calls: Mutex::new(vec![]), launched: json!({"app": "notepad", "pid": 10}), secure_ref: None, - front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false} + front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false, unnamed: false} } fn front(&self) -> Option { self.front.lock().unwrap().clone() @@ -1328,6 +1343,9 @@ mod tests { if args[0] == "get" && self.secure_ref == Some(args[1].as_str()) { return Err("AMBIGUOUS_TARGET".into()); } + if args[0] == "press" && self.unnamed && args.iter().any(|a| a == "--window-id") { + return Err("APP_NOT_FOUND".into()); + } Ok(match args[0].as_str() { "list-windows" => { let mut w = self.windows.lock().unwrap(); @@ -1725,6 +1743,23 @@ mod tests { assert_eq!(fake.front().as_deref(), Some("w-2")); } + #[test] + fn a_store_app_agent_desktop_cannot_name_gets_its_keys_once_it_is_in_front() { + // Brought in front, and already in front: the key is pressed once, in that window. + for already in [false, true] { + let fake = Fake { unnamed: true, ..Fake::new() }; + if already { + *fake.front.lock().unwrap() = Some("w-1".into()); + } + let (pc, fake, _) = keys_auto(fake); + let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "1"}))); + assert_eq!(o.reply["ok"], true, "{o:?}"); + let presses: Vec> = fake.calls_from("list-windows").into_iter().filter(|a| a[0] == "press").collect(); + assert_eq!(presses.last().unwrap(), &args(&["press", "1"]), "{already}"); + assert_eq!(fake.front().as_deref(), Some(if already { "w-1" } else { "w-2" }), "{already}: the window in front is back"); + } + } + #[test] fn a_window_already_in_front_gets_its_keys_there_and_nothing_moves() { let fake = Fake::new(); diff --git a/app/tests/pc_keys_windows.rs b/app/tests/pc_keys_windows.rs index 6d03e6a..6ffb405 100644 --- a/app/tests/pc_keys_windows.rs +++ b/app/tests/pc_keys_windows.rs @@ -30,6 +30,13 @@ fn text(reply: &Value) -> String { .map(|e| format!("{} {}", e["name"].as_str().unwrap_or(""), e["value"].as_str().unwrap_or(""))).collect::>().join(" | ") } +/// Every element's value of a reading, joined: the typed text without the window's labels (a label +/// such as "Ctrl+Shift+X" contains "hi"). +fn values(reply: &Value) -> String { + reply["item"]["elements"].as_array().into_iter().flatten() + .filter_map(|e| e["value"].as_str()).collect::>().join(" | ") +} + /// Closed however the test ends (by name too: a program may hand over to another process). struct Closed(Vec, Vec); @@ -100,7 +107,7 @@ fn keys_go_behind_or_in_front_for_a_moment_and_the_window_in_front_stays() { } let read = act(&pc, "read", json!({"window": notepad})); println!("notepad after behind: {}", text(&read).chars().take(300).collect::()); - assert!(text(&read).contains("hi") && text(&read).contains("ok"), "the letters reached Notepad behind"); + assert!(values(&read).contains("hi") && values(&read).contains("ok"), "the letters reached Notepad behind"); // A combination goes in front for a moment, then the window in front is back. let r = act(&pc, "keys", json!({"window": notepad, "keys": "ctrl+a"})); @@ -108,9 +115,9 @@ fn keys_go_behind_or_in_front_for_a_moment_and_the_window_in_front_stays() { assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "ctrl+a: the window in front is back"); let r = act(&pc, "keys", json!({"window": notepad, "keys": "x"})); assert_eq!(r["ok"], true, "x: {r}"); - let after = text(&act(&pc, "read", json!({"window": notepad}))); - println!("notepad after ctrl+a, x: {}", after.chars().take(300).collect::()); - assert!(!after.contains("hi"), "ctrl+a selected the text and x replaced it"); + let after = values(&act(&pc, "read", json!({"window": notepad}))); + println!("notepad after ctrl+a, x: {after:?}"); + assert!(after.contains('x') && !after.contains("hi"), "ctrl+a selected the text and x replaced it"); // The Calculator: in front for a moment when it does not take keys behind (the Store app). let calc = act(&pc, "run", json!({"program": "calc"}));