fix(v2): pcControl keys reach a Store app agent-desktop cannot name by its window

On the game PC agent-desktop's press --window-id answers APP_NOT_FOUND for
the new Calculator ("not found with exact process identity", nothing sent),
though focus-window brings it in front. The keys now go to the window in
front while it is still that window. The Windows test checks Notepad's typed
text by element values only (a label such as Ctrl+Shift+X contains "hi").

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-19 10:55:19 +09:00
parent 378ccfafad
commit 5767ffa5ae
3 changed files with 55 additions and 9 deletions

View File

@@ -256,7 +256,11 @@ agent-desktop and decides who may.
- **In front for a moment**: a combination, any other key, and every key for a window that does
not take keys behind. The window is brought to the front (`focus-window`, then `press`) and the
window that was in front before is brought back right after (`focus-window` on it, also when
`press` failed; one that cannot be brought back is left, with a warning in the log). A key the
`press` failed; one that cannot be brought back is left, with a warning in the log). For a
window whose process agent-desktop cannot find from the window (the new Calculator, a Store
app: `press --window-id` is `APP_NOT_FOUND` and sends nothing; game PC 2026-11-19) the keys
are pressed without naming the window (nothing was sent before), into the window in front,
only while that is still the window (checked just before). A key the
person types in that moment may go to the window. When Windows does not let the window come
forward (its foreground lock), no key is sent ("그 창을 앞으로 가져오지 못해 키를 보내지
않았습니다"). A window already in front gets the keys there and nothing else moves.

View File

@@ -749,8 +749,7 @@ impl Pc {
let front = self.desktop.foreground();
if front.as_deref() == Some(w.id.as_str()) {
crate::debug_log!("conversation", "pc keys way=front-already");
self.cmd(&args(&["press", keys, "--window-id", &w.id]), None, "keys")?;
return Ok(());
return self.press(w, keys);
}
if behind_key(keys).is_some() && self.desktop.takes_keys_behind(&w.id) {
crate::debug_log!("conversation", "pc keys way=behind");
@@ -762,7 +761,7 @@ impl Pc {
}
crate::debug_log!("conversation", "pc keys way=front");
let pressed = self.cmd(&args(&["focus-window", "--window-id", &w.id]), None, "keys")
.and_then(|_| self.cmd(&args(&["press", keys, "--window-id", &w.id]), None, "keys"));
.and_then(|_| self.press(w, keys));
if let Some(back) = front {
if self.desktop.run(&args(&["focus-window", "--window-id", &back]), None, self.stop).is_err() {
crate::warn_log!("conversation", "Warning: PC control could not bring back the window that was in front");
@@ -771,6 +770,19 @@ impl Pc {
pressed.map(|_| ())
}
/// `press` in window `w`, which is in front. agent-desktop cannot find some Store apps' process
/// by the window (the new Calculator: APP_NOT_FOUND, nothing sent), so then the keys go to the
/// window in front while it is still `w`.
fn press(&self, w: &Win, keys: &str) -> Result<(), Fail> {
match self.desktop.run(&args(&["press", keys, "--window-id", &w.id]), None, self.stop) {
Err(code) if code == "APP_NOT_FOUND" && self.desktop.foreground().as_deref() == Some(w.id.as_str()) => {
crate::debug_log!("conversation", "pc keys press=in-front (window's process not found)");
self.cmd(&args(&["press", keys]), None, "keys").map(|_| ())
}
r => r.map(|_| ()).map_err(|code| failed(&code, "keys")),
}
}
fn cmd(&self, args: &[String], stdin: Option<&str>, action: &str) -> Result<Value, Fail> {
self.desktop.run(args, stdin, self.stop).map_err(|code| failed(&code, action))
}
@@ -1285,6 +1297,9 @@ mod tests {
front: Mutex<Option<String>>,
fullscreen: bool,
behind: bool,
/// agent-desktop cannot find the window's process by its identity (the new Calculator, a
/// Store app): `press --window-id` is APP_NOT_FOUND, nothing sent.
unnamed: bool,
}
impl Fake {
@@ -1295,7 +1310,7 @@ mod tests {
{"id": "w-3", "title": "Calc", "app_name": "CalculatorApp.exe", "pid": 12, "is_focused": false, "minimized": true}])])),
processes: vec!["explorer.exe".into()], elevated: false, secure: false, fail: None,
calls: Mutex::new(vec![]), launched: json!({"app": "notepad", "pid": 10}), secure_ref: None,
front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false}
front: Mutex::new(Some("w-2".into())), fullscreen: false, behind: false, unnamed: false}
}
fn front(&self) -> Option<String> {
self.front.lock().unwrap().clone()
@@ -1328,6 +1343,9 @@ mod tests {
if args[0] == "get" && self.secure_ref == Some(args[1].as_str()) {
return Err("AMBIGUOUS_TARGET".into());
}
if args[0] == "press" && self.unnamed && args.iter().any(|a| a == "--window-id") {
return Err("APP_NOT_FOUND".into());
}
Ok(match args[0].as_str() {
"list-windows" => {
let mut w = self.windows.lock().unwrap();
@@ -1725,6 +1743,23 @@ mod tests {
assert_eq!(fake.front().as_deref(), Some("w-2"));
}
#[test]
fn a_store_app_agent_desktop_cannot_name_gets_its_keys_once_it_is_in_front() {
// Brought in front, and already in front: the key is pressed once, in that window.
for already in [false, true] {
let fake = Fake { unnamed: true, ..Fake::new() };
if already {
*fake.front.lock().unwrap() = Some("w-1".into());
}
let (pc, fake, _) = keys_auto(fake);
let o = pc.act(&action("keys", OWNER, json!({"window": "notepad", "keys": "1"})));
assert_eq!(o.reply["ok"], true, "{o:?}");
let presses: Vec<Vec<String>> = fake.calls_from("list-windows").into_iter().filter(|a| a[0] == "press").collect();
assert_eq!(presses.last().unwrap(), &args(&["press", "1"]), "{already}");
assert_eq!(fake.front().as_deref(), Some(if already { "w-1" } else { "w-2" }), "{already}: the window in front is back");
}
}
#[test]
fn a_window_already_in_front_gets_its_keys_there_and_nothing_moves() {
let fake = Fake::new();

View File

@@ -30,6 +30,13 @@ fn text(reply: &Value) -> String {
.map(|e| format!("{} {}", e["name"].as_str().unwrap_or(""), e["value"].as_str().unwrap_or(""))).collect::<Vec<_>>().join(" | ")
}
/// Every element's value of a reading, joined: the typed text without the window's labels (a label
/// such as "Ctrl+Shift+X" contains "hi").
fn values(reply: &Value) -> String {
reply["item"]["elements"].as_array().into_iter().flatten()
.filter_map(|e| e["value"].as_str()).collect::<Vec<_>>().join(" | ")
}
/// Closed however the test ends (by name too: a program may hand over to another process).
struct Closed(Vec<String>, Vec<std::process::Child>);
@@ -100,7 +107,7 @@ fn keys_go_behind_or_in_front_for_a_moment_and_the_window_in_front_stays() {
}
let read = act(&pc, "read", json!({"window": notepad}));
println!("notepad after behind: {}", text(&read).chars().take(300).collect::<String>());
assert!(text(&read).contains("hi") && text(&read).contains("ok"), "the letters reached Notepad behind");
assert!(values(&read).contains("hi") && values(&read).contains("ok"), "the letters reached Notepad behind");
// A combination goes in front for a moment, then the window in front is back.
let r = act(&pc, "keys", json!({"window": notepad, "keys": "ctrl+a"}));
@@ -108,9 +115,9 @@ fn keys_go_behind_or_in_front_for_a_moment_and_the_window_in_front_stays() {
assert_eq!(probe.foreground().as_deref(), Some(front.as_str()), "ctrl+a: the window in front is back");
let r = act(&pc, "keys", json!({"window": notepad, "keys": "x"}));
assert_eq!(r["ok"], true, "x: {r}");
let after = text(&act(&pc, "read", json!({"window": notepad})));
println!("notepad after ctrl+a, x: {}", after.chars().take(300).collect::<String>());
assert!(!after.contains("hi"), "ctrl+a selected the text and x replaced it");
let after = values(&act(&pc, "read", json!({"window": notepad})));
println!("notepad after ctrl+a, x: {after:?}");
assert!(after.contains('x') && !after.contains("hi"), "ctrl+a selected the text and x replaced it");
// The Calculator: in front for a moment when it does not take keys behind (the Store app).
let calc = act(&pc, "run", json!({"program": "calc"}));