On the shared screen a tab brought to the front makes Chrome bring its window forward and take the keyboard focus from the game or program the user is in (seen on the game PC). The botBrowser tool now opens each person's page in a window of its own, behind and at the place and size of the bot browser's window (Target.createTarget with newWindow and background), and closes it again if the connection to it fails. While the browser runs on the virtual monitor, the app looks for new Chrome windows every 200 ms and settles each like the first (tool window, fitted to the work area, without activating it), the newest in front of the older ones, so the shared screen shows the page the bot opened last. Checked on the game PC: the bot's window loading a public page comes in front of the first, both out of the taskbar and Alt+Tab, the focus staying in Discord also with the user idle for hours. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
238 lines
12 KiB
Rust
238 lines
12 KiB
Rust
//! The bot browser with a real Chrome (admin.spec.md, "Bot browser"). Runs only with
|
|
//! `JARVIS_TEST_CHROME=<chrome executable>`; needs the internet for the public page. On Ubuntu also
|
|
//! set `JARVIS_CHROME_NO_SANDBOX=1`.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
use jarvis_app::browser::BotBrowser;
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
|
|
async fn titles(client: &reqwest::Client, port: u16) -> Vec<(String, String)> {
|
|
let text = client.get(format!("http://127.0.0.1:{port}/json/list")).send().await.unwrap().text().await.unwrap();
|
|
let list: serde_json::Value = serde_json::from_str(&text).unwrap();
|
|
list.as_array().unwrap().iter()
|
|
.map(|t| (t["url"].as_str().unwrap_or("").to_owned(), t["title"].as_str().unwrap_or("").to_owned())).collect()
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_bot_browser_reaches_public_pages_never_this_pc_and_stops() {
|
|
let Ok(exe) = std::env::var("JARVIS_TEST_CHROME") else {
|
|
eprintln!("skipped: set JARVIS_TEST_CHROME");
|
|
return;
|
|
};
|
|
// A page on this PC the browser must never reach.
|
|
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let local = l.local_addr().unwrap().port();
|
|
let hits = Arc::new(Mutex::new(0));
|
|
let h2 = hits.clone();
|
|
tokio::spawn(async move {
|
|
while let Ok((mut c, _)) = l.accept().await {
|
|
*h2.lock().unwrap() += 1;
|
|
let mut b = [0u8; 1024];
|
|
let _ = c.read(&mut b).await;
|
|
let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n<title>secret local</title>xx").await;
|
|
}
|
|
});
|
|
|
|
let profile = std::env::temp_dir().join(format!("jarvis-bot-browser-{}", std::process::id()));
|
|
let b = BotBrowser::start(std::path::Path::new(&exe), &profile, None).await.expect("chrome starts");
|
|
assert!(b.running());
|
|
let client = reqwest::Client::builder().no_proxy().build().unwrap();
|
|
for url in [format!("http://127.0.0.1:{local}/"), format!("http://localhost:{local}/"), "https://example.com/".into()] {
|
|
client.put(format!("http://127.0.0.1:{}/json/new?{url}", b.cdp_port)).send().await.unwrap();
|
|
}
|
|
let mut got = vec![];
|
|
for _ in 0..40 {
|
|
got = titles(&client, b.cdp_port).await;
|
|
if got.iter().any(|(_, t)| t == "Example Domain") {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
}
|
|
eprintln!("tabs: {got:?}");
|
|
assert!(got.iter().any(|(_, t)| t == "Example Domain"), "the public page loads through the proxy");
|
|
assert!(!got.iter().any(|(_, t)| t.contains("secret local")));
|
|
assert_eq!(*hits.lock().unwrap(), 0, "nothing reached this PC");
|
|
|
|
let port = b.cdp_port;
|
|
let t0 = std::time::Instant::now();
|
|
b.stop().await;
|
|
eprintln!("stopped in {:?}", t0.elapsed());
|
|
assert!(!b.running());
|
|
assert!(client.get(format!("http://127.0.0.1:{port}/json/version")).send().await.is_err(), "CDP is closed");
|
|
let _ = std::fs::remove_dir_all(&profile);
|
|
}
|
|
|
|
/// The install step downloads the pinned Chrome (about 200 MB) and unpacks a runnable browser.
|
|
/// Runs only with `JARVIS_TEST_CHROME_INSTALL=1`.
|
|
#[tokio::test]
|
|
async fn the_browser_step_installs_a_chrome_that_starts() {
|
|
if std::env::var("JARVIS_TEST_CHROME_INSTALL").as_deref() != Ok("1") {
|
|
eprintln!("skipped: set JARVIS_TEST_CHROME_INSTALL=1");
|
|
return;
|
|
}
|
|
let data = std::env::temp_dir().join(format!("jarvis-chrome-install-{}", std::process::id()));
|
|
let layout = jarvis_app::install::Layout { data: data.clone(), src: std::path::PathBuf::new() };
|
|
let steps = jarvis_app::install::steps().into_iter().filter(|s| s.id() == "browser").collect();
|
|
let installer = jarvis_app::install::Installer::new(layout.clone(), steps);
|
|
assert!(installer.run_all().await, "{}", installer.status());
|
|
assert!(installer.done("browser"));
|
|
assert!(!data.join("cache").join("downloads").read_dir().map(|mut d| d.next().is_some()).unwrap_or(false),
|
|
"the archive is removed after unpacking");
|
|
let b = BotBrowser::start(&layout.chrome_exe(), &jarvis_app::browser::profile_dir(&data), None).await.expect("starts");
|
|
assert!(b.running());
|
|
b.stop().await;
|
|
let _ = std::fs::remove_dir_all(&data);
|
|
}
|
|
|
|
/// The process whose window has the keyboard focus (Windows).
|
|
#[cfg(windows)]
|
|
fn foreground_pid() -> u32 {
|
|
use windows_sys::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
|
|
let mut pid = 0;
|
|
// SAFETY: plain queries; `pid` outlives the call.
|
|
unsafe { GetWindowThreadProcessId(GetForegroundWindow(), &mut pid) };
|
|
pid
|
|
}
|
|
|
|
/// The visible top-level Chrome windows of process `pid`, top first: (title, tool window).
|
|
#[cfg(windows)]
|
|
fn chrome_windows(pid: u32) -> Vec<(String, bool)> {
|
|
use windows_sys::Win32::Foundation::{HWND, LPARAM};
|
|
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
|
unsafe extern "system" fn each(h: HWND, l: LPARAM) -> windows_sys::core::BOOL {
|
|
// SAFETY: `l` is the `(pid, Vec)` below, alive through EnumWindows; plain queries on `h`.
|
|
unsafe {
|
|
let (pid, out) = &mut *(l as *mut (u32, Vec<(String, bool)>));
|
|
let (mut p, mut class, mut title) = (0u32, [0u16; 64], [0u16; 256]);
|
|
GetWindowThreadProcessId(h, &mut p);
|
|
let n = GetClassNameW(h, class.as_mut_ptr(), 64) as usize;
|
|
if p == *pid && IsWindowVisible(h) != 0 && GetWindow(h, GW_OWNER).is_null()
|
|
&& String::from_utf16_lossy(&class[..n]) == "Chrome_WidgetWin_1" {
|
|
let t = GetWindowTextW(h, title.as_mut_ptr(), 256) as usize;
|
|
let tool = GetWindowLongPtrW(h, GWL_EXSTYLE) as u32 & WS_EX_TOOLWINDOW != 0;
|
|
out.push((String::from_utf16_lossy(&title[..t]), tool));
|
|
}
|
|
}
|
|
1
|
|
}
|
|
let mut found = (pid, vec![]);
|
|
// SAFETY: `each` writes only into `found`, which outlives the call.
|
|
unsafe { EnumWindows(Some(each), &mut found as *mut _ as LPARAM) };
|
|
found.1
|
|
}
|
|
|
|
/// One call on the browser's CDP connection.
|
|
async fn cdp<S>(s: &mut S, id: u64, method: &str, params: serde_json::Value) -> serde_json::Value
|
|
where S: futures_util::Sink<tokio_tungstenite::tungstenite::Message> + futures_util::Stream<Item =
|
|
Result<tokio_tungstenite::tungstenite::Message, tokio_tungstenite::tungstenite::Error>> + Unpin,
|
|
S::Error: std::fmt::Debug {
|
|
use futures_util::{SinkExt, StreamExt};
|
|
let msg = serde_json::json!({"id": id, "method": method, "params": params});
|
|
s.send(tokio_tungstenite::tungstenite::Message::text(msg.to_string())).await.unwrap();
|
|
loop {
|
|
let v: serde_json::Value = serde_json::from_str(s.next().await.unwrap().unwrap().to_text().unwrap()).unwrap();
|
|
if v["id"] == id {
|
|
break v["result"].clone();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// With a screen, Chrome starts without a window and the app opens exactly one, at the screen's
|
|
/// place and size, without taking the focus. The bot's page then opens the way the botBrowser tool
|
|
/// opens it (a window of its own behind, at that window's place): also without taking the focus,
|
|
/// and on Windows the app makes it a tool window in front of the first. Needs a display on Linux
|
|
/// (`DISPLAY`); on Windows the screen is the virtual monitor (its driver installed), on the desktop
|
|
/// only meanwhile.
|
|
#[tokio::test]
|
|
async fn on_a_screen_the_bot_browser_has_one_window_at_the_monitor() {
|
|
let Ok(exe) = std::env::var("JARVIS_TEST_CHROME") else {
|
|
eprintln!("skipped: set JARVIS_TEST_CHROME");
|
|
return;
|
|
};
|
|
if cfg!(unix) && std::env::var_os("DISPLAY").is_none() {
|
|
eprintln!("skipped: no DISPLAY");
|
|
return;
|
|
}
|
|
let profile = std::env::temp_dir().join(format!("jarvis-bot-browser-screen-{}", std::process::id()));
|
|
let screen = if cfg!(windows) {
|
|
let r = jarvis_app::vdisplay::attach().expect("the virtual monitor goes on");
|
|
eprintln!("virtual monitor at {r:?}");
|
|
r
|
|
} else {
|
|
jarvis_app::vdisplay::Rect { x: 40, y: 30, w: 1280, h: 720 }
|
|
};
|
|
#[cfg(windows)]
|
|
let focused = foreground_pid();
|
|
let b = BotBrowser::start(std::path::Path::new(&exe), &profile, Some(screen)).await.expect("chrome starts");
|
|
#[cfg(windows)]
|
|
assert_eq!(foreground_pid(), focused, "opening the window leaves the focus where it was");
|
|
let client = reqwest::Client::builder().no_proxy().build().unwrap();
|
|
let pages = |l: serde_json::Value| l.as_array().unwrap().iter().filter(|t| t["type"] == "page")
|
|
.map(|t| t["id"].as_str().unwrap().to_owned()).collect::<Vec<_>>();
|
|
let list: serde_json::Value = client.get(format!("http://127.0.0.1:{}/json/list", b.cdp_port)).send().await.unwrap()
|
|
.text().await.map(|t| serde_json::from_str(&t).unwrap()).unwrap();
|
|
let first = pages(list);
|
|
assert_eq!(first.len(), 1, "one page in one window");
|
|
|
|
let ws: serde_json::Value = client.get(format!("http://127.0.0.1:{}/json/version", b.cdp_port)).send().await.unwrap()
|
|
.text().await.map(|t| serde_json::from_str(&t).unwrap()).unwrap();
|
|
let (mut s, _) = tokio_tungstenite::connect_async(ws["webSocketDebuggerUrl"].as_str().unwrap()).await.unwrap();
|
|
// As the botBrowser tool does it.
|
|
let at = cdp(&mut s, 1, "Browser.getWindowForTarget", serde_json::json!({"targetId": first[0]})).await["bounds"].clone();
|
|
let bot = cdp(&mut s, 2, "Target.createTarget", serde_json::json!({"url": "about:blank",
|
|
"newWindow": true, "background": true, "left": at["left"], "top": at["top"], "width": at["width"],
|
|
"height": at["height"]})).await["targetId"].as_str().unwrap().to_owned();
|
|
// A page loading in it, as the tool's `open` does.
|
|
let (mut page, _) = tokio_tungstenite::connect_async(format!("ws://127.0.0.1:{}/devtools/page/{bot}", b.cdp_port))
|
|
.await.unwrap();
|
|
cdp(&mut page, 1, "Page.navigate", serde_json::json!({"url": "https://example.com/"})).await;
|
|
let mut loaded = false;
|
|
for _ in 0..40 {
|
|
loaded = titles(&client, b.cdp_port).await.iter().any(|(_, t)| t == "Example Domain");
|
|
if loaded {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
}
|
|
assert!(loaded, "the bot's page loads in its window");
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
#[cfg(windows)]
|
|
{
|
|
assert_eq!(foreground_pid(), focused, "the bot's window leaves the focus where it was");
|
|
let w = chrome_windows(b.pid());
|
|
eprintln!("chrome windows, top first: {w:?}");
|
|
assert_eq!(w.len(), 2, "{w:?}");
|
|
assert!(w[0].0.starts_with("Example Domain"), "the bot's window is in front: {w:?}");
|
|
assert!(w.iter().all(|(_, tool)| *tool), "no window in the taskbar or Alt+Tab: {w:?}");
|
|
}
|
|
|
|
// Each page's window over CDP: a window each, at the screen's rectangle.
|
|
let mut windows = std::collections::HashSet::new();
|
|
for (i, id) in [first[0].clone(), bot].iter().enumerate() {
|
|
let r = cdp(&mut s, 10 + i as u64, "Browser.getWindowForTarget", serde_json::json!({"targetId": id})).await;
|
|
eprintln!("window: {r}");
|
|
windows.insert(r["windowId"].as_i64().unwrap());
|
|
let b = &r["bounds"];
|
|
let (left, top, width, height) = (b["left"].as_i64().unwrap(), b["top"].as_i64().unwrap(),
|
|
b["width"].as_i64().unwrap(), b["height"].as_i64().unwrap());
|
|
let (x, y, w, h) = (screen.x as i64, screen.y as i64, screen.w as i64, screen.h as i64);
|
|
if cfg!(windows) {
|
|
// Inside the monitor, filling it but for the monitor's own taskbar if Windows gave it one.
|
|
assert!(left >= x && top >= y && left + width <= x + w && top + height <= y + h, "{b}");
|
|
assert!(width * height * 10 >= w * h * 8, "{b}");
|
|
} else {
|
|
assert_eq!((left, top, width, height), (x, y, w, h));
|
|
}
|
|
}
|
|
assert_eq!(windows.len(), 2, "the bot's page has a window of its own");
|
|
if std::env::var_os("JARVIS_TEST_HOLD").is_some() {
|
|
// Time for a look at the screen (the Windows check takes a screenshot meanwhile).
|
|
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
|
|
}
|
|
b.stop().await;
|
|
jarvis_app::vdisplay::detach();
|
|
let _ = std::fs::remove_dir_all(&profile);
|
|
}
|