Files
javis_bot/app/tests/bot_browser.rs
javis-bot decd7ae477 fix(v2): YouTube no longer stops 45 s in, consent and the bot's video keep their tools
From the live conversation on 0.1.40 (2026-11-18):

- The bot browser told pages a program drives it (navigator.webdriver), and
  YouTube stopped every video about 45 s in with "문제가 발생했습니다"
  (ump.spsrejectfailure, reproduced twice on the game PC). Chrome now starts
  with --disable-blink-features=AutomationControlled, as the M0 video check
  already did; the live Chrome test checks webdriver is false.
- "네." to "계산기를 실행해도 될까요?" got webSearch and botBrowser from the
  router, so the consent call was refused and the calculator never started.
  Waiting tools: pcControl stays on offer while a request asked in the
  previous reply waits, botBrowser while it holds the asker's page.
- "소리 반으로 줄여 줘" for the bot's video worked on the person's own Edge
  window. botBrowser gets a volume action (media elements and YouTube's player),
  the page shows its volume, its description names its actions first (the part
  the planner sees), pcControl's says the bot's browser is not among its
  windows, and a turn whose plan has no step while a tool holds something for
  the asker gets the one-step nudge for it.

Evals (qwen3.6 on the game PC): volume 0/3 before, 8/8 after; "네." consent
10/10; pcControl 10/10; botBrowser 16/18 (the NewJeans play case fails 0/6
before this change too). Unit: Python 2542 passed, 13 failed (the same 13 as
before); app 348 passed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 00:47:38 +09:00

247 lines
13 KiB
Rust

//! The bot browser with a real Chrome (admin.spec.md, "Bot browser"). Runs only with
//! `JARVIS_TEST_CHROME=<chrome executable>`; needs the internet for the public page. On Ubuntu also
//! set `JARVIS_CHROME_NO_SANDBOX=1`.
use std::sync::{Arc, Mutex};
use std::time::Duration;
use jarvis_app::browser::BotBrowser;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
async fn titles(client: &reqwest::Client, port: u16) -> Vec<(String, String)> {
let text = client.get(format!("http://127.0.0.1:{port}/json/list")).send().await.unwrap().text().await.unwrap();
let list: serde_json::Value = serde_json::from_str(&text).unwrap();
list.as_array().unwrap().iter()
.map(|t| (t["url"].as_str().unwrap_or("").to_owned(), t["title"].as_str().unwrap_or("").to_owned())).collect()
}
#[tokio::test]
async fn the_bot_browser_reaches_public_pages_never_this_pc_and_stops() {
let Ok(exe) = std::env::var("JARVIS_TEST_CHROME") else {
eprintln!("skipped: set JARVIS_TEST_CHROME");
return;
};
// A page on this PC the browser must never reach.
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let local = l.local_addr().unwrap().port();
let hits = Arc::new(Mutex::new(0));
let h2 = hits.clone();
tokio::spawn(async move {
while let Ok((mut c, _)) = l.accept().await {
*h2.lock().unwrap() += 1;
let mut b = [0u8; 1024];
let _ = c.read(&mut b).await;
let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n<title>secret local</title>xx").await;
}
});
let profile = std::env::temp_dir().join(format!("jarvis-bot-browser-{}", std::process::id()));
let b = BotBrowser::start(std::path::Path::new(&exe), &profile, None).await.expect("chrome starts");
assert!(b.running());
let client = reqwest::Client::builder().no_proxy().build().unwrap();
for url in [format!("http://127.0.0.1:{local}/"), format!("http://localhost:{local}/"), "https://example.com/".into()] {
client.put(format!("http://127.0.0.1:{}/json/new?{url}", b.cdp_port)).send().await.unwrap();
}
let mut got = vec![];
for _ in 0..40 {
got = titles(&client, b.cdp_port).await;
if got.iter().any(|(_, t)| t == "Example Domain") {
break;
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
eprintln!("tabs: {got:?}");
assert!(got.iter().any(|(_, t)| t == "Example Domain"), "the public page loads through the proxy");
assert!(!got.iter().any(|(_, t)| t.contains("secret local")));
assert_eq!(*hits.lock().unwrap(), 0, "nothing reached this PC");
// Pages are not told a program drives the browser: with navigator.webdriver YouTube stopped
// every video about 45 s in ("문제가 발생했습니다", ump.spsrejectfailure; live 2026-11-18).
let list: serde_json::Value = serde_json::from_str(&client.get(format!("http://127.0.0.1:{}/json/list", b.cdp_port))
.send().await.unwrap().text().await.unwrap()).unwrap();
let page = list.as_array().unwrap().iter().find(|t| t["title"] == "Example Domain").unwrap();
let (mut s, _) = tokio_tungstenite::connect_async(page["webSocketDebuggerUrl"].as_str().unwrap()).await.unwrap();
let wd = cdp(&mut s, 1, "Runtime.evaluate", serde_json::json!({"expression": "navigator.webdriver"})).await;
assert_eq!(wd["result"]["value"], false, "{wd}");
let port = b.cdp_port;
let t0 = std::time::Instant::now();
b.stop().await;
eprintln!("stopped in {:?}", t0.elapsed());
assert!(!b.running());
assert!(client.get(format!("http://127.0.0.1:{port}/json/version")).send().await.is_err(), "CDP is closed");
let _ = std::fs::remove_dir_all(&profile);
}
/// The install step downloads the pinned Chrome (about 200 MB) and unpacks a runnable browser.
/// Runs only with `JARVIS_TEST_CHROME_INSTALL=1`.
#[tokio::test]
async fn the_browser_step_installs_a_chrome_that_starts() {
if std::env::var("JARVIS_TEST_CHROME_INSTALL").as_deref() != Ok("1") {
eprintln!("skipped: set JARVIS_TEST_CHROME_INSTALL=1");
return;
}
let data = std::env::temp_dir().join(format!("jarvis-chrome-install-{}", std::process::id()));
let layout = jarvis_app::install::Layout { data: data.clone(), src: std::path::PathBuf::new() };
let steps = jarvis_app::install::steps().into_iter().filter(|s| s.id() == "browser").collect();
let installer = jarvis_app::install::Installer::new(layout.clone(), steps);
assert!(installer.run_all().await, "{}", installer.status());
assert!(installer.done("browser"));
assert!(!data.join("cache").join("downloads").read_dir().map(|mut d| d.next().is_some()).unwrap_or(false),
"the archive is removed after unpacking");
let b = BotBrowser::start(&layout.chrome_exe(), &jarvis_app::browser::profile_dir(&data), None).await.expect("starts");
assert!(b.running());
b.stop().await;
let _ = std::fs::remove_dir_all(&data);
}
/// The process whose window has the keyboard focus (Windows).
#[cfg(windows)]
fn foreground_pid() -> u32 {
use windows_sys::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
let mut pid = 0;
// SAFETY: plain queries; `pid` outlives the call.
unsafe { GetWindowThreadProcessId(GetForegroundWindow(), &mut pid) };
pid
}
/// The visible top-level Chrome windows of process `pid`, top first: (title, tool window).
#[cfg(windows)]
fn chrome_windows(pid: u32) -> Vec<(String, bool)> {
use windows_sys::Win32::Foundation::{HWND, LPARAM};
use windows_sys::Win32::UI::WindowsAndMessaging::*;
unsafe extern "system" fn each(h: HWND, l: LPARAM) -> windows_sys::core::BOOL {
// SAFETY: `l` is the `(pid, Vec)` below, alive through EnumWindows; plain queries on `h`.
unsafe {
let (pid, out) = &mut *(l as *mut (u32, Vec<(String, bool)>));
let (mut p, mut class, mut title) = (0u32, [0u16; 64], [0u16; 256]);
GetWindowThreadProcessId(h, &mut p);
let n = GetClassNameW(h, class.as_mut_ptr(), 64) as usize;
if p == *pid && IsWindowVisible(h) != 0 && GetWindow(h, GW_OWNER).is_null()
&& String::from_utf16_lossy(&class[..n]) == "Chrome_WidgetWin_1" {
let t = GetWindowTextW(h, title.as_mut_ptr(), 256) as usize;
let tool = GetWindowLongPtrW(h, GWL_EXSTYLE) as u32 & WS_EX_TOOLWINDOW != 0;
out.push((String::from_utf16_lossy(&title[..t]), tool));
}
}
1
}
let mut found = (pid, vec![]);
// SAFETY: `each` writes only into `found`, which outlives the call.
unsafe { EnumWindows(Some(each), &mut found as *mut _ as LPARAM) };
found.1
}
/// One call on the browser's CDP connection.
async fn cdp<S>(s: &mut S, id: u64, method: &str, params: serde_json::Value) -> serde_json::Value
where S: futures_util::Sink<tokio_tungstenite::tungstenite::Message> + futures_util::Stream<Item =
Result<tokio_tungstenite::tungstenite::Message, tokio_tungstenite::tungstenite::Error>> + Unpin,
S::Error: std::fmt::Debug {
use futures_util::{SinkExt, StreamExt};
let msg = serde_json::json!({"id": id, "method": method, "params": params});
s.send(tokio_tungstenite::tungstenite::Message::text(msg.to_string())).await.unwrap();
loop {
let v: serde_json::Value = serde_json::from_str(s.next().await.unwrap().unwrap().to_text().unwrap()).unwrap();
if v["id"] == id {
break v["result"].clone();
}
}
}
/// With a screen, Chrome starts without a window and the app opens exactly one, at the screen's
/// place and size, without taking the focus. The bot's page then opens the way the botBrowser tool
/// opens it (a window of its own behind, at that window's place): also without taking the focus,
/// and on Windows the app makes it a tool window in front of the first. Needs a display on Linux
/// (`DISPLAY`); on Windows the screen is the virtual monitor (its driver installed), on the desktop
/// only meanwhile.
#[tokio::test]
async fn on_a_screen_the_bot_browser_has_one_window_at_the_monitor() {
let Ok(exe) = std::env::var("JARVIS_TEST_CHROME") else {
eprintln!("skipped: set JARVIS_TEST_CHROME");
return;
};
if cfg!(unix) && std::env::var_os("DISPLAY").is_none() {
eprintln!("skipped: no DISPLAY");
return;
}
let profile = std::env::temp_dir().join(format!("jarvis-bot-browser-screen-{}", std::process::id()));
let screen = if cfg!(windows) {
let r = jarvis_app::vdisplay::attach().expect("the virtual monitor goes on");
eprintln!("virtual monitor at {r:?}");
r
} else {
jarvis_app::vdisplay::Rect { x: 40, y: 30, w: 1280, h: 720 }
};
#[cfg(windows)]
let focused = foreground_pid();
let b = BotBrowser::start(std::path::Path::new(&exe), &profile, Some(screen)).await.expect("chrome starts");
#[cfg(windows)]
assert_eq!(foreground_pid(), focused, "opening the window leaves the focus where it was");
let client = reqwest::Client::builder().no_proxy().build().unwrap();
let pages = |l: serde_json::Value| l.as_array().unwrap().iter().filter(|t| t["type"] == "page")
.map(|t| t["id"].as_str().unwrap().to_owned()).collect::<Vec<_>>();
let list: serde_json::Value = client.get(format!("http://127.0.0.1:{}/json/list", b.cdp_port)).send().await.unwrap()
.text().await.map(|t| serde_json::from_str(&t).unwrap()).unwrap();
let first = pages(list);
assert_eq!(first.len(), 1, "one page in one window");
let ws: serde_json::Value = client.get(format!("http://127.0.0.1:{}/json/version", b.cdp_port)).send().await.unwrap()
.text().await.map(|t| serde_json::from_str(&t).unwrap()).unwrap();
let (mut s, _) = tokio_tungstenite::connect_async(ws["webSocketDebuggerUrl"].as_str().unwrap()).await.unwrap();
// As the botBrowser tool does it.
let at = cdp(&mut s, 1, "Browser.getWindowForTarget", serde_json::json!({"targetId": first[0]})).await["bounds"].clone();
let bot = cdp(&mut s, 2, "Target.createTarget", serde_json::json!({"url": "about:blank",
"newWindow": true, "background": true, "left": at["left"], "top": at["top"], "width": at["width"],
"height": at["height"]})).await["targetId"].as_str().unwrap().to_owned();
// A page loading in it, as the tool's `open` does.
let (mut page, _) = tokio_tungstenite::connect_async(format!("ws://127.0.0.1:{}/devtools/page/{bot}", b.cdp_port))
.await.unwrap();
cdp(&mut page, 1, "Page.navigate", serde_json::json!({"url": "https://example.com/"})).await;
let mut loaded = false;
for _ in 0..40 {
loaded = titles(&client, b.cdp_port).await.iter().any(|(_, t)| t == "Example Domain");
if loaded {
break;
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
assert!(loaded, "the bot's page loads in its window");
tokio::time::sleep(Duration::from_secs(1)).await;
#[cfg(windows)]
{
assert_eq!(foreground_pid(), focused, "the bot's window leaves the focus where it was");
let w = chrome_windows(b.pid());
eprintln!("chrome windows, top first: {w:?}");
assert_eq!(w.len(), 2, "{w:?}");
assert!(w[0].0.starts_with("Example Domain"), "the bot's window is in front: {w:?}");
assert!(w.iter().all(|(_, tool)| *tool), "no window in the taskbar or Alt+Tab: {w:?}");
}
// Each page's window over CDP: a window each, at the screen's rectangle.
let mut windows = std::collections::HashSet::new();
for (i, id) in [first[0].clone(), bot].iter().enumerate() {
let r = cdp(&mut s, 10 + i as u64, "Browser.getWindowForTarget", serde_json::json!({"targetId": id})).await;
eprintln!("window: {r}");
windows.insert(r["windowId"].as_i64().unwrap());
let b = &r["bounds"];
let (left, top, width, height) = (b["left"].as_i64().unwrap(), b["top"].as_i64().unwrap(),
b["width"].as_i64().unwrap(), b["height"].as_i64().unwrap());
let (x, y, w, h) = (screen.x as i64, screen.y as i64, screen.w as i64, screen.h as i64);
if cfg!(windows) {
// Inside the monitor, filling it but for the monitor's own taskbar if Windows gave it one.
assert!(left >= x && top >= y && left + width <= x + w && top + height <= y + h, "{b}");
assert!(width * height * 10 >= w * h * 8, "{b}");
} else {
assert_eq!((left, top, width, height), (x, y, w, h));
}
}
assert_eq!(windows.len(), 2, "the bot's page has a window of its own");
if std::env::var_os("JARVIS_TEST_HOLD").is_some() {
// Time for a look at the screen (the Windows check takes a screenshot meanwhile).
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
}
b.stop().await;
jarvis_app::vdisplay::detach();
let _ = std::fs::remove_dir_all(&profile);
}