Files
javis_bot/app/src/browser.rs
javis-bot 24d0798bae feat(app): the bot's own Chrome in the data folder, reaching only public addresses
- install step browser: Chrome for Testing 157 (pinned SHA-256) into runtime/chrome
- services start it headless with its own profile when features.browser is on; CDP on a free
  loopback port passed to the worker as JARVIS_BROWSER_CDP; a failure leaves the services running
- every Chrome connection goes through the app's loopback egress proxy: one lookup, public
  addresses only (403 otherwise, no host logged), QUIC off, WebRTC only through the proxy
- stop: Browser.close, then kill after 5 s; dashboard shows the browser's state
- the worker's runtime environment follows the settings step, not every step being installed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 22:01:04 +09:00

426 lines
19 KiB
Rust

//! The bot browser (`app/src/admin.spec.md`, "Bot browser"): the bot's own Chrome in the data
//! folder, driven over CDP, and the egress proxy every Chrome connection goes through so it can
//! only reach public addresses.
use std::net::{IpAddr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10);
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const HEAD_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_HEAD: usize = 64 * 1024;
const READY_TIMEOUT: Duration = Duration::from_secs(30);
const CLOSE_WAIT: Duration = Duration::from_secs(5);
/// Whether an address is on the public internet: not this PC, the local network or a special range.
pub fn is_public(ip: &IpAddr) -> bool {
match ip {
IpAddr::V4(v4) => {
let o = v4.octets();
!(v4.is_private() || v4.is_loopback() || v4.is_link_local() || v4.is_broadcast()
|| v4.is_documentation() || v4.is_unspecified() || v4.is_multicast()
|| o[0] == 0 // "this network"
|| (o[0] == 100 && (64..128).contains(&o[1])) // CGNAT 100.64/10
|| (o[0] == 192 && o[1] == 0 && o[2] == 0) // IETF 192.0.0/24
|| (o[0] == 198 && (18..20).contains(&o[1])) // benchmarking 198.18/15
|| o[0] >= 240) // reserved
}
IpAddr::V6(v6) => {
if let Some(v4) = v6.to_ipv4_mapped() {
return is_public(&IpAddr::V4(v4));
}
let s = v6.segments();
!(v6.is_loopback() || v6.is_unspecified() || v6.is_multicast()
|| (s[0] & 0xfe00) == 0xfc00 // unique local fc00::/7
|| (s[0] & 0xffc0) == 0xfe80 // link local fe80::/10
|| (s[0] == 0x2001 && s[1] == 0x0db8) // documentation
|| (s[0] == 0x0064 && s[1] == 0xff9b) // NAT64 (could reach IPv4 private)
|| s[0] == 0x2002 // 6to4 (embeds any IPv4)
|| (s[0] == 0x2001 && s[1] == 0)) // Teredo
}
}
}
// ---- egress proxy ---------------------------------------------------------------------------------
/// A loopback HTTP proxy that connects only to addresses `allow` accepts (`is_public` for Chrome).
pub struct EgressProxy {
pub port: u16,
task: tokio::task::JoinHandle<()>,
}
impl EgressProxy {
pub async fn start(allow: fn(&IpAddr) -> bool) -> std::io::Result<Self> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let port = listener.local_addr()?.port();
let task = tokio::spawn(async move {
while let Ok((client, _)) = listener.accept().await {
tokio::spawn(async move {
let _ = serve(client, allow).await;
});
}
});
Ok(Self { port, task })
}
pub fn stop(&self) {
self.task.abort();
}
}
impl Drop for EgressProxy {
fn drop(&mut self) {
self.task.abort();
}
}
/// The request head (up to the blank line) and whatever came after it.
async fn read_head(s: &mut TcpStream) -> std::io::Result<Option<(String, Vec<u8>)>> {
let mut buf = Vec::new();
let mut chunk = [0u8; 8192];
loop {
if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
let rest = buf.split_off(end + 4);
return Ok(String::from_utf8(buf).ok().map(|h| (h, rest)));
}
if buf.len() > MAX_HEAD {
return Ok(None);
}
let n = s.read(&mut chunk).await?;
if n == 0 {
return Ok(None);
}
buf.extend_from_slice(&chunk[..n]);
}
}
/// `host:port` with an optional `[..]` around an IPv6 address.
fn split_host_port(s: &str) -> Option<(String, u16)> {
let (host, port) = s.rsplit_once(':')?;
let host = host.strip_prefix('[').and_then(|h| h.strip_suffix(']')).unwrap_or(host);
if host.is_empty() {
return None;
}
Some((host.to_owned(), port.parse().ok()?))
}
/// Resolve once, require every address to be allowed, and connect only to those.
async fn connect_allowed(host: &str, port: u16, allow: fn(&IpAddr) -> bool) -> Result<TcpStream, &'static str> {
let addrs: Vec<SocketAddr> = match host.parse::<IpAddr>() {
Ok(ip) => vec![SocketAddr::new(ip, port)],
Err(_) => match tokio::time::timeout(RESOLVE_TIMEOUT, tokio::net::lookup_host((host, port))).await {
Ok(Ok(a)) => a.collect(),
_ => return Err("name not resolved"),
},
};
if addrs.is_empty() || !addrs.iter().all(|a| allow(&a.ip())) {
return Err("not a public address");
}
for a in &addrs {
if let Ok(Ok(s)) = tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(a)).await {
return Ok(s);
}
}
Err("could not connect")
}
async fn reply(client: &mut TcpStream, status: &str) -> std::io::Result<()> {
client.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await
}
async fn serve(mut client: TcpStream, allow: fn(&IpAddr) -> bool) -> std::io::Result<()> {
let Ok(Ok(Some((head, rest)))) = tokio::time::timeout(HEAD_TIMEOUT, read_head(&mut client)).await else {
return reply(&mut client, "400 Bad Request").await;
};
let mut lines = head.split("\r\n");
let mut first = lines.next().unwrap_or("").split(' ');
let (method, target, version) = (first.next().unwrap_or(""), first.next().unwrap_or(""), first.next().unwrap_or(""));
if method.eq_ignore_ascii_case("CONNECT") {
let Some((host, port)) = split_host_port(target) else { return reply(&mut client, "400 Bad Request").await };
let mut up = match connect_allowed(&host, port, allow).await {
Ok(s) => s,
Err(why) => return refused(&mut client, why).await,
};
client.write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n").await?;
up.write_all(&rest).await?;
tokio::io::copy_bidirectional(&mut client, &mut up).await?;
return Ok(());
}
let Ok(url) = reqwest::Url::parse(target) else { return reply(&mut client, "400 Bad Request").await };
let (Some(host), Some(port)) = (url.host_str(), url.port_or_known_default()) else {
return reply(&mut client, "400 Bad Request").await;
};
if url.scheme() != "http" || !version.starts_with("HTTP/") {
return reply(&mut client, "400 Bad Request").await;
}
let host = host.trim_start_matches('[').trim_end_matches(']').to_owned();
let mut up = match connect_allowed(&host, port, allow).await {
Ok(s) => s,
Err(why) => return refused(&mut client, why).await,
};
// Origin form, no proxy headers, one request per connection (the next may be for another host).
let path = match url.query() {
Some(q) => format!("{}?{q}", url.path()),
None => url.path().to_owned(),
};
let mut out = format!("{method} {path} {version}\r\n");
for l in lines.filter(|l| !l.is_empty()) {
let name = l.split(':').next().unwrap_or("").trim().to_ascii_lowercase();
if !matches!(name.as_str(), "proxy-connection" | "proxy-authorization" | "connection" | "keep-alive") {
out.push_str(l);
out.push_str("\r\n");
}
}
out.push_str("Connection: close\r\n\r\n");
up.write_all(out.as_bytes()).await?;
up.write_all(&rest).await?;
tokio::io::copy_bidirectional(&mut client, &mut up).await?;
Ok(())
}
async fn refused(client: &mut TcpStream, why: &str) -> std::io::Result<()> {
// Never the host or the address: they are what the bot was asked to open.
crate::debug_log!("browser", "proxy refused a connection: {why}");
reply(client, "403 Forbidden").await
}
// ---- Chrome ---------------------------------------------------------------------------------------
/// The arguments the bot's Chrome runs with (headless, its profile, CDP on a free port, every
/// connection through the proxy on `proxy_port`).
pub fn chrome_args(profile: &Path, proxy_port: u16, no_sandbox: bool) -> Vec<String> {
let mut a = vec![
"--headless=new".to_owned(),
format!("--user-data-dir={}", profile.display()),
"--remote-debugging-port=0".into(),
format!("--proxy-server=http://127.0.0.1:{proxy_port}"),
"--proxy-bypass-list=<-loopback>".into(),
"--disable-quic".into(),
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp".into(),
"--no-first-run".into(),
"--no-default-browser-check".into(),
"--disable-sync".into(),
"--disable-background-networking".into(),
"--disable-component-update".into(),
"--disable-default-apps".into(),
"--disable-features=Translate,MediaRouter,OptimizationHints".into(),
"--window-size=1280,800".into(),
];
if no_sandbox {
a.push("--no-sandbox".into());
}
a.push("about:blank".into());
a
}
pub struct BotBrowser {
child: Mutex<Option<tokio::process::Child>>,
proxy: EgressProxy,
pub cdp_port: u16,
}
impl BotBrowser {
/// Start Chrome `exe` with profile folder `profile` and wait until CDP answers.
pub async fn start(exe: &Path, profile: &Path) -> Result<Self, String> {
let proxy = EgressProxy::start(is_public).await.map_err(|e| format!("proxy: {e}"))?;
std::fs::create_dir_all(profile).map_err(|e| e.to_string())?;
let port_file = profile.join("DevToolsActivePort");
let _ = std::fs::remove_file(&port_file);
let no_sandbox = cfg!(not(windows)) && std::env::var("JARVIS_CHROME_NO_SANDBOX").is_ok_and(|v| v == "1");
let mut cmd = tokio::process::Command::new(exe);
cmd.args(chrome_args(profile, proxy.port, no_sandbox)).stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
#[cfg(windows)]
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
let mut child = cmd.spawn().map_err(|e| format!("Chrome을 실행하지 못했습니다: {e}"))?;
let until = tokio::time::Instant::now() + READY_TIMEOUT;
let cdp_port = loop {
if let Some(p) = std::fs::read_to_string(&port_file).ok()
.and_then(|t| t.lines().next().and_then(|l| l.trim().parse::<u16>().ok())) {
if version(p).await.is_some() {
break p;
}
}
if let Ok(Some(status)) = child.try_wait() {
return Err(format!("Chrome이 바로 끝났습니다 ({status})"));
}
if tokio::time::Instant::now() > until {
kill_tree(&mut child).await;
return Err("Chrome이 30초 안에 준비되지 않았습니다".into());
}
tokio::time::sleep(Duration::from_millis(200)).await;
};
crate::info_log!("browser", "bot browser ready (CDP on 127.0.0.1:{cdp_port})");
Ok(Self { child: Mutex::new(Some(child)), proxy, cdp_port })
}
pub fn cdp_url(&self) -> String {
format!("http://127.0.0.1:{}", self.cdp_port)
}
/// Whether Chrome still runs.
pub fn running(&self) -> bool {
self.child.lock().unwrap().as_mut().is_some_and(|c| matches!(c.try_wait(), Ok(None)))
}
/// `Browser.close`, then up to 5 s for Chrome to end, else it is killed; the proxy ends too.
pub async fn stop(&self) {
let child = self.child.lock().unwrap().take();
if let Some(mut child) = child {
if let Some(ws) = version(self.cdp_port).await.and_then(|v| v["webSocketDebuggerUrl"].as_str().map(str::to_owned)) {
let _ = close_over_cdp(&ws).await;
}
if !matches!(tokio::time::timeout(CLOSE_WAIT, child.wait()).await, Ok(Ok(_))) {
crate::warn_log!("browser", "Chrome did not close in {} s; killing it", CLOSE_WAIT.as_secs());
kill_tree(&mut child).await;
}
crate::info_log!("browser", "bot browser stopped");
}
self.proxy.stop();
}
}
/// CDP's `/json/version`, if it answers.
async fn version(port: u16) -> Option<serde_json::Value> {
let client = reqwest::Client::builder().no_proxy().timeout(Duration::from_secs(2)).build().ok()?;
let text = client.get(format!("http://127.0.0.1:{port}/json/version")).send().await.ok()?.text().await.ok()?;
serde_json::from_str(&text).ok()
}
async fn close_over_cdp(ws: &str) -> Result<(), String> {
use futures_util::SinkExt;
let (mut s, _) = tokio::time::timeout(Duration::from_secs(2), tokio_tungstenite::connect_async(ws)).await
.map_err(|_| "timeout".to_string())?.map_err(|e| e.to_string())?;
s.send(tokio_tungstenite::tungstenite::Message::text(r#"{"id":1,"method":"Browser.close"}"#)).await
.map_err(|e| e.to_string())
}
async fn kill_tree(child: &mut tokio::process::Child) {
if let Some(pid) = child.id() {
#[cfg(unix)]
// SAFETY: a plain signal to Chrome's own process group (made with process_group(0)).
unsafe {
libc::kill(-(pid as i32), libc::SIGKILL);
}
#[cfg(windows)]
{
let mut tk = tokio::process::Command::new("taskkill");
tk.args(["/T", "/F", "/PID", &pid.to_string()]).stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()).creation_flags(0x0800_0000);
let _ = tk.status().await;
}
}
let _ = child.kill().await;
}
/// Where the bot browser's profile lives in the data folder.
pub fn profile_dir(data: &Path) -> PathBuf {
data.join("browser").join("profile")
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
#[test]
fn only_public_addresses_are_public() {
let public = ["8.8.8.8", "1.1.1.1", "142.250.1.1", "2606:4700:4700::1111", "2a00:1450:4001::1"];
let not = ["127.0.0.1", "10.0.0.1", "172.16.5.4", "192.168.10.9", "169.254.169.254", "100.64.0.1",
"0.0.0.0", "0.1.2.3", "192.0.0.8", "192.0.2.1", "198.18.0.1", "203.0.113.5", "224.0.0.1", "240.0.0.1",
"255.255.255.255", "::1", "::", "fc00::1", "fd12::1", "fe80::1", "ff02::1", "2001:db8::1",
"::ffff:127.0.0.1", "::ffff:192.168.0.1", "64:ff9b::a00:1", "2002:c0a8:1::1", "2001:0:4136:e378::1"];
for a in public {
assert!(is_public(&a.parse().unwrap()), "{a} should be public");
}
for a in not {
assert!(!is_public(&a.parse().unwrap()), "{a} should not be public");
}
}
/// A local server recording each connection's first bytes and answering `answer`.
async fn server(answer: &'static [u8]) -> (u16, Arc<Mutex<Vec<String>>>) {
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = l.local_addr().unwrap().port();
let seen = Arc::new(Mutex::new(Vec::new()));
let s2 = seen.clone();
tokio::spawn(async move {
while let Ok((mut c, _)) = l.accept().await {
let seen = s2.clone();
tokio::spawn(async move {
let mut buf = vec![0u8; 4096];
let n = c.read(&mut buf).await.unwrap_or(0);
seen.lock().unwrap().push(String::from_utf8_lossy(&buf[..n]).into_owned());
let _ = c.write_all(answer).await;
});
}
});
(port, seen)
}
async fn ask(proxy: u16, req: String) -> String {
let mut c = TcpStream::connect(("127.0.0.1", proxy)).await.unwrap();
c.write_all(req.as_bytes()).await.unwrap();
let mut out = Vec::new();
let _ = tokio::time::timeout(Duration::from_secs(5), c.read_to_end(&mut out)).await;
String::from_utf8_lossy(&out).into_owned()
}
fn loopback(ip: &IpAddr) -> bool {
ip.is_loopback()
}
#[tokio::test]
async fn the_proxy_refuses_this_pc_and_the_local_network() {
let (port, seen) = server(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nhi").await;
let p = EgressProxy::start(is_public).await.unwrap();
for req in [format!("CONNECT 127.0.0.1:{port} HTTP/1.1\r\nHost: x\r\n\r\n"),
format!("CONNECT localhost:{port} HTTP/1.1\r\n\r\n"),
format!("CONNECT [::1]:{port} HTTP/1.1\r\n\r\n"),
format!("GET http://127.0.0.1:{port}/a HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"),
format!("GET http://localhost:{port}/a HTTP/1.1\r\n\r\n"),
"CONNECT 192.168.10.1:443 HTTP/1.1\r\n\r\n".to_string(),
"GET http://10.0.0.1/ HTTP/1.1\r\n\r\n".to_string()] {
let r = ask(p.port, req.clone()).await;
assert!(r.starts_with("HTTP/1.1 403"), "{req:?} -> {r:?}");
}
assert!(seen.lock().unwrap().is_empty());
for bad in ["nonsense\r\n\r\n", "GET /relative HTTP/1.1\r\n\r\n", "GET ftp://a.example/ HTTP/1.1\r\n\r\n",
"CONNECT nohost HTTP/1.1\r\n\r\n"] {
assert!(ask(p.port, bad.into()).await.starts_with("HTTP/1.1 400"), "{bad:?}");
}
}
#[tokio::test]
async fn the_proxy_tunnels_and_forwards_to_allowed_addresses() {
let (port, seen) = server(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nhi").await;
let p = EgressProxy::start(loopback).await.unwrap();
let r = ask(p.port, format!("CONNECT 127.0.0.1:{port} HTTP/1.1\r\n\r\nping")).await;
assert!(r.starts_with("HTTP/1.1 200 Connection Established\r\n\r\nHTTP/1.1 200 OK"), "{r:?}");
let r = ask(p.port, format!("GET http://127.0.0.1:{port}/a/b?c=1 HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n\
Proxy-Connection: keep-alive\r\nConnection: keep-alive\r\nAccept: */*\r\n\r\n")).await;
assert!(r.ends_with("hi"), "{r:?}");
let seen = seen.lock().unwrap().clone();
assert_eq!(seen[0], "ping");
assert_eq!(seen[1], format!("GET /a/b?c=1 HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAccept: */*\r\nConnection: close\r\n\r\n"));
}
#[test]
fn chrome_goes_through_the_proxy_with_its_own_profile() {
let a = chrome_args(Path::new("/d/browser/profile"), 4321, false);
for want in ["--headless=new", "--user-data-dir=/d/browser/profile", "--remote-debugging-port=0",
"--proxy-server=http://127.0.0.1:4321", "--proxy-bypass-list=<-loopback>", "--disable-quic"] {
assert!(a.iter().any(|x| x == want), "{want}");
}
assert!(!a.iter().any(|x| x == "--no-sandbox"));
assert!(chrome_args(Path::new("/p"), 1, true).iter().any(|x| x == "--no-sandbox"));
}
}