From aebf183950b26284ce4fcb9012f7cdd189118f67 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Mon, 15 Jun 2026 10:41:57 +0900 Subject: [PATCH] feat: browser-control server on host (real input) + remote-bot routing + ignore env backups - control-server.mjs runs chrome-control.mjs LOCALLY on the browser host, so a remote bot's controlBrowser (BROWSER_CONTROL_URL) drives real xdotool input on THIS screen instead of the bot machine. Published on the LAN. - controlBrowser tool posts to BROWSER_CONTROL_URL when set, else runs locally. - Drop hard depends_on ollama so a browser-host doesn't start Ollama. - gitignore .env.bak*/*.bak (a backup with tokens had been left untracked). --- .gitignore | 5 ++- bot/scripts/stream-test/control-server.mjs | 48 +++++++++++++++++++++ docker-compose.yml | 16 ++++++- docker/supervisord.conf | 13 ++++++ src/jarvis/tools/builtin/control_browser.py | 31 +++++++++---- 5 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 bot/scripts/stream-test/control-server.mjs diff --git a/.gitignore b/.gitignore index b7f8759..5090bd9 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,7 @@ dist/ qt.conf # Auto-generated version file (created at build time) -src/jarvis/_version.py \ No newline at end of file +src/jarvis/_version.py +# never commit env backups (contain tokens) +.env.bak* +*.bak diff --git a/bot/scripts/stream-test/control-server.mjs b/bot/scripts/stream-test/control-server.mjs new file mode 100644 index 0000000..072335d --- /dev/null +++ b/bot/scripts/stream-test/control-server.mjs @@ -0,0 +1,48 @@ +// Browser-control HTTP endpoint for the BROWSER HOST. +// +// The on-screen Chrome, the X display (:1), xdotool (real cursor/keyboard) and +// the broadcast capture all live on THIS machine. A remote `bot` on another PC +// therefore cannot drive them directly β€” it must send a command here, where +// chrome-control.mjs runs LOCALLY (real input lands on this host's screen, +// visible on its VNC / Go-Live). +// +// POST /control body: {"action":"navigate","url":"naver.com", ...} +// GET /health +// +// Internal-network use only (no auth, per deployment decision). Bind/port: +// BROWSER_CONTROL_BIND (default 0.0.0.0), BROWSER_CONTROL_PORT (default 8777) +import http from 'node:http'; +import { execFile } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const PORT = parseInt(process.env.BROWSER_CONTROL_PORT || '8777', 10); +const BIND = process.env.BROWSER_CONTROL_BIND || '0.0.0.0'; +const SCRIPT = join(dirname(fileURLToPath(import.meta.url)), 'chrome-control.mjs'); + +const server = http.createServer((req, res) => { + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, host: 'browser' })); + return; + } + if (req.method !== 'POST') { + res.writeHead(405); res.end('POST /control'); + return; + } + let body = ''; + req.on('data', (c) => { body += c; if (body.length > 1e6) req.destroy(); }); + req.on('end', () => { + // Run the action LOCALLY: chrome-control.mjs uses CDP + xdotool on this + // host, so the cursor really moves and text is typed on this screen. + execFile('node', [SCRIPT, body || '{}'], { timeout: 95_000, env: process.env }, (err, stdout, stderr) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + const out = (stdout || '').trim(); + res.end(out || JSON.stringify({ ok: false, error: String((stderr || '').trim() || err?.message || 'no output') })); + }); + }); +}); + +server.listen(PORT, BIND, () => { + console.log(`[control-server] listening on ${BIND}:${PORT} (browser host)`); +}); diff --git a/docker-compose.yml b/docker-compose.yml index 414725d..ecec0be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,8 +91,17 @@ services: # Where the bot drives Chrome. Loopback for full/browser; on a remote bot # set CDP_HOST to the browser host's LAN IP (e.g. 192.168.10.9). CDP_HOST: ${CDP_HOST:-127.0.0.1} - depends_on: - - ollama + # Browser-control endpoint. The browser host serves it (BIND/PORT); a + # remote bot sets BROWSER_CONTROL_URL=http://:8777 so its + # controlBrowser tool posts there instead of running node locally. Empty + # on full/browser β†’ the tool runs chrome-control.mjs locally. + BROWSER_CONTROL_BIND: ${BROWSER_CONTROL_BIND:-0.0.0.0} + BROWSER_CONTROL_PORT: ${BROWSER_CONTROL_PORT:-8777} + BROWSER_CONTROL_URL: ${BROWSER_CONTROL_URL:-} + # No hard depends_on ollama: a browser-host (`docker compose up -d javis`) + # must NOT pull in Ollama. Full/bot layouts start it with a plain + # `docker compose up -d` (all services); the bridge tolerates Ollama warming + # up lazily, so start order doesn't matter. # GPU: accelerates Whisper STT (and anything else CUDA) in this container. # Verified: faster-whisper float16 works on the RTX 5050 (sm_120). devices: @@ -110,6 +119,9 @@ services: # Chrome CDP for a remote bot (JARVIS_ROLE=bot). Loopback by default; for a # LAN browser-host set CDP_PUBLISH_BIND=0.0.0.0 (internal network, no auth). - "${CDP_PUBLISH_BIND:-127.0.0.1}:${CDP_PORT:-9222}:9222" # Chrome CDP + # Browser-control endpoint a remote bot posts to (real xdotool input runs + # on THIS host). Published on the LAN for the browser-host layout. + - "${CDP_PUBLISH_BIND:-127.0.0.1}:${BROWSER_CONTROL_PORT:-8777}:8777" # control-server # The brain bridge is NOT published: it binds the container's loopback # (BRIDGE_HOST=127.0.0.1) and is only consumed by the bot in this same # container, so it needs no host port and stays unreachable off-container. diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 5ea64fb..9b49315 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -91,6 +91,19 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 +[program:control-server] +; Browser-control HTTP endpoint on the BROWSER HOST. A remote `bot` posts +; commands here so xdotool / CDP run on THIS machine (real input on this +; screen). Only meaningful in full/browser roles. Internal network only. +command=/app/docker/run-if-role.sh full,browser node /app/bot/scripts/stream-test/control-server.mjs +directory=/app/bot +priority=360 +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + [program:bot] command=/app/docker/run-if-role.sh full,bot /app/docker/run-bot.sh directory=/app/bot diff --git a/src/jarvis/tools/builtin/control_browser.py b/src/jarvis/tools/builtin/control_browser.py index 9c13498..84de475 100644 --- a/src/jarvis/tools/builtin/control_browser.py +++ b/src/jarvis/tools/builtin/control_browser.py @@ -101,15 +101,30 @@ class ControlBrowserTool(Tool): debug_log(f" πŸ–±οΈ controlBrowser {command[:120]}", "tools") # Human-input actions need time for the visible cursor move + char typing. timeout = 25 if action in _READ_ACTIONS else 90 + # Split deployment: when the browser (Chrome + X + xdotool) lives on a + # different machine, send the command to its control-server so the REAL + # input lands on that host's screen. Otherwise run chrome-control.mjs + # locally (all-in-one / browser-host layout). + control_url = os.environ.get("BROWSER_CONTROL_URL", "").strip() try: - proc = subprocess.run( - ["node", str(_NODE_SCRIPT), command], - capture_output=True, - text=True, - timeout=timeout, - env={**os.environ, "CDP_PORT": os.environ.get("CDP_PORT", "9222")}, - ) - data = json.loads((proc.stdout or "").strip() or "{}") + if control_url: + import urllib.request + req = urllib.request.Request( + control_url.rstrip("/") + "/control", + data=command.encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads((resp.read().decode("utf-8") or "").strip() or "{}") + else: + proc = subprocess.run( + ["node", str(_NODE_SCRIPT), command], + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, "CDP_PORT": os.environ.get("CDP_PORT", "9222")}, + ) + data = json.loads((proc.stdout or "").strip() or "{}") except Exception as e: return ToolExecutionResult(success=False, reply_text=f"λΈŒλΌμš°μ € μ œμ–΄μ— μ‹€νŒ¨ν–ˆμŠ΅λ‹ˆλ‹€: {e}")