From 109dbc7e16fc9cc6081d1ea3e355d0807558e0d8 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sun, 14 Jun 2026 02:33:59 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20robust=20controlBrowser=20=E2=80=94=20r?= =?UTF-8?q?eal=20xdotool=20input,=20tabs/back/forward/popups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Install xdotool + wmctrl (late Docker layer, preserves the melo cache) so the on-screen Chrome gets real X input (visible cursor, char-by-char typing) instead of synthetic events; falls back to the Playwright API if absent. - Fix active-tab detection (probe document.visibilityState instead of assuming tab 0) so sequential ops target the right tab. - Add back / forward / refresh; new/switch/close tabs via real keyboard (Ctrl+T / Ctrl+ / Ctrl+W) when xdotool is present. - Auto-dismiss native JS dialogs; closePopups clears blank/popup tabs. - Report broadcast (Go-Live) state in status from the turn context. Co-Authored-By: Claude Opus 4.7 --- Dockerfile | 8 + bot/scripts/stream-test/chrome-control.mjs | 213 +++++++++++--------- src/jarvis/tools/builtin/control_browser.py | 13 +- 3 files changed, 140 insertions(+), 94 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc6a873..64ee9ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,6 +65,14 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \ COPY docker/setup-melo.sh /app/docker/setup-melo.sh RUN bash /app/docker/setup-melo.sh +# --- Human input + window management for the on-screen Chrome control tool. +# Placed AFTER the heavy melo layer so it doesn't bust that cache. xdotool +# injects real X pointer/keyboard events (visible cursor, char-by-char +# typing) into the broadcast; wmctrl lists/moves windows. --- +RUN apt-get update && apt-get install -y --no-install-recommends \ + xdotool wmctrl \ + && rm -rf /var/lib/apt/lists/* + # --- Discord bot deps (cache layer on lockfile) --- COPY bot/package.json bot/bun.lock /app/bot/ RUN cd /app/bot && bun install --frozen-lockfile || bun install diff --git a/bot/scripts/stream-test/chrome-control.mjs b/bot/scripts/stream-test/chrome-control.mjs index 3746002..e6031d5 100644 --- a/bot/scripts/stream-test/chrome-control.mjs +++ b/bot/scripts/stream-test/chrome-control.mjs @@ -4,182 +4,213 @@ // // Connects to the on-screen Chrome over CDP (so every action is visible on the // broadcast) and performs ONE command, printing a JSON result on stdout for the -// Python `controlBrowser` tool to wrap. READ actions use the CDP/DOM API; -// ACTION actions (navigate/click/type/scroll) drive real input via xdotool -// (human.mjs) so the cursor visibly moves and text is typed one char at a time, -// exactly as a person would — never synthetic DOM events. +// Python `controlBrowser` tool to wrap. +// +// Input style: when xdotool is available the pointer/keyboard ACTIONS +// (navigate via the omnibox, click, type, scroll, tab keys) are driven as REAL +// X input — the cursor visibly moves and text is typed one character at a time, +// exactly as a person would. If xdotool is missing it falls back to the +// Playwright/CDP API so the action still happens (just without a visible +// cursor). READ actions always use the CDP/DOM API. // // Commands (json): { "action": "", ...params } -// status -> broadcast/browser readiness + tab list -// listTabs -> [{index,url,title,active}] -// navigate {url, human?} -> open url in the active tab -// newTab {url?} -> open a new tab (optionally to url) -// closeTab {index} -> close tab by index -// activateTab{index} -> bring tab to front -// closePopups -> close blank/popup tabs, keep the active one -// click {selector, human?} -> click an element (human cursor by default) -// type {text, selector?, human?}-> type text char-by-char (human by default) -// scroll {dir, notches?} -> wheel scroll (dir: "down"|"up") -// pressKey {key} -> press a single key (xdotool keysym) -// screenshot {path} -> save a PNG of the active tab +// status | listTabs +// navigate {url} | back | forward | refresh +// newTab {url?} | closeTab {index?} | activateTab {index} | closePopups +// click {selector} | type {text, selector?} | scroll {dir, notches?} +// pressKey {key} | screenshot {path} import { chromium } from 'playwright'; -import { - humanClick, - humanType, - pressKey, - humanScroll, - navigateOmnibox, -} from './human.mjs'; +import { execFileSync } from 'node:child_process'; +import * as human from './human.mjs'; const CDP = process.env.CDP_PORT || '9222'; const CDP_HOST = process.env.CDP_HOST || '127.0.0.1'; const out = (o) => process.stdout.write(JSON.stringify(o)); +const HAS_XDOTOOL = (() => { + try { execFileSync('which', ['xdotool'], { stdio: 'ignore' }); return true; } + catch { return false; } +})(); + let cmd; -try { - cmd = JSON.parse(process.argv[2] || '{}'); -} catch (e) { - out({ ok: false, error: `bad command json: ${e?.message || e}` }); - process.exit(1); -} +try { cmd = JSON.parse(process.argv[2] || '{}'); } +catch (e) { out({ ok: false, error: `bad command json: ${e?.message || e}` }); process.exit(1); } const action = String(cmd.action || '').trim(); -if (!action) { - out({ ok: false, error: 'no action' }); - process.exit(1); +if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); } + +const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`); + +// The genuinely-active tab is the one whose document is visible. Playwright has +// no "active page" accessor over CDP, so probe visibilityState (fixes treating +// tab 0 as active and breaking sequential ops on a specific tab). +async function pickActive(pages) { + for (const p of pages) { + try { if (await p.evaluate(() => document.visibilityState === 'visible')) return p; } + catch { /* page may be closing */ } + } + return pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0]; } -const tabInfo = async (pages, active) => { +async function tabInfo(pages, active) { const list = []; for (let i = 0; i < pages.length; i++) { const p = pages[i]; - list.push({ - index: i, - url: p.url(), - title: await p.title().catch(() => ''), - active: p === active, - }); + list.push({ index: i, url: p.url(), title: await p.title().catch(() => ''), active: p === active }); } return list; -}; +} let b; try { b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`); const ctx = b.contexts()[0]; if (!ctx) throw new Error('no browser context (is Chrome running?)'); - const pages = ctx.pages(); - // The "active" tab: the most recently fronted page. Playwright has no direct - // accessor, so we treat the first visible page as active and let activateTab - // change focus explicitly. - let page = pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0] || (await ctx.newPage()); + let pages = ctx.pages(); + if (!pages.length) pages = [await ctx.newPage()]; + let page = await pickActive(pages); page.setDefaultTimeout(20000); + // Auto-dismiss native JS dialogs (alert/confirm/beforeunload) so a popup can + // never wedge the page. + page.on('dialog', (d) => d.dismiss().catch(() => {})); + + const front = async (p) => { await p.bringToFront().catch(() => {}); }; + const reload = async () => { await page.reload({ waitUntil: 'domcontentloaded' }).catch(() => {}); }; switch (action) { - case 'status': { - out({ ok: true, browserOpen: true, tabCount: pages.length, tabs: await tabInfo(pages, page) }); + case 'status': + case 'listTabs': + await front(page); + out({ ok: true, browserOpen: true, xdotool: HAS_XDOTOOL, tabCount: pages.length, tabs: await tabInfo(pages, page) }); break; - } - case 'listTabs': { - out({ ok: true, tabs: await tabInfo(pages, page) }); - break; - } + case 'navigate': { const url = String(cmd.url || '').trim(); if (!url) throw new Error('navigate: no url'); - const target = /^https?:\/\//i.test(url) ? url : `https://${url}`; - await page.bringToFront().catch(() => {}); - if (cmd.human === false) { - await page.goto(target, { waitUntil: 'domcontentloaded' }); + await front(page); + if (HAS_XDOTOOL && cmd.human !== false) { + try { await human.navigateOmnibox(norm(url)); await page.waitForLoadState('domcontentloaded').catch(() => {}); } + catch { await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); } } else { - // Type the URL into the omnibox char-by-char with a real cursor. - await navigateOmnibox(target); - await page.waitForLoadState('domcontentloaded').catch(() => {}); + await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); } - out({ ok: true, url: page.url(), title: await page.title().catch(() => '') }); + out({ ok: true, url: page.url(), title: await page.title().catch(() => ''), input: HAS_XDOTOOL ? 'human' : 'api' }); break; } + + case 'back': await front(page); await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break; + case 'forward': await front(page); await page.goForward({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break; + case 'refresh': await front(page); await reload(); out({ ok: true, url: page.url() }); break; + case 'newTab': { - const np = await ctx.newPage(); - await np.bringToFront().catch(() => {}); - if (cmd.url) { - const target = /^https?:\/\//i.test(cmd.url) ? cmd.url : `https://${cmd.url}`; - await np.goto(target, { waitUntil: 'domcontentloaded' }).catch(() => {}); + let np; + if (HAS_XDOTOOL && cmd.human !== false) { + await front(page); + try { await human.pressKey('ctrl+t'); } catch { /* fall through */ } + await page.waitForTimeout(500); + const after = ctx.pages(); + np = after[after.length - 1]; } - out({ ok: true, index: ctx.pages().length - 1, url: np.url() }); - break; - } - case 'closeTab': { - const idx = Number(cmd.index); - if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index'); - await pages[idx].close(); - out({ ok: true, closed: idx, remaining: ctx.pages().length }); + if (!np) np = await ctx.newPage(); // API fallback / no xdotool + await front(np); + if (cmd.url) { + if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(norm(cmd.url)); } catch { await np.goto(norm(cmd.url)).catch(() => {}); } } + else await np.goto(norm(cmd.url), { waitUntil: 'domcontentloaded' }).catch(() => {}); + } + out({ ok: true, index: ctx.pages().indexOf(np), url: np.url(), input: HAS_XDOTOOL ? 'human' : 'api' }); break; } + case 'activateTab': { const idx = Number(cmd.index); if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index'); - await pages[idx].bringToFront(); + // Real keyboard: Ctrl+<1..8> selects that tab, Ctrl+9 the last. + if (HAS_XDOTOOL && cmd.human !== false && idx < 8) { + await front(pages[0]); + try { await human.pressKey(`ctrl+${idx + 1}`); } catch { await pages[idx].bringToFront(); } + } else { + await pages[idx].bringToFront(); + } out({ ok: true, active: idx, url: pages[idx].url() }); break; } + + case 'closeTab': { + const idx = Number.isInteger(Number(cmd.index)) ? Number(cmd.index) : pages.indexOf(page); + if (idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index'); + if (HAS_XDOTOOL && cmd.human !== false && idx < 8) { + await front(pages[0]); + try { await human.pressKey(`ctrl+${idx + 1}`); await human.pressKey('ctrl+w'); } + catch { await pages[idx].close(); } + } else { + await pages[idx].close(); + } + out({ ok: true, closed: idx, remaining: ctx.pages().length }); + break; + } + case 'closePopups': { - // Close throwaway/popup tabs (about:blank or anything that isn't the - // active content tab), keeping the active one. + // Close popup / blank / extra tabs, keeping the active content tab. let closed = 0; for (const p of pages) { if (p === page) continue; const u = p.url(); - if (!u || u === 'about:blank' || cmd.all) { - await p.close().catch(() => {}); - closed++; - } + if (cmd.all || !u || u === 'about:blank') { await p.close().catch(() => {}); closed++; } } out({ ok: true, closed, remaining: ctx.pages().length }); break; } + case 'click': { const selector = String(cmd.selector || '').trim(); if (!selector) throw new Error('click: no selector'); - await page.bringToFront().catch(() => {}); + await front(page); const locator = page.locator(selector).first(); - if (cmd.human === false) await locator.click(); - else await humanClick(page, locator); + if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click(); } } + else await locator.click(); out({ ok: true }); break; } + case 'type': { const text = String(cmd.text ?? ''); - await page.bringToFront().catch(() => {}); + await front(page); if (cmd.selector) { const locator = page.locator(String(cmd.selector)).first(); - if (cmd.human === false) { await locator.fill(text); out({ ok: true }); break; } - await humanClick(page, locator); // focus the field with a real click first + if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click().catch(() => {}); } } + else { await locator.fill(text); out({ ok: true, input: 'api' }); break; } } - await humanType(text); - out({ ok: true }); + if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanType(text); } catch { await page.keyboard.type(text, { delay: 80 }); } } + else await page.keyboard.type(text, { delay: 80 }); + out({ ok: true, input: HAS_XDOTOOL ? 'human' : 'api' }); break; } + case 'scroll': { const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1; - await humanScroll(page, dir, Number(cmd.notches) || 5); + if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanScroll(page, dir, Number(cmd.notches) || 5); } catch { await page.mouse.wheel(0, dir * 600); } } + else await page.mouse.wheel(0, dir * (Number(cmd.notches) || 5) * 120); out({ ok: true }); break; } + case 'pressKey': { const key = String(cmd.key || '').trim(); if (!key) throw new Error('pressKey: no key'); - await pressKey(key); + if (HAS_XDOTOOL) { try { await human.pressKey(key); } catch { await page.keyboard.press(key); } } + else await page.keyboard.press(key); out({ ok: true }); break; } + case 'screenshot': { const path = String(cmd.path || '').trim(); if (!path) throw new Error('screenshot: no path'); - await page.bringToFront().catch(() => {}); + await front(page); await page.screenshot({ path }); out({ ok: true, path }); break; } + default: out({ ok: false, error: `unknown action: ${action}` }); await b.close(); diff --git a/src/jarvis/tools/builtin/control_browser.py b/src/jarvis/tools/builtin/control_browser.py index 6006244..5e244f9 100644 --- a/src/jarvis/tools/builtin/control_browser.py +++ b/src/jarvis/tools/builtin/control_browser.py @@ -57,9 +57,10 @@ class ControlBrowserTool(Tool): "action": { "type": "string", "enum": [ - "status", "listTabs", "navigate", "newTab", "closeTab", - "activateTab", "closePopups", "click", "type", "scroll", - "pressKey", "screenshot", + "status", "listTabs", "navigate", "back", "forward", + "refresh", "newTab", "closeTab", "activateTab", + "closePopups", "click", "type", "scroll", "pressKey", + "screenshot", ], "description": "What to do in the browser.", }, @@ -116,6 +117,12 @@ class ControlBrowserTool(Tool): # Return a factual summary of what ACTUALLY happened so the reply engine # describes the real outcome and doesn't confabulate. summary = self._summarise(action, args, data) + if action == "status": + # The broadcast (Go-Live) state lives in the bot runtime, surfaced + # to the tool via the turn context — report it alongside the tabs. + bc = getattr(context, "broadcasting", None) + state = "켜짐" if bc else ("꺼짐" if bc is not None else "알 수 없음") + summary = f"📡 방송: {state}\n{summary}" return ToolExecutionResult(success=True, reply_text=summary) @staticmethod