feat: robust controlBrowser — real xdotool input, tabs/back/forward/popups

- Install xdotool + wmctrl (late Docker layer, preserves the melo cache) so
  the on-screen Chrome gets real X input (visible cursor, char-by-char typing)
  instead of synthetic events; falls back to the Playwright API if absent.
- Fix active-tab detection (probe document.visibilityState instead of assuming
  tab 0) so sequential ops target the right tab.
- Add back / forward / refresh; new/switch/close tabs via real keyboard
  (Ctrl+T / Ctrl+<n> / Ctrl+W) when xdotool is present.
- Auto-dismiss native JS dialogs; closePopups clears blank/popup tabs.
- Report broadcast (Go-Live) state in status from the turn context.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Dockerfile

@@ -65,6 +65,14 @@ RUN ls -d /opt/venv/lib/python*/site-packages/nvidia/cublas/lib \
 COPY docker/setup-melo.sh /app/docker/setup-melo.sh
 RUN bash /app/docker/setup-melo.sh
 
+# --- Human input + window management for the on-screen Chrome control tool.
+#     Placed AFTER the heavy melo layer so it doesn't bust that cache. xdotool
+#     injects real X pointer/keyboard events (visible cursor, char-by-char
+#     typing) into the broadcast; wmctrl lists/moves windows. ---
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      xdotool wmctrl \
+    && rm -rf /var/lib/apt/lists/*
+
 # --- Discord bot deps (cache layer on lockfile) ---
 COPY bot/package.json bot/bun.lock /app/bot/
 RUN cd /app/bot && bun install --frozen-lockfile || bun install

bot/scripts/stream-test/chrome-control.mjs

@@ -4,182 +4,213 @@
 //
 // Connects to the on-screen Chrome over CDP (so every action is visible on the
 // broadcast) and performs ONE command, printing a JSON result on stdout for the
-// Python `controlBrowser` tool to wrap. READ actions use the CDP/DOM API;
+// Python `controlBrowser` tool to wrap.
-// ACTION actions (navigate/click/type/scroll) drive real input via xdotool
+//
-// (human.mjs) so the cursor visibly moves and text is typed one char at a time,
+// Input style: when xdotool is available the pointer/keyboard ACTIONS
-// exactly as a person would — never synthetic DOM events.
+// (navigate via the omnibox, click, type, scroll, tab keys) are driven as REAL
+// X input — the cursor visibly moves and text is typed one character at a time,
+// exactly as a person would. If xdotool is missing it falls back to the
+// Playwright/CDP API so the action still happens (just without a visible
+// cursor). READ actions always use the CDP/DOM API.
 //
 // Commands (json): { "action": "<name>", ...params }
-//   status                              -> broadcast/browser readiness + tab list
+//   status | listTabs
-//   listTabs                            -> [{index,url,title,active}]
+//   navigate {url} | back | forward | refresh
-//   navigate   {url, human?}            -> open url in the active tab
+//   newTab {url?} | closeTab {index?} | activateTab {index} | closePopups
-//   newTab     {url?}                   -> open a new tab (optionally to url)
+//   click {selector} | type {text, selector?} | scroll {dir, notches?}
-//   closeTab   {index}                  -> close tab by index
+//   pressKey {key} | screenshot {path}
-//   activateTab{index}                  -> bring tab to front
-//   closePopups                         -> close blank/popup tabs, keep the active one
-//   click      {selector, human?}       -> click an element (human cursor by default)
-//   type       {text, selector?, human?}-> type text char-by-char (human by default)
-//   scroll     {dir, notches?}          -> wheel scroll (dir: "down"|"up")
-//   pressKey   {key}                    -> press a single key (xdotool keysym)
-//   screenshot {path}                   -> save a PNG of the active tab
 import { chromium } from 'playwright';
-import {
+import { execFileSync } from 'node:child_process';
-  humanClick,
+import * as human from './human.mjs';
-  humanType,
-  pressKey,
-  humanScroll,
-  navigateOmnibox,
-} from './human.mjs';
 
 const CDP = process.env.CDP_PORT || '9222';
 const CDP_HOST = process.env.CDP_HOST || '127.0.0.1';
 const out = (o) => process.stdout.write(JSON.stringify(o));
 
+const HAS_XDOTOOL = (() => {
+  try { execFileSync('which', ['xdotool'], { stdio: 'ignore' }); return true; }
+  catch { return false; }
+})();
+
 let cmd;
-try {
+try { cmd = JSON.parse(process.argv[2] || '{}'); }
-  cmd = JSON.parse(process.argv[2] || '{}');
+catch (e) { out({ ok: false, error: `bad command json: ${e?.message || e}` }); process.exit(1); }
-} catch (e) {
-  out({ ok: false, error: `bad command json: ${e?.message || e}` });
-  process.exit(1);
-}
 const action = String(cmd.action || '').trim();
-if (!action) {
+if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); }
-  out({ ok: false, error: 'no action' });
+
-  process.exit(1);
+const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`);
+
+// The genuinely-active tab is the one whose document is visible. Playwright has
+// no "active page" accessor over CDP, so probe visibilityState (fixes treating
+// tab 0 as active and breaking sequential ops on a specific tab).
+async function pickActive(pages) {
+  for (const p of pages) {
+    try { if (await p.evaluate(() => document.visibilityState === 'visible')) return p; }
+    catch { /* page may be closing */ }
+  }
+  return pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0];
 }
 
-const tabInfo = async (pages, active) => {
+async function tabInfo(pages, active) {
   const list = [];
   for (let i = 0; i < pages.length; i++) {
     const p = pages[i];
-    list.push({
+    list.push({ index: i, url: p.url(), title: await p.title().catch(() => ''), active: p === active });
-      index: i,
-      url: p.url(),
-      title: await p.title().catch(() => ''),
-      active: p === active,
-    });
   }
   return list;
-};
+}
 
 let b;
 try {
   b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`);
   const ctx = b.contexts()[0];
   if (!ctx) throw new Error('no browser context (is Chrome running?)');
-  const pages = ctx.pages();
+  let pages = ctx.pages();
-  // The "active" tab: the most recently fronted page. Playwright has no direct
+  if (!pages.length) pages = [await ctx.newPage()];
-  // accessor, so we treat the first visible page as active and let activateTab
+  let page = await pickActive(pages);
-  // change focus explicitly.
-  let page = pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0] || (await ctx.newPage());
   page.setDefaultTimeout(20000);
+  // Auto-dismiss native JS dialogs (alert/confirm/beforeunload) so a popup can
+  // never wedge the page.
+  page.on('dialog', (d) => d.dismiss().catch(() => {}));
+
+  const front = async (p) => { await p.bringToFront().catch(() => {}); };
+  const reload = async () => { await page.reload({ waitUntil: 'domcontentloaded' }).catch(() => {}); };
 
   switch (action) {
-    case 'status': {
+    case 'status':
-      out({ ok: true, browserOpen: true, tabCount: pages.length, tabs: await tabInfo(pages, page) });
+    case 'listTabs':
+      await front(page);
+      out({ ok: true, browserOpen: true, xdotool: HAS_XDOTOOL, tabCount: pages.length, tabs: await tabInfo(pages, page) });
       break;
-    }
+
-    case 'listTabs': {
-      out({ ok: true, tabs: await tabInfo(pages, page) });
-      break;
-    }
     case 'navigate': {
       const url = String(cmd.url || '').trim();
       if (!url) throw new Error('navigate: no url');
-      const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
+      await front(page);
-      await page.bringToFront().catch(() => {});
+      if (HAS_XDOTOOL && cmd.human !== false) {
-      if (cmd.human === false) {
+        try { await human.navigateOmnibox(norm(url)); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
-        await page.goto(target, { waitUntil: 'domcontentloaded' });
+        catch { await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); }
       } else {
-        // Type the URL into the omnibox char-by-char with a real cursor.
+        await page.goto(norm(url), { waitUntil: 'domcontentloaded' });
-        await navigateOmnibox(target);
-        await page.waitForLoadState('domcontentloaded').catch(() => {});
       }
-      out({ ok: true, url: page.url(), title: await page.title().catch(() => '') });
+      out({ ok: true, url: page.url(), title: await page.title().catch(() => ''), input: HAS_XDOTOOL ? 'human' : 'api' });
       break;
     }
+
+    case 'back': await front(page); await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break;
+    case 'forward': await front(page); await page.goForward({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break;
+    case 'refresh': await front(page); await reload(); out({ ok: true, url: page.url() }); break;
+
     case 'newTab': {
-      const np = await ctx.newPage();
+      let np;
-      await np.bringToFront().catch(() => {});
+      if (HAS_XDOTOOL && cmd.human !== false) {
-      if (cmd.url) {
+        await front(page);
-        const target = /^https?:\/\//i.test(cmd.url) ? cmd.url : `https://${cmd.url}`;
+        try { await human.pressKey('ctrl+t'); } catch { /* fall through */ }
-        await np.goto(target, { waitUntil: 'domcontentloaded' }).catch(() => {});
+        await page.waitForTimeout(500);
+        const after = ctx.pages();
+        np = after[after.length - 1];
       }
-      out({ ok: true, index: ctx.pages().length - 1, url: np.url() });
+      if (!np) np = await ctx.newPage(); // API fallback / no xdotool
-      break;
+      await front(np);
-    }
+      if (cmd.url) {
-    case 'closeTab': {
+        if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(norm(cmd.url)); } catch { await np.goto(norm(cmd.url)).catch(() => {}); } }
-      const idx = Number(cmd.index);
+        else await np.goto(norm(cmd.url), { waitUntil: 'domcontentloaded' }).catch(() => {});
-      if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
+      }
-      await pages[idx].close();
+      out({ ok: true, index: ctx.pages().indexOf(np), url: np.url(), input: HAS_XDOTOOL ? 'human' : 'api' });
-      out({ ok: true, closed: idx, remaining: ctx.pages().length });
       break;
     }
+
     case 'activateTab': {
       const idx = Number(cmd.index);
       if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index');
-      await pages[idx].bringToFront();
+      // Real keyboard: Ctrl+<1..8> selects that tab, Ctrl+9 the last.
+      if (HAS_XDOTOOL && cmd.human !== false && idx < 8) {
+        await front(pages[0]);
+        try { await human.pressKey(`ctrl+${idx + 1}`); } catch { await pages[idx].bringToFront(); }
+      } else {
+        await pages[idx].bringToFront();
+      }
       out({ ok: true, active: idx, url: pages[idx].url() });
       break;
     }
+
+    case 'closeTab': {
+      const idx = Number.isInteger(Number(cmd.index)) ? Number(cmd.index) : pages.indexOf(page);
+      if (idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
+      if (HAS_XDOTOOL && cmd.human !== false && idx < 8) {
+        await front(pages[0]);
+        try { await human.pressKey(`ctrl+${idx + 1}`); await human.pressKey('ctrl+w'); }
+        catch { await pages[idx].close(); }
+      } else {
+        await pages[idx].close();
+      }
+      out({ ok: true, closed: idx, remaining: ctx.pages().length });
+      break;
+    }
+
     case 'closePopups': {
-      // Close throwaway/popup tabs (about:blank or anything that isn't the
+      // Close popup / blank / extra tabs, keeping the active content tab.
-      // active content tab), keeping the active one.
       let closed = 0;
       for (const p of pages) {
         if (p === page) continue;
         const u = p.url();
-        if (!u || u === 'about:blank' || cmd.all) {
+        if (cmd.all || !u || u === 'about:blank') { await p.close().catch(() => {}); closed++; }
-          await p.close().catch(() => {});
-          closed++;
-        }
       }
       out({ ok: true, closed, remaining: ctx.pages().length });
       break;
     }
+
     case 'click': {
       const selector = String(cmd.selector || '').trim();
       if (!selector) throw new Error('click: no selector');
-      await page.bringToFront().catch(() => {});
+      await front(page);
       const locator = page.locator(selector).first();
-      if (cmd.human === false) await locator.click();
+      if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click(); } }
-      else await humanClick(page, locator);
+      else await locator.click();
       out({ ok: true });
       break;
     }
+
     case 'type': {
       const text = String(cmd.text ?? '');
-      await page.bringToFront().catch(() => {});
+      await front(page);
       if (cmd.selector) {
         const locator = page.locator(String(cmd.selector)).first();
-        if (cmd.human === false) { await locator.fill(text); out({ ok: true }); break; }
+        if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click().catch(() => {}); } }
-        await humanClick(page, locator); // focus the field with a real click first
+        else { await locator.fill(text); out({ ok: true, input: 'api' }); break; }
       }
-      await humanType(text);
+      if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanType(text); } catch { await page.keyboard.type(text, { delay: 80 }); } }
-      out({ ok: true });
+      else await page.keyboard.type(text, { delay: 80 });
+      out({ ok: true, input: HAS_XDOTOOL ? 'human' : 'api' });
       break;
     }
+
     case 'scroll': {
       const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1;
-      await humanScroll(page, dir, Number(cmd.notches) || 5);
+      if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanScroll(page, dir, Number(cmd.notches) || 5); } catch { await page.mouse.wheel(0, dir * 600); } }
+      else await page.mouse.wheel(0, dir * (Number(cmd.notches) || 5) * 120);
       out({ ok: true });
       break;
     }
+
     case 'pressKey': {
       const key = String(cmd.key || '').trim();
       if (!key) throw new Error('pressKey: no key');
-      await pressKey(key);
+      if (HAS_XDOTOOL) { try { await human.pressKey(key); } catch { await page.keyboard.press(key); } }
+      else await page.keyboard.press(key);
       out({ ok: true });
       break;
     }
+
     case 'screenshot': {
       const path = String(cmd.path || '').trim();
       if (!path) throw new Error('screenshot: no path');
-      await page.bringToFront().catch(() => {});
+      await front(page);
       await page.screenshot({ path });
       out({ ok: true, path });
       break;
     }
+
     default:
       out({ ok: false, error: `unknown action: ${action}` });
       await b.close();

src/jarvis/tools/builtin/control_browser.py

@@ -57,9 +57,10 @@ class ControlBrowserTool(Tool):
                 "action": {
                     "type": "string",
                     "enum": [
-                        "status", "listTabs", "navigate", "newTab", "closeTab",
+                        "status", "listTabs", "navigate", "back", "forward",
-                        "activateTab", "closePopups", "click", "type", "scroll",
+                        "refresh", "newTab", "closeTab", "activateTab",
-                        "pressKey", "screenshot",
+                        "closePopups", "click", "type", "scroll", "pressKey",
+                        "screenshot",
                     ],
                     "description": "What to do in the browser.",
                 },
@@ -116,6 +117,12 @@ class ControlBrowserTool(Tool):
         # Return a factual summary of what ACTUALLY happened so the reply engine
         # describes the real outcome and doesn't confabulate.
         summary = self._summarise(action, args, data)
+        if action == "status":
+            # The broadcast (Go-Live) state lives in the bot runtime, surfaced
+            # to the tool via the turn context — report it alongside the tabs.
+            bc = getattr(context, "broadcasting", None)
+            state = "켜짐" if bc else ("꺼짐" if bc is not None else "알 수 없음")
+            summary = f"📡 방송: {state}\n{summary}"
         return ToolExecutionResult(success=True, reply_text=summary)
 
     @staticmethod