feat: controlBrowser tool — human-operable on-screen Chrome

Adds a general browser-control tool (navigate to any site, list/open/close/
switch tabs, close popups, click, type, scroll, screenshot) for the Go-Live
Chrome, on top of the existing CDP + xdotool human-input layer (visible
cursor, char-by-char typing). Closes the gap where "open Naver" had no tool
and the model confabulated success. Also adds a system-prompt rule against
claiming actions no tool actually performed.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

bot/scripts/stream-test/chrome-control.mjs

@@ -0,0 +1,193 @@
+// Human-operable Chrome control for the on-screen (Go-Live) browser.
+//
+//   node chrome-control.mjs '<json-command>'
+//
+// Connects to the on-screen Chrome over CDP (so every action is visible on the
+// broadcast) and performs ONE command, printing a JSON result on stdout for the
+// Python `controlBrowser` tool to wrap. READ actions use the CDP/DOM API;
+// ACTION actions (navigate/click/type/scroll) drive real input via xdotool
+// (human.mjs) so the cursor visibly moves and text is typed one char at a time,
+// exactly as a person would — never synthetic DOM events.
+//
+// Commands (json): { "action": "<name>", ...params }
+//   status                              -> broadcast/browser readiness + tab list
+//   listTabs                            -> [{index,url,title,active}]
+//   navigate   {url, human?}            -> open url in the active tab
+//   newTab     {url?}                   -> open a new tab (optionally to url)
+//   closeTab   {index}                  -> close tab by index
+//   activateTab{index}                  -> bring tab to front
+//   closePopups                         -> close blank/popup tabs, keep the active one
+//   click      {selector, human?}       -> click an element (human cursor by default)
+//   type       {text, selector?, human?}-> type text char-by-char (human by default)
+//   scroll     {dir, notches?}          -> wheel scroll (dir: "down"|"up")
+//   pressKey   {key}                    -> press a single key (xdotool keysym)
+//   screenshot {path}                   -> save a PNG of the active tab
+import { chromium } from 'playwright';
+import {
+  humanClick,
+  humanType,
+  pressKey,
+  humanScroll,
+  navigateOmnibox,
+} from './human.mjs';
+
+const CDP = process.env.CDP_PORT || '9222';
+const CDP_HOST = process.env.CDP_HOST || '127.0.0.1';
+const out = (o) => process.stdout.write(JSON.stringify(o));
+
+let cmd;
+try {
+  cmd = JSON.parse(process.argv[2] || '{}');
+} catch (e) {
+  out({ ok: false, error: `bad command json: ${e?.message || e}` });
+  process.exit(1);
+}
+const action = String(cmd.action || '').trim();
+if (!action) {
+  out({ ok: false, error: 'no action' });
+  process.exit(1);
+}
+
+const tabInfo = async (pages, active) => {
+  const list = [];
+  for (let i = 0; i < pages.length; i++) {
+    const p = pages[i];
+    list.push({
+      index: i,
+      url: p.url(),
+      title: await p.title().catch(() => ''),
+      active: p === active,
+    });
+  }
+  return list;
+};
+
+let b;
+try {
+  b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`);
+  const ctx = b.contexts()[0];
+  if (!ctx) throw new Error('no browser context (is Chrome running?)');
+  const pages = ctx.pages();
+  // The "active" tab: the most recently fronted page. Playwright has no direct
+  // accessor, so we treat the first visible page as active and let activateTab
+  // change focus explicitly.
+  let page = pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0] || (await ctx.newPage());
+  page.setDefaultTimeout(20000);
+
+  switch (action) {
+    case 'status': {
+      out({ ok: true, browserOpen: true, tabCount: pages.length, tabs: await tabInfo(pages, page) });
+      break;
+    }
+    case 'listTabs': {
+      out({ ok: true, tabs: await tabInfo(pages, page) });
+      break;
+    }
+    case 'navigate': {
+      const url = String(cmd.url || '').trim();
+      if (!url) throw new Error('navigate: no url');
+      const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
+      await page.bringToFront().catch(() => {});
+      if (cmd.human === false) {
+        await page.goto(target, { waitUntil: 'domcontentloaded' });
+      } else {
+        // Type the URL into the omnibox char-by-char with a real cursor.
+        await navigateOmnibox(target);
+        await page.waitForLoadState('domcontentloaded').catch(() => {});
+      }
+      out({ ok: true, url: page.url(), title: await page.title().catch(() => '') });
+      break;
+    }
+    case 'newTab': {
+      const np = await ctx.newPage();
+      await np.bringToFront().catch(() => {});
+      if (cmd.url) {
+        const target = /^https?:\/\//i.test(cmd.url) ? cmd.url : `https://${cmd.url}`;
+        await np.goto(target, { waitUntil: 'domcontentloaded' }).catch(() => {});
+      }
+      out({ ok: true, index: ctx.pages().length - 1, url: np.url() });
+      break;
+    }
+    case 'closeTab': {
+      const idx = Number(cmd.index);
+      if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
+      await pages[idx].close();
+      out({ ok: true, closed: idx, remaining: ctx.pages().length });
+      break;
+    }
+    case 'activateTab': {
+      const idx = Number(cmd.index);
+      if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index');
+      await pages[idx].bringToFront();
+      out({ ok: true, active: idx, url: pages[idx].url() });
+      break;
+    }
+    case 'closePopups': {
+      // Close throwaway/popup tabs (about:blank or anything that isn't the
+      // active content tab), keeping the active one.
+      let closed = 0;
+      for (const p of pages) {
+        if (p === page) continue;
+        const u = p.url();
+        if (!u || u === 'about:blank' || cmd.all) {
+          await p.close().catch(() => {});
+          closed++;
+        }
+      }
+      out({ ok: true, closed, remaining: ctx.pages().length });
+      break;
+    }
+    case 'click': {
+      const selector = String(cmd.selector || '').trim();
+      if (!selector) throw new Error('click: no selector');
+      await page.bringToFront().catch(() => {});
+      const locator = page.locator(selector).first();
+      if (cmd.human === false) await locator.click();
+      else await humanClick(page, locator);
+      out({ ok: true });
+      break;
+    }
+    case 'type': {
+      const text = String(cmd.text ?? '');
+      await page.bringToFront().catch(() => {});
+      if (cmd.selector) {
+        const locator = page.locator(String(cmd.selector)).first();
+        if (cmd.human === false) { await locator.fill(text); out({ ok: true }); break; }
+        await humanClick(page, locator); // focus the field with a real click first
+      }
+      await humanType(text);
+      out({ ok: true });
+      break;
+    }
+    case 'scroll': {
+      const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1;
+      await humanScroll(page, dir, Number(cmd.notches) || 5);
+      out({ ok: true });
+      break;
+    }
+    case 'pressKey': {
+      const key = String(cmd.key || '').trim();
+      if (!key) throw new Error('pressKey: no key');
+      await pressKey(key);
+      out({ ok: true });
+      break;
+    }
+    case 'screenshot': {
+      const path = String(cmd.path || '').trim();
+      if (!path) throw new Error('screenshot: no path');
+      await page.bringToFront().catch(() => {});
+      await page.screenshot({ path });
+      out({ ok: true, path });
+      break;
+    }
+    default:
+      out({ ok: false, error: `unknown action: ${action}` });
+      await b.close();
+      process.exit(1);
+  }
+  await b.close();
+} catch (e) {
+  try { await b?.close(); } catch { /* ignore */ }
+  out({ ok: false, error: String(e?.message || e) });
+  process.exit(1);
+}

src/jarvis/system_prompt.py

@@ -36,6 +36,10 @@ _SYSTEM_PROMPT_TEMPLATE: str = (
     "butler clichés, and never address the user as 'sir', 'madam', 'my liege', or similar. "
     "Never stack multiple jokes in one reply. "
     "Be concise, conversational, and actionable. "
+    "Never claim you performed an action — opening a website, navigating the browser, "
+    "playing or showing something on screen, changing a setting, sending a message — unless a "
+    "tool actually did it this turn and reported success. If you have no tool for what was asked, "
+    "or the tool failed, say so plainly; do not narrate a success that did not happen. "
     "Never answer with a bare greeting like 'Hey there!', 'Hi!', 'Hello, how can I help you?', "
     "'I hope you have a relaxing time today', or 'I'm here and ready to chat'. Always engage "
     "with the user's actual prompt, and when the 'Information the user has shared…' section is "

src/jarvis/tools/builtin/control_browser.py

@@ -0,0 +1,139 @@
+"""Operate the on-screen (Go-Live) Chrome like a person would.
+
+Only meaningful in screen-share mode (``STREAM_BROWSER`` true): it drives the
+on-screen Chrome via the Node CDP helper (``chrome-control.mjs``) so every
+action is visible on the broadcast. Pointer moves, clicks and typing are real
+xdotool input (visible cursor, char-by-char typing), never synthetic DOM
+events.
+
+This is the general browser-control tool (navigate to any site, manage
+windows/tabs, click, type, scroll, screenshot). ``browseAndPlay`` remains the
+YouTube-playback shortcut.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from ..base import Tool, ToolContext
+from ..types import ToolExecutionResult
+from ...debug import debug_log
+
+# .../owner/src/jarvis/tools/builtin/control_browser.py -> parents[4] == .../owner
+_REPO_ROOT = Path(__file__).resolve().parents[4]
+_NODE_SCRIPT = _REPO_ROOT / "bot" / "scripts" / "stream-test" / "chrome-control.mjs"
+
+# Actions that don't change anything (safe, fast, no human-input latency).
+_READ_ACTIONS = {"status", "listTabs"}
+
+
+class ControlBrowserTool(Tool):
+    """Drive the on-screen Chrome: navigate, manage tabs, click, type, scroll."""
+
+    @property
+    def name(self) -> str:
+        return "controlBrowser"
+
+    @property
+    def description(self) -> str:
+        return (
+            "Operate the on-screen Chrome shown on the broadcast like a person: open a "
+            "website/URL, list/open/close/switch tabs, close popups, click an element, "
+            "type text, scroll, or screenshot. Use whenever the user asks to go to a site "
+            "(e.g. 'open Naver'), check what's on screen, or interact with the page. "
+            "Only available in screen-share mode. Do NOT claim you did any of this unless "
+            "this tool returns success."
+        )
+
+    @property
+    def inputSchema(self) -> Dict[str, Any]:
+        return {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": [
+                        "status", "listTabs", "navigate", "newTab", "closeTab",
+                        "activateTab", "closePopups", "click", "type", "scroll",
+                        "pressKey", "screenshot",
+                    ],
+                    "description": "What to do in the browser.",
+                },
+                "url": {"type": "string", "description": "Target URL/site for navigate/newTab (e.g. 'naver.com')."},
+                "index": {"type": "integer", "description": "Tab index for closeTab/activateTab (from listTabs)."},
+                "selector": {"type": "string", "description": "CSS selector for click/type."},
+                "text": {"type": "string", "description": "Text to type."},
+                "key": {"type": "string", "description": "Key to press, e.g. 'Return', 'Escape'."},
+                "dir": {"type": "string", "description": "Scroll direction: 'down' or 'up'."},
+            },
+            "required": ["action"],
+        }
+
+    def run(self, args: Optional[Dict[str, Any]], context: ToolContext) -> ToolExecutionResult:
+        cfg = context.cfg
+        if not getattr(cfg, "stream_browser", True):
+            return ToolExecutionResult(
+                success=False,
+                reply_text="화면 공유 모드(STREAM_BROWSER=true)에서만 브라우저를 제어할 수 있습니다.",
+            )
+        if not _NODE_SCRIPT.exists():
+            return ToolExecutionResult(success=False, reply_text="브라우저 제어 도구를 찾을 수 없습니다.")
+        if not args or not isinstance(args, dict):
+            return ToolExecutionResult(success=False, reply_text="수행할 동작(action)을 알려주세요.")
+
+        action = str(args.get("action", "")).strip()
+        if not action:
+            return ToolExecutionResult(success=False, reply_text="수행할 동작(action)을 알려주세요.")
+
+        # Pass the whole arg dict through as the command (the Node side reads the
+        # fields it needs per action).
+        command = json.dumps(args, ensure_ascii=False)
+        context.user_print(f"🖱️ 브라우저 제어: {action}")
+        debug_log(f"    🖱️ controlBrowser {command[:120]}", "tools")
+        # Human-input actions need time for the visible cursor move + char typing.
+        timeout = 25 if action in _READ_ACTIONS else 90
+        try:
+            proc = subprocess.run(
+                ["node", str(_NODE_SCRIPT), command],
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+                env={**os.environ, "CDP_PORT": os.environ.get("CDP_PORT", "9222")},
+            )
+            data = json.loads((proc.stdout or "").strip() or "{}")
+        except Exception as e:
+            return ToolExecutionResult(success=False, reply_text=f"브라우저 제어에 실패했습니다: {e}")
+
+        if not data.get("ok"):
+            return ToolExecutionResult(
+                success=False, reply_text=f"브라우저 제어에 실패했습니다: {data.get('error', 'unknown')}"
+            )
+
+        # Return a factual summary of what ACTUALLY happened so the reply engine
+        # describes the real outcome and doesn't confabulate.
+        summary = self._summarise(action, args, data)
+        return ToolExecutionResult(success=True, reply_text=summary)
+
+    @staticmethod
+    def _summarise(action: str, args: Dict[str, Any], data: Dict[str, Any]) -> str:
+        if action == "navigate":
+            return f"브라우저에서 {data.get('url', args.get('url'))} 로 이동했습니다."
+        if action in ("status", "listTabs"):
+            tabs = data.get("tabs", [])
+            lines = "\n".join(f"  [{t['index']}] {t.get('title') or t['url']}" for t in tabs) or "  (탭 없음)"
+            return f"브라우저 탭 {len(tabs)}개:\n{lines}"
+        if action == "newTab":
+            return f"새 탭을 열었습니다 (index {data.get('index')})."
+        if action == "closeTab":
+            return f"탭 {data.get('closed')}번을 닫았습니다 (남은 탭 {data.get('remaining')}개)."
+        if action == "activateTab":
+            return f"탭 {data.get('active')}번으로 전환했습니다."
+        if action == "closePopups":
+            return f"팝업/빈 탭 {data.get('closed')}개를 닫았습니다."
+        if action == "screenshot":
+            return f"화면을 캡처했습니다: {data.get('path')}"
+        return "완료했습니다."

src/jarvis/tools/registry.py

@@ -21,6 +21,7 @@ from .builtin.weather import WeatherTool
 from .builtin.stop import StopTool
 from .builtin.tool_search import ToolSearchTool
 from .builtin.browse_and_play import BrowseAndPlayTool
+from .builtin.control_browser import ControlBrowserTool
 from .builtin.set_broadcast import SetBroadcastTool
 from .types import ToolExecutionResult
 from ..config import Settings
@@ -42,6 +43,7 @@ BUILTIN_TOOLS = {
     "stop": StopTool(),
     "toolSearchTool": ToolSearchTool(),
     "browseAndPlay": BrowseAndPlayTool(),
+    "controlBrowser": ControlBrowserTool(),
     "setBroadcast": SetBroadcastTool(),
 }