From 3d1e56f60fa92f932289414d3901444858680682 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Sun, 14 Jun 2026 02:22:36 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20controlBrowser=20tool=20=E2=80=94=20hum?= =?UTF-8?q?an-operable=20on-screen=20Chrome?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a general browser-control tool (navigate to any site, list/open/close/ switch tabs, close popups, click, type, scroll, screenshot) for the Go-Live Chrome, on top of the existing CDP + xdotool human-input layer (visible cursor, char-by-char typing). Closes the gap where "open Naver" had no tool and the model confabulated success. Also adds a system-prompt rule against claiming actions no tool actually performed. Co-Authored-By: Claude Opus 4.7 --- bot/scripts/stream-test/chrome-control.mjs | 193 ++++++++++++++++++++ src/jarvis/system_prompt.py | 4 + src/jarvis/tools/builtin/control_browser.py | 139 ++++++++++++++ src/jarvis/tools/registry.py | 2 + 4 files changed, 338 insertions(+) create mode 100644 bot/scripts/stream-test/chrome-control.mjs create mode 100644 src/jarvis/tools/builtin/control_browser.py diff --git a/bot/scripts/stream-test/chrome-control.mjs b/bot/scripts/stream-test/chrome-control.mjs new file mode 100644 index 0000000..3746002 --- /dev/null +++ b/bot/scripts/stream-test/chrome-control.mjs @@ -0,0 +1,193 @@ +// Human-operable Chrome control for the on-screen (Go-Live) browser. +// +// node chrome-control.mjs '' +// +// Connects to the on-screen Chrome over CDP (so every action is visible on the +// broadcast) and performs ONE command, printing a JSON result on stdout for the +// Python `controlBrowser` tool to wrap. READ actions use the CDP/DOM API; +// ACTION actions (navigate/click/type/scroll) drive real input via xdotool +// (human.mjs) so the cursor visibly moves and text is typed one char at a time, +// exactly as a person would — never synthetic DOM events. +// +// Commands (json): { "action": "", ...params } +// status -> broadcast/browser readiness + tab list +// listTabs -> [{index,url,title,active}] +// navigate {url, human?} -> open url in the active tab +// newTab {url?} -> open a new tab (optionally to url) +// closeTab {index} -> close tab by index +// activateTab{index} -> bring tab to front +// closePopups -> close blank/popup tabs, keep the active one +// click {selector, human?} -> click an element (human cursor by default) +// type {text, selector?, human?}-> type text char-by-char (human by default) +// scroll {dir, notches?} -> wheel scroll (dir: "down"|"up") +// pressKey {key} -> press a single key (xdotool keysym) +// screenshot {path} -> save a PNG of the active tab +import { chromium } from 'playwright'; +import { + humanClick, + humanType, + pressKey, + humanScroll, + navigateOmnibox, +} from './human.mjs'; + +const CDP = process.env.CDP_PORT || '9222'; +const CDP_HOST = process.env.CDP_HOST || '127.0.0.1'; +const out = (o) => process.stdout.write(JSON.stringify(o)); + +let cmd; +try { + cmd = JSON.parse(process.argv[2] || '{}'); +} catch (e) { + out({ ok: false, error: `bad command json: ${e?.message || e}` }); + process.exit(1); +} +const action = String(cmd.action || '').trim(); +if (!action) { + out({ ok: false, error: 'no action' }); + process.exit(1); +} + +const tabInfo = async (pages, active) => { + const list = []; + for (let i = 0; i < pages.length; i++) { + const p = pages[i]; + list.push({ + index: i, + url: p.url(), + title: await p.title().catch(() => ''), + active: p === active, + }); + } + return list; +}; + +let b; +try { + b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`); + const ctx = b.contexts()[0]; + if (!ctx) throw new Error('no browser context (is Chrome running?)'); + const pages = ctx.pages(); + // The "active" tab: the most recently fronted page. Playwright has no direct + // accessor, so we treat the first visible page as active and let activateTab + // change focus explicitly. + let page = pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0] || (await ctx.newPage()); + page.setDefaultTimeout(20000); + + switch (action) { + case 'status': { + out({ ok: true, browserOpen: true, tabCount: pages.length, tabs: await tabInfo(pages, page) }); + break; + } + case 'listTabs': { + out({ ok: true, tabs: await tabInfo(pages, page) }); + break; + } + case 'navigate': { + const url = String(cmd.url || '').trim(); + if (!url) throw new Error('navigate: no url'); + const target = /^https?:\/\//i.test(url) ? url : `https://${url}`; + await page.bringToFront().catch(() => {}); + if (cmd.human === false) { + await page.goto(target, { waitUntil: 'domcontentloaded' }); + } else { + // Type the URL into the omnibox char-by-char with a real cursor. + await navigateOmnibox(target); + await page.waitForLoadState('domcontentloaded').catch(() => {}); + } + out({ ok: true, url: page.url(), title: await page.title().catch(() => '') }); + break; + } + case 'newTab': { + const np = await ctx.newPage(); + await np.bringToFront().catch(() => {}); + if (cmd.url) { + const target = /^https?:\/\//i.test(cmd.url) ? cmd.url : `https://${cmd.url}`; + await np.goto(target, { waitUntil: 'domcontentloaded' }).catch(() => {}); + } + out({ ok: true, index: ctx.pages().length - 1, url: np.url() }); + break; + } + case 'closeTab': { + const idx = Number(cmd.index); + if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index'); + await pages[idx].close(); + out({ ok: true, closed: idx, remaining: ctx.pages().length }); + break; + } + case 'activateTab': { + const idx = Number(cmd.index); + if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index'); + await pages[idx].bringToFront(); + out({ ok: true, active: idx, url: pages[idx].url() }); + break; + } + case 'closePopups': { + // Close throwaway/popup tabs (about:blank or anything that isn't the + // active content tab), keeping the active one. + let closed = 0; + for (const p of pages) { + if (p === page) continue; + const u = p.url(); + if (!u || u === 'about:blank' || cmd.all) { + await p.close().catch(() => {}); + closed++; + } + } + out({ ok: true, closed, remaining: ctx.pages().length }); + break; + } + case 'click': { + const selector = String(cmd.selector || '').trim(); + if (!selector) throw new Error('click: no selector'); + await page.bringToFront().catch(() => {}); + const locator = page.locator(selector).first(); + if (cmd.human === false) await locator.click(); + else await humanClick(page, locator); + out({ ok: true }); + break; + } + case 'type': { + const text = String(cmd.text ?? ''); + await page.bringToFront().catch(() => {}); + if (cmd.selector) { + const locator = page.locator(String(cmd.selector)).first(); + if (cmd.human === false) { await locator.fill(text); out({ ok: true }); break; } + await humanClick(page, locator); // focus the field with a real click first + } + await humanType(text); + out({ ok: true }); + break; + } + case 'scroll': { + const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1; + await humanScroll(page, dir, Number(cmd.notches) || 5); + out({ ok: true }); + break; + } + case 'pressKey': { + const key = String(cmd.key || '').trim(); + if (!key) throw new Error('pressKey: no key'); + await pressKey(key); + out({ ok: true }); + break; + } + case 'screenshot': { + const path = String(cmd.path || '').trim(); + if (!path) throw new Error('screenshot: no path'); + await page.bringToFront().catch(() => {}); + await page.screenshot({ path }); + out({ ok: true, path }); + break; + } + default: + out({ ok: false, error: `unknown action: ${action}` }); + await b.close(); + process.exit(1); + } + await b.close(); +} catch (e) { + try { await b?.close(); } catch { /* ignore */ } + out({ ok: false, error: String(e?.message || e) }); + process.exit(1); +} diff --git a/src/jarvis/system_prompt.py b/src/jarvis/system_prompt.py index 94db85a..a6d10f6 100644 --- a/src/jarvis/system_prompt.py +++ b/src/jarvis/system_prompt.py @@ -36,6 +36,10 @@ _SYSTEM_PROMPT_TEMPLATE: str = ( "butler clichés, and never address the user as 'sir', 'madam', 'my liege', or similar. " "Never stack multiple jokes in one reply. " "Be concise, conversational, and actionable. " + "Never claim you performed an action — opening a website, navigating the browser, " + "playing or showing something on screen, changing a setting, sending a message — unless a " + "tool actually did it this turn and reported success. If you have no tool for what was asked, " + "or the tool failed, say so plainly; do not narrate a success that did not happen. " "Never answer with a bare greeting like 'Hey there!', 'Hi!', 'Hello, how can I help you?', " "'I hope you have a relaxing time today', or 'I'm here and ready to chat'. Always engage " "with the user's actual prompt, and when the 'Information the user has shared…' section is " diff --git a/src/jarvis/tools/builtin/control_browser.py b/src/jarvis/tools/builtin/control_browser.py new file mode 100644 index 0000000..6006244 --- /dev/null +++ b/src/jarvis/tools/builtin/control_browser.py @@ -0,0 +1,139 @@ +"""Operate the on-screen (Go-Live) Chrome like a person would. + +Only meaningful in screen-share mode (``STREAM_BROWSER`` true): it drives the +on-screen Chrome via the Node CDP helper (``chrome-control.mjs``) so every +action is visible on the broadcast. Pointer moves, clicks and typing are real +xdotool input (visible cursor, char-by-char typing), never synthetic DOM +events. + +This is the general browser-control tool (navigate to any site, manage +windows/tabs, click, type, scroll, screenshot). ``browseAndPlay`` remains the +YouTube-playback shortcut. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from typing import Dict, Any, Optional + +from ..base import Tool, ToolContext +from ..types import ToolExecutionResult +from ...debug import debug_log + +# .../owner/src/jarvis/tools/builtin/control_browser.py -> parents[4] == .../owner +_REPO_ROOT = Path(__file__).resolve().parents[4] +_NODE_SCRIPT = _REPO_ROOT / "bot" / "scripts" / "stream-test" / "chrome-control.mjs" + +# Actions that don't change anything (safe, fast, no human-input latency). +_READ_ACTIONS = {"status", "listTabs"} + + +class ControlBrowserTool(Tool): + """Drive the on-screen Chrome: navigate, manage tabs, click, type, scroll.""" + + @property + def name(self) -> str: + return "controlBrowser" + + @property + def description(self) -> str: + return ( + "Operate the on-screen Chrome shown on the broadcast like a person: open a " + "website/URL, list/open/close/switch tabs, close popups, click an element, " + "type text, scroll, or screenshot. Use whenever the user asks to go to a site " + "(e.g. 'open Naver'), check what's on screen, or interact with the page. " + "Only available in screen-share mode. Do NOT claim you did any of this unless " + "this tool returns success." + ) + + @property + def inputSchema(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "status", "listTabs", "navigate", "newTab", "closeTab", + "activateTab", "closePopups", "click", "type", "scroll", + "pressKey", "screenshot", + ], + "description": "What to do in the browser.", + }, + "url": {"type": "string", "description": "Target URL/site for navigate/newTab (e.g. 'naver.com')."}, + "index": {"type": "integer", "description": "Tab index for closeTab/activateTab (from listTabs)."}, + "selector": {"type": "string", "description": "CSS selector for click/type."}, + "text": {"type": "string", "description": "Text to type."}, + "key": {"type": "string", "description": "Key to press, e.g. 'Return', 'Escape'."}, + "dir": {"type": "string", "description": "Scroll direction: 'down' or 'up'."}, + }, + "required": ["action"], + } + + def run(self, args: Optional[Dict[str, Any]], context: ToolContext) -> ToolExecutionResult: + cfg = context.cfg + if not getattr(cfg, "stream_browser", True): + return ToolExecutionResult( + success=False, + reply_text="화면 공유 모드(STREAM_BROWSER=true)에서만 브라우저를 제어할 수 있습니다.", + ) + if not _NODE_SCRIPT.exists(): + return ToolExecutionResult(success=False, reply_text="브라우저 제어 도구를 찾을 수 없습니다.") + if not args or not isinstance(args, dict): + return ToolExecutionResult(success=False, reply_text="수행할 동작(action)을 알려주세요.") + + action = str(args.get("action", "")).strip() + if not action: + return ToolExecutionResult(success=False, reply_text="수행할 동작(action)을 알려주세요.") + + # Pass the whole arg dict through as the command (the Node side reads the + # fields it needs per action). + command = json.dumps(args, ensure_ascii=False) + context.user_print(f"🖱️ 브라우저 제어: {action}") + debug_log(f" 🖱️ controlBrowser {command[:120]}", "tools") + # Human-input actions need time for the visible cursor move + char typing. + timeout = 25 if action in _READ_ACTIONS else 90 + try: + proc = subprocess.run( + ["node", str(_NODE_SCRIPT), command], + capture_output=True, + text=True, + timeout=timeout, + env={**os.environ, "CDP_PORT": os.environ.get("CDP_PORT", "9222")}, + ) + data = json.loads((proc.stdout or "").strip() or "{}") + except Exception as e: + return ToolExecutionResult(success=False, reply_text=f"브라우저 제어에 실패했습니다: {e}") + + if not data.get("ok"): + return ToolExecutionResult( + success=False, reply_text=f"브라우저 제어에 실패했습니다: {data.get('error', 'unknown')}" + ) + + # Return a factual summary of what ACTUALLY happened so the reply engine + # describes the real outcome and doesn't confabulate. + summary = self._summarise(action, args, data) + return ToolExecutionResult(success=True, reply_text=summary) + + @staticmethod + def _summarise(action: str, args: Dict[str, Any], data: Dict[str, Any]) -> str: + if action == "navigate": + return f"브라우저에서 {data.get('url', args.get('url'))} 로 이동했습니다." + if action in ("status", "listTabs"): + tabs = data.get("tabs", []) + lines = "\n".join(f" [{t['index']}] {t.get('title') or t['url']}" for t in tabs) or " (탭 없음)" + return f"브라우저 탭 {len(tabs)}개:\n{lines}" + if action == "newTab": + return f"새 탭을 열었습니다 (index {data.get('index')})." + if action == "closeTab": + return f"탭 {data.get('closed')}번을 닫았습니다 (남은 탭 {data.get('remaining')}개)." + if action == "activateTab": + return f"탭 {data.get('active')}번으로 전환했습니다." + if action == "closePopups": + return f"팝업/빈 탭 {data.get('closed')}개를 닫았습니다." + if action == "screenshot": + return f"화면을 캡처했습니다: {data.get('path')}" + return "완료했습니다." diff --git a/src/jarvis/tools/registry.py b/src/jarvis/tools/registry.py index 2616348..6a8baa2 100644 --- a/src/jarvis/tools/registry.py +++ b/src/jarvis/tools/registry.py @@ -21,6 +21,7 @@ from .builtin.weather import WeatherTool from .builtin.stop import StopTool from .builtin.tool_search import ToolSearchTool from .builtin.browse_and_play import BrowseAndPlayTool +from .builtin.control_browser import ControlBrowserTool from .builtin.set_broadcast import SetBroadcastTool from .types import ToolExecutionResult from ..config import Settings @@ -42,6 +43,7 @@ BUILTIN_TOOLS = { "stop": StopTool(), "toolSearchTool": ToolSearchTool(), "browseAndPlay": BrowseAndPlayTool(), + "controlBrowser": ControlBrowserTool(), "setBroadcast": SetBroadcastTool(), }