feat: controlBrowser tool — human-operable on-screen Chrome
Adds a general browser-control tool (navigate to any site, list/open/close/ switch tabs, close popups, click, type, scroll, screenshot) for the Go-Live Chrome, on top of the existing CDP + xdotool human-input layer (visible cursor, char-by-char typing). Closes the gap where "open Naver" had no tool and the model confabulated success. Also adds a system-prompt rule against claiming actions no tool actually performed. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
193
bot/scripts/stream-test/chrome-control.mjs
Normal file
193
bot/scripts/stream-test/chrome-control.mjs
Normal file
@@ -0,0 +1,193 @@
|
||||
// Human-operable Chrome control for the on-screen (Go-Live) browser.
|
||||
//
|
||||
// node chrome-control.mjs '<json-command>'
|
||||
//
|
||||
// Connects to the on-screen Chrome over CDP (so every action is visible on the
|
||||
// broadcast) and performs ONE command, printing a JSON result on stdout for the
|
||||
// Python `controlBrowser` tool to wrap. READ actions use the CDP/DOM API;
|
||||
// ACTION actions (navigate/click/type/scroll) drive real input via xdotool
|
||||
// (human.mjs) so the cursor visibly moves and text is typed one char at a time,
|
||||
// exactly as a person would — never synthetic DOM events.
|
||||
//
|
||||
// Commands (json): { "action": "<name>", ...params }
|
||||
// status -> broadcast/browser readiness + tab list
|
||||
// listTabs -> [{index,url,title,active}]
|
||||
// navigate {url, human?} -> open url in the active tab
|
||||
// newTab {url?} -> open a new tab (optionally to url)
|
||||
// closeTab {index} -> close tab by index
|
||||
// activateTab{index} -> bring tab to front
|
||||
// closePopups -> close blank/popup tabs, keep the active one
|
||||
// click {selector, human?} -> click an element (human cursor by default)
|
||||
// type {text, selector?, human?}-> type text char-by-char (human by default)
|
||||
// scroll {dir, notches?} -> wheel scroll (dir: "down"|"up")
|
||||
// pressKey {key} -> press a single key (xdotool keysym)
|
||||
// screenshot {path} -> save a PNG of the active tab
|
||||
import { chromium } from 'playwright';
|
||||
import {
|
||||
humanClick,
|
||||
humanType,
|
||||
pressKey,
|
||||
humanScroll,
|
||||
navigateOmnibox,
|
||||
} from './human.mjs';
|
||||
|
||||
const CDP = process.env.CDP_PORT || '9222';
|
||||
const CDP_HOST = process.env.CDP_HOST || '127.0.0.1';
|
||||
const out = (o) => process.stdout.write(JSON.stringify(o));
|
||||
|
||||
let cmd;
|
||||
try {
|
||||
cmd = JSON.parse(process.argv[2] || '{}');
|
||||
} catch (e) {
|
||||
out({ ok: false, error: `bad command json: ${e?.message || e}` });
|
||||
process.exit(1);
|
||||
}
|
||||
const action = String(cmd.action || '').trim();
|
||||
if (!action) {
|
||||
out({ ok: false, error: 'no action' });
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const tabInfo = async (pages, active) => {
|
||||
const list = [];
|
||||
for (let i = 0; i < pages.length; i++) {
|
||||
const p = pages[i];
|
||||
list.push({
|
||||
index: i,
|
||||
url: p.url(),
|
||||
title: await p.title().catch(() => ''),
|
||||
active: p === active,
|
||||
});
|
||||
}
|
||||
return list;
|
||||
};
|
||||
|
||||
let b;
|
||||
try {
|
||||
b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`);
|
||||
const ctx = b.contexts()[0];
|
||||
if (!ctx) throw new Error('no browser context (is Chrome running?)');
|
||||
const pages = ctx.pages();
|
||||
// The "active" tab: the most recently fronted page. Playwright has no direct
|
||||
// accessor, so we treat the first visible page as active and let activateTab
|
||||
// change focus explicitly.
|
||||
let page = pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0] || (await ctx.newPage());
|
||||
page.setDefaultTimeout(20000);
|
||||
|
||||
switch (action) {
|
||||
case 'status': {
|
||||
out({ ok: true, browserOpen: true, tabCount: pages.length, tabs: await tabInfo(pages, page) });
|
||||
break;
|
||||
}
|
||||
case 'listTabs': {
|
||||
out({ ok: true, tabs: await tabInfo(pages, page) });
|
||||
break;
|
||||
}
|
||||
case 'navigate': {
|
||||
const url = String(cmd.url || '').trim();
|
||||
if (!url) throw new Error('navigate: no url');
|
||||
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
||||
await page.bringToFront().catch(() => {});
|
||||
if (cmd.human === false) {
|
||||
await page.goto(target, { waitUntil: 'domcontentloaded' });
|
||||
} else {
|
||||
// Type the URL into the omnibox char-by-char with a real cursor.
|
||||
await navigateOmnibox(target);
|
||||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||||
}
|
||||
out({ ok: true, url: page.url(), title: await page.title().catch(() => '') });
|
||||
break;
|
||||
}
|
||||
case 'newTab': {
|
||||
const np = await ctx.newPage();
|
||||
await np.bringToFront().catch(() => {});
|
||||
if (cmd.url) {
|
||||
const target = /^https?:\/\//i.test(cmd.url) ? cmd.url : `https://${cmd.url}`;
|
||||
await np.goto(target, { waitUntil: 'domcontentloaded' }).catch(() => {});
|
||||
}
|
||||
out({ ok: true, index: ctx.pages().length - 1, url: np.url() });
|
||||
break;
|
||||
}
|
||||
case 'closeTab': {
|
||||
const idx = Number(cmd.index);
|
||||
if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
|
||||
await pages[idx].close();
|
||||
out({ ok: true, closed: idx, remaining: ctx.pages().length });
|
||||
break;
|
||||
}
|
||||
case 'activateTab': {
|
||||
const idx = Number(cmd.index);
|
||||
if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index');
|
||||
await pages[idx].bringToFront();
|
||||
out({ ok: true, active: idx, url: pages[idx].url() });
|
||||
break;
|
||||
}
|
||||
case 'closePopups': {
|
||||
// Close throwaway/popup tabs (about:blank or anything that isn't the
|
||||
// active content tab), keeping the active one.
|
||||
let closed = 0;
|
||||
for (const p of pages) {
|
||||
if (p === page) continue;
|
||||
const u = p.url();
|
||||
if (!u || u === 'about:blank' || cmd.all) {
|
||||
await p.close().catch(() => {});
|
||||
closed++;
|
||||
}
|
||||
}
|
||||
out({ ok: true, closed, remaining: ctx.pages().length });
|
||||
break;
|
||||
}
|
||||
case 'click': {
|
||||
const selector = String(cmd.selector || '').trim();
|
||||
if (!selector) throw new Error('click: no selector');
|
||||
await page.bringToFront().catch(() => {});
|
||||
const locator = page.locator(selector).first();
|
||||
if (cmd.human === false) await locator.click();
|
||||
else await humanClick(page, locator);
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
case 'type': {
|
||||
const text = String(cmd.text ?? '');
|
||||
await page.bringToFront().catch(() => {});
|
||||
if (cmd.selector) {
|
||||
const locator = page.locator(String(cmd.selector)).first();
|
||||
if (cmd.human === false) { await locator.fill(text); out({ ok: true }); break; }
|
||||
await humanClick(page, locator); // focus the field with a real click first
|
||||
}
|
||||
await humanType(text);
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
case 'scroll': {
|
||||
const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1;
|
||||
await humanScroll(page, dir, Number(cmd.notches) || 5);
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
case 'pressKey': {
|
||||
const key = String(cmd.key || '').trim();
|
||||
if (!key) throw new Error('pressKey: no key');
|
||||
await pressKey(key);
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
case 'screenshot': {
|
||||
const path = String(cmd.path || '').trim();
|
||||
if (!path) throw new Error('screenshot: no path');
|
||||
await page.bringToFront().catch(() => {});
|
||||
await page.screenshot({ path });
|
||||
out({ ok: true, path });
|
||||
break;
|
||||
}
|
||||
default:
|
||||
out({ ok: false, error: `unknown action: ${action}` });
|
||||
await b.close();
|
||||
process.exit(1);
|
||||
}
|
||||
await b.close();
|
||||
} catch (e) {
|
||||
try { await b?.close(); } catch { /* ignore */ }
|
||||
out({ ok: false, error: String(e?.message || e) });
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -36,6 +36,10 @@ _SYSTEM_PROMPT_TEMPLATE: str = (
|
||||
"butler clichés, and never address the user as 'sir', 'madam', 'my liege', or similar. "
|
||||
"Never stack multiple jokes in one reply. "
|
||||
"Be concise, conversational, and actionable. "
|
||||
"Never claim you performed an action — opening a website, navigating the browser, "
|
||||
"playing or showing something on screen, changing a setting, sending a message — unless a "
|
||||
"tool actually did it this turn and reported success. If you have no tool for what was asked, "
|
||||
"or the tool failed, say so plainly; do not narrate a success that did not happen. "
|
||||
"Never answer with a bare greeting like 'Hey there!', 'Hi!', 'Hello, how can I help you?', "
|
||||
"'I hope you have a relaxing time today', or 'I'm here and ready to chat'. Always engage "
|
||||
"with the user's actual prompt, and when the 'Information the user has shared…' section is "
|
||||
|
||||
139
src/jarvis/tools/builtin/control_browser.py
Normal file
139
src/jarvis/tools/builtin/control_browser.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""Operate the on-screen (Go-Live) Chrome like a person would.
|
||||
|
||||
Only meaningful in screen-share mode (``STREAM_BROWSER`` true): it drives the
|
||||
on-screen Chrome via the Node CDP helper (``chrome-control.mjs``) so every
|
||||
action is visible on the broadcast. Pointer moves, clicks and typing are real
|
||||
xdotool input (visible cursor, char-by-char typing), never synthetic DOM
|
||||
events.
|
||||
|
||||
This is the general browser-control tool (navigate to any site, manage
|
||||
windows/tabs, click, type, scroll, screenshot). ``browseAndPlay`` remains the
|
||||
YouTube-playback shortcut.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from ..base import Tool, ToolContext
|
||||
from ..types import ToolExecutionResult
|
||||
from ...debug import debug_log
|
||||
|
||||
# .../owner/src/jarvis/tools/builtin/control_browser.py -> parents[4] == .../owner
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
_NODE_SCRIPT = _REPO_ROOT / "bot" / "scripts" / "stream-test" / "chrome-control.mjs"
|
||||
|
||||
# Actions that don't change anything (safe, fast, no human-input latency).
|
||||
_READ_ACTIONS = {"status", "listTabs"}
|
||||
|
||||
|
||||
class ControlBrowserTool(Tool):
|
||||
"""Drive the on-screen Chrome: navigate, manage tabs, click, type, scroll."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "controlBrowser"
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return (
|
||||
"Operate the on-screen Chrome shown on the broadcast like a person: open a "
|
||||
"website/URL, list/open/close/switch tabs, close popups, click an element, "
|
||||
"type text, scroll, or screenshot. Use whenever the user asks to go to a site "
|
||||
"(e.g. 'open Naver'), check what's on screen, or interact with the page. "
|
||||
"Only available in screen-share mode. Do NOT claim you did any of this unless "
|
||||
"this tool returns success."
|
||||
)
|
||||
|
||||
@property
|
||||
def inputSchema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"status", "listTabs", "navigate", "newTab", "closeTab",
|
||||
"activateTab", "closePopups", "click", "type", "scroll",
|
||||
"pressKey", "screenshot",
|
||||
],
|
||||
"description": "What to do in the browser.",
|
||||
},
|
||||
"url": {"type": "string", "description": "Target URL/site for navigate/newTab (e.g. 'naver.com')."},
|
||||
"index": {"type": "integer", "description": "Tab index for closeTab/activateTab (from listTabs)."},
|
||||
"selector": {"type": "string", "description": "CSS selector for click/type."},
|
||||
"text": {"type": "string", "description": "Text to type."},
|
||||
"key": {"type": "string", "description": "Key to press, e.g. 'Return', 'Escape'."},
|
||||
"dir": {"type": "string", "description": "Scroll direction: 'down' or 'up'."},
|
||||
},
|
||||
"required": ["action"],
|
||||
}
|
||||
|
||||
def run(self, args: Optional[Dict[str, Any]], context: ToolContext) -> ToolExecutionResult:
|
||||
cfg = context.cfg
|
||||
if not getattr(cfg, "stream_browser", True):
|
||||
return ToolExecutionResult(
|
||||
success=False,
|
||||
reply_text="화면 공유 모드(STREAM_BROWSER=true)에서만 브라우저를 제어할 수 있습니다.",
|
||||
)
|
||||
if not _NODE_SCRIPT.exists():
|
||||
return ToolExecutionResult(success=False, reply_text="브라우저 제어 도구를 찾을 수 없습니다.")
|
||||
if not args or not isinstance(args, dict):
|
||||
return ToolExecutionResult(success=False, reply_text="수행할 동작(action)을 알려주세요.")
|
||||
|
||||
action = str(args.get("action", "")).strip()
|
||||
if not action:
|
||||
return ToolExecutionResult(success=False, reply_text="수행할 동작(action)을 알려주세요.")
|
||||
|
||||
# Pass the whole arg dict through as the command (the Node side reads the
|
||||
# fields it needs per action).
|
||||
command = json.dumps(args, ensure_ascii=False)
|
||||
context.user_print(f"🖱️ 브라우저 제어: {action}")
|
||||
debug_log(f" 🖱️ controlBrowser {command[:120]}", "tools")
|
||||
# Human-input actions need time for the visible cursor move + char typing.
|
||||
timeout = 25 if action in _READ_ACTIONS else 90
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["node", str(_NODE_SCRIPT), command],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env={**os.environ, "CDP_PORT": os.environ.get("CDP_PORT", "9222")},
|
||||
)
|
||||
data = json.loads((proc.stdout or "").strip() or "{}")
|
||||
except Exception as e:
|
||||
return ToolExecutionResult(success=False, reply_text=f"브라우저 제어에 실패했습니다: {e}")
|
||||
|
||||
if not data.get("ok"):
|
||||
return ToolExecutionResult(
|
||||
success=False, reply_text=f"브라우저 제어에 실패했습니다: {data.get('error', 'unknown')}"
|
||||
)
|
||||
|
||||
# Return a factual summary of what ACTUALLY happened so the reply engine
|
||||
# describes the real outcome and doesn't confabulate.
|
||||
summary = self._summarise(action, args, data)
|
||||
return ToolExecutionResult(success=True, reply_text=summary)
|
||||
|
||||
@staticmethod
|
||||
def _summarise(action: str, args: Dict[str, Any], data: Dict[str, Any]) -> str:
|
||||
if action == "navigate":
|
||||
return f"브라우저에서 {data.get('url', args.get('url'))} 로 이동했습니다."
|
||||
if action in ("status", "listTabs"):
|
||||
tabs = data.get("tabs", [])
|
||||
lines = "\n".join(f" [{t['index']}] {t.get('title') or t['url']}" for t in tabs) or " (탭 없음)"
|
||||
return f"브라우저 탭 {len(tabs)}개:\n{lines}"
|
||||
if action == "newTab":
|
||||
return f"새 탭을 열었습니다 (index {data.get('index')})."
|
||||
if action == "closeTab":
|
||||
return f"탭 {data.get('closed')}번을 닫았습니다 (남은 탭 {data.get('remaining')}개)."
|
||||
if action == "activateTab":
|
||||
return f"탭 {data.get('active')}번으로 전환했습니다."
|
||||
if action == "closePopups":
|
||||
return f"팝업/빈 탭 {data.get('closed')}개를 닫았습니다."
|
||||
if action == "screenshot":
|
||||
return f"화면을 캡처했습니다: {data.get('path')}"
|
||||
return "완료했습니다."
|
||||
@@ -21,6 +21,7 @@ from .builtin.weather import WeatherTool
|
||||
from .builtin.stop import StopTool
|
||||
from .builtin.tool_search import ToolSearchTool
|
||||
from .builtin.browse_and_play import BrowseAndPlayTool
|
||||
from .builtin.control_browser import ControlBrowserTool
|
||||
from .builtin.set_broadcast import SetBroadcastTool
|
||||
from .types import ToolExecutionResult
|
||||
from ..config import Settings
|
||||
@@ -42,6 +43,7 @@ BUILTIN_TOOLS = {
|
||||
"stop": StopTool(),
|
||||
"toolSearchTool": ToolSearchTool(),
|
||||
"browseAndPlay": BrowseAndPlayTool(),
|
||||
"controlBrowser": ControlBrowserTool(),
|
||||
"setBroadcast": SetBroadcastTool(),
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user