diff --git a/bot/scripts/stream-test/chrome-control.mjs b/bot/scripts/stream-test/chrome-control.mjs index fd7f633..d1e8558 100644 --- a/bot/scripts/stream-test/chrome-control.mjs +++ b/bot/scripts/stream-test/chrome-control.mjs @@ -17,6 +17,7 @@ // status | listTabs // navigate {url} | back | forward | refresh // newTab {url?} | closeTab {index?} | activateTab {index} | closePopups +// moveMouse {selector | site} (hover the real cursor, no click) // click {selector} | type {text, selector?} | scroll {dir, notches?} // pressKey {key} | screenshot {path} import { chromium } from 'playwright'; @@ -40,6 +41,15 @@ if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); } const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`); +// Per-site homepage + search-box selector, shared by `search` and `moveMouse`. +const SITES = { + naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' }, + google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' }, + daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' }, + youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' }, + bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' }, +}; + // The genuinely-active tab is the one whose document is visible. Playwright has // no "active page" accessor over CDP, so probe visibilityState (fixes treating // tab 0 as active and breaking sequential ops on a specific tab). @@ -103,13 +113,6 @@ try { const q = String(cmd.query || '').trim(); if (!q) throw new Error('search: no query'); const siteKey = String(cmd.site || 'google').toLowerCase(); - const SITES = { - naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' }, - google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' }, - daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' }, - youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' }, - bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' }, - }; const s = SITES[siteKey] || SITES.google; await front(page); // 1) Go to the homepage. @@ -205,6 +208,26 @@ try { break; } + case 'moveMouse': { + // Move/hover the REAL cursor onto an element WITHOUT clicking. Target is a + // CSS selector, or site=naver/google/... for that site's search box. + // Only meaningful with xdotool (the visible cursor); with no xdotool there + // is no cursor to move, so report that rather than faking success. + const siteKey = String(cmd.site || '').toLowerCase(); + const selector = String(cmd.selector || '').trim() || (SITES[siteKey] ? SITES[siteKey].box : ''); + if (!selector) throw new Error('moveMouse: no selector or known site'); + if (!(HAS_XDOTOOL && cmd.human !== false)) { + out({ ok: false, error: 'no xdotool: cannot move the visible cursor on this host' }); + break; + } + await front(page); + const locator = page.locator(selector).first(); + await locator.waitFor({ state: 'visible', timeout: 10000 }).catch(() => {}); + await human.humanHover(page, locator); + out({ ok: true, target: cmd.selector || siteKey, input: 'human' }); + break; + } + case 'click': { const selector = String(cmd.selector || '').trim(); if (!selector) throw new Error('click: no selector'); diff --git a/src/jarvis/tools/builtin/control_browser.py b/src/jarvis/tools/builtin/control_browser.py index 84de475..56a1041 100644 --- a/src/jarvis/tools/builtin/control_browser.py +++ b/src/jarvis/tools/builtin/control_browser.py @@ -45,9 +45,16 @@ class ControlBrowserTool(Tool): "Use this (NOT webSearch) whenever the user wants something done or shown IN the " "browser on screen: open a website or URL, search on a specific site (action " "'search' with site=naver/google/daum/youtube/bing), go back/forward, refresh, " - "manage tabs (list/new/close/switch), close popups, click, type, scroll, or " - "screenshot. webSearch only returns text and shows nothing on screen; this tool " - "actually navigates the visible browser. Only available in screen-share mode. " + "manage tabs (list/new/close/switch), close popups, move the mouse onto an element, " + "click, type, scroll, or screenshot. webSearch only returns text and shows nothing " + "on screen; this tool actually navigates the visible browser. " + "Cursor behaviour matters: 'search' and 'type' (with a selector) move the REAL mouse " + "cursor to the on-page box, click it, then type one character at a time — use these " + "when the user wants to search or type on a page. 'navigate' only types the URL into " + "the address bar (no mouse movement). 'moveMouse' moves/hovers the visible cursor " + "onto an element (selector, or site=naver/... for that site's search box) WITHOUT " + "clicking — use it when the user just asks to move the mouse somewhere. " + "Only available in screen-share mode. " "Never claim you did any of this unless this tool returns success." ) @@ -61,16 +68,16 @@ class ControlBrowserTool(Tool): "enum": [ "status", "listTabs", "navigate", "search", "back", "forward", "refresh", "newTab", "closeTab", "activateTab", - "closePopups", "click", "type", "scroll", "pressKey", + "closePopups", "moveMouse", "click", "type", "scroll", "pressKey", "screenshot", ], "description": "What to do in the browser.", }, "url": {"type": "string", "description": "Target URL/site for navigate/newTab (e.g. 'naver.com')."}, "query": {"type": "string", "description": "Search text for action 'search'."}, - "site": {"type": "string", "description": "Search site for action 'search': naver, google, daum, youtube, bing."}, + "site": {"type": "string", "description": "Site for action 'search' or 'moveMouse': naver, google, daum, youtube, bing. For moveMouse it targets that site's search box."}, "index": {"type": "integer", "description": "Tab index for closeTab/activateTab (from listTabs)."}, - "selector": {"type": "string", "description": "CSS selector for click/type."}, + "selector": {"type": "string", "description": "CSS selector for click/type/moveMouse."}, "text": {"type": "string", "description": "Text to type."}, "key": {"type": "string", "description": "Key to press, e.g. 'Return', 'Escape'."}, "dir": {"type": "string", "description": "Scroll direction: 'down' or 'up'."}, @@ -164,6 +171,9 @@ class ControlBrowserTool(Tool): return f"탭 {data.get('active')}번으로 전환했습니다." if action == "closePopups": return f"팝업/빈 탭 {data.get('closed')}개를 닫았습니다." + if action == "moveMouse": + target = data.get("target") or args.get("selector") or args.get("site") or "대상" + return f"마우스 커서를 {target} 위치로 옮겼습니다." if action == "screenshot": return f"화면을 캡처했습니다: {data.get('path')}" return "완료했습니다." diff --git a/tests/test_control_browser.py b/tests/test_control_browser.py new file mode 100644 index 0000000..f84c718 --- /dev/null +++ b/tests/test_control_browser.py @@ -0,0 +1,37 @@ +"""Tests for the controlBrowser tool's action surface. + +These are deterministic schema/summary checks — they do not drive a real +browser. The actual cursor movement is exercised live on the browser host +(xdotool + CDP), which these tests cannot reach. +""" + +import pytest + +from jarvis.tools.builtin.control_browser import ControlBrowserTool + + +@pytest.fixture +def tool(): + return ControlBrowserTool() + + +def test_movemouse_is_an_exposed_action(tool): + # A weak model confabulated "moved the mouse" because no move/hover action + # existed to call. The cursor-move capability must be a real action so the + # request "move the mouse to the search box" maps to a tool call. + enum = tool.inputSchema["properties"]["action"]["enum"] + assert "moveMouse" in enum + + +def test_movemouse_summary_reports_the_target(tool): + summary = tool._summarise("moveMouse", {"site": "naver"}, {"ok": True, "target": "naver"}) + assert "마우스" in summary and "naver" in summary + + +def test_description_distinguishes_cursor_paths(tool): + # The model must know navigate is address-bar only (no mouse) while + # search/type/moveMouse move the real cursor — that distinction is the + # whole point of the fix. + desc = tool.description + assert "moveMouse" in desc + assert "address bar" in desc # navigate is described as address-bar typing