fix(controlBrowser): never report moveMouse/search success without a real move

moveMouse returned ok:true even when humanHover did nothing (no on-screen box)
or the selector never matched — recreating the "claims it moved but didn't"
bug. Now: humanHover returns a boolean (and brings the element into view first);
moveMouse returns ok:false when the target isn't found or has no on-screen box,
and when site=naver/... whose box isn't on the current page it navigates to the
site home first before hovering. search now reports input=human|api-fallback|api
so a silent fallback to cursor-less DOM input is visible, and the tool surfaces
that note in the reply instead of implying a human-like search happened.

bot/scripts/stream-test/chrome-control.mjs

@@ -17,6 +17,7 @@
 //   status | listTabs
 //   navigate {url} | back | forward | refresh
 //   newTab {url?} | closeTab {index?} | activateTab {index} | closePopups
+//   moveMouse {selector | site}  (hover the real cursor, no click)
 //   click {selector} | type {text, selector?} | scroll {dir, notches?}
 //   pressKey {key} | screenshot {path}
 import { chromium } from 'playwright';
@@ -40,6 +41,15 @@ if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); }
 
 const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`);
 
+// Per-site homepage + search-box selector, shared by `search` and `moveMouse`.
+const SITES = {
+  naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' },
+  google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' },
+  daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' },
+  youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' },
+  bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' },
+};
+
 // The genuinely-active tab is the one whose document is visible. Playwright has
 // no "active page" accessor over CDP, so probe visibilityState (fixes treating
 // tab 0 as active and breaking sequential ops on a specific tab).
@@ -103,13 +113,6 @@ try {
       const q = String(cmd.query || '').trim();
       if (!q) throw new Error('search: no query');
       const siteKey = String(cmd.site || 'google').toLowerCase();
-      const SITES = {
-        naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' },
-        google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' },
-        daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' },
-        youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' },
-        bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' },
-      };
       const s = SITES[siteKey] || SITES.google;
       await front(page);
       // 1) Go to the homepage.
@@ -122,23 +125,31 @@ try {
       // 2) Click the on-page search box, type the query, submit.
       const box = page.locator(s.box).first();
       await box.waitFor({ state: 'visible', timeout: 15000 }).catch(() => {});
+      // Report which input path actually ran: 'human' = real xdotool cursor
+      // move + char typing; 'api-fallback' = the humanClick path threw and we
+      // fell back to cursor-less DOM click/fill; 'api' = no xdotool at all. This
+      // makes "did the cursor really move" verifiable from the result.
+      let searchInput;
       if (HAS_XDOTOOL && cmd.human !== false) {
         try {
           await human.humanClick(page, box);
           await human.humanType(q);
           await human.pressKey('Return');
+          searchInput = 'human';
         } catch {
+          searchInput = 'api-fallback';
           await box.click().catch(() => {});
           await box.fill(q).catch(() => {});
           await page.keyboard.press('Enter').catch(() => {});
         }
       } else {
+        searchInput = 'api';
         await box.click().catch(() => {});
         await box.fill(q);
         await page.keyboard.press('Enter');
       }
       await page.waitForLoadState('domcontentloaded').catch(() => {});
-      out({ ok: true, site: SITES[siteKey] ? siteKey : 'google', query: q, url: page.url(), title: await page.title().catch(() => '') });
+      out({ ok: true, site: SITES[siteKey] ? siteKey : 'google', query: q, url: page.url(), title: await page.title().catch(() => ''), input: searchInput });
       break;
     }
 
@@ -205,6 +216,45 @@ try {
       break;
     }
 
+    case 'moveMouse': {
+      // Move/hover the REAL cursor onto an element WITHOUT clicking. Target is a
+      // CSS selector, or site=naver/google/... for that site's search box.
+      // Only meaningful with xdotool (the visible cursor); with no xdotool there
+      // is no cursor to move, so report that rather than faking success. Every
+      // failure to actually move (no xdotool, selector never matches, element
+      // has no on-screen box) returns ok:false — we must never claim the cursor
+      // moved when it did not (the exact bug the user reported).
+      const siteKey = String(cmd.site || '').toLowerCase();
+      const selector = String(cmd.selector || '').trim() || (SITES[siteKey] ? SITES[siteKey].box : '');
+      if (!selector) throw new Error('moveMouse: no selector or known site');
+      if (!(HAS_XDOTOOL && cmd.human !== false)) {
+        out({ ok: false, error: 'no xdotool: cannot move the visible cursor on this host' });
+        break;
+      }
+      await front(page);
+      let locator = page.locator(selector).first();
+      let visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false);
+      // A named site whose search box isn't on the current page: go to its home
+      // first (real omnibox), then target the box there.
+      if (!visible && SITES[siteKey]) {
+        try { await human.navigateOmnibox(SITES[siteKey].home); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
+        catch { await page.goto(SITES[siteKey].home, { waitUntil: 'domcontentloaded' }).catch(() => {}); }
+        locator = page.locator(SITES[siteKey].box).first();
+        visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false);
+      }
+      if (!visible) {
+        out({ ok: false, error: `moveMouse: target not found (${cmd.selector || siteKey})` });
+        break;
+      }
+      const moved = await human.humanHover(page, locator);
+      if (!moved) {
+        out({ ok: false, error: 'moveMouse: element has no on-screen box; cursor not moved' });
+        break;
+      }
+      out({ ok: true, target: cmd.selector || siteKey, input: 'human' });
+      break;
+    }
+
     case 'click': {
       const selector = String(cmd.selector || '').trim();
       if (!selector) throw new Error('click: no selector');

bot/scripts/stream-test/human.mjs

@@ -136,14 +136,18 @@ export async function navigateOmnibox(text) {
 }
 
 // Move the real cursor over an element (hover, no click) - e.g. to reveal a
-// video player's controls or to focus it for a keyboard shortcut.
+// video player's controls or to focus it for a keyboard shortcut. Returns true
+// only if the element had an on-screen box and the cursor was actually moved;
+// returns false when there is nothing to move to (so callers must not report
+// success). Brings the element into view with a real wheel scroll first.
 export async function humanHover(page, locator) {
-  const box = await locator.boundingBox().catch(() => null);
-  if (!box) return;
+  const box = await bringIntoView(page, locator);
+  if (!box) return false;
   const g = await page.evaluate(() => ({ sx: window.screenX, sy: window.screenY, ow: window.outerWidth, oh: window.outerHeight, iw: window.innerWidth, ih: window.innerHeight }));
   const bx = Math.max(0, Math.round((g.ow - g.iw) / 2));
   const oy = g.sy + Math.max(0, g.oh - g.ih - bx);
   await humanMove(Math.round(g.sx + bx + box.x + box.width * 0.5), Math.round(oy + box.y + box.height * 0.4));
+  return true;
 }
 
 export { sleep, rand };

src/jarvis/tools/builtin/control_browser.py

@@ -45,9 +45,16 @@ class ControlBrowserTool(Tool):
             "Use this (NOT webSearch) whenever the user wants something done or shown IN the "
             "browser on screen: open a website or URL, search on a specific site (action "
             "'search' with site=naver/google/daum/youtube/bing), go back/forward, refresh, "
-            "manage tabs (list/new/close/switch), close popups, click, type, scroll, or "
-            "screenshot. webSearch only returns text and shows nothing on screen; this tool "
-            "actually navigates the visible browser. Only available in screen-share mode. "
+            "manage tabs (list/new/close/switch), close popups, move the mouse onto an element, "
+            "click, type, scroll, or screenshot. webSearch only returns text and shows nothing "
+            "on screen; this tool actually navigates the visible browser. "
+            "Cursor behaviour matters: 'search' and 'type' (with a selector) move the REAL mouse "
+            "cursor to the on-page box, click it, then type one character at a time — use these "
+            "when the user wants to search or type on a page. 'navigate' only types the URL into "
+            "the address bar (no mouse movement). 'moveMouse' moves/hovers the visible cursor "
+            "onto an element (selector, or site=naver/... for that site's search box) WITHOUT "
+            "clicking — use it when the user just asks to move the mouse somewhere. "
+            "Only available in screen-share mode. "
             "Never claim you did any of this unless this tool returns success."
         )
 
@@ -61,16 +68,16 @@ class ControlBrowserTool(Tool):
                     "enum": [
                         "status", "listTabs", "navigate", "search", "back",
                         "forward", "refresh", "newTab", "closeTab", "activateTab",
-                        "closePopups", "click", "type", "scroll", "pressKey",
+                        "closePopups", "moveMouse", "click", "type", "scroll", "pressKey",
                         "screenshot",
                     ],
                     "description": "What to do in the browser.",
                 },
                 "url": {"type": "string", "description": "Target URL/site for navigate/newTab (e.g. 'naver.com')."},
                 "query": {"type": "string", "description": "Search text for action 'search'."},
-                "site": {"type": "string", "description": "Search site for action 'search': naver, google, daum, youtube, bing."},
+                "site": {"type": "string", "description": "Site for action 'search' or 'moveMouse': naver, google, daum, youtube, bing. For moveMouse it targets that site's search box."},
                 "index": {"type": "integer", "description": "Tab index for closeTab/activateTab (from listTabs)."},
-                "selector": {"type": "string", "description": "CSS selector for click/type."},
+                "selector": {"type": "string", "description": "CSS selector for click/type/moveMouse."},
                 "text": {"type": "string", "description": "Text to type."},
                 "key": {"type": "string", "description": "Key to press, e.g. 'Return', 'Escape'."},
                 "dir": {"type": "string", "description": "Scroll direction: 'down' or 'up'."},
@@ -149,7 +156,12 @@ class ControlBrowserTool(Tool):
         if action == "navigate":
             return f"브라우저에서 {data.get('url', args.get('url'))} 로 이동했습니다."
         if action == "search":
-            return f"{data.get('site', '')}에서 '{data.get('query', args.get('query'))}'를 검색해 화면에 띄웠습니다."
+            base = f"{data.get('site', '')}에서 '{data.get('query', args.get('query'))}'를 검색해 화면에 띄웠습니다."
+            # Flag when the real cursor path didn't run, so a silent fallback to
+            # cursor-less DOM input is visible rather than reported as "human".
+            if data.get("input") in ("api", "api-fallback"):
+                base += " (참고: 실제 마우스 커서 이동 없이 처리됨)"
+            return base
         if action in ("back", "forward", "refresh"):
             return f"브라우저: {action} 완료 ({data.get('url', '')})."
         if action in ("status", "listTabs"):
@@ -164,6 +176,9 @@ class ControlBrowserTool(Tool):
             return f"탭 {data.get('active')}번으로 전환했습니다."
         if action == "closePopups":
             return f"팝업/빈 탭 {data.get('closed')}개를 닫았습니다."
+        if action == "moveMouse":
+            target = data.get("target") or args.get("selector") or args.get("site") or "대상"
+            return f"마우스 커서를 {target} 위치로 옮겼습니다."
         if action == "screenshot":
             return f"화면을 캡처했습니다: {data.get('path')}"
         return "완료했습니다."

tests/test_control_browser.py

@@ -0,0 +1,56 @@
+"""Tests for the controlBrowser tool's action surface.
+
+These are deterministic schema/summary checks — they do not drive a real
+browser. The actual cursor movement is exercised live on the browser host
+(xdotool + CDP), which these tests cannot reach.
+"""
+
+import pytest
+
+from jarvis.tools.builtin.control_browser import ControlBrowserTool
+
+
+@pytest.fixture
+def tool():
+    return ControlBrowserTool()
+
+
+def test_movemouse_is_an_exposed_action(tool):
+    # A weak model confabulated "moved the mouse" because no move/hover action
+    # existed to call. The cursor-move capability must be a real action so the
+    # request "move the mouse to the search box" maps to a tool call.
+    enum = tool.inputSchema["properties"]["action"]["enum"]
+    assert "moveMouse" in enum
+
+
+def test_movemouse_summary_reports_the_target(tool):
+    summary = tool._summarise("moveMouse", {"site": "naver"}, {"ok": True, "target": "naver"})
+    assert "마우스" in summary and "naver" in summary
+
+
+def test_description_distinguishes_cursor_paths(tool):
+    # The model must know navigate is address-bar only (no mouse) while
+    # search/type/moveMouse move the real cursor — that distinction is the
+    # whole point of the fix.
+    desc = tool.description
+    assert "moveMouse" in desc
+    assert "address bar" in desc  # navigate is described as address-bar typing
+
+
+def test_search_summary_flags_cursorless_fallback(tool):
+    # When the real xdotool cursor path didn't run, the summary must say so
+    # rather than implying a human-like search happened.
+    human = tool._summarise("search", {"query": "날씨"}, {"ok": True, "site": "naver", "query": "날씨", "input": "human"})
+    assert "참고: 실제 마우스" not in human
+
+    fell_back = tool._summarise("search", {"query": "날씨"}, {"ok": True, "site": "naver", "query": "날씨", "input": "api-fallback"})
+    assert "실제 마우스 커서 이동 없이" in fell_back
+
+
+def test_movemouse_summary_only_runs_on_success(tool):
+    # _summarise is only called on ok:true; an ok:false (target not found / no
+    # xdotool) is handled by run() as a failure reply, so a failed move can no
+    # longer be reported as "moved". Sanity-check the success summary names a
+    # target rather than a placeholder when one is present.
+    summary = tool._summarise("moveMouse", {"selector": "#query"}, {"ok": True, "target": "#query"})
+    assert "#query" in summary