// Human-operable Chrome control for the on-screen (Go-Live) browser. // // node chrome-control.mjs '' // // Connects to the on-screen Chrome over CDP (so every action is visible on the // broadcast) and performs ONE command, printing a JSON result on stdout for the // Python `controlBrowser` tool to wrap. // // Input style: when xdotool is available the pointer/keyboard ACTIONS // (navigate via the omnibox, click, type, scroll, tab keys) are driven as REAL // X input — the cursor visibly moves and text is typed one character at a time, // exactly as a person would. If xdotool is missing it falls back to the // Playwright/CDP API so the action still happens (just without a visible // cursor). READ actions always use the CDP/DOM API. // // Commands (json): { "action": "", ...params } // status | listTabs // navigate {url} | back | forward | refresh // newTab {url?} | closeTab {index?} | activateTab {index} | closePopups // moveMouse {selector | site} (hover the real cursor, no click) // click {selector} | type {text, selector?} | scroll {dir, notches?} // pressKey {key} | screenshot {path} import { chromium } from 'playwright'; import { execFileSync } from 'node:child_process'; import * as human from './human.mjs'; const CDP = process.env.CDP_PORT || '9222'; const CDP_HOST = process.env.CDP_HOST || '127.0.0.1'; const out = (o) => process.stdout.write(JSON.stringify(o)); const HAS_XDOTOOL = (() => { try { execFileSync('which', ['xdotool'], { stdio: 'ignore' }); return true; } catch { return false; } })(); let cmd; try { cmd = JSON.parse(process.argv[2] || '{}'); } catch (e) { out({ ok: false, error: `bad command json: ${e?.message || e}` }); process.exit(1); } const action = String(cmd.action || '').trim(); if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); } const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`); // Per-site homepage + search-box selector, shared by `search` and `moveMouse`. const SITES = { naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' }, google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' }, daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' }, youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' }, bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' }, }; // The genuinely-active tab is the one whose document is visible. Playwright has // no "active page" accessor over CDP, so probe visibilityState (fixes treating // tab 0 as active and breaking sequential ops on a specific tab). async function pickActive(pages) { for (const p of pages) { try { if (await p.evaluate(() => document.visibilityState === 'visible')) return p; } catch { /* page may be closing */ } } return pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0]; } async function tabInfo(pages, active) { const list = []; for (let i = 0; i < pages.length; i++) { const p = pages[i]; list.push({ index: i, url: p.url(), title: await p.title().catch(() => ''), active: p === active }); } return list; } let b; try { b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`); const ctx = b.contexts()[0]; if (!ctx) throw new Error('no browser context (is Chrome running?)'); let pages = ctx.pages(); if (!pages.length) pages = [await ctx.newPage()]; let page = await pickActive(pages); page.setDefaultTimeout(20000); // Auto-dismiss native JS dialogs (alert/confirm/beforeunload) so a popup can // never wedge the page. page.on('dialog', (d) => d.dismiss().catch(() => {})); const front = async (p) => { await p.bringToFront().catch(() => {}); }; const reload = async () => { await page.reload({ waitUntil: 'domcontentloaded' }).catch(() => {}); }; switch (action) { case 'status': case 'listTabs': await front(page); out({ ok: true, browserOpen: true, xdotool: HAS_XDOTOOL, tabCount: pages.length, tabs: await tabInfo(pages, page) }); break; case 'navigate': { const url = String(cmd.url || '').trim(); if (!url) throw new Error('navigate: no url'); await front(page); if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(norm(url)); await page.waitForLoadState('domcontentloaded').catch(() => {}); } catch { await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); } } else { await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); } out({ ok: true, url: page.url(), title: await page.title().catch(() => ''), input: HAS_XDOTOOL ? 'human' : 'api' }); break; } case 'search': { // Search like a PERSON: open the site's main page, click its search box, // type the query char-by-char, press Enter — NOT a direct results-URL. const q = String(cmd.query || '').trim(); if (!q) throw new Error('search: no query'); const siteKey = String(cmd.site || 'google').toLowerCase(); const s = SITES[siteKey] || SITES.google; await front(page); // 1) Go to the homepage. if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(s.home); await page.waitForLoadState('domcontentloaded').catch(() => {}); } catch { await page.goto(s.home, { waitUntil: 'domcontentloaded' }); } } else { await page.goto(s.home, { waitUntil: 'domcontentloaded' }); } // 2) Click the on-page search box, type the query, submit. const box = page.locator(s.box).first(); await box.waitFor({ state: 'visible', timeout: 15000 }).catch(() => {}); // Report which input path actually ran: 'human' = real xdotool cursor // move + char typing; 'api-fallback' = the humanClick path threw and we // fell back to cursor-less DOM click/fill; 'api' = no xdotool at all. This // makes "did the cursor really move" verifiable from the result. let searchInput; if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, box); await human.humanType(q); await human.pressKey('Return'); searchInput = 'human'; } catch { searchInput = 'api-fallback'; await box.click().catch(() => {}); await box.fill(q).catch(() => {}); await page.keyboard.press('Enter').catch(() => {}); } } else { searchInput = 'api'; await box.click().catch(() => {}); await box.fill(q); await page.keyboard.press('Enter'); } await page.waitForLoadState('domcontentloaded').catch(() => {}); out({ ok: true, site: SITES[siteKey] ? siteKey : 'google', query: q, url: page.url(), title: await page.title().catch(() => ''), input: searchInput }); break; } case 'back': await front(page); await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break; case 'forward': await front(page); await page.goForward({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break; case 'refresh': await front(page); await reload(); out({ ok: true, url: page.url() }); break; case 'newTab': { let np; if (HAS_XDOTOOL && cmd.human !== false) { await front(page); try { await human.pressKey('ctrl+t'); } catch { /* fall through */ } await page.waitForTimeout(500); const after = ctx.pages(); np = after[after.length - 1]; } if (!np) np = await ctx.newPage(); // API fallback / no xdotool await front(np); if (cmd.url) { if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(norm(cmd.url)); } catch { await np.goto(norm(cmd.url)).catch(() => {}); } } else await np.goto(norm(cmd.url), { waitUntil: 'domcontentloaded' }).catch(() => {}); } out({ ok: true, index: ctx.pages().indexOf(np), url: np.url(), input: HAS_XDOTOOL ? 'human' : 'api' }); break; } case 'activateTab': { const idx = Number(cmd.index); if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index'); // Real keyboard: Ctrl+<1..8> selects that tab, Ctrl+9 the last. if (HAS_XDOTOOL && cmd.human !== false && idx < 8) { await front(pages[0]); try { await human.pressKey(`ctrl+${idx + 1}`); } catch { await pages[idx].bringToFront(); } } else { await pages[idx].bringToFront(); } out({ ok: true, active: idx, url: pages[idx].url() }); break; } case 'closeTab': { const idx = Number.isInteger(Number(cmd.index)) ? Number(cmd.index) : pages.indexOf(page); if (idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index'); if (HAS_XDOTOOL && cmd.human !== false && idx < 8) { await front(pages[0]); try { await human.pressKey(`ctrl+${idx + 1}`); await human.pressKey('ctrl+w'); } catch { await pages[idx].close(); } } else { await pages[idx].close(); } out({ ok: true, closed: idx, remaining: ctx.pages().length }); break; } case 'closePopups': { // Close popup / blank / extra tabs, keeping the active content tab. let closed = 0; for (const p of pages) { if (p === page) continue; const u = p.url(); if (cmd.all || !u || u === 'about:blank') { await p.close().catch(() => {}); closed++; } } out({ ok: true, closed, remaining: ctx.pages().length }); break; } case 'moveMouse': { // Move/hover the REAL cursor onto an element WITHOUT clicking. Target is a // CSS selector, or site=naver/google/... for that site's search box. // Only meaningful with xdotool (the visible cursor); with no xdotool there // is no cursor to move, so report that rather than faking success. Every // failure to actually move (no xdotool, selector never matches, element // has no on-screen box) returns ok:false — we must never claim the cursor // moved when it did not (the exact bug the user reported). const siteKey = String(cmd.site || '').toLowerCase(); const selector = String(cmd.selector || '').trim() || (SITES[siteKey] ? SITES[siteKey].box : ''); if (!selector) throw new Error('moveMouse: no selector or known site'); if (!(HAS_XDOTOOL && cmd.human !== false)) { out({ ok: false, error: 'no xdotool: cannot move the visible cursor on this host' }); break; } await front(page); let locator = page.locator(selector).first(); let visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false); // A named site whose search box isn't on the current page: go to its home // first (real omnibox), then target the box there. if (!visible && SITES[siteKey]) { try { await human.navigateOmnibox(SITES[siteKey].home); await page.waitForLoadState('domcontentloaded').catch(() => {}); } catch { await page.goto(SITES[siteKey].home, { waitUntil: 'domcontentloaded' }).catch(() => {}); } locator = page.locator(SITES[siteKey].box).first(); visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false); } if (!visible) { out({ ok: false, error: `moveMouse: target not found (${cmd.selector || siteKey})` }); break; } const moved = await human.humanHover(page, locator); if (!moved) { out({ ok: false, error: 'moveMouse: element has no on-screen box; cursor not moved' }); break; } out({ ok: true, target: cmd.selector || siteKey, input: 'human' }); break; } case 'click': { const selector = String(cmd.selector || '').trim(); if (!selector) throw new Error('click: no selector'); await front(page); const locator = page.locator(selector).first(); if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click(); } } else await locator.click(); out({ ok: true }); break; } case 'type': { const text = String(cmd.text ?? ''); await front(page); if (cmd.selector) { const locator = page.locator(String(cmd.selector)).first(); if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click().catch(() => {}); } } else { await locator.fill(text); out({ ok: true, input: 'api' }); break; } } if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanType(text); } catch { await page.keyboard.type(text, { delay: 80 }); } } else await page.keyboard.type(text, { delay: 80 }); out({ ok: true, input: HAS_XDOTOOL ? 'human' : 'api' }); break; } case 'scroll': { const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1; if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanScroll(page, dir, Number(cmd.notches) || 5); } catch { await page.mouse.wheel(0, dir * 600); } } else await page.mouse.wheel(0, dir * (Number(cmd.notches) || 5) * 120); out({ ok: true }); break; } case 'pressKey': { const key = String(cmd.key || '').trim(); if (!key) throw new Error('pressKey: no key'); if (HAS_XDOTOOL) { try { await human.pressKey(key); } catch { await page.keyboard.press(key); } } else await page.keyboard.press(key); out({ ok: true }); break; } case 'screenshot': { const path = String(cmd.path || '').trim(); if (!path) throw new Error('screenshot: no path'); await front(page); await page.screenshot({ path }); out({ ok: true, path }); break; } default: out({ ok: false, error: `unknown action: ${action}` }); await b.close(); process.exit(1); } await b.close(); } catch (e) { try { await b?.close(); } catch { /* ignore */ } out({ ok: false, error: String(e?.message || e) }); process.exit(1); }