ffc16665e5
Some checks failed
Release / semantic-release (push) Successful in 19s
tests / Unit tests (Linux, Python 3.11) (push) Failing after 5m15s
Release / build-linux (push) Failing after 7m7s
Release / build-windows (push) Has been cancelled
Release / build-macos (arm64, macos-latest) (push) Has been cancelled
Release / build-macos (x64, macos-15-intel) (push) Has been cancelled
Release / release-main (push) Has been cancelled
Release / release-develop (push) Has been cancelled
moveMouse returned ok:true even when humanHover did nothing (no on-screen box) or the selector never matched — recreating the "claims it moved but didn't" bug. Now: humanHover returns a boolean (and brings the element into view first); moveMouse returns ok:false when the target isn't found or has no on-screen box, and when site=naver/... whose box isn't on the current page it navigates to the site home first before hovering. search now reports input=human|api-fallback|api so a silent fallback to cursor-less DOM input is visible, and the tool surfaces that note in the reply instead of implying a human-like search happened.
320 lines
14 KiB
JavaScript
320 lines
14 KiB
JavaScript
// Human-operable Chrome control for the on-screen (Go-Live) browser.
|
|
//
|
|
// node chrome-control.mjs '<json-command>'
|
|
//
|
|
// Connects to the on-screen Chrome over CDP (so every action is visible on the
|
|
// broadcast) and performs ONE command, printing a JSON result on stdout for the
|
|
// Python `controlBrowser` tool to wrap.
|
|
//
|
|
// Input style: when xdotool is available the pointer/keyboard ACTIONS
|
|
// (navigate via the omnibox, click, type, scroll, tab keys) are driven as REAL
|
|
// X input — the cursor visibly moves and text is typed one character at a time,
|
|
// exactly as a person would. If xdotool is missing it falls back to the
|
|
// Playwright/CDP API so the action still happens (just without a visible
|
|
// cursor). READ actions always use the CDP/DOM API.
|
|
//
|
|
// Commands (json): { "action": "<name>", ...params }
|
|
// status | listTabs
|
|
// navigate {url} | back | forward | refresh
|
|
// newTab {url?} | closeTab {index?} | activateTab {index} | closePopups
|
|
// moveMouse {selector | site} (hover the real cursor, no click)
|
|
// click {selector} | type {text, selector?} | scroll {dir, notches?}
|
|
// pressKey {key} | screenshot {path}
|
|
import { chromium } from 'playwright';
|
|
import { execFileSync } from 'node:child_process';
|
|
import * as human from './human.mjs';
|
|
|
|
const CDP = process.env.CDP_PORT || '9222';
|
|
const CDP_HOST = process.env.CDP_HOST || '127.0.0.1';
|
|
const out = (o) => process.stdout.write(JSON.stringify(o));
|
|
|
|
const HAS_XDOTOOL = (() => {
|
|
try { execFileSync('which', ['xdotool'], { stdio: 'ignore' }); return true; }
|
|
catch { return false; }
|
|
})();
|
|
|
|
let cmd;
|
|
try { cmd = JSON.parse(process.argv[2] || '{}'); }
|
|
catch (e) { out({ ok: false, error: `bad command json: ${e?.message || e}` }); process.exit(1); }
|
|
const action = String(cmd.action || '').trim();
|
|
if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); }
|
|
|
|
const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`);
|
|
|
|
// Per-site homepage + search-box selector, shared by `search` and `moveMouse`.
|
|
const SITES = {
|
|
naver: { home: 'https://www.naver.com', box: '#query, input[name="query"]' },
|
|
google: { home: 'https://www.google.com', box: 'textarea[name="q"], input[name="q"]' },
|
|
daum: { home: 'https://www.daum.net', box: '#q, input[name="q"]' },
|
|
youtube: { home: 'https://www.youtube.com', box: 'input#search, input[name="search_query"]' },
|
|
bing: { home: 'https://www.bing.com', box: '#sb_form_q, input[name="q"]' },
|
|
};
|
|
|
|
// The genuinely-active tab is the one whose document is visible. Playwright has
|
|
// no "active page" accessor over CDP, so probe visibilityState (fixes treating
|
|
// tab 0 as active and breaking sequential ops on a specific tab).
|
|
async function pickActive(pages) {
|
|
for (const p of pages) {
|
|
try { if (await p.evaluate(() => document.visibilityState === 'visible')) return p; }
|
|
catch { /* page may be closing */ }
|
|
}
|
|
return pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0];
|
|
}
|
|
|
|
async function tabInfo(pages, active) {
|
|
const list = [];
|
|
for (let i = 0; i < pages.length; i++) {
|
|
const p = pages[i];
|
|
list.push({ index: i, url: p.url(), title: await p.title().catch(() => ''), active: p === active });
|
|
}
|
|
return list;
|
|
}
|
|
|
|
let b;
|
|
try {
|
|
b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`);
|
|
const ctx = b.contexts()[0];
|
|
if (!ctx) throw new Error('no browser context (is Chrome running?)');
|
|
let pages = ctx.pages();
|
|
if (!pages.length) pages = [await ctx.newPage()];
|
|
let page = await pickActive(pages);
|
|
page.setDefaultTimeout(20000);
|
|
// Auto-dismiss native JS dialogs (alert/confirm/beforeunload) so a popup can
|
|
// never wedge the page.
|
|
page.on('dialog', (d) => d.dismiss().catch(() => {}));
|
|
|
|
const front = async (p) => { await p.bringToFront().catch(() => {}); };
|
|
const reload = async () => { await page.reload({ waitUntil: 'domcontentloaded' }).catch(() => {}); };
|
|
|
|
switch (action) {
|
|
case 'status':
|
|
case 'listTabs':
|
|
await front(page);
|
|
out({ ok: true, browserOpen: true, xdotool: HAS_XDOTOOL, tabCount: pages.length, tabs: await tabInfo(pages, page) });
|
|
break;
|
|
|
|
case 'navigate': {
|
|
const url = String(cmd.url || '').trim();
|
|
if (!url) throw new Error('navigate: no url');
|
|
await front(page);
|
|
if (HAS_XDOTOOL && cmd.human !== false) {
|
|
try { await human.navigateOmnibox(norm(url)); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
|
|
catch { await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); }
|
|
} else {
|
|
await page.goto(norm(url), { waitUntil: 'domcontentloaded' });
|
|
}
|
|
out({ ok: true, url: page.url(), title: await page.title().catch(() => ''), input: HAS_XDOTOOL ? 'human' : 'api' });
|
|
break;
|
|
}
|
|
|
|
case 'search': {
|
|
// Search like a PERSON: open the site's main page, click its search box,
|
|
// type the query char-by-char, press Enter — NOT a direct results-URL.
|
|
const q = String(cmd.query || '').trim();
|
|
if (!q) throw new Error('search: no query');
|
|
const siteKey = String(cmd.site || 'google').toLowerCase();
|
|
const s = SITES[siteKey] || SITES.google;
|
|
await front(page);
|
|
// 1) Go to the homepage.
|
|
if (HAS_XDOTOOL && cmd.human !== false) {
|
|
try { await human.navigateOmnibox(s.home); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
|
|
catch { await page.goto(s.home, { waitUntil: 'domcontentloaded' }); }
|
|
} else {
|
|
await page.goto(s.home, { waitUntil: 'domcontentloaded' });
|
|
}
|
|
// 2) Click the on-page search box, type the query, submit.
|
|
const box = page.locator(s.box).first();
|
|
await box.waitFor({ state: 'visible', timeout: 15000 }).catch(() => {});
|
|
// Report which input path actually ran: 'human' = real xdotool cursor
|
|
// move + char typing; 'api-fallback' = the humanClick path threw and we
|
|
// fell back to cursor-less DOM click/fill; 'api' = no xdotool at all. This
|
|
// makes "did the cursor really move" verifiable from the result.
|
|
let searchInput;
|
|
if (HAS_XDOTOOL && cmd.human !== false) {
|
|
try {
|
|
await human.humanClick(page, box);
|
|
await human.humanType(q);
|
|
await human.pressKey('Return');
|
|
searchInput = 'human';
|
|
} catch {
|
|
searchInput = 'api-fallback';
|
|
await box.click().catch(() => {});
|
|
await box.fill(q).catch(() => {});
|
|
await page.keyboard.press('Enter').catch(() => {});
|
|
}
|
|
} else {
|
|
searchInput = 'api';
|
|
await box.click().catch(() => {});
|
|
await box.fill(q);
|
|
await page.keyboard.press('Enter');
|
|
}
|
|
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
|
out({ ok: true, site: SITES[siteKey] ? siteKey : 'google', query: q, url: page.url(), title: await page.title().catch(() => ''), input: searchInput });
|
|
break;
|
|
}
|
|
|
|
case 'back': await front(page); await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break;
|
|
case 'forward': await front(page); await page.goForward({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break;
|
|
case 'refresh': await front(page); await reload(); out({ ok: true, url: page.url() }); break;
|
|
|
|
case 'newTab': {
|
|
let np;
|
|
if (HAS_XDOTOOL && cmd.human !== false) {
|
|
await front(page);
|
|
try { await human.pressKey('ctrl+t'); } catch { /* fall through */ }
|
|
await page.waitForTimeout(500);
|
|
const after = ctx.pages();
|
|
np = after[after.length - 1];
|
|
}
|
|
if (!np) np = await ctx.newPage(); // API fallback / no xdotool
|
|
await front(np);
|
|
if (cmd.url) {
|
|
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(norm(cmd.url)); } catch { await np.goto(norm(cmd.url)).catch(() => {}); } }
|
|
else await np.goto(norm(cmd.url), { waitUntil: 'domcontentloaded' }).catch(() => {});
|
|
}
|
|
out({ ok: true, index: ctx.pages().indexOf(np), url: np.url(), input: HAS_XDOTOOL ? 'human' : 'api' });
|
|
break;
|
|
}
|
|
|
|
case 'activateTab': {
|
|
const idx = Number(cmd.index);
|
|
if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index');
|
|
// Real keyboard: Ctrl+<1..8> selects that tab, Ctrl+9 the last.
|
|
if (HAS_XDOTOOL && cmd.human !== false && idx < 8) {
|
|
await front(pages[0]);
|
|
try { await human.pressKey(`ctrl+${idx + 1}`); } catch { await pages[idx].bringToFront(); }
|
|
} else {
|
|
await pages[idx].bringToFront();
|
|
}
|
|
out({ ok: true, active: idx, url: pages[idx].url() });
|
|
break;
|
|
}
|
|
|
|
case 'closeTab': {
|
|
const idx = Number.isInteger(Number(cmd.index)) ? Number(cmd.index) : pages.indexOf(page);
|
|
if (idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
|
|
if (HAS_XDOTOOL && cmd.human !== false && idx < 8) {
|
|
await front(pages[0]);
|
|
try { await human.pressKey(`ctrl+${idx + 1}`); await human.pressKey('ctrl+w'); }
|
|
catch { await pages[idx].close(); }
|
|
} else {
|
|
await pages[idx].close();
|
|
}
|
|
out({ ok: true, closed: idx, remaining: ctx.pages().length });
|
|
break;
|
|
}
|
|
|
|
case 'closePopups': {
|
|
// Close popup / blank / extra tabs, keeping the active content tab.
|
|
let closed = 0;
|
|
for (const p of pages) {
|
|
if (p === page) continue;
|
|
const u = p.url();
|
|
if (cmd.all || !u || u === 'about:blank') { await p.close().catch(() => {}); closed++; }
|
|
}
|
|
out({ ok: true, closed, remaining: ctx.pages().length });
|
|
break;
|
|
}
|
|
|
|
case 'moveMouse': {
|
|
// Move/hover the REAL cursor onto an element WITHOUT clicking. Target is a
|
|
// CSS selector, or site=naver/google/... for that site's search box.
|
|
// Only meaningful with xdotool (the visible cursor); with no xdotool there
|
|
// is no cursor to move, so report that rather than faking success. Every
|
|
// failure to actually move (no xdotool, selector never matches, element
|
|
// has no on-screen box) returns ok:false — we must never claim the cursor
|
|
// moved when it did not (the exact bug the user reported).
|
|
const siteKey = String(cmd.site || '').toLowerCase();
|
|
const selector = String(cmd.selector || '').trim() || (SITES[siteKey] ? SITES[siteKey].box : '');
|
|
if (!selector) throw new Error('moveMouse: no selector or known site');
|
|
if (!(HAS_XDOTOOL && cmd.human !== false)) {
|
|
out({ ok: false, error: 'no xdotool: cannot move the visible cursor on this host' });
|
|
break;
|
|
}
|
|
await front(page);
|
|
let locator = page.locator(selector).first();
|
|
let visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false);
|
|
// A named site whose search box isn't on the current page: go to its home
|
|
// first (real omnibox), then target the box there.
|
|
if (!visible && SITES[siteKey]) {
|
|
try { await human.navigateOmnibox(SITES[siteKey].home); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
|
|
catch { await page.goto(SITES[siteKey].home, { waitUntil: 'domcontentloaded' }).catch(() => {}); }
|
|
locator = page.locator(SITES[siteKey].box).first();
|
|
visible = await locator.waitFor({ state: 'visible', timeout: 8000 }).then(() => true).catch(() => false);
|
|
}
|
|
if (!visible) {
|
|
out({ ok: false, error: `moveMouse: target not found (${cmd.selector || siteKey})` });
|
|
break;
|
|
}
|
|
const moved = await human.humanHover(page, locator);
|
|
if (!moved) {
|
|
out({ ok: false, error: 'moveMouse: element has no on-screen box; cursor not moved' });
|
|
break;
|
|
}
|
|
out({ ok: true, target: cmd.selector || siteKey, input: 'human' });
|
|
break;
|
|
}
|
|
|
|
case 'click': {
|
|
const selector = String(cmd.selector || '').trim();
|
|
if (!selector) throw new Error('click: no selector');
|
|
await front(page);
|
|
const locator = page.locator(selector).first();
|
|
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click(); } }
|
|
else await locator.click();
|
|
out({ ok: true });
|
|
break;
|
|
}
|
|
|
|
case 'type': {
|
|
const text = String(cmd.text ?? '');
|
|
await front(page);
|
|
if (cmd.selector) {
|
|
const locator = page.locator(String(cmd.selector)).first();
|
|
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click().catch(() => {}); } }
|
|
else { await locator.fill(text); out({ ok: true, input: 'api' }); break; }
|
|
}
|
|
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanType(text); } catch { await page.keyboard.type(text, { delay: 80 }); } }
|
|
else await page.keyboard.type(text, { delay: 80 });
|
|
out({ ok: true, input: HAS_XDOTOOL ? 'human' : 'api' });
|
|
break;
|
|
}
|
|
|
|
case 'scroll': {
|
|
const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1;
|
|
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanScroll(page, dir, Number(cmd.notches) || 5); } catch { await page.mouse.wheel(0, dir * 600); } }
|
|
else await page.mouse.wheel(0, dir * (Number(cmd.notches) || 5) * 120);
|
|
out({ ok: true });
|
|
break;
|
|
}
|
|
|
|
case 'pressKey': {
|
|
const key = String(cmd.key || '').trim();
|
|
if (!key) throw new Error('pressKey: no key');
|
|
if (HAS_XDOTOOL) { try { await human.pressKey(key); } catch { await page.keyboard.press(key); } }
|
|
else await page.keyboard.press(key);
|
|
out({ ok: true });
|
|
break;
|
|
}
|
|
|
|
case 'screenshot': {
|
|
const path = String(cmd.path || '').trim();
|
|
if (!path) throw new Error('screenshot: no path');
|
|
await front(page);
|
|
await page.screenshot({ path });
|
|
out({ ok: true, path });
|
|
break;
|
|
}
|
|
|
|
default:
|
|
out({ ok: false, error: `unknown action: ${action}` });
|
|
await b.close();
|
|
process.exit(1);
|
|
}
|
|
await b.close();
|
|
} catch (e) {
|
|
try { await b?.close(); } catch { /* ignore */ }
|
|
out({ ok: false, error: String(e?.message || e) });
|
|
process.exit(1);
|
|
}
|