feat: robust controlBrowser — real xdotool input, tabs/back/forward/popups
- Install xdotool + wmctrl (late Docker layer, preserves the melo cache) so the on-screen Chrome gets real X input (visible cursor, char-by-char typing) instead of synthetic events; falls back to the Playwright API if absent. - Fix active-tab detection (probe document.visibilityState instead of assuming tab 0) so sequential ops target the right tab. - Add back / forward / refresh; new/switch/close tabs via real keyboard (Ctrl+T / Ctrl+<n> / Ctrl+W) when xdotool is present. - Auto-dismiss native JS dialogs; closePopups clears blank/popup tabs. - Report broadcast (Go-Live) state in status from the turn context. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
javis-bot
@@ -4,182 +4,213 @@
|
||||
//
|
||||
// Connects to the on-screen Chrome over CDP (so every action is visible on the
|
||||
// broadcast) and performs ONE command, printing a JSON result on stdout for the
|
||||
// Python `controlBrowser` tool to wrap. READ actions use the CDP/DOM API;
|
||||
// ACTION actions (navigate/click/type/scroll) drive real input via xdotool
|
||||
// (human.mjs) so the cursor visibly moves and text is typed one char at a time,
|
||||
// exactly as a person would — never synthetic DOM events.
|
||||
// Python `controlBrowser` tool to wrap.
|
||||
//
|
||||
// Input style: when xdotool is available the pointer/keyboard ACTIONS
|
||||
// (navigate via the omnibox, click, type, scroll, tab keys) are driven as REAL
|
||||
// X input — the cursor visibly moves and text is typed one character at a time,
|
||||
// exactly as a person would. If xdotool is missing it falls back to the
|
||||
// Playwright/CDP API so the action still happens (just without a visible
|
||||
// cursor). READ actions always use the CDP/DOM API.
|
||||
//
|
||||
// Commands (json): { "action": "<name>", ...params }
|
||||
// status -> broadcast/browser readiness + tab list
|
||||
// listTabs -> [{index,url,title,active}]
|
||||
// navigate {url, human?} -> open url in the active tab
|
||||
// newTab {url?} -> open a new tab (optionally to url)
|
||||
// closeTab {index} -> close tab by index
|
||||
// activateTab{index} -> bring tab to front
|
||||
// closePopups -> close blank/popup tabs, keep the active one
|
||||
// click {selector, human?} -> click an element (human cursor by default)
|
||||
// type {text, selector?, human?}-> type text char-by-char (human by default)
|
||||
// scroll {dir, notches?} -> wheel scroll (dir: "down"|"up")
|
||||
// pressKey {key} -> press a single key (xdotool keysym)
|
||||
// screenshot {path} -> save a PNG of the active tab
|
||||
// status | listTabs
|
||||
// navigate {url} | back | forward | refresh
|
||||
// newTab {url?} | closeTab {index?} | activateTab {index} | closePopups
|
||||
// click {selector} | type {text, selector?} | scroll {dir, notches?}
|
||||
// pressKey {key} | screenshot {path}
|
||||
import { chromium } from 'playwright';
|
||||
import {
|
||||
humanClick,
|
||||
humanType,
|
||||
pressKey,
|
||||
humanScroll,
|
||||
navigateOmnibox,
|
||||
} from './human.mjs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import * as human from './human.mjs';
|
||||
|
||||
const CDP = process.env.CDP_PORT || '9222';
|
||||
const CDP_HOST = process.env.CDP_HOST || '127.0.0.1';
|
||||
const out = (o) => process.stdout.write(JSON.stringify(o));
|
||||
|
||||
const HAS_XDOTOOL = (() => {
|
||||
try { execFileSync('which', ['xdotool'], { stdio: 'ignore' }); return true; }
|
||||
catch { return false; }
|
||||
})();
|
||||
|
||||
let cmd;
|
||||
try {
|
||||
cmd = JSON.parse(process.argv[2] || '{}');
|
||||
} catch (e) {
|
||||
out({ ok: false, error: `bad command json: ${e?.message || e}` });
|
||||
process.exit(1);
|
||||
}
|
||||
try { cmd = JSON.parse(process.argv[2] || '{}'); }
|
||||
catch (e) { out({ ok: false, error: `bad command json: ${e?.message || e}` }); process.exit(1); }
|
||||
const action = String(cmd.action || '').trim();
|
||||
if (!action) {
|
||||
out({ ok: false, error: 'no action' });
|
||||
process.exit(1);
|
||||
if (!action) { out({ ok: false, error: 'no action' }); process.exit(1); }
|
||||
|
||||
const norm = (u) => (/^https?:\/\//i.test(u) ? u : `https://${u}`);
|
||||
|
||||
// The genuinely-active tab is the one whose document is visible. Playwright has
|
||||
// no "active page" accessor over CDP, so probe visibilityState (fixes treating
|
||||
// tab 0 as active and breaking sequential ops on a specific tab).
|
||||
async function pickActive(pages) {
|
||||
for (const p of pages) {
|
||||
try { if (await p.evaluate(() => document.visibilityState === 'visible')) return p; }
|
||||
catch { /* page may be closing */ }
|
||||
}
|
||||
return pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0];
|
||||
}
|
||||
|
||||
const tabInfo = async (pages, active) => {
|
||||
async function tabInfo(pages, active) {
|
||||
const list = [];
|
||||
for (let i = 0; i < pages.length; i++) {
|
||||
const p = pages[i];
|
||||
list.push({
|
||||
index: i,
|
||||
url: p.url(),
|
||||
title: await p.title().catch(() => ''),
|
||||
active: p === active,
|
||||
});
|
||||
list.push({ index: i, url: p.url(), title: await p.title().catch(() => ''), active: p === active });
|
||||
}
|
||||
return list;
|
||||
};
|
||||
}
|
||||
|
||||
let b;
|
||||
try {
|
||||
b = await chromium.connectOverCDP(`http://${CDP_HOST}:${CDP}`);
|
||||
const ctx = b.contexts()[0];
|
||||
if (!ctx) throw new Error('no browser context (is Chrome running?)');
|
||||
const pages = ctx.pages();
|
||||
// The "active" tab: the most recently fronted page. Playwright has no direct
|
||||
// accessor, so we treat the first visible page as active and let activateTab
|
||||
// change focus explicitly.
|
||||
let page = pages.find((p) => p.url() && p.url() !== 'about:blank') || pages[0] || (await ctx.newPage());
|
||||
let pages = ctx.pages();
|
||||
if (!pages.length) pages = [await ctx.newPage()];
|
||||
let page = await pickActive(pages);
|
||||
page.setDefaultTimeout(20000);
|
||||
// Auto-dismiss native JS dialogs (alert/confirm/beforeunload) so a popup can
|
||||
// never wedge the page.
|
||||
page.on('dialog', (d) => d.dismiss().catch(() => {}));
|
||||
|
||||
const front = async (p) => { await p.bringToFront().catch(() => {}); };
|
||||
const reload = async () => { await page.reload({ waitUntil: 'domcontentloaded' }).catch(() => {}); };
|
||||
|
||||
switch (action) {
|
||||
case 'status': {
|
||||
out({ ok: true, browserOpen: true, tabCount: pages.length, tabs: await tabInfo(pages, page) });
|
||||
case 'status':
|
||||
case 'listTabs':
|
||||
await front(page);
|
||||
out({ ok: true, browserOpen: true, xdotool: HAS_XDOTOOL, tabCount: pages.length, tabs: await tabInfo(pages, page) });
|
||||
break;
|
||||
}
|
||||
case 'listTabs': {
|
||||
out({ ok: true, tabs: await tabInfo(pages, page) });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'navigate': {
|
||||
const url = String(cmd.url || '').trim();
|
||||
if (!url) throw new Error('navigate: no url');
|
||||
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
||||
await page.bringToFront().catch(() => {});
|
||||
if (cmd.human === false) {
|
||||
await page.goto(target, { waitUntil: 'domcontentloaded' });
|
||||
await front(page);
|
||||
if (HAS_XDOTOOL && cmd.human !== false) {
|
||||
try { await human.navigateOmnibox(norm(url)); await page.waitForLoadState('domcontentloaded').catch(() => {}); }
|
||||
catch { await page.goto(norm(url), { waitUntil: 'domcontentloaded' }); }
|
||||
} else {
|
||||
// Type the URL into the omnibox char-by-char with a real cursor.
|
||||
await navigateOmnibox(target);
|
||||
await page.waitForLoadState('domcontentloaded').catch(() => {});
|
||||
await page.goto(norm(url), { waitUntil: 'domcontentloaded' });
|
||||
}
|
||||
out({ ok: true, url: page.url(), title: await page.title().catch(() => '') });
|
||||
out({ ok: true, url: page.url(), title: await page.title().catch(() => ''), input: HAS_XDOTOOL ? 'human' : 'api' });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'back': await front(page); await page.goBack({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break;
|
||||
case 'forward': await front(page); await page.goForward({ waitUntil: 'domcontentloaded' }).catch(() => {}); out({ ok: true, url: page.url() }); break;
|
||||
case 'refresh': await front(page); await reload(); out({ ok: true, url: page.url() }); break;
|
||||
|
||||
case 'newTab': {
|
||||
const np = await ctx.newPage();
|
||||
await np.bringToFront().catch(() => {});
|
||||
if (cmd.url) {
|
||||
const target = /^https?:\/\//i.test(cmd.url) ? cmd.url : `https://${cmd.url}`;
|
||||
await np.goto(target, { waitUntil: 'domcontentloaded' }).catch(() => {});
|
||||
let np;
|
||||
if (HAS_XDOTOOL && cmd.human !== false) {
|
||||
await front(page);
|
||||
try { await human.pressKey('ctrl+t'); } catch { /* fall through */ }
|
||||
await page.waitForTimeout(500);
|
||||
const after = ctx.pages();
|
||||
np = after[after.length - 1];
|
||||
}
|
||||
out({ ok: true, index: ctx.pages().length - 1, url: np.url() });
|
||||
break;
|
||||
}
|
||||
case 'closeTab': {
|
||||
const idx = Number(cmd.index);
|
||||
if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
|
||||
await pages[idx].close();
|
||||
out({ ok: true, closed: idx, remaining: ctx.pages().length });
|
||||
if (!np) np = await ctx.newPage(); // API fallback / no xdotool
|
||||
await front(np);
|
||||
if (cmd.url) {
|
||||
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.navigateOmnibox(norm(cmd.url)); } catch { await np.goto(norm(cmd.url)).catch(() => {}); } }
|
||||
else await np.goto(norm(cmd.url), { waitUntil: 'domcontentloaded' }).catch(() => {});
|
||||
}
|
||||
out({ ok: true, index: ctx.pages().indexOf(np), url: np.url(), input: HAS_XDOTOOL ? 'human' : 'api' });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'activateTab': {
|
||||
const idx = Number(cmd.index);
|
||||
if (!Number.isInteger(idx) || idx < 0 || idx >= pages.length) throw new Error('activateTab: bad index');
|
||||
await pages[idx].bringToFront();
|
||||
// Real keyboard: Ctrl+<1..8> selects that tab, Ctrl+9 the last.
|
||||
if (HAS_XDOTOOL && cmd.human !== false && idx < 8) {
|
||||
await front(pages[0]);
|
||||
try { await human.pressKey(`ctrl+${idx + 1}`); } catch { await pages[idx].bringToFront(); }
|
||||
} else {
|
||||
await pages[idx].bringToFront();
|
||||
}
|
||||
out({ ok: true, active: idx, url: pages[idx].url() });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'closeTab': {
|
||||
const idx = Number.isInteger(Number(cmd.index)) ? Number(cmd.index) : pages.indexOf(page);
|
||||
if (idx < 0 || idx >= pages.length) throw new Error('closeTab: bad index');
|
||||
if (HAS_XDOTOOL && cmd.human !== false && idx < 8) {
|
||||
await front(pages[0]);
|
||||
try { await human.pressKey(`ctrl+${idx + 1}`); await human.pressKey('ctrl+w'); }
|
||||
catch { await pages[idx].close(); }
|
||||
} else {
|
||||
await pages[idx].close();
|
||||
}
|
||||
out({ ok: true, closed: idx, remaining: ctx.pages().length });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'closePopups': {
|
||||
// Close throwaway/popup tabs (about:blank or anything that isn't the
|
||||
// active content tab), keeping the active one.
|
||||
// Close popup / blank / extra tabs, keeping the active content tab.
|
||||
let closed = 0;
|
||||
for (const p of pages) {
|
||||
if (p === page) continue;
|
||||
const u = p.url();
|
||||
if (!u || u === 'about:blank' || cmd.all) {
|
||||
await p.close().catch(() => {});
|
||||
closed++;
|
||||
}
|
||||
if (cmd.all || !u || u === 'about:blank') { await p.close().catch(() => {}); closed++; }
|
||||
}
|
||||
out({ ok: true, closed, remaining: ctx.pages().length });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'click': {
|
||||
const selector = String(cmd.selector || '').trim();
|
||||
if (!selector) throw new Error('click: no selector');
|
||||
await page.bringToFront().catch(() => {});
|
||||
await front(page);
|
||||
const locator = page.locator(selector).first();
|
||||
if (cmd.human === false) await locator.click();
|
||||
else await humanClick(page, locator);
|
||||
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click(); } }
|
||||
else await locator.click();
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'type': {
|
||||
const text = String(cmd.text ?? '');
|
||||
await page.bringToFront().catch(() => {});
|
||||
await front(page);
|
||||
if (cmd.selector) {
|
||||
const locator = page.locator(String(cmd.selector)).first();
|
||||
if (cmd.human === false) { await locator.fill(text); out({ ok: true }); break; }
|
||||
await humanClick(page, locator); // focus the field with a real click first
|
||||
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanClick(page, locator); } catch { await locator.click().catch(() => {}); } }
|
||||
else { await locator.fill(text); out({ ok: true, input: 'api' }); break; }
|
||||
}
|
||||
await humanType(text);
|
||||
out({ ok: true });
|
||||
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanType(text); } catch { await page.keyboard.type(text, { delay: 80 }); } }
|
||||
else await page.keyboard.type(text, { delay: 80 });
|
||||
out({ ok: true, input: HAS_XDOTOOL ? 'human' : 'api' });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'scroll': {
|
||||
const dir = String(cmd.dir || 'down').toLowerCase() === 'up' ? -1 : 1;
|
||||
await humanScroll(page, dir, Number(cmd.notches) || 5);
|
||||
if (HAS_XDOTOOL && cmd.human !== false) { try { await human.humanScroll(page, dir, Number(cmd.notches) || 5); } catch { await page.mouse.wheel(0, dir * 600); } }
|
||||
else await page.mouse.wheel(0, dir * (Number(cmd.notches) || 5) * 120);
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'pressKey': {
|
||||
const key = String(cmd.key || '').trim();
|
||||
if (!key) throw new Error('pressKey: no key');
|
||||
await pressKey(key);
|
||||
if (HAS_XDOTOOL) { try { await human.pressKey(key); } catch { await page.keyboard.press(key); } }
|
||||
else await page.keyboard.press(key);
|
||||
out({ ok: true });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'screenshot': {
|
||||
const path = String(cmd.path || '').trim();
|
||||
if (!path) throw new Error('screenshot: no path');
|
||||
await page.bringToFront().catch(() => {});
|
||||
await front(page);
|
||||
await page.screenshot({ path });
|
||||
out({ ok: true, path });
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
out({ ok: false, error: `unknown action: ${action}` });
|
||||
await b.close();
|
||||
|
||||
Reference in New Issue
Block a user