// Human-like interaction helpers: drive the REAL X mouse/keyboard via xdotool // so the cursor visibly moves and is captured by the screen stream, using // Playwright only to LOCATE elements and read state. This is the default // interaction mode for the browse scenarios. // // Note: only the user-visible browsing actions are real input (cursor move, // click, scroll, char-by-char typing). Behind-the-scenes control (window // fullscreen, play, quality, autoplay toggle, page navigation, and click // fallbacks) intentionally uses the CDP/DOM API for reliability. import { execFile } from 'node:child_process'; const DISPLAY = process.env.VNC_DISPLAY || ':1'; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const rand = (a, b) => a + Math.random() * (b - a); const xdo = (args) => new Promise((res, rej) => execFile('xdotool', args, { env: { ...process.env, DISPLAY } }, (e, so) => (e ? rej(e) : res(so || ''))), ); let cur = { x: 960, y: 540 }; const easeInOut = (t) => (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2); async function contentOrigin(page) { const g = await page.evaluate(() => ({ sx: window.screenX, sy: window.screenY, ow: window.outerWidth, oh: window.outerHeight, iw: window.innerWidth, ih: window.innerHeight, })); const bx = Math.max(0, Math.round((g.ow - g.iw) / 2)); const topInset = Math.max(0, g.oh - g.ih - bx); return { ox: g.sx + bx, oy: g.sy + topInset }; } // Smoothly move the real cursor to a screen point with eased, slightly jittered steps. export async function humanMove(toX, toY) { const steps = Math.max(12, Math.min(48, Math.round(Math.hypot(toX - cur.x, toY - cur.y) / 22))); const cmd = []; for (let i = 1; i <= steps; i++) { const t = easeInOut(i / steps); const jx = i < steps ? rand(-1.5, 1.5) : 0; const jy = i < steps ? rand(-1.5, 1.5) : 0; cmd.push('mousemove', String(Math.round(cur.x + (toX - cur.x) * t + jx)), String(Math.round(cur.y + (toY - cur.y) * t + jy)), 'sleep', rand(0.006, 0.018).toFixed(3)); } await xdo(cmd); cur = { x: toX, y: toY }; await sleep(rand(40, 130)); } export async function humanClickXY(sx, sy) { await humanMove(sx, sy); await sleep(rand(60, 170)); await xdo(['click', '1']); await sleep(rand(130, 300)); } // Locate a Playwright element, move the real cursor into it (random offset), click. export async function humanClick(page, locator) { await locator.scrollIntoViewIfNeeded().catch(() => {}); await sleep(rand(150, 380)); const box = await locator.boundingBox(); if (!box) { await locator.click({ timeout: 5000 }).catch(() => {}); return; } const { ox, oy } = await contentOrigin(page); const sx = Math.round(ox + box.x + box.width * rand(0.35, 0.65)); const sy = Math.round(oy + box.y + box.height * rand(0.35, 0.65)); await humanClickXY(sx, sy); } // Type text one character at a time at a human, slightly irregular pace. export async function humanType(text) { await sleep(rand(220, 420)); // let focus settle so the 1st char isn't dropped for (const ch of text) { await xdo(['type', '--clearmodifiers', '--', ch]); await sleep(rand(70, 200)); if (Math.random() < 0.12) await sleep(rand(150, 400)); // occasional pause } } export async function pressKey(key) { await xdo(['key', '--clearmodifiers', key]); await sleep(rand(120, 280)); } // Gradual wheel scroll (dir>0 = down). Optionally hover over an element first. export async function humanScroll(page, dir, notches, overLocator) { if (overLocator) { const box = await overLocator.boundingBox().catch(() => null); if (box) { const { ox, oy } = await contentOrigin(page); await humanMove(Math.round(ox + box.x + box.width / 2), Math.round(oy + box.y + box.height / 2)); } } const button = dir > 0 ? '5' : '4'; for (let i = 0; i < notches; i++) { await xdo(['click', button]); await sleep(rand(40, 115)); if (i % 6 === 5) await sleep(rand(250, 600)); // pause like reading } await sleep(rand(250, 600)); } export { sleep, rand };