javis_bot/bot/scripts/stream-test/human.mjs

// Human-like interaction helpers: drive the REAL X mouse/keyboard via xdotool
// so the cursor visibly moves and is captured by the screen stream, using
// Playwright only to LOCATE elements and read state. This is the default
// interaction mode for the browse scenarios.
//
// Note: only the user-visible browsing actions are real input (cursor move,
// click, scroll, char-by-char typing). Behind-the-scenes control (window
// fullscreen, play, quality, autoplay toggle, page navigation, and click
// fallbacks) intentionally uses the CDP/DOM API for reliability.
import { execFile } from 'node:child_process';

const DISPLAY = process.env.VNC_DISPLAY || ':1';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const rand = (a, b) => a + Math.random() * (b - a);
const xdo = (args) =>
  new Promise((res, rej) =>
    execFile('xdotool', args, { env: { ...process.env, DISPLAY } }, (e, so) => (e ? rej(e) : res(so || ''))),
  );

let cur = { x: 960, y: 540 };
const easeInOut = (t) => (t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2);

async function contentOrigin(page) {
  const g = await page.evaluate(() => ({
    sx: window.screenX, sy: window.screenY,
    ow: window.outerWidth, oh: window.outerHeight,
    iw: window.innerWidth, ih: window.innerHeight,
  }));
  const bx = Math.max(0, Math.round((g.ow - g.iw) / 2));
  const topInset = Math.max(0, g.oh - g.ih - bx);
  return { ox: g.sx + bx, oy: g.sy + topInset };
}

// Smoothly move the real cursor to a screen point with eased, slightly jittered steps.
export async function humanMove(toX, toY) {
  const steps = Math.max(12, Math.min(48, Math.round(Math.hypot(toX - cur.x, toY - cur.y) / 22)));
  const cmd = [];
  for (let i = 1; i <= steps; i++) {
    const t = easeInOut(i / steps);
    const jx = i < steps ? rand(-1.5, 1.5) : 0;
    const jy = i < steps ? rand(-1.5, 1.5) : 0;
    cmd.push('mousemove', String(Math.round(cur.x + (toX - cur.x) * t + jx)),
                          String(Math.round(cur.y + (toY - cur.y) * t + jy)),
             'sleep', rand(0.006, 0.018).toFixed(3));
  }
  await xdo(cmd);
  cur = { x: toX, y: toY };
  await sleep(rand(40, 130));
}

export async function humanClickXY(sx, sy) {
  await humanMove(sx, sy);
  await sleep(rand(60, 170));
  await xdo(['click', '1']);
  await sleep(rand(130, 300));
}

// Locate a Playwright element, move the real cursor into it (random offset), click.
export async function humanClick(page, locator) {
  await locator.scrollIntoViewIfNeeded().catch(() => {});
  await sleep(rand(150, 380));
  const box = await locator.boundingBox();
  if (!box) { await locator.click({ timeout: 5000 }).catch(() => {}); return; }
  const { ox, oy } = await contentOrigin(page);
  const sx = Math.round(ox + box.x + box.width * rand(0.35, 0.65));
  const sy = Math.round(oy + box.y + box.height * rand(0.35, 0.65));
  await humanClickXY(sx, sy);
}

// Type text one character at a time at a human, slightly irregular pace.
export async function humanType(text) {
  await sleep(rand(220, 420)); // let focus settle so the 1st char isn't dropped
  for (const ch of text) {
    await xdo(['type', '--clearmodifiers', '--', ch]);
    await sleep(rand(70, 200));
    if (Math.random() < 0.12) await sleep(rand(150, 400)); // occasional pause
  }
}

export async function pressKey(key) {
  await xdo(['key', '--clearmodifiers', key]);
  await sleep(rand(120, 280));
}

// Gradual wheel scroll (dir>0 = down). Optionally hover over an element first.
export async function humanScroll(page, dir, notches, overLocator) {
  if (overLocator) {
    const box = await overLocator.boundingBox().catch(() => null);
    if (box) {
      const { ox, oy } = await contentOrigin(page);
      await humanMove(Math.round(ox + box.x + box.width / 2), Math.round(oy + box.y + box.height / 2));
    }
  }
  const button = dir > 0 ? '5' : '4';
  for (let i = 0; i < notches; i++) {
    await xdo(['click', button]);
    await sleep(rand(40, 115));
    if (i % 6 === 5) await sleep(rand(250, 600)); // pause like reading
  }
  await sleep(rand(250, 600));
}

export { sleep, rand };