tkrmagid/javis_bot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(stream-test): drive the whole browse scenario with real input

Browse Source 
Make every action real keyboard/mouse via xdotool, not just the visible
browsing: address-bar navigation (Ctrl+L + char-by-char typing), the YouTube
settings gear -> 화질 -> 1080p menu (real clicks, verified hd1080), the autoplay
toggle, the play button, and fullscreen via the real 'f' key (F11 isn't honored
by this WM; 'f' yields true 1080p fullscreen without pausing). CDP/DOM API is
now used only to read state for verification.
This commit is contained in:
javis-bot
2026-06-10 14:11:58 +09:00
parent 1e30a49562
commit 2cdd159fc1
3 changed files with 91 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
bot/scripts/stream-test/scenario.mjs
View File

@@ -1,68 +1,97 @@
// Browse scenario driven with human-like real mouse/keyboard (see human.mjs).
// Connects to a Chrome already running with --remote-debugging-port (default
// 9222) on the streamed X display, and performs:
// YouTube search -> open IU live concert -> 1080p -> watch 20s -> fullscreen
// 20s -> Naver search 아이유 -> scroll -> 나무위키 -> scroll.
// Browse scenario driven ENTIRELY with real mouse/keyboard input via xdotool
// (see human.mjs). Connects to a Chrome already running with
// --remote-debugging-port (default 9222) on the streamed X display.
//
// Real input (visible on the stream): search/Naver typing, clicking the video,
// the fullscreen button, scrolling, entering 나무위키.
// API-driven (behind the scenes): window fullscreen, play, quality, autoplay
// toggle, page navigation, and click fallbacks.
// All ACTIONS are real input: address-bar navigation (Ctrl+L + typing),
// search typing, clicking the video, the settings gear -> 화질 -> 1080p menu,
// the autoplay toggle, the play button, fullscreen via the 'f' key, scrolling,
// and entering 나무위키. The CDP/DOM API is used ONLY to read state for
// verification (paused/quality/fullscreen) and as a rare click fallback when an
// element has no on-screen box.
import { chromium } from 'playwright';
import { humanClick, humanType, pressKey, humanScroll, sleep } from './human.mjs';
import { humanClick, humanType, humanKey, humanHover, navigateOmnibox, humanScroll, sleep } from './human.mjs';
const CDP = process.env.CDP_PORT || '9222';
const VID = process.env.TEST_VIDEO_ID || 'X_am71G6Vy4'; // IU HEREH WORLD TOUR (live, 1080p+)
const VID = process.env.TEST_VIDEO_ID || 'X_am71G6Vy4';
const SEARCH = process.env.TEST_YT_QUERY || '내손을잡아';
const NAVER_Q = process.env.TEST_NAVER_QUERY || '아이유';
const b = await chromium.connectOverCDP(`http://localhost:${CDP}`);
const ctx = b.contexts()[0];
const page = ctx.pages()[0];
const s = await ctx.newCDPSession(page);
page.setDefaultTimeout(25000);
const winState = async (st) => { const { windowId } = await s.send('Browser.getWindowForTarget'); await s.send('Browser.setWindowBounds', { windowId, bounds: { windowState: st } }); };
const ensurePlaying = () => page.evaluate(() => { const v = document.querySelector('video'); const p = document.getElementById('movie_player'); try { p && p.playVideo && p.playVideo(); } catch {} if (v && v.paused) v.play().catch(() => {}); });
const autoplayOff = () => page.evaluate(() => { const btn = document.querySelector('.ytp-autonav-toggle-button'); if (btn && btn.getAttribute('aria-checked') === 'true') btn.click(); });
const read = (fn) => page.evaluate(fn);
const playerLoc = () => page.locator('#movie_player');
await page.evaluate(() => { if (document.fullscreenElement) document.exitFullscreen?.(); });
await winState('normal');
await page.goto('https://www.youtube.com', { waitUntil: 'domcontentloaded' }); await sleep(2500);
// 1) open YouTube by typing the URL in the address bar
await navigateOmnibox('https://www.youtube.com'); await sleep(3000);
// 2) really type the search and submit
await humanClick(page, page.locator('input#search, input[name=search_query]').first());
await humanType(SEARCH);
await pressKey('Return');
await humanKey('Return');
await sleep(3800);
// 3) click the IU concert result with the real mouse
let link = page.locator(`a#video-title[href*="${VID}"], a[href*="${VID}"]`).first();
if (!(await link.count().catch(() => 0))) link = page.locator('ytd-video-renderer a#video-title, ytd-rich-item-renderer a#video-title').first();
await humanClick(page, link);
await sleep(3000);
if (!/watch/.test(page.url())) await page.goto('https://www.youtube.com/watch?v=' + VID, { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#movie_player', { timeout: 25000 }); await sleep(2500);
for (let i = 0; i < 8; i++) { const ad = page.locator('.ytp-ad-skip-button, .ytp-ad-skip-button-modern, .ytp-skip-ad-button'); if (await ad.count().catch(() => 0)) { await ad.first().click({ timeout: 1500 }).catch(() => {}); await sleep(1200); } else break; }
await sleep(3500);
await page.waitForSelector('#movie_player', { timeout: 25000 }); await sleep(2000);
for (let i = 0; i < 8; i++) { const ad = page.locator('.ytp-ad-skip-button, .ytp-ad-skip-button-modern, .ytp-skip-ad-button'); if (await ad.count().catch(() => 0)) { await humanClick(page, ad.first()); await sleep(1200); } else break; }
await ensurePlaying(); await sleep(1200);
await page.evaluate(() => { const p = document.getElementById('movie_player'); try { const L = p.getAvailableQualityLevels ? p.getAvailableQualityLevels() : []; const w = L.includes('hd1080') ? 'hd1080' : (L[0]); p.setPlaybackQualityRange && p.setPlaybackQualityRange(w, w); } catch {} });
await autoplayOff();
// 4) if paused, press play with the real mouse
if (await read(() => document.querySelector('video')?.paused)) {
const big = page.locator('.ytp-large-play-button, .ytp-play-button').first();
await humanClick(page, big);
}
await sleep(1500);
// 5) set 1080p through the real settings menu (gear -> 화질 -> 1080p), verify
async function setQuality1080() {
for (let attempt = 0; attempt < 2; attempt++) {
await humanHover(page, playerLoc());
await humanClick(page, page.locator('.ytp-settings-button')); await sleep(900);
let qrow = page.locator('.ytp-menuitem', { hasText: /화질|Quality/ }).first();
if (!(await qrow.count().catch(() => 0))) qrow = page.locator('.ytp-panel-menu .ytp-menuitem').last();
await humanClick(page, qrow); await sleep(900);
const item = page.locator('.ytp-menuitem', { hasText: /1080/ }).first();
if (await item.count().catch(() => 0)) await humanClick(page, item);
await sleep(2000);
const q = await read(() => document.getElementById('movie_player')?.getPlaybackQuality?.());
if (q && /1080/.test(q)) return q;
}
return null;
}
console.log('QUALITY', await setQuality1080());
// 6) turn off autoplay with a real click if it is on
const auto = page.locator('.ytp-autonav-toggle-button');
if ((await auto.count().catch(() => 0)) && (await auto.getAttribute('aria-checked').catch(() => null)) === 'true') {
await humanHover(page, playerLoc());
await humanClick(page, auto);
}
console.log('STEP watch-1080-windowed'); await sleep(20000);
await winState('fullscreen'); await sleep(1200);
await humanClick(page, page.locator('.ytp-fullscreen-button'));
await sleep(800); await ensurePlaying();
console.log('STEP fullscreen'); await sleep(20000);
// 7) fullscreen with the real 'f' key (hover the player to focus it), 20s
await humanHover(page, playerLoc());
await humanKey('f'); await sleep(1500);
if (!(await read(() => !!document.fullscreenElement))) { await humanHover(page, playerLoc()); await humanKey('f'); await sleep(1200); }
console.log('STEP fullscreen', await read(() => ({ full: !!document.fullscreenElement, h: window.innerHeight }))); await sleep(20000);
await page.evaluate(() => { if (document.fullscreenElement) document.exitFullscreen?.(); });
await winState('normal'); await sleep(1500);
// 8) exit fullscreen with real 'f'
await humanKey('f'); await sleep(1500);
await page.goto('https://www.naver.com', { waitUntil: 'domcontentloaded' }); await sleep(2500);
// 9) Naver via the address bar, then really type the query
await navigateOmnibox('https://www.naver.com'); await sleep(2800);
await humanClick(page, page.locator('input#query').first());
await humanType(NAVER_Q);
await pressKey('Return');
await humanKey('Return');
await sleep(2800);
await humanScroll(page, +1, 18);
console.log('STEP naver-scrolled');
// 10) enter 나무위키 with a real click, then scroll
const namu = page.locator('a[href*="namu.wiki"]').first();
if (await namu.count().catch(() => 0)) {
await humanClick(page, namu);