Files
javis_bot/scripts/m0/m0lib/video.py
javis-bot 457b8636fa feat(m0): measure a YouTube video's dropped frames next to Jarvis
User decision (2026-11-11): the 95 % GPU cap is off by default because
gamemode_tune showed it did not lower nvidia-smi's utilisation and cost
6 to 10 % of the LLM's output. The worry behind it was a video stuttering
next to Jarvis, so the new on-demand stage video_drops measures that:

- A public Blender upload plays at 1080p60 in a throwaway Chrome/Edge
  profile (removed afterwards), driven over DevTools with a small
  standard-library WebSocket client, so no package is added to the
  already built venvs.
- Each window counts the video's dropped frames (the player's own
  counter) and the gaps over 50/100 ms between shown frames
  (requestVideoFrameCallback); windows with an ad, a pause, the end or a
  new video are kept but not counted.
- Rows: video alone, Jarvis idle (normal placement with the STT room and
  1 GB for other programs), Jarvis busy (LLM answering, STT transcribing
  ten speakers back to back) without and with the cap in A B B A order,
  then the video alone again. The cap pauses the engine and the STT
  worker only, never the browser.
- run_m0_video.bat runs just this stage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 00:55:26 +09:00

513 lines
23 KiB
Python

"""Stage part: does Jarvis make a video stutter, and does the 95 % GPU cap help?
User decision (2026-11-11): the cap's default is off, because gamemode_tune showed it did not lower
nvidia-smi's utilisation and cost 6 to 10 % of the LLM's output. What the user cared about is a
video next to Jarvis, so that is measured directly: a YouTube video plays in a throwaway Chrome/Edge
profile (never the user's) and the kit counts its dropped frames and long gaps between shown frames
with Jarvis off, Jarvis loaded and idle, and Jarvis busy (the LLM answering, the STT transcribing ten
speakers back to back) without and with the cap. The busy rows run in the order A B B A (no cap,
cap, cap, no cap) so a slow drift does not count for one side.
The browser is driven through the Chrome DevTools Protocol with a small standard-library WebSocket
client (the kit's venvs are already built, so no package is added). Nothing from the page is saved
except the video's own counters, its quality and whether an ad was showing.
"""
from __future__ import annotations
import base64
import json
import os
import shutil
import socket
import struct
import subprocess
import time
import urllib.parse
from pathlib import Path
from typing import Any, Callable
from m0lib import gpucap, llm
from m0lib.common import IS_WINDOWS, StageResult, gpu_memory_mib, say
from m0lib.llm_client import ChatClient, http_json
# Blender's own upload (CC BY 3.0, 10:34, 4K and 1080p at 60 fps): no music claim, rarely ads.
VIDEO_URL = "https://www.youtube.com/watch?v=aqz-KE-bpKQ"
QUALITY = "hd1080"
QUIET_S = 60 # the video alone, and Jarvis loaded but idle
SETTLE_S = 5 # after a quality change or a seek, before a window starts
MIN_LEFT_S = 240 # a window that could reach the end starts the video again first
GAP_MS = (50, 100) # a gap between shown frames this long is a visible hitch at 60 fps
ROUNDS = ("busy", "busy_cap95", "busy_cap95_2", "busy_2")
# --------------------------------------------------------------------------
# A minimal WebSocket client (RFC 6455) for one DevTools page.
def encode_frame(payload: bytes, opcode: int = 1, mask: bytes | None = None) -> bytes:
"""One final client frame; a client must mask what it sends."""
mask = mask or os.urandom(4)
n = len(payload)
if n < 126:
head = bytes([0x80 | opcode, 0x80 | n])
elif n < 1 << 16:
head = bytes([0x80 | opcode, 0x80 | 126]) + struct.pack(">H", n)
else:
head = bytes([0x80 | opcode, 0x80 | 127]) + struct.pack(">Q", n)
return head + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
class Cdp:
"""One DevTools WebSocket: ``call`` sends a command and returns its result, skipping events."""
def __init__(self, ws_url: str, timeout: float = 60, sock: socket.socket | None = None):
self._buf = b""
self._id = 0
if sock is not None: # tests hand in one end of a socket pair, already "upgraded"
self.sock = sock
return
u = urllib.parse.urlparse(ws_url)
self.sock = socket.create_connection((u.hostname, u.port), timeout=timeout)
key = base64.b64encode(os.urandom(16)).decode()
# No Origin header: Chrome only checks --remote-allow-origins when one is sent.
self.sock.sendall((f"GET {u.path} HTTP/1.1\r\nHost: {u.hostname}:{u.port}\r\nUpgrade: websocket\r\n"
f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n"
).encode())
while b"\r\n\r\n" not in self._buf:
self._more()
head, self._buf = self._buf.split(b"\r\n\r\n", 1)
if b" 101 " not in head.split(b"\r\n", 1)[0]:
raise ConnectionError(f"DevTools refused the WebSocket: {head[:200]!r}")
def _more(self) -> None:
got = self.sock.recv(65536)
if not got:
raise ConnectionError("DevTools closed the connection")
self._buf += got
def _take(self, n: int) -> bytes:
while len(self._buf) < n:
self._more()
out, self._buf = self._buf[:n], self._buf[n:]
return out
def _frame(self) -> tuple[bool, int, bytes]:
b0, b1 = self._take(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", self._take(2))[0]
elif n == 127:
n = struct.unpack(">Q", self._take(8))[0]
mask = self._take(4) if b1 & 0x80 else None
data = self._take(n)
if mask:
data = bytes(b ^ mask[i % 4] for i, b in enumerate(data))
return bool(b0 & 0x80), b0 & 0x0F, data
def recv(self) -> str:
"""The next whole text message; answers pings, raises on close."""
parts: list[bytes] = []
while True:
fin, op, data = self._frame()
if op == 8:
raise ConnectionError("DevTools closed the WebSocket")
if op == 9:
self.sock.sendall(encode_frame(data, opcode=10))
continue
if op == 10:
continue
parts.append(data)
if fin:
return b"".join(parts).decode("utf-8", "replace")
def call(self, method: str, params: dict | None = None) -> dict:
self._id += 1
mine = self._id
self.sock.sendall(encode_frame(json.dumps({"id": mine, "method": method, "params": params or {}}).encode()))
while True:
msg = json.loads(self.recv())
if msg.get("id") != mine:
continue # an event, or the answer to a command that timed out earlier
if "error" in msg:
raise RuntimeError(f"{method}: {msg['error']}")
return msg.get("result") or {}
def evaluate(self, expression: str) -> Any:
res = self.call("Runtime.evaluate", {"expression": expression, "returnByValue": True})
if res.get("exceptionDetails"):
d = res["exceptionDetails"]
raise RuntimeError(f"page script failed: {(d.get('exception') or {}).get('description') or d.get('text')}")
return (res.get("result") or {}).get("value")
def close(self) -> None:
try:
self.sock.close()
except OSError:
pass
# --------------------------------------------------------------------------
# The page: what runs inside YouTube's watch page.
# Counts shown frames (requestVideoFrameCallback runs once per frame put on screen), the long gaps
# between them, buffering and ads; installed again if YouTube swaps the <video> element.
_INSTALL = """(() => {
const v = document.querySelector('video.html5-main-video') || document.querySelector('video');
if (!v || !v.requestVideoFrameCallback) return false;
if (window.__m0 && window.__m0.v === v) return true;
const m = window.__m0 = {v, shown: 0, last: null, maxGap: 0, gaps: [0, 0], waits: 0, ad: false};
v.addEventListener('waiting', () => { m.waits++; });
const tick = (now) => {
if (m.last !== null) {
const g = now - m.last;
if (g > m.maxGap) m.maxGap = g;
if (g > %(g0)d) m.gaps[0]++;
if (g > %(g1)d) m.gaps[1]++;
}
m.last = now; m.shown++;
const p = document.getElementById('movie_player');
if (p && p.classList.contains('ad-showing')) m.ad = true;
v.requestVideoFrameCallback(tick);
};
v.requestVideoFrameCallback(tick);
return true;
})()""" % {"g0": GAP_MS[0], "g1": GAP_MS[1]}
_RESET = "(() => { const m = window.__m0; m.shown = 0; m.last = null; m.maxGap = 0; m.gaps = [0, 0]; " \
"m.waits = 0; m.ad = false; return true; })()"
_SNAPSHOT = """(() => {
const m = window.__m0, v = m.v, p = document.getElementById('movie_player');
const q = v.getVideoPlaybackQuality();
return {dropped: q.droppedVideoFrames, total: q.totalVideoFrames, shown: m.shown, max_gap_ms: m.maxGap,
gaps: m.gaps, waits: m.waits, ad: m.ad || !!(p && p.classList.contains('ad-showing')),
paused: v.paused, ended: v.ended, time: v.currentTime, duration: v.duration,
width: v.videoWidth, height: v.videoHeight,
quality: p && p.getPlaybackQuality ? p.getPlaybackQuality() : null};
})()"""
# One step of getting the video going: skip an ad when it can be skipped, press play.
_STEP = """(() => {
const p = document.getElementById('movie_player');
const v = document.querySelector('video.html5-main-video');
if (!p || !v) return {state: 'loading'};
if (p.classList.contains('ad-showing')) {
const b = document.querySelector('.ytp-skip-ad-button, .ytp-ad-skip-button, .ytp-ad-skip-button-modern');
if (b) b.click();
return {state: 'ad', skipped: !!b};
}
if (v.paused && p.playVideo) p.playVideo();
return {state: v.readyState >= 3 && !v.paused && v.currentTime > 1 ? 'playing' : 'starting',
levels: p.getAvailableQualityLevels ? p.getAvailableQualityLevels() : []};
})()"""
def _js(value: Any) -> str:
return json.dumps(value)
class VideoPage:
"""The watch page in the throwaway browser."""
def __init__(self, cdp: Cdp, proc: subprocess.Popen | None = None, profile: Path | None = None):
self.cdp, self.proc, self.profile = cdp, proc, profile
def start(self, quality: str = QUALITY, timeout: float = 180) -> dict:
"""Wait for the video to play past any ad, then ask for ``quality``. Returns what happened."""
out: dict = {"started": False, "ad_seen": False, "ad_skipped": False, "quality_asked": quality}
t0 = time.monotonic()
while time.monotonic() - t0 < timeout:
st = self.cdp.evaluate(_STEP) or {}
out["ad_seen"] |= st.get("state") == "ad"
out["ad_skipped"] |= bool(st.get("skipped"))
if st.get("state") == "playing":
out["started"] = True
out["levels"] = st.get("levels")
break
time.sleep(0.5)
out["start_s"] = round(time.monotonic() - t0, 1)
if not out["started"]:
return out
if not self.cdp.evaluate(_INSTALL):
out["started"] = False
out["error"] = "the page has no video that reports its shown frames"
return out
# The player's own quality setting (Stats for nerds shows the same).
self.cdp.evaluate("(() => { const p = document.getElementById('movie_player'); "
f"if (p && p.setPlaybackQualityRange) p.setPlaybackQualityRange({_js(quality)}, "
f"{_js(quality)}); return true; }})()")
t1 = time.monotonic()
while time.monotonic() - t1 < 30 and (self.snapshot().get("quality") != quality):
time.sleep(1)
time.sleep(SETTLE_S)
snap = self.snapshot()
out.update(quality=snap.get("quality"), width=snap.get("width"), height=snap.get("height"))
return out
def keep_room(self, min_left: float = MIN_LEFT_S) -> None:
"""Start the video again when a window could run into its end (YouTube then plays the next one)."""
snap = self.snapshot()
if (snap.get("duration") or 0) - (snap.get("time") or 0) < min_left:
self.cdp.evaluate("(() => { document.getElementById('movie_player').seekTo(5, true); return true; })()")
time.sleep(SETTLE_S)
self.cdp.evaluate(_INSTALL)
def reset(self) -> None:
self.cdp.evaluate(_RESET)
def snapshot(self) -> dict:
return self.cdp.evaluate(_SNAPSHOT) or {}
def close(self) -> None:
try:
self.cdp.call("Browser.close")
except Exception:
pass
self.cdp.close()
if self.proc:
try:
self.proc.wait(timeout=15)
except subprocess.TimeoutExpired:
self.proc.kill()
if self.profile:
shutil.rmtree(self.profile, ignore_errors=True)
def find_browser() -> str | None:
if IS_WINDOWS:
cands = [os.path.expandvars(p) for p in (
r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe",
r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe")]
return next((p for p in cands if os.path.exists(p)), None)
return next((shutil.which(n) for n in ("google-chrome", "chromium", "chromium-browser") if shutil.which(n)), None)
def open_video_page(work: Path, url: str = VIDEO_URL, extra_args: tuple[str, ...] = ()) -> VideoPage | None:
"""Open ``url`` in a new throwaway profile with DevTools on a free port. None without a browser."""
exe = find_browser()
if not exe:
return None
profile = work / "cache" / "video" / "browser-profile"
shutil.rmtree(profile, ignore_errors=True)
profile.mkdir(parents=True)
# Kept running and drawing even if another window covers part of it; muted (nobody listens).
proc = subprocess.Popen([exe, f"--user-data-dir={profile}", "--remote-debugging-port=0", "--no-first-run",
"--no-default-browser-check", "--autoplay-policy=no-user-gesture-required",
"--mute-audio", "--start-maximized", "--disable-backgrounding-occluded-windows",
"--disable-renderer-backgrounding", "--disable-background-timer-throttling",
*extra_args, url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
port_file = profile / "DevToolsActivePort"
t0 = time.monotonic()
while time.monotonic() - t0 < 60:
if port_file.exists() and port_file.read_text(encoding="utf-8").strip():
port = int(port_file.read_text(encoding="utf-8").split()[0])
for _ in range(30):
code, pages = http_json("GET", f"http://127.0.0.1:{port}/json/list", timeout=5)
page = next((p for p in (pages or []) if p.get("type") == "page" and "youtube" in p.get("url", "")),
None) if code == 200 else None
if page:
return VideoPage(Cdp(page["webSocketDebuggerUrl"]), proc, profile)
time.sleep(1)
break
if proc.poll() is not None:
break
time.sleep(0.5)
proc.kill()
raise RuntimeError("the browser started without DevTools or without the video page")
# --------------------------------------------------------------------------
# One measured window and the comparison.
def window_stats(before: dict, after: dict, seconds: float) -> dict:
"""What the video did between two snapshots (``after`` also carries the shown-frame counters
since the reset just before ``before``). A window with an ad, a pause, the end or a reload of the
counters is kept but not counted (``valid`` False, with the reason)."""
out: dict = {"seconds": round(seconds, 1)}
reasons = []
if after.get("total", 0) < before.get("total", 0) or after.get("dropped", 0) < before.get("dropped", 0):
reasons.append("the video's counters started again (another video was loaded)")
for key, why in (("ad", "an ad was showing"), ("paused", "the video was paused"), ("ended", "the video ended")):
if after.get(key):
reasons.append(why)
total = after.get("total", 0) - before.get("total", 0)
dropped = after.get("dropped", 0) - before.get("dropped", 0)
out.update(frames=total, dropped=dropped, shown=after.get("shown"),
dropped_share=round(dropped / total, 4) if total > 0 else None,
dropped_per_min=round(dropped / seconds * 60, 1) if seconds > 0 else None,
fps=round(total / seconds, 1) if seconds > 0 else None,
gaps_over_ms={str(GAP_MS[0]): (after.get("gaps") or [0, 0])[0],
str(GAP_MS[1]): (after.get("gaps") or [0, 0])[1]},
max_gap_ms=round(after.get("max_gap_ms") or 0, 1), buffering=after.get("waits", 0),
quality=after.get("quality"), height=after.get("height"))
if total <= 0 and not reasons:
reasons.append("the video showed no frames")
out["valid"] = not reasons
if reasons:
out["invalid_because"] = reasons
return out
def compare(rows: list[dict]) -> dict:
"""The busy rows without and with the cap, added up over the valid windows."""
out = {}
for name, capped in (("no_cap", False), ("cap95", True)):
valid = [x["video"] for x in rows
if x.get("jarvis") == "busy" and bool(x.get("gpu_cap")) == capped and (x.get("video") or {}).get("valid")]
frames = sum(v["frames"] for v in valid)
dropped = sum(v["dropped"] for v in valid)
secs = sum(v["seconds"] for v in valid)
out[name] = {"windows": len(valid), "frames": frames, "dropped": dropped,
"dropped_share": round(dropped / frames, 4) if frames else None,
"dropped_per_min": round(dropped / secs * 60, 1) if secs else None,
f"gaps_over_{GAP_MS[0]}ms": sum(v["gaps_over_ms"][str(GAP_MS[0])] for v in valid),
f"gaps_over_{GAP_MS[1]}ms": sum(v["gaps_over_ms"][str(GAP_MS[1])] for v in valid)}
return out
def measure_window(page: VideoPage, work: Callable[[], dict] | None = None,
seconds: float | None = None) -> tuple[dict, dict]:
"""The video's stats over ``work()`` (or ``seconds`` of nothing), with the GPU's use meanwhile."""
page.keep_room()
page.reset()
before = page.snapshot()
t0 = time.perf_counter()
done: dict = {}
with gpucap.UtilSampler() as util:
if work:
done = work()
else:
time.sleep(QUIET_S if seconds is None else seconds)
after = page.snapshot()
stats = window_stats(before, after, time.perf_counter() - t0)
stats["gpu_util"] = util.summary()
return stats, done
# --------------------------------------------------------------------------
# The stage.
ROOM_LEVEL = next(lv for lv in llm.TUNED if lv["key"] == "normal_room_stt_gpu")
def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_python: str, url: str = VIDEO_URL,
quality: str = QUALITY) -> None:
"""Video alone, next to Jarvis idle, next to Jarvis busy without and with the cap, alone again."""
base = f"http://127.0.0.1:{llm.PORT}/v1"
rows: list[dict] = []
r["video_rows"] = rows
r["video_source"] = {"url": url, "quality_asked": quality}
say(" 브라우저(새 임시 프로필)로 유튜브 영상을 틉니다. 끝날 때까지 PC에 손대지 말아 주세요.")
try:
page = open_video_page(s.log.parent.parent)
except Exception as e:
r.error("browser", e)
return
if page is None:
r.error("browser", "Chrome이나 Edge를 찾지 못했습니다")
return
stt = None
try:
start = page.start(quality)
r["video_source"].update(start)
if not start["started"]:
r.error("video", start.get("error") or "the video did not start playing")
return
if start.get("quality") != quality:
r.note(f"the player stayed at {start.get('quality')} instead of {quality}")
def row(key: str, jarvis: str, cap: float | None = None) -> dict:
x = {"key": key, "jarvis": jarvis, "gpu_cap": cap}
used = gpu_memory_mib()
if used:
x["vram_free_mib"] = used[1] - used[0]
rows.append(x)
return x
def keep(x: dict, stats: dict) -> None:
x["video"] = stats
if not stats["valid"]:
r.note(f"{x['key']}: not counted ({'; '.join(stats['invalid_because'])})")
v = stats
say(f" {x['key']}: 끊긴 프레임 {v['dropped']}/{v['frames']}, {GAP_MS[1]}ms 넘는 멈춤 "
f"{v['gaps_over_ms'][str(GAP_MS[1])]}번")
r.save()
say(" - 영상만")
x = row("video_only", "off")
keep(x, measure_window(page)[0])
say(" - Jarvis 켜기 (평소 배치, STT 자리와 여유 1GB)")
s.stop()
stt = llm.SttLoad(stt_python, s.log.parent.parent, "cuda", s.log.with_name("stt_load.log"))
load = stt.start()
r["stt_load"] = load
if load.get("status") != "loaded":
r.error("video stt", "; ".join(load.get("errors") or []) or "the STT did not load")
return
lv = dict(ROOM_LEVEL)
place = {"context": lv.get("context", context)}
room = llm.stt_room(r, s, preset, place, lv, base)
r["stt_room"] = room
sets = list(lv["sets"]) + [f"expert-vram={room['expert_vram_gib']}"]
r["placement"] = {"sets": sets, "context": place["context"], "parallel": lv["parallel"], "vision": lv["vision"]}
launch = s.launch(preset, place["context"], parallel=lv["parallel"], vision=lv["vision"], extra_sets=sets,
timeout=3600)
if not launch.get("ready"):
r.error("video llm", "the normal placement did not start")
return
c = ChatClient(base)
r["server"] = {"model": c.model}
c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time
say(" - Jarvis 켜 두고 쉬는 중")
x = row("jarvis_idle", "idle")
keep(x, measure_window(page)[0])
stt.go()
for key in ROUNDS:
cap = llm.GPU_CAP if "cap95" in key else None
say(f" - Jarvis가 답하고 STT가 쉬지 않고 전사하는 중{' (GPU 95% 제한)' if cap else ''}")
x = row(key, "busy", cap)
pids = llm.cap_pids(s, stt) if cap else []
if cap and not pids:
r.error(f"video {key} gpu cap", "found no engine process to pause")
def work() -> dict:
with gpucap.DutyCap(pids, cap or 1.0) as duty:
bench = llm.quick_bench(c)
return {"bench": bench, "duty": duty.summary()}
stats, done = measure_window(page, work)
bench = done.get("bench") or {}
x["llm"] = {k: bench.get(k) for k in ("decode_tok_s", "voice_first_sentence", "context")}
for err in bench.get("errors") or []:
r.error(f"video {key}", err)
if cap:
x["gpu_cap"] = done["duty"]
if x["gpu_cap"]["left_paused"]:
r.error(f"video {key} gpu cap", f"{x['gpu_cap']['left_paused']} process(es) are still paused")
keep(x, stats)
busy = stt.finish()
r["stt_busy"] = {k: busy.get(k) for k in ("status", "batch10_s", "single_s", "errors", "busy_share")
if k in busy}
for err in llm.stt_gaps(busy):
r.error("video stt", err)
stt.close()
stt = None
s.stop()
say(" - Jarvis 끄고 영상만 다시")
x = row("video_only_again", "off")
keep(x, measure_window(page)[0])
r["comparison"] = compare(rows)
except Exception as e:
r.error("video", e)
finally:
if stt:
stt.close()
s.stop()
page.close()