When loading the page again did not bring a player back, keep_room still moved the missing player back to the start, the page script threw and the stage stopped at the first window. Now a failed reload (or a refused navigation) is recorded as "the video did not play again", only a playing video is moved back (and the script checks the player exists), so the window is measured as it is, not counted, and tried again up to the limit before the stage goes on. The reload waits RELOAD_S for the video. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
639 lines
30 KiB
Python
639 lines
30 KiB
Python
"""Stage part: does Jarvis make a video stutter, and does the 95 % GPU cap help?
|
|
|
|
User decision (2026-11-11): the cap's default is off, because gamemode_tune showed it did not lower
|
|
nvidia-smi's utilisation and cost 6 to 10 % of the LLM's output. What the user cared about is a
|
|
video next to Jarvis, so that is measured directly: a YouTube video plays in a throwaway Chrome/Edge
|
|
profile (never the user's) and the kit counts its dropped frames and long gaps between shown frames
|
|
with Jarvis off, Jarvis loaded and idle, and Jarvis busy (the LLM answering, the STT transcribing ten
|
|
speakers back to back) without and with the cap. The busy rows run in the order A B B A (no cap,
|
|
cap, cap, no cap) so a slow drift does not count for one side.
|
|
|
|
The browser is driven through the Chrome DevTools Protocol with a small standard-library WebSocket
|
|
client (the kit's venvs are already built, so no package is added). Nothing from the page is saved
|
|
except the video's own counters, its quality and whether an ad was showing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import time
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from m0lib import gpucap, llm
|
|
from m0lib.common import IS_WINDOWS, StageResult, gpu_memory_mib, say
|
|
from m0lib.llm_client import ChatClient, http_json
|
|
|
|
# Blender's own upload (CC BY 3.0, 10:34, 4K and 1080p at 60 fps): no music claim, rarely ads.
|
|
VIDEO_URL = "https://www.youtube.com/watch?v=aqz-KE-bpKQ"
|
|
QUALITY = "hd1080"
|
|
# What a counted window must have played: the asked quality at its height and the source's 60 fps (the
|
|
# player's frame counter includes dropped frames, so a busy GPU does not lower it; a 30 fps format does).
|
|
WANT = {"quality": QUALITY, "height": 1080, "fps": 60}
|
|
MIN_FPS_SHARE = 0.8
|
|
QUIET_S = 60 # the video alone, and Jarvis loaded but idle
|
|
SETTLE_S = 5 # after a quality change or a seek, before a window starts
|
|
MIN_LEFT_S = 240 # a window that could reach the end starts the video again first
|
|
GAP_MS = (50, 100) # a gap between shown frames this long is a visible hitch at 60 fps
|
|
ROUNDS = ("busy", "busy_cap95", "busy_cap95_2", "busy_2")
|
|
# A window spoilt by the player (an error, a reload, a pause, an ad, buffering) is measured again, at most
|
|
# this many times in all; one spoilt by a lower quality or frame rate is not (a busy GPU can cause that).
|
|
ATTEMPTS = 3
|
|
REPLAY_S = 30 # pressing play on a stopped player, before the page is loaded again
|
|
RELOAD_S = 120 # the page loaded again, until its video plays
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# A minimal WebSocket client (RFC 6455) for one DevTools page.
|
|
|
|
def encode_frame(payload: bytes, opcode: int = 1, mask: bytes | None = None) -> bytes:
|
|
"""One final client frame; a client must mask what it sends."""
|
|
mask = mask or os.urandom(4)
|
|
n = len(payload)
|
|
if n < 126:
|
|
head = bytes([0x80 | opcode, 0x80 | n])
|
|
elif n < 1 << 16:
|
|
head = bytes([0x80 | opcode, 0x80 | 126]) + struct.pack(">H", n)
|
|
else:
|
|
head = bytes([0x80 | opcode, 0x80 | 127]) + struct.pack(">Q", n)
|
|
return head + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
|
|
|
|
|
|
class Cdp:
|
|
"""One DevTools WebSocket: ``call`` sends a command and returns its result, skipping events."""
|
|
|
|
def __init__(self, ws_url: str, timeout: float = 60, sock: socket.socket | None = None):
|
|
self._buf = b""
|
|
self._id = 0
|
|
if sock is not None: # tests hand in one end of a socket pair, already "upgraded"
|
|
self.sock = sock
|
|
return
|
|
u = urllib.parse.urlparse(ws_url)
|
|
self.sock = socket.create_connection((u.hostname, u.port), timeout=timeout)
|
|
key = base64.b64encode(os.urandom(16)).decode()
|
|
# No Origin header: Chrome only checks --remote-allow-origins when one is sent.
|
|
self.sock.sendall((f"GET {u.path} HTTP/1.1\r\nHost: {u.hostname}:{u.port}\r\nUpgrade: websocket\r\n"
|
|
f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n"
|
|
).encode())
|
|
while b"\r\n\r\n" not in self._buf:
|
|
self._more()
|
|
head, self._buf = self._buf.split(b"\r\n\r\n", 1)
|
|
if b" 101 " not in head.split(b"\r\n", 1)[0]:
|
|
raise ConnectionError(f"DevTools refused the WebSocket: {head[:200]!r}")
|
|
|
|
def _more(self) -> None:
|
|
got = self.sock.recv(65536)
|
|
if not got:
|
|
raise ConnectionError("DevTools closed the connection")
|
|
self._buf += got
|
|
|
|
def _take(self, n: int) -> bytes:
|
|
while len(self._buf) < n:
|
|
self._more()
|
|
out, self._buf = self._buf[:n], self._buf[n:]
|
|
return out
|
|
|
|
def _frame(self) -> tuple[bool, int, bytes]:
|
|
b0, b1 = self._take(2)
|
|
n = b1 & 0x7F
|
|
if n == 126:
|
|
n = struct.unpack(">H", self._take(2))[0]
|
|
elif n == 127:
|
|
n = struct.unpack(">Q", self._take(8))[0]
|
|
mask = self._take(4) if b1 & 0x80 else None
|
|
data = self._take(n)
|
|
if mask:
|
|
data = bytes(b ^ mask[i % 4] for i, b in enumerate(data))
|
|
return bool(b0 & 0x80), b0 & 0x0F, data
|
|
|
|
def recv(self) -> str:
|
|
"""The next whole text message; answers pings, raises on close."""
|
|
parts: list[bytes] = []
|
|
while True:
|
|
fin, op, data = self._frame()
|
|
if op == 8:
|
|
raise ConnectionError("DevTools closed the WebSocket")
|
|
if op == 9:
|
|
self.sock.sendall(encode_frame(data, opcode=10))
|
|
continue
|
|
if op == 10:
|
|
continue
|
|
parts.append(data)
|
|
if fin:
|
|
return b"".join(parts).decode("utf-8", "replace")
|
|
|
|
def call(self, method: str, params: dict | None = None) -> dict:
|
|
self._id += 1
|
|
mine = self._id
|
|
self.sock.sendall(encode_frame(json.dumps({"id": mine, "method": method, "params": params or {}}).encode()))
|
|
while True:
|
|
msg = json.loads(self.recv())
|
|
if msg.get("id") != mine:
|
|
continue # an event, or the answer to a command that timed out earlier
|
|
if "error" in msg:
|
|
raise RuntimeError(f"{method}: {msg['error']}")
|
|
return msg.get("result") or {}
|
|
|
|
def evaluate(self, expression: str) -> Any:
|
|
res = self.call("Runtime.evaluate", {"expression": expression, "returnByValue": True})
|
|
if res.get("exceptionDetails"):
|
|
d = res["exceptionDetails"]
|
|
raise RuntimeError(f"page script failed: {(d.get('exception') or {}).get('description') or d.get('text')}")
|
|
return (res.get("result") or {}).get("value")
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
self.sock.close()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The page: what runs inside YouTube's watch page.
|
|
|
|
# Counts shown frames (requestVideoFrameCallback runs once per frame put on screen), the long gaps
|
|
# between them, buffering and ads; installed again if YouTube swaps the <video> element.
|
|
_INSTALL = """(() => {
|
|
const v = document.querySelector('video.html5-main-video') || document.querySelector('video');
|
|
if (!v || !v.requestVideoFrameCallback) return false;
|
|
if (window.__m0 && window.__m0.v === v) return true;
|
|
const m = window.__m0 = {v, shown: 0, last: null, maxGap: 0, gaps: [0, 0], waits: 0, pauses: 0, seeks: 0,
|
|
ad: false, minH: v.videoHeight, maxH: v.videoHeight};
|
|
v.addEventListener('waiting', () => { m.waits++; });
|
|
v.addEventListener('pause', () => { m.pauses++; });
|
|
v.addEventListener('seeking', () => { m.seeks++; });
|
|
const tick = (now) => {
|
|
if (m.last !== null) {
|
|
const g = now - m.last;
|
|
if (g > m.maxGap) m.maxGap = g;
|
|
if (g > %(g0)d) m.gaps[0]++;
|
|
if (g > %(g1)d) m.gaps[1]++;
|
|
}
|
|
m.last = now; m.shown++;
|
|
m.minH = Math.min(m.minH, v.videoHeight); m.maxH = Math.max(m.maxH, v.videoHeight);
|
|
const p = document.getElementById('movie_player');
|
|
if (p && p.classList.contains('ad-showing')) m.ad = true;
|
|
v.requestVideoFrameCallback(tick);
|
|
};
|
|
v.requestVideoFrameCallback(tick);
|
|
return true;
|
|
})()""" % {"g0": GAP_MS[0], "g1": GAP_MS[1]}
|
|
|
|
_RESET = "(() => { const m = window.__m0; if (!m) return false; m.shown = 0; m.last = null; m.maxGap = 0; m.gaps = [0, 0]; " \
|
|
"m.waits = 0; m.pauses = 0; m.seeks = 0; m.ad = false; m.minH = m.maxH = m.v.videoHeight; return true; })()"
|
|
|
|
# ``lost`` when the counters are gone (the page was loaded again, or YouTube swapped the <video>);
|
|
# ``error`` when the player shows its own error screen (only its first line is kept, without the IDs).
|
|
_SNAPSHOT = """(() => {
|
|
const m = window.__m0, p = document.getElementById('movie_player');
|
|
const e = p && p.querySelector('.ytp-error');
|
|
const error = !!(e && e.offsetParent !== null);
|
|
const reason = error ? (e.querySelector('.ytp-error-content-wrap-reason') || e).textContent : null;
|
|
if (!m || !m.v.isConnected) return {lost: true, error, error_text: reason};
|
|
const v = m.v, q = v.getVideoPlaybackQuality();
|
|
return {dropped: q.droppedVideoFrames, total: q.totalVideoFrames, shown: m.shown, max_gap_ms: m.maxGap,
|
|
gaps: m.gaps, waits: m.waits, pauses: m.pauses, seeks: m.seeks, min_height: m.minH, max_height: m.maxH,
|
|
ad: m.ad || !!(p && p.classList.contains('ad-showing')),
|
|
ad_now: !!(p && p.classList.contains('ad-showing')), error, error_text: reason,
|
|
paused: v.paused, ended: v.ended, time: v.currentTime, duration: v.duration,
|
|
width: v.videoWidth, height: v.videoHeight,
|
|
quality: p && p.getPlaybackQuality ? p.getPlaybackQuality() : null};
|
|
})()"""
|
|
|
|
# One step of getting the video going: skip an ad when it can be skipped, press play.
|
|
_STEP = """(() => {
|
|
const p = document.getElementById('movie_player');
|
|
const v = document.querySelector('video.html5-main-video');
|
|
if (!p || !v) return {state: 'loading'};
|
|
if (p.classList.contains('ad-showing')) {
|
|
const b = document.querySelector('.ytp-skip-ad-button, .ytp-ad-skip-button, .ytp-ad-skip-button-modern');
|
|
if (b) b.click();
|
|
return {state: 'ad', skipped: !!b};
|
|
}
|
|
if (v.paused && p.playVideo) p.playVideo();
|
|
return {state: v.readyState >= 3 && !v.paused && v.currentTime > 1 ? 'playing' : 'starting',
|
|
levels: p.getAvailableQualityLevels ? p.getAvailableQualityLevels() : []};
|
|
})()"""
|
|
|
|
|
|
def _js(value: Any) -> str:
|
|
return json.dumps(value)
|
|
|
|
|
|
class VideoPage:
|
|
"""The watch page in the throwaway browser."""
|
|
|
|
def __init__(self, cdp: Cdp, proc: subprocess.Popen | None = None, profile: Path | None = None,
|
|
url: str = VIDEO_URL):
|
|
self.cdp, self.proc, self.profile, self.url = cdp, proc, profile, url
|
|
self.quality = QUALITY
|
|
|
|
def start(self, quality: str = QUALITY, timeout: float = 180) -> dict:
|
|
"""Wait for the video to play past any ad, then ask for ``quality``. Returns what happened."""
|
|
self.quality = quality
|
|
out: dict = {"started": False, "ad_seen": False, "ad_skipped": False, "quality_asked": quality}
|
|
t0 = time.monotonic()
|
|
while time.monotonic() - t0 < timeout:
|
|
st = self.cdp.evaluate(_STEP) or {}
|
|
out["ad_seen"] |= st.get("state") == "ad"
|
|
out["ad_skipped"] |= bool(st.get("skipped"))
|
|
if st.get("state") == "playing":
|
|
out["started"] = True
|
|
out["levels"] = st.get("levels")
|
|
break
|
|
time.sleep(0.5)
|
|
out["start_s"] = round(time.monotonic() - t0, 1)
|
|
if not out["started"]:
|
|
return out
|
|
if not self.cdp.evaluate(_INSTALL):
|
|
out["started"] = False
|
|
out["error"] = "the page has no video that reports its shown frames"
|
|
return out
|
|
# The player's own quality setting (Stats for nerds shows the same).
|
|
self.cdp.evaluate("(() => { const p = document.getElementById('movie_player'); "
|
|
f"if (p && p.setPlaybackQualityRange) p.setPlaybackQualityRange({_js(quality)}, "
|
|
f"{_js(quality)}); return true; }})()")
|
|
t1 = time.monotonic()
|
|
while time.monotonic() - t1 < 30 and (self.snapshot().get("quality") != quality):
|
|
time.sleep(1)
|
|
time.sleep(SETTLE_S)
|
|
snap = self.snapshot()
|
|
out.update(quality=snap.get("quality"), width=snap.get("width"), height=snap.get("height"))
|
|
return out
|
|
|
|
def reload(self) -> bool:
|
|
"""Load the watch page again (its counters go with it) and get the video going as at the start.
|
|
False when it did not play again; the window is then measured as it is and not counted."""
|
|
try:
|
|
self.cdp.call("Page.navigate", {"url": self.url})
|
|
except RuntimeError:
|
|
return False
|
|
time.sleep(SETTLE_S)
|
|
return bool(self.start(self.quality, timeout=RELOAD_S)["started"])
|
|
|
|
def keep_room(self, min_left: float = MIN_LEFT_S) -> list[str]:
|
|
"""Before a window: get a stopped, broken or reloaded player going again (pressing play first,
|
|
loading the page again if that is not enough), and start the video again when the window could
|
|
run into its end (YouTube then plays the next one). Returns what had to be done."""
|
|
did: list[str] = []
|
|
snap = self.snapshot()
|
|
if snap.get("error"):
|
|
did.append(f"the player showed an error: {_short(snap.get('error_text'))}")
|
|
reload = bool(snap.get("lost") or snap.get("error"))
|
|
if not reload and (snap.get("paused") or snap.get("ended") or snap.get("ad_now") or not snap.get("height")):
|
|
did.append("pressed play")
|
|
reload = not self.start(self.quality, timeout=REPLAY_S)["started"]
|
|
if reload:
|
|
did.append("loaded the page again")
|
|
if not self.reload():
|
|
did.append("the video did not play again")
|
|
if did:
|
|
snap = self.snapshot()
|
|
# Only a playing video is moved back; a missing player is left to the window, which then does
|
|
# not count and is tried again.
|
|
if not snap.get("lost") and snap.get("duration") and snap["duration"] - (snap.get("time") or 0) < min_left:
|
|
self.cdp.evaluate("(() => { const p = document.getElementById('movie_player'); "
|
|
"if (p && p.seekTo) p.seekTo(5, true); return true; })()")
|
|
time.sleep(SETTLE_S)
|
|
self.cdp.evaluate(_INSTALL)
|
|
return did
|
|
|
|
def reset(self) -> None:
|
|
self.cdp.evaluate(_RESET)
|
|
|
|
def snapshot(self) -> dict:
|
|
"""The counters; ``lost`` too when the page script could not run (the page was being loaded)."""
|
|
try:
|
|
return self.cdp.evaluate(_SNAPSHOT) or {}
|
|
except RuntimeError as e:
|
|
return {"lost": True, "script_error": _short(str(e))}
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
self.cdp.call("Browser.close")
|
|
except Exception:
|
|
pass
|
|
self.cdp.close()
|
|
if self.proc:
|
|
try:
|
|
self.proc.wait(timeout=15)
|
|
except subprocess.TimeoutExpired:
|
|
self.proc.kill()
|
|
if self.profile:
|
|
shutil.rmtree(self.profile, ignore_errors=True)
|
|
|
|
|
|
def _short(text: Any, n: int = 120) -> str:
|
|
"""One line of a player message, without its bracketed IDs (YouTube adds a playback ID)."""
|
|
return re.sub(r"\s+", " ", re.sub(r"[(\[][^)\]]*[)\]]", "", str(text or ""))).strip()[:n]
|
|
|
|
|
|
def find_browser() -> str | None:
|
|
if IS_WINDOWS:
|
|
cands = [os.path.expandvars(p) for p in (
|
|
r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
|
|
r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe",
|
|
r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
|
|
r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe")]
|
|
return next((p for p in cands if os.path.exists(p)), None)
|
|
return next((shutil.which(n) for n in ("google-chrome", "chromium", "chromium-browser") if shutil.which(n)), None)
|
|
|
|
|
|
def open_video_page(work: Path, url: str = VIDEO_URL, extra_args: tuple[str, ...] = ()) -> VideoPage | None:
|
|
"""Open ``url`` in a new throwaway profile with DevTools on a free port. None without a browser."""
|
|
exe = find_browser()
|
|
if not exe:
|
|
return None
|
|
profile = work / "cache" / "video" / "browser-profile"
|
|
shutil.rmtree(profile, ignore_errors=True)
|
|
profile.mkdir(parents=True)
|
|
# Kept running and drawing even if another window covers part of it; muted (nobody listens).
|
|
proc = subprocess.Popen([exe, f"--user-data-dir={profile}", "--remote-debugging-port=0", "--no-first-run",
|
|
"--no-default-browser-check", "--autoplay-policy=no-user-gesture-required",
|
|
"--mute-audio", "--start-maximized", "--disable-backgrounding-occluded-windows",
|
|
"--disable-renderer-backgrounding", "--disable-background-timer-throttling",
|
|
*extra_args, url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
port_file = profile / "DevToolsActivePort"
|
|
t0 = time.monotonic()
|
|
while time.monotonic() - t0 < 60:
|
|
if port_file.exists() and port_file.read_text(encoding="utf-8").strip():
|
|
port = int(port_file.read_text(encoding="utf-8").split()[0])
|
|
for _ in range(30):
|
|
code, pages = http_json("GET", f"http://127.0.0.1:{port}/json/list", timeout=5)
|
|
page = next((p for p in (pages or []) if p.get("type") == "page" and "youtube" in p.get("url", "")),
|
|
None) if code == 200 else None
|
|
if page:
|
|
return VideoPage(Cdp(page["webSocketDebuggerUrl"]), proc, profile, url)
|
|
time.sleep(1)
|
|
break
|
|
if proc.poll() is not None:
|
|
break
|
|
time.sleep(0.5)
|
|
proc.kill()
|
|
raise RuntimeError("the browser started without DevTools or without the video page")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# One measured window and the comparison.
|
|
|
|
def window_stats(before: dict, after: dict, seconds: float, want: dict | None = None) -> dict:
|
|
"""What the video did between two snapshots (``after`` also carries the counters kept since the
|
|
reset just before ``before``). A window is kept but not counted (``valid`` False, with the
|
|
reasons) when something other than the GPU could have changed it: an ad, a pause or a seek at
|
|
any point, buffering, the end, a reload of the counters, or (with ``want``) another quality,
|
|
height or a lower frame rate than the source's."""
|
|
out: dict = {"seconds": round(seconds, 1)}
|
|
if before.get("lost") or after.get("lost"):
|
|
# The page was loaded again (by hand or by YouTube): its counters do not relate to ``before``.
|
|
return {**out, "frames": None, "dropped": None, "gaps_over_ms": {str(g): None for g in GAP_MS},
|
|
"valid": False, "retry": True,
|
|
"invalid_because": ["the page was loaded again, so its counters were gone"]
|
|
+ ([f"the player showed an error: {_short(after.get('error_text'))}"]
|
|
if after.get("error") else [])}
|
|
reasons = []
|
|
if after.get("error"):
|
|
reasons.append(f"the player showed an error: {_short(after.get('error_text'))}")
|
|
if after.get("total", 0) < before.get("total", 0) or after.get("dropped", 0) < before.get("dropped", 0):
|
|
reasons.append("the video's counters started again (another video was loaded)")
|
|
for key, why in (("ad", "an ad was showing"), ("paused", "the video was paused"), ("ended", "the video ended")):
|
|
if after.get(key):
|
|
reasons.append(why)
|
|
for key, why in (("pauses", "the video was paused during the window"),
|
|
("seeks", "the video jumped during the window"), ("waits", "the video was buffering")):
|
|
if after.get(key):
|
|
reasons.append(f"{why} ({after[key]}x)")
|
|
total = after.get("total", 0) - before.get("total", 0)
|
|
dropped = after.get("dropped", 0) - before.get("dropped", 0)
|
|
out.update(frames=total, dropped=dropped, shown=after.get("shown"),
|
|
dropped_share=round(dropped / total, 4) if total > 0 else None,
|
|
dropped_per_min=round(dropped / seconds * 60, 1) if seconds > 0 else None,
|
|
fps=round(total / seconds, 1) if seconds > 0 else None,
|
|
gaps_over_ms={str(GAP_MS[0]): (after.get("gaps") or [0, 0])[0],
|
|
str(GAP_MS[1]): (after.get("gaps") or [0, 0])[1]},
|
|
max_gap_ms=round(after.get("max_gap_ms") or 0, 1), buffering=after.get("waits", 0),
|
|
quality=after.get("quality"), height=after.get("height"))
|
|
if total <= 0 and not reasons:
|
|
reasons.append("the video showed no frames")
|
|
# Everything so far comes from the player, not from the GPU: measuring again is fair.
|
|
out["retry"] = bool(reasons)
|
|
if want:
|
|
got = {before.get("quality"), after.get("quality")}
|
|
if got != {want["quality"]}:
|
|
reasons.append(f"the quality was {'/'.join(sorted(str(q) for q in got))}, not {want['quality']}")
|
|
heights = {before.get("height"), after.get("min_height"), after.get("max_height"), after.get("height")}
|
|
heights.discard(None)
|
|
if heights != {want["height"]}:
|
|
reasons.append(f"the picture was {'/'.join(str(h) for h in sorted(heights))} lines high, "
|
|
f"not {want['height']}")
|
|
if out["fps"] is not None and total > 0 and out["fps"] < want["fps"] * MIN_FPS_SHARE:
|
|
reasons.append(f"the video played {out['fps']} fps, not {want['fps']}")
|
|
out["valid"] = not reasons
|
|
if reasons:
|
|
out["invalid_because"] = reasons
|
|
return out
|
|
|
|
|
|
def compare(rows: list[dict]) -> dict:
|
|
"""The busy rows without and with the cap, added up over the valid windows."""
|
|
out = {}
|
|
for name, capped in (("no_cap", False), ("cap95", True)):
|
|
valid = [x["video"] for x in rows
|
|
if x.get("jarvis") == "busy" and bool(x.get("gpu_cap")) == capped and (x.get("video") or {}).get("valid")]
|
|
frames = sum(v["frames"] for v in valid)
|
|
dropped = sum(v["dropped"] for v in valid)
|
|
secs = sum(v["seconds"] for v in valid)
|
|
out[name] = {"windows": len(valid), "frames": frames, "dropped": dropped,
|
|
"dropped_share": round(dropped / frames, 4) if frames else None,
|
|
"dropped_per_min": round(dropped / secs * 60, 1) if secs else None,
|
|
f"gaps_over_{GAP_MS[0]}ms": sum(v["gaps_over_ms"][str(GAP_MS[0])] for v in valid),
|
|
f"gaps_over_{GAP_MS[1]}ms": sum(v["gaps_over_ms"][str(GAP_MS[1])] for v in valid)}
|
|
return out
|
|
|
|
|
|
def cap_problems(duty: dict, pids: list[int]) -> list[str]:
|
|
"""Why a capped window did not run under the cap it claims: nothing to pause, no pause cycle,
|
|
or a process that could not be paused or resumed at once."""
|
|
out = []
|
|
if not pids:
|
|
out.append("the cap found no engine process to pause")
|
|
elif not duty.get("cycles"):
|
|
out.append("the cap paused nothing during the window")
|
|
if duty.get("failed_processes"):
|
|
out.append(f"{duty['failed_processes']} process(es) could not be paused or resumed at once")
|
|
if duty.get("left_paused"):
|
|
out.append(f"{duty['left_paused']} process(es) are still paused")
|
|
return out
|
|
|
|
|
|
def measure_window(page: VideoPage, work: Callable[[], dict] | None = None,
|
|
seconds: float | None = None, want: dict | None = WANT) -> tuple[dict, dict]:
|
|
"""The video's stats over ``work()`` (or ``seconds`` of nothing), with the GPU's use meanwhile.
|
|
A window the player spoilt is measured again (``work()`` too), up to ``ATTEMPTS`` in all; the
|
|
spoilt ones are listed under ``earlier_attempts`` and what was done to the player under ``player_fixes``."""
|
|
earlier: list[dict] = []
|
|
fixes: list[str] = []
|
|
while True:
|
|
fixes += page.keep_room()
|
|
page.reset()
|
|
before = page.snapshot()
|
|
t0 = time.perf_counter()
|
|
done: dict = {}
|
|
with gpucap.UtilSampler() as util:
|
|
if work:
|
|
done = work()
|
|
else:
|
|
time.sleep(QUIET_S if seconds is None else seconds)
|
|
after = page.snapshot()
|
|
stats = window_stats(before, after, time.perf_counter() - t0, want)
|
|
stats["gpu_util"] = util.summary()
|
|
if stats["valid"] or not stats.get("retry") or len(earlier) + 1 >= ATTEMPTS:
|
|
break
|
|
earlier.append({"seconds": stats["seconds"], "invalid_because": stats["invalid_because"]})
|
|
if earlier:
|
|
stats["earlier_attempts"] = earlier
|
|
if fixes:
|
|
stats["player_fixes"] = fixes
|
|
return stats, done
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The stage.
|
|
|
|
ROOM_LEVEL = next(lv for lv in llm.TUNED if lv["key"] == "normal_room_stt_gpu")
|
|
|
|
|
|
def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_python: str, url: str = VIDEO_URL,
|
|
quality: str = QUALITY) -> None:
|
|
"""Video alone, next to Jarvis idle, next to Jarvis busy without and with the cap, alone again."""
|
|
base = f"http://127.0.0.1:{llm.PORT}/v1"
|
|
rows: list[dict] = []
|
|
r["video_rows"] = rows
|
|
r["video_source"] = {"url": url, "quality_asked": quality}
|
|
say(" 브라우저(새 임시 프로필)로 유튜브 영상을 틉니다. 끝날 때까지 PC에 손대지 말아 주세요.")
|
|
say(" 영상이 멈추거나 오류가 떠도 키트가 알아서 다시 틀고 그 구간을 다시 잽니다. 새로고침하지 않으셔도 됩니다.")
|
|
try:
|
|
page = open_video_page(s.log.parent.parent)
|
|
except Exception as e:
|
|
r.error("browser", e)
|
|
return
|
|
if page is None:
|
|
r.error("browser", "Chrome이나 Edge를 찾지 못했습니다")
|
|
return
|
|
stt = None
|
|
try:
|
|
start = page.start(quality)
|
|
r["video_source"].update(start)
|
|
if not start["started"]:
|
|
r.error("video", start.get("error") or "the video did not start playing")
|
|
return
|
|
if start.get("quality") != quality or start.get("height") != WANT["height"]:
|
|
# Every window then plays something else; they are kept but none is counted.
|
|
r.error("video", f"the player stayed at {start.get('quality')} ({start.get('height')} lines) "
|
|
f"instead of {quality}")
|
|
|
|
def row(key: str, jarvis: str, cap: float | None = None) -> dict:
|
|
x = {"key": key, "jarvis": jarvis, "gpu_cap": cap}
|
|
used = gpu_memory_mib()
|
|
if used:
|
|
x["vram_free_mib"] = used[1] - used[0]
|
|
rows.append(x)
|
|
return x
|
|
|
|
def keep(x: dict, stats: dict) -> None:
|
|
x["video"] = stats
|
|
if not stats["valid"]:
|
|
r.note(f"{x['key']}: not counted ({'; '.join(stats['invalid_because'])})")
|
|
for a in stats.get("earlier_attempts") or []:
|
|
r.note(f"{x['key']}: measured again after ({'; '.join(a['invalid_because'])})")
|
|
v = stats
|
|
say(f" {x['key']}: 끊긴 프레임 {v['dropped']}/{v['frames']}, {GAP_MS[1]}ms 넘는 멈춤 "
|
|
f"{v['gaps_over_ms'][str(GAP_MS[1])]}번")
|
|
r.save()
|
|
|
|
say(" - 영상만")
|
|
x = row("video_only", "off")
|
|
keep(x, measure_window(page)[0])
|
|
|
|
say(" - Jarvis 켜기 (평소 배치, STT 자리와 여유 1GB)")
|
|
s.stop()
|
|
stt = llm.SttLoad(stt_python, s.log.parent.parent, "cuda", s.log.with_name("stt_load.log"))
|
|
load = stt.start()
|
|
r["stt_load"] = load
|
|
if load.get("status") != "loaded":
|
|
r.error("video stt", "; ".join(load.get("errors") or []) or "the STT did not load")
|
|
return
|
|
lv = dict(ROOM_LEVEL)
|
|
place = {"context": lv.get("context", context)}
|
|
room = llm.stt_room(r, s, preset, place, lv, base)
|
|
r["stt_room"] = room
|
|
sets = list(lv["sets"]) + [f"expert-vram={room['expert_vram_gib']}"]
|
|
r["placement"] = {"sets": sets, "context": place["context"], "parallel": lv["parallel"], "vision": lv["vision"]}
|
|
launch = s.launch(preset, place["context"], parallel=lv["parallel"], vision=lv["vision"], extra_sets=sets,
|
|
timeout=3600)
|
|
if not launch.get("ready"):
|
|
r.error("video llm", "the normal placement did not start")
|
|
return
|
|
c = ChatClient(base)
|
|
r["server"] = {"model": c.model}
|
|
c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time
|
|
|
|
say(" - Jarvis 켜 두고 쉬는 중")
|
|
x = row("jarvis_idle", "idle")
|
|
keep(x, measure_window(page)[0])
|
|
|
|
stt.go()
|
|
for key in ROUNDS:
|
|
cap = llm.GPU_CAP if "cap95" in key else None
|
|
say(f" - Jarvis가 답하고 STT가 쉬지 않고 전사하는 중{' (GPU 95% 제한)' if cap else ''}")
|
|
x = row(key, "busy", cap)
|
|
pids = llm.cap_pids(s, stt) if cap else []
|
|
|
|
def work() -> dict:
|
|
with gpucap.DutyCap(pids, cap or 1.0) as duty:
|
|
bench = llm.quick_bench(c)
|
|
return {"bench": bench, "duty": duty.summary()}
|
|
|
|
stats, done = measure_window(page, work)
|
|
bench = done.get("bench") or {}
|
|
x["llm"] = {k: bench.get(k) for k in ("decode_tok_s", "voice_first_sentence", "context")}
|
|
for err in bench.get("errors") or []:
|
|
r.error(f"video {key}", err)
|
|
if cap:
|
|
x["gpu_cap"] = done["duty"]
|
|
problems = cap_problems(x["gpu_cap"], pids)
|
|
for why in problems:
|
|
r.error(f"video {key} gpu cap", why)
|
|
if problems:
|
|
# Not a capped window, so it must not count for the cap.
|
|
stats["valid"] = False
|
|
stats["invalid_because"] = stats.get("invalid_because", []) + problems
|
|
keep(x, stats)
|
|
busy = stt.finish()
|
|
r["stt_busy"] = {k: busy.get(k) for k in ("status", "batch10_s", "single_s", "errors", "busy_share")
|
|
if k in busy}
|
|
for err in llm.stt_gaps(busy):
|
|
r.error("video stt", err)
|
|
stt.close()
|
|
stt = None
|
|
s.stop()
|
|
|
|
say(" - Jarvis 끄고 영상만 다시")
|
|
x = row("video_only_again", "off")
|
|
keep(x, measure_window(page)[0])
|
|
r["comparison"] = compare(rows)
|
|
except Exception as e:
|
|
r.error("video", e)
|
|
finally:
|
|
if stt:
|
|
stt.close()
|
|
s.stop()
|
|
page.close()
|