fix(m0): judge game frames by what the screen showed and drop spoilt windows
- Frame rate, 1 % low and hitches come from msBetweenDisplayChange of the frames that reached the screen; frames never shown are counted apart and the presented rate is kept as presented_fps. A game presenting 60 fps while the screen changes every 167 ms now shows as the stutter it is. - A window whose capture failed, a busy window where the LLM finished no answer, and every busy window when the STT load failed are not counted. - A yielding window counts only when every Jarvis process got both its CPU and its GPU priority lowered. - The "game hardly drawing" floor applies only with Jarvis off or idle, so a game Jarvis slows that far still counts as harm. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -41,7 +41,7 @@ PRESENTMON_SHA256 = "b2a706bc6ad475749e3b7e3409263aa1e6906d45bdcf993f6dbc0f66018
|
||||
WINDOW_S = 60
|
||||
SETTLE_S = 5 # Jarvis busy (or yielding) before a window starts
|
||||
HITCH_MS = (50, 100) # a frame this long is a visible hitch
|
||||
MIN_FRAMES = 300 # under this in a minute (5 fps) the game was not really drawing
|
||||
MIN_FRAMES = 300 # under this in a minute (5 fps) with Jarvis off or idle, the game was not really drawing
|
||||
ROUNDS = ("busy", "busy_yield", "busy_yield_2", "busy_2")
|
||||
# The foreground window belongs to one of these when the user did not get back to the game.
|
||||
NOT_A_GAME = {"windowsterminal.exe", "openconsole.exe", "conhost.exe", "cmd.exe", "powershell.exe",
|
||||
@@ -183,12 +183,15 @@ class Yield:
|
||||
|
||||
|
||||
def yield_problems(summary: dict) -> list[str]:
|
||||
"""Why a yielding window did not yield: no process, nothing lowered, or a priority not put back."""
|
||||
"""Why a yielding window did not yield: no process, a CPU or GPU priority left normal on any of
|
||||
Jarvis's processes (yielding with some of them is not the yield being measured), or one not put back."""
|
||||
out = []
|
||||
if not summary.get("processes"):
|
||||
n = summary.get("processes") or 0
|
||||
if not n:
|
||||
out.append("yield found no Jarvis process")
|
||||
elif not summary.get("cpu_lowered") and not summary.get("gpu_lowered"):
|
||||
out.append("yield lowered no priority")
|
||||
for key, what in (("cpu_lowered", "CPU"), ("gpu_lowered", "GPU")):
|
||||
if n and summary.get(key, 0) < n:
|
||||
out.append(f"yield lowered the {what} priority of only {summary.get(key, 0)} of {n} processes")
|
||||
if summary.get("not_restored"):
|
||||
out.append(f"{summary['not_restored']} process(es) kept the lowered priority")
|
||||
return out
|
||||
@@ -252,26 +255,37 @@ def _num(value: str) -> float | None:
|
||||
return None
|
||||
|
||||
|
||||
def frame_stats(rows: list[dict], seconds: float) -> dict:
|
||||
"""Frame rate, 1 % low (the 99th-percentile frame time as a rate) and hitches of the game's main
|
||||
swap chain (the one with the most frames)."""
|
||||
def frame_stats(rows: list[dict], seconds: float, min_frames: int = 0) -> dict:
|
||||
"""What the screen showed of the game's main swap chain (the one with the most frames): the rate of
|
||||
shown frames, the 1 % low (the 99th-percentile time between screen updates as a rate) and hitches,
|
||||
all from the time each shown frame stayed before the next (msBetweenDisplayChange). A frame the
|
||||
game presented but the screen never showed is counted in ``not_shown``; the rate the game
|
||||
presented at is kept apart (``presented_fps``), since a game can present at 60 fps while the
|
||||
screen changes far less often. ``min_frames`` (per minute) marks a window where the game was
|
||||
hardly drawing; it is only given with Jarvis off or idle, so a game Jarvis slows down that much
|
||||
still counts as the harm it is."""
|
||||
chains: dict[str, list[dict]] = {}
|
||||
for x in rows:
|
||||
chains.setdefault(x.get("swapchainaddress", ""), []).append(x)
|
||||
main = max(chains.values(), key=len) if chains else []
|
||||
ms = [v for v in (_num(x.get("msbetweenpresents")) for x in main[1:]) if v is not None and v > 0]
|
||||
out: dict[str, Any] = {"seconds": round(seconds, 1), "frames": len(ms)}
|
||||
main = (max(chains.values(), key=len) if chains else [])[1:] # the first reaches back before the capture
|
||||
presents = [v for v in (_num(x.get("msbetweenpresents")) for x in main) if v is not None and v > 0]
|
||||
shown = [x for x in main if x.get("dropped") != "1"]
|
||||
ms = [v for v in (_num(x.get("msbetweendisplaychange")) for x in shown) if v is not None and v > 0][1:]
|
||||
out: dict[str, Any] = {"seconds": round(seconds, 1), "frames": len(ms), "presented": len(presents),
|
||||
"not_shown": len(main) - len(shown)}
|
||||
if presents:
|
||||
out["presented_fps"] = round(len(presents) / (sum(presents) / 1000), 1)
|
||||
if not ms:
|
||||
return {**out, "valid": False, "invalid_because": ["the game drew no frames"]}
|
||||
why = "the game drew no frames" if not presents else "PresentMon saw none of the frames reach the screen"
|
||||
return {**out, "valid": False, "invalid_because": [why]}
|
||||
srt = sorted(ms)
|
||||
p99 = srt[min(len(srt) - 1, int(len(srt) * 0.99))]
|
||||
out.update(fps=round(len(ms) / (sum(ms) / 1000), 1), low1_fps=round(1000 / p99, 1),
|
||||
p99_ms=round(p99, 2), max_ms=round(srt[-1], 1),
|
||||
hitches_over_ms={str(h): sum(v > h for v in ms) for h in HITCH_MS},
|
||||
not_shown=sum(x.get("dropped") == "1" for x in main[1:]))
|
||||
hitches_over_ms={str(h): sum(v > h for v in ms) for h in HITCH_MS})
|
||||
reasons = []
|
||||
if len(ms) < MIN_FRAMES * seconds / 60:
|
||||
reasons.append(f"the game drew only {len(ms)} frames")
|
||||
if len(ms) < min_frames * seconds / 60:
|
||||
reasons.append(f"the screen showed only {len(ms)} of the game's frames")
|
||||
out["valid"] = not reasons
|
||||
if reasons:
|
||||
out["invalid_because"] = reasons
|
||||
@@ -361,6 +375,11 @@ def ask_for_a_game() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _spoil(x: dict, why: str) -> None:
|
||||
x["game"]["valid"] = False
|
||||
x["game"]["invalid_because"] = x["game"].get("invalid_because", []) + [why]
|
||||
|
||||
|
||||
def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_python: str) -> None:
|
||||
work = s.log.parent.parent
|
||||
pm_log = s.log.with_name("presentmon.log")
|
||||
@@ -393,14 +412,17 @@ def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_pyt
|
||||
x["vram_free_mib"] = used[1] - used[0]
|
||||
with gpucap.UtilSampler() as util:
|
||||
frames, err = capture(pm, game, WINDOW_S, work / "cache" / "frames.csv", pm_log)
|
||||
g = frame_stats(frames, WINDOW_S)
|
||||
g = frame_stats(frames, WINDOW_S, MIN_FRAMES if kind in ("off", "idle") else 0)
|
||||
g["gpu_util"] = util.summary()
|
||||
now = foreground_process()
|
||||
if now is not None and now[0] != game:
|
||||
g["valid"] = False
|
||||
g["invalid_because"] = g.get("invalid_because", []) + ["the game was not in front at the end"]
|
||||
if err:
|
||||
# Frames from a capture that failed or was cut off do not cover the window.
|
||||
r.error(f"frames {key}", err)
|
||||
g["valid"] = False
|
||||
g["invalid_because"] = g.get("invalid_because", []) + [f"the capture failed ({err})"]
|
||||
x["game"] = g
|
||||
rows.append(x)
|
||||
if not g["valid"]:
|
||||
@@ -450,20 +472,27 @@ def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_pyt
|
||||
x["llm"] = busy.summary()
|
||||
for err in x["llm"]["errors"]:
|
||||
r.error(f"frames {key} llm", err)
|
||||
if not x["llm"]["decode_tok_s"].get("n"):
|
||||
# Jarvis was not busy, so the window measured the game next to an idle Jarvis.
|
||||
r.error(f"frames {key} llm", "no answer finished during the window")
|
||||
_spoil(x, "the LLM finished no answer during the window")
|
||||
if yielding:
|
||||
x["yield"] = y.summary()
|
||||
problems = yield_problems(x["yield"])
|
||||
for why in problems:
|
||||
r.error(f"frames {key} yield", why)
|
||||
if problems:
|
||||
x["game"]["valid"] = False
|
||||
x["game"]["invalid_because"] = x["game"].get("invalid_because", []) + problems
|
||||
for why in problems:
|
||||
_spoil(x, why)
|
||||
r.save()
|
||||
busy_stt = stt.finish()
|
||||
r["stt_busy"] = {k: busy_stt.get(k) for k in ("status", "single_s", "errors", "final_wait_s")
|
||||
if k in busy_stt}
|
||||
for err in llm.stt_gaps(busy_stt, llm._FINALS_ONLY):
|
||||
r.error("frames stt", err)
|
||||
# The STT's part of the load is not known to have run, in any busy window.
|
||||
for x in rows:
|
||||
if x["kind"].startswith("busy"):
|
||||
_spoil(x, f"the STT load failed ({err})")
|
||||
stt.close()
|
||||
stt = None
|
||||
s.stop()
|
||||
|
||||
Reference in New Issue
Block a user