fix(m0): judge game frames by what the screen showed and drop spoilt windows

- Frame rate, 1 % low and hitches come from msBetweenDisplayChange of the
  frames that reached the screen; frames never shown are counted apart and
  the presented rate is kept as presented_fps. A game presenting 60 fps
  while the screen changes every 167 ms now shows as the stutter it is.
- A window whose capture failed, a busy window where the LLM finished no
  answer, and every busy window when the STT load failed are not counted.
- A yielding window counts only when every Jarvis process got both its
  CPU and its GPU priority lowered.
- The "game hardly drawing" floor applies only with Jarvis off or idle, so
  a game Jarvis slows that far still counts as harm.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-11 09:12:15 +09:00
parent dff18c5c7e
commit cec52165b2
4 changed files with 116 additions and 39 deletions

View File

@@ -41,7 +41,7 @@ PRESENTMON_SHA256 = "b2a706bc6ad475749e3b7e3409263aa1e6906d45bdcf993f6dbc0f66018
WINDOW_S = 60
SETTLE_S = 5 # Jarvis busy (or yielding) before a window starts
HITCH_MS = (50, 100) # a frame this long is a visible hitch
MIN_FRAMES = 300 # under this in a minute (5 fps) the game was not really drawing
MIN_FRAMES = 300 # under this in a minute (5 fps) with Jarvis off or idle, the game was not really drawing
ROUNDS = ("busy", "busy_yield", "busy_yield_2", "busy_2")
# The foreground window belongs to one of these when the user did not get back to the game.
NOT_A_GAME = {"windowsterminal.exe", "openconsole.exe", "conhost.exe", "cmd.exe", "powershell.exe",
@@ -183,12 +183,15 @@ class Yield:
def yield_problems(summary: dict) -> list[str]:
"""Why a yielding window did not yield: no process, nothing lowered, or a priority not put back."""
"""Why a yielding window did not yield: no process, a CPU or GPU priority left normal on any of
Jarvis's processes (yielding with some of them is not the yield being measured), or one not put back."""
out = []
if not summary.get("processes"):
n = summary.get("processes") or 0
if not n:
out.append("yield found no Jarvis process")
elif not summary.get("cpu_lowered") and not summary.get("gpu_lowered"):
out.append("yield lowered no priority")
for key, what in (("cpu_lowered", "CPU"), ("gpu_lowered", "GPU")):
if n and summary.get(key, 0) < n:
out.append(f"yield lowered the {what} priority of only {summary.get(key, 0)} of {n} processes")
if summary.get("not_restored"):
out.append(f"{summary['not_restored']} process(es) kept the lowered priority")
return out
@@ -252,26 +255,37 @@ def _num(value: str) -> float | None:
return None
def frame_stats(rows: list[dict], seconds: float) -> dict:
"""Frame rate, 1 % low (the 99th-percentile frame time as a rate) and hitches of the game's main
swap chain (the one with the most frames)."""
def frame_stats(rows: list[dict], seconds: float, min_frames: int = 0) -> dict:
"""What the screen showed of the game's main swap chain (the one with the most frames): the rate of
shown frames, the 1 % low (the 99th-percentile time between screen updates as a rate) and hitches,
all from the time each shown frame stayed before the next (msBetweenDisplayChange). A frame the
game presented but the screen never showed is counted in ``not_shown``; the rate the game
presented at is kept apart (``presented_fps``), since a game can present at 60 fps while the
screen changes far less often. ``min_frames`` (per minute) marks a window where the game was
hardly drawing; it is only given with Jarvis off or idle, so a game Jarvis slows down that much
still counts as the harm it is."""
chains: dict[str, list[dict]] = {}
for x in rows:
chains.setdefault(x.get("swapchainaddress", ""), []).append(x)
main = max(chains.values(), key=len) if chains else []
ms = [v for v in (_num(x.get("msbetweenpresents")) for x in main[1:]) if v is not None and v > 0]
out: dict[str, Any] = {"seconds": round(seconds, 1), "frames": len(ms)}
main = (max(chains.values(), key=len) if chains else [])[1:] # the first reaches back before the capture
presents = [v for v in (_num(x.get("msbetweenpresents")) for x in main) if v is not None and v > 0]
shown = [x for x in main if x.get("dropped") != "1"]
ms = [v for v in (_num(x.get("msbetweendisplaychange")) for x in shown) if v is not None and v > 0][1:]
out: dict[str, Any] = {"seconds": round(seconds, 1), "frames": len(ms), "presented": len(presents),
"not_shown": len(main) - len(shown)}
if presents:
out["presented_fps"] = round(len(presents) / (sum(presents) / 1000), 1)
if not ms:
return {**out, "valid": False, "invalid_because": ["the game drew no frames"]}
why = "the game drew no frames" if not presents else "PresentMon saw none of the frames reach the screen"
return {**out, "valid": False, "invalid_because": [why]}
srt = sorted(ms)
p99 = srt[min(len(srt) - 1, int(len(srt) * 0.99))]
out.update(fps=round(len(ms) / (sum(ms) / 1000), 1), low1_fps=round(1000 / p99, 1),
p99_ms=round(p99, 2), max_ms=round(srt[-1], 1),
hitches_over_ms={str(h): sum(v > h for v in ms) for h in HITCH_MS},
not_shown=sum(x.get("dropped") == "1" for x in main[1:]))
hitches_over_ms={str(h): sum(v > h for v in ms) for h in HITCH_MS})
reasons = []
if len(ms) < MIN_FRAMES * seconds / 60:
reasons.append(f"the game drew only {len(ms)} frames")
if len(ms) < min_frames * seconds / 60:
reasons.append(f"the screen showed only {len(ms)} of the game's frames")
out["valid"] = not reasons
if reasons:
out["invalid_because"] = reasons
@@ -361,6 +375,11 @@ def ask_for_a_game() -> bool:
return True
def _spoil(x: dict, why: str) -> None:
x["game"]["valid"] = False
x["game"]["invalid_because"] = x["game"].get("invalid_because", []) + [why]
def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_python: str) -> None:
work = s.log.parent.parent
pm_log = s.log.with_name("presentmon.log")
@@ -393,14 +412,17 @@ def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_pyt
x["vram_free_mib"] = used[1] - used[0]
with gpucap.UtilSampler() as util:
frames, err = capture(pm, game, WINDOW_S, work / "cache" / "frames.csv", pm_log)
g = frame_stats(frames, WINDOW_S)
g = frame_stats(frames, WINDOW_S, MIN_FRAMES if kind in ("off", "idle") else 0)
g["gpu_util"] = util.summary()
now = foreground_process()
if now is not None and now[0] != game:
g["valid"] = False
g["invalid_because"] = g.get("invalid_because", []) + ["the game was not in front at the end"]
if err:
# Frames from a capture that failed or was cut off do not cover the window.
r.error(f"frames {key}", err)
g["valid"] = False
g["invalid_because"] = g.get("invalid_because", []) + [f"the capture failed ({err})"]
x["game"] = g
rows.append(x)
if not g["valid"]:
@@ -450,20 +472,27 @@ def measure(r: StageResult, s: "llm.Strata", preset: dict, context: int, stt_pyt
x["llm"] = busy.summary()
for err in x["llm"]["errors"]:
r.error(f"frames {key} llm", err)
if not x["llm"]["decode_tok_s"].get("n"):
# Jarvis was not busy, so the window measured the game next to an idle Jarvis.
r.error(f"frames {key} llm", "no answer finished during the window")
_spoil(x, "the LLM finished no answer during the window")
if yielding:
x["yield"] = y.summary()
problems = yield_problems(x["yield"])
for why in problems:
r.error(f"frames {key} yield", why)
if problems:
x["game"]["valid"] = False
x["game"]["invalid_because"] = x["game"].get("invalid_because", []) + problems
for why in problems:
_spoil(x, why)
r.save()
busy_stt = stt.finish()
r["stt_busy"] = {k: busy_stt.get(k) for k in ("status", "single_s", "errors", "final_wait_s")
if k in busy_stt}
for err in llm.stt_gaps(busy_stt, llm._FINALS_ONLY):
r.error("frames stt", err)
# The STT's part of the load is not known to have run, in any busy window.
for x in rows:
if x["kind"].startswith("busy"):
_spoil(x, f"the STT load failed ({err})")
stt.close()
stt = None
s.stop()