fix(m0): never leave a process paused when a resume fails under the GPU cap

A resume that failed was only counted, and the process stayed suspended
after the cap ended. DutyCap now keeps what it still owes per process,
does not stack another suspend on it, retries the resume every period
and at the end, treats a raising call as a failure, and reports any
process it could not resume as left_paused (an error for the row).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-10 23:56:57 +09:00
parent d0938c10f9
commit ba662ae199
3 changed files with 107 additions and 9 deletions

View File

@@ -150,8 +150,12 @@ def resume_pid(pid: int) -> bool:
class DutyCap:
"""Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs; every
paused process is resumed before the block ends, also on an error."""
"""Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs. A process
whose resume fails stays owed: it is not paused again on top (each suspend adds one to its
suspend count), its resume is retried every period and at the end, and one that still cannot
be resumed is reported as ``left_paused``."""
RESUME_TRIES_AT_END = 5
def __init__(self, pids: list[int], cap: float, period: float | None = None,
suspend: Callable[[int], bool] | None = None, resume: Callable[[int], bool] | None = None):
@@ -161,6 +165,7 @@ class DutyCap:
self.paused_s = 0.0
self.cycles = 0
self.failed: set[int] = set()
self.owed: dict[int, int] = {} # pid -> suspends not yet undone
self._stop = threading.Event()
self._t: threading.Thread | None = None
self._t0 = 0.0
@@ -173,17 +178,45 @@ class DutyCap:
self._t.start()
return self
@staticmethod
def _call(fn: Callable[[int], bool], pid: int) -> bool:
try:
return bool(fn(pid))
except Exception:
return False
def _resume(self, pid: int) -> None:
if self._call(self.resume, pid):
return
self.failed.add(pid)
self.owed[pid] = self.owed.get(pid, 0) + 1
def _settle_owed(self) -> None:
for pid, n in list(self.owed.items()):
while n and self._call(self.resume, pid):
n -= 1
if n:
self.owed[pid] = n
else:
del self.owed[pid]
def _loop(self) -> None:
while not self._stop.wait(self.period - self.pause):
held = [p for p in self.pids if self.suspend(p)]
self.failed.update(p for p in self.pids if p not in held)
self._settle_owed()
held = []
for p in self.pids:
if p in self.owed:
continue # still paused from a resume that failed
if self._call(self.suspend, p):
held.append(p)
else:
self.failed.add(p)
s0 = time.perf_counter()
try:
time.sleep(self.pause)
finally:
for p in held:
if not self.resume(p):
self.failed.add(p)
self._resume(p)
self.paused_s += time.perf_counter() - s0
self.cycles += 1
@@ -191,6 +224,12 @@ class DutyCap:
self._stop.set()
if self._t:
self._t.join(timeout=10)
for _ in range(self.RESUME_TRIES_AT_END):
if not self.owed:
break
self._settle_owed()
if self.owed:
time.sleep(0.05)
self._t1 = time.perf_counter()
def summary(self) -> dict:
@@ -198,4 +237,4 @@ class DutyCap:
return {"cap": self.cap, "period_s": self.period, "pause_s": round(self.pause, 4),
"processes": len(self.pids), "cycles": self.cycles,
"paused_share": round(self.paused_s / span, 3) if span > 0 else None,
"failed_processes": len(self.failed)}
"failed_processes": len(self.failed), "left_paused": len(self.owed)}

View File

@@ -514,9 +514,13 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels
row["gpu_util"] = util.summary(cap)
if cap:
row["gpu_cap"] = duty.summary()
if row["gpu_cap"]["failed_processes"]:
if row["gpu_cap"]["left_paused"]:
r.error(f"level {lv['key']} gpu cap",
f"{row['gpu_cap']['failed_processes']} process(es) could not be paused")
f"{row['gpu_cap']['left_paused']} process(es) are still paused")
elif row["gpu_cap"]["failed_processes"]:
r.error(f"level {lv['key']} gpu cap",
f"{row['gpu_cap']['failed_processes']} process(es) could not be "
"paused or resumed at once")
for err in row["errors"]:
r.error(f"level {lv['key']}", err)
except Exception as e:

View File

@@ -1151,3 +1151,58 @@ def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp
r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, [])
assert r["status"] == "partial" and not paused
assert any("no engine process" in e["error"] for e in r["errors"])
class _CountingProcs:
"""NtSuspendProcess/NtResumeProcess on fake processes: each suspend adds one to the process's
suspend count and each successful resume takes one off; ``fail`` says when a resume fails."""
def __init__(self, fail):
self.count: dict[int, int] = {}
self.fail = fail
self.resumes = 0
def suspend(self, pid):
self.count[pid] = self.count.get(pid, 0) + 1
return True
def resume(self, pid):
self.resumes += 1
if self.fail(pid, self.resumes):
return False
self.count[pid] -= 1
return True
def test_a_resume_that_fails_once_is_retried_and_nothing_is_left_paused():
import time as _time
procs = _CountingProcs(fail=lambda pid, n: pid == 1 and n == 1)
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
_time.sleep(0.2)
s = d.summary()
assert all(v == 0 for v in procs.count.values())
assert s["failed_processes"] == 1 and s["left_paused"] == 0
def test_a_process_that_cannot_be_resumed_is_reported_as_left_paused():
import time as _time
procs = _CountingProcs(fail=lambda pid, n: pid == 1)
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
_time.sleep(0.2)
assert procs.count[1] == 1 and procs.count[2] == 0 # paused once, never stacked
assert d.summary()["left_paused"] == 1
def test_a_suspend_or_resume_that_raises_does_not_end_the_cap_with_processes_paused():
import time as _time
procs = _CountingProcs(fail=lambda pid, n: False)
def resume(pid):
if procs.resumes == 0:
procs.resumes += 1
raise OSError("access denied")
return procs.resume(pid)
with gpucap.DutyCap([1], 0.9, period=0.02, suspend=procs.suspend, resume=resume) as d:
_time.sleep(0.2)
assert procs.count[1] == 0 and d.summary()["left_paused"] == 0