fix(m0): never leave a process paused when a resume fails under the GPU cap
A resume that failed was only counted, and the process stayed suspended after the cap ended. DutyCap now keeps what it still owes per process, does not stack another suspend on it, retries the resume every period and at the end, treats a raising call as a failure, and reports any process it could not resume as left_paused (an error for the row). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -150,8 +150,12 @@ def resume_pid(pid: int) -> bool:
|
||||
|
||||
|
||||
class DutyCap:
|
||||
"""Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs; every
|
||||
paused process is resumed before the block ends, also on an error."""
|
||||
"""Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs. A process
|
||||
whose resume fails stays owed: it is not paused again on top (each suspend adds one to its
|
||||
suspend count), its resume is retried every period and at the end, and one that still cannot
|
||||
be resumed is reported as ``left_paused``."""
|
||||
|
||||
RESUME_TRIES_AT_END = 5
|
||||
|
||||
def __init__(self, pids: list[int], cap: float, period: float | None = None,
|
||||
suspend: Callable[[int], bool] | None = None, resume: Callable[[int], bool] | None = None):
|
||||
@@ -161,6 +165,7 @@ class DutyCap:
|
||||
self.paused_s = 0.0
|
||||
self.cycles = 0
|
||||
self.failed: set[int] = set()
|
||||
self.owed: dict[int, int] = {} # pid -> suspends not yet undone
|
||||
self._stop = threading.Event()
|
||||
self._t: threading.Thread | None = None
|
||||
self._t0 = 0.0
|
||||
@@ -173,17 +178,45 @@ class DutyCap:
|
||||
self._t.start()
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _call(fn: Callable[[int], bool], pid: int) -> bool:
|
||||
try:
|
||||
return bool(fn(pid))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _resume(self, pid: int) -> None:
|
||||
if self._call(self.resume, pid):
|
||||
return
|
||||
self.failed.add(pid)
|
||||
self.owed[pid] = self.owed.get(pid, 0) + 1
|
||||
|
||||
def _settle_owed(self) -> None:
|
||||
for pid, n in list(self.owed.items()):
|
||||
while n and self._call(self.resume, pid):
|
||||
n -= 1
|
||||
if n:
|
||||
self.owed[pid] = n
|
||||
else:
|
||||
del self.owed[pid]
|
||||
|
||||
def _loop(self) -> None:
|
||||
while not self._stop.wait(self.period - self.pause):
|
||||
held = [p for p in self.pids if self.suspend(p)]
|
||||
self.failed.update(p for p in self.pids if p not in held)
|
||||
self._settle_owed()
|
||||
held = []
|
||||
for p in self.pids:
|
||||
if p in self.owed:
|
||||
continue # still paused from a resume that failed
|
||||
if self._call(self.suspend, p):
|
||||
held.append(p)
|
||||
else:
|
||||
self.failed.add(p)
|
||||
s0 = time.perf_counter()
|
||||
try:
|
||||
time.sleep(self.pause)
|
||||
finally:
|
||||
for p in held:
|
||||
if not self.resume(p):
|
||||
self.failed.add(p)
|
||||
self._resume(p)
|
||||
self.paused_s += time.perf_counter() - s0
|
||||
self.cycles += 1
|
||||
|
||||
@@ -191,6 +224,12 @@ class DutyCap:
|
||||
self._stop.set()
|
||||
if self._t:
|
||||
self._t.join(timeout=10)
|
||||
for _ in range(self.RESUME_TRIES_AT_END):
|
||||
if not self.owed:
|
||||
break
|
||||
self._settle_owed()
|
||||
if self.owed:
|
||||
time.sleep(0.05)
|
||||
self._t1 = time.perf_counter()
|
||||
|
||||
def summary(self) -> dict:
|
||||
@@ -198,4 +237,4 @@ class DutyCap:
|
||||
return {"cap": self.cap, "period_s": self.period, "pause_s": round(self.pause, 4),
|
||||
"processes": len(self.pids), "cycles": self.cycles,
|
||||
"paused_share": round(self.paused_s / span, 3) if span > 0 else None,
|
||||
"failed_processes": len(self.failed)}
|
||||
"failed_processes": len(self.failed), "left_paused": len(self.owed)}
|
||||
|
||||
@@ -514,9 +514,13 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels
|
||||
row["gpu_util"] = util.summary(cap)
|
||||
if cap:
|
||||
row["gpu_cap"] = duty.summary()
|
||||
if row["gpu_cap"]["failed_processes"]:
|
||||
if row["gpu_cap"]["left_paused"]:
|
||||
r.error(f"level {lv['key']} gpu cap",
|
||||
f"{row['gpu_cap']['failed_processes']} process(es) could not be paused")
|
||||
f"{row['gpu_cap']['left_paused']} process(es) are still paused")
|
||||
elif row["gpu_cap"]["failed_processes"]:
|
||||
r.error(f"level {lv['key']} gpu cap",
|
||||
f"{row['gpu_cap']['failed_processes']} process(es) could not be "
|
||||
"paused or resumed at once")
|
||||
for err in row["errors"]:
|
||||
r.error(f"level {lv['key']}", err)
|
||||
except Exception as e:
|
||||
|
||||
@@ -1151,3 +1151,58 @@ def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp
|
||||
r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, [])
|
||||
assert r["status"] == "partial" and not paused
|
||||
assert any("no engine process" in e["error"] for e in r["errors"])
|
||||
|
||||
|
||||
class _CountingProcs:
|
||||
"""NtSuspendProcess/NtResumeProcess on fake processes: each suspend adds one to the process's
|
||||
suspend count and each successful resume takes one off; ``fail`` says when a resume fails."""
|
||||
|
||||
def __init__(self, fail):
|
||||
self.count: dict[int, int] = {}
|
||||
self.fail = fail
|
||||
self.resumes = 0
|
||||
|
||||
def suspend(self, pid):
|
||||
self.count[pid] = self.count.get(pid, 0) + 1
|
||||
return True
|
||||
|
||||
def resume(self, pid):
|
||||
self.resumes += 1
|
||||
if self.fail(pid, self.resumes):
|
||||
return False
|
||||
self.count[pid] -= 1
|
||||
return True
|
||||
|
||||
|
||||
def test_a_resume_that_fails_once_is_retried_and_nothing_is_left_paused():
|
||||
import time as _time
|
||||
procs = _CountingProcs(fail=lambda pid, n: pid == 1 and n == 1)
|
||||
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
|
||||
_time.sleep(0.2)
|
||||
s = d.summary()
|
||||
assert all(v == 0 for v in procs.count.values())
|
||||
assert s["failed_processes"] == 1 and s["left_paused"] == 0
|
||||
|
||||
|
||||
def test_a_process_that_cannot_be_resumed_is_reported_as_left_paused():
|
||||
import time as _time
|
||||
procs = _CountingProcs(fail=lambda pid, n: pid == 1)
|
||||
with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d:
|
||||
_time.sleep(0.2)
|
||||
assert procs.count[1] == 1 and procs.count[2] == 0 # paused once, never stacked
|
||||
assert d.summary()["left_paused"] == 1
|
||||
|
||||
|
||||
def test_a_suspend_or_resume_that_raises_does_not_end_the_cap_with_processes_paused():
|
||||
import time as _time
|
||||
procs = _CountingProcs(fail=lambda pid, n: False)
|
||||
|
||||
def resume(pid):
|
||||
if procs.resumes == 0:
|
||||
procs.resumes += 1
|
||||
raise OSError("access denied")
|
||||
return procs.resume(pid)
|
||||
|
||||
with gpucap.DutyCap([1], 0.9, period=0.02, suspend=procs.suspend, resume=resume) as d:
|
||||
_time.sleep(0.2)
|
||||
assert procs.count[1] == 0 and d.summary()["left_paused"] == 0
|
||||
|
||||
Reference in New Issue
Block a user