From ba662ae199635d8767a58f2494af533c7d35183a Mon Sep 17 00:00:00 2001 From: javis-bot Date: Tue, 10 Nov 2026 23:56:57 +0900 Subject: [PATCH] fix(m0): never leave a process paused when a resume fails under the GPU cap A resume that failed was only counted, and the process stayed suspended after the cap ended. DutyCap now keeps what it still owes per process, does not stack another suspend on it, retries the resume every period and at the end, treats a raising call as a failure, and reports any process it could not resume as left_paused (an error for the row). Co-Authored-By: Claude Opus 5.5 --- scripts/m0/m0lib/gpucap.py | 53 +++++++++++++++++++++++++++++++----- scripts/m0/m0lib/llm.py | 8 ++++-- tests/test_m0_kit.py | 55 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/scripts/m0/m0lib/gpucap.py b/scripts/m0/m0lib/gpucap.py index 4ef53a3..0d8418a 100644 --- a/scripts/m0/m0lib/gpucap.py +++ b/scripts/m0/m0lib/gpucap.py @@ -150,8 +150,12 @@ def resume_pid(pid: int) -> bool: class DutyCap: - """Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs; every - paused process is resumed before the block ends, also on an error.""" + """Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs. A process + whose resume fails stays owed: it is not paused again on top (each suspend adds one to its + suspend count), its resume is retried every period and at the end, and one that still cannot + be resumed is reported as ``left_paused``.""" + + RESUME_TRIES_AT_END = 5 def __init__(self, pids: list[int], cap: float, period: float | None = None, suspend: Callable[[int], bool] | None = None, resume: Callable[[int], bool] | None = None): @@ -161,6 +165,7 @@ class DutyCap: self.paused_s = 0.0 self.cycles = 0 self.failed: set[int] = set() + self.owed: dict[int, int] = {} # pid -> suspends not yet undone self._stop = threading.Event() self._t: threading.Thread | None = None self._t0 = 0.0 @@ -173,17 +178,45 @@ class DutyCap: self._t.start() return self + @staticmethod + def _call(fn: Callable[[int], bool], pid: int) -> bool: + try: + return bool(fn(pid)) + except Exception: + return False + + def _resume(self, pid: int) -> None: + if self._call(self.resume, pid): + return + self.failed.add(pid) + self.owed[pid] = self.owed.get(pid, 0) + 1 + + def _settle_owed(self) -> None: + for pid, n in list(self.owed.items()): + while n and self._call(self.resume, pid): + n -= 1 + if n: + self.owed[pid] = n + else: + del self.owed[pid] + def _loop(self) -> None: while not self._stop.wait(self.period - self.pause): - held = [p for p in self.pids if self.suspend(p)] - self.failed.update(p for p in self.pids if p not in held) + self._settle_owed() + held = [] + for p in self.pids: + if p in self.owed: + continue # still paused from a resume that failed + if self._call(self.suspend, p): + held.append(p) + else: + self.failed.add(p) s0 = time.perf_counter() try: time.sleep(self.pause) finally: for p in held: - if not self.resume(p): - self.failed.add(p) + self._resume(p) self.paused_s += time.perf_counter() - s0 self.cycles += 1 @@ -191,6 +224,12 @@ class DutyCap: self._stop.set() if self._t: self._t.join(timeout=10) + for _ in range(self.RESUME_TRIES_AT_END): + if not self.owed: + break + self._settle_owed() + if self.owed: + time.sleep(0.05) self._t1 = time.perf_counter() def summary(self) -> dict: @@ -198,4 +237,4 @@ class DutyCap: return {"cap": self.cap, "period_s": self.period, "pause_s": round(self.pause, 4), "processes": len(self.pids), "cycles": self.cycles, "paused_share": round(self.paused_s / span, 3) if span > 0 else None, - "failed_processes": len(self.failed)} + "failed_processes": len(self.failed), "left_paused": len(self.owed)} diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 6a5e4d5..e8d87fb 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -514,9 +514,13 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels row["gpu_util"] = util.summary(cap) if cap: row["gpu_cap"] = duty.summary() - if row["gpu_cap"]["failed_processes"]: + if row["gpu_cap"]["left_paused"]: r.error(f"level {lv['key']} gpu cap", - f"{row['gpu_cap']['failed_processes']} process(es) could not be paused") + f"{row['gpu_cap']['left_paused']} process(es) are still paused") + elif row["gpu_cap"]["failed_processes"]: + r.error(f"level {lv['key']} gpu cap", + f"{row['gpu_cap']['failed_processes']} process(es) could not be " + "paused or resumed at once") for err in row["errors"]: r.error(f"level {lv['key']}", err) except Exception as e: diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index d11dd4f..9be61b9 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -1151,3 +1151,58 @@ def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, []) assert r["status"] == "partial" and not paused assert any("no engine process" in e["error"] for e in r["errors"]) + + +class _CountingProcs: + """NtSuspendProcess/NtResumeProcess on fake processes: each suspend adds one to the process's + suspend count and each successful resume takes one off; ``fail`` says when a resume fails.""" + + def __init__(self, fail): + self.count: dict[int, int] = {} + self.fail = fail + self.resumes = 0 + + def suspend(self, pid): + self.count[pid] = self.count.get(pid, 0) + 1 + return True + + def resume(self, pid): + self.resumes += 1 + if self.fail(pid, self.resumes): + return False + self.count[pid] -= 1 + return True + + +def test_a_resume_that_fails_once_is_retried_and_nothing_is_left_paused(): + import time as _time + procs = _CountingProcs(fail=lambda pid, n: pid == 1 and n == 1) + with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d: + _time.sleep(0.2) + s = d.summary() + assert all(v == 0 for v in procs.count.values()) + assert s["failed_processes"] == 1 and s["left_paused"] == 0 + + +def test_a_process_that_cannot_be_resumed_is_reported_as_left_paused(): + import time as _time + procs = _CountingProcs(fail=lambda pid, n: pid == 1) + with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d: + _time.sleep(0.2) + assert procs.count[1] == 1 and procs.count[2] == 0 # paused once, never stacked + assert d.summary()["left_paused"] == 1 + + +def test_a_suspend_or_resume_that_raises_does_not_end_the_cap_with_processes_paused(): + import time as _time + procs = _CountingProcs(fail=lambda pid, n: False) + + def resume(pid): + if procs.resumes == 0: + procs.resumes += 1 + raise OSError("access denied") + return procs.resume(pid) + + with gpucap.DutyCap([1], 0.9, period=0.02, suspend=procs.suspend, resume=resume) as d: + _time.sleep(0.2) + assert procs.count[1] == 0 and d.summary()["left_paused"] == 0