diff --git a/scripts/m0/m0lib/gpucap.py b/scripts/m0/m0lib/gpucap.py index 4ef53a3..0d8418a 100644 --- a/scripts/m0/m0lib/gpucap.py +++ b/scripts/m0/m0lib/gpucap.py @@ -150,8 +150,12 @@ def resume_pid(pid: int) -> bool: class DutyCap: - """Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs; every - paused process is resumed before the block ends, also on an error.""" + """Pause ``pids`` together for (1 - cap) of every ``period`` s while the block runs. A process + whose resume fails stays owed: it is not paused again on top (each suspend adds one to its + suspend count), its resume is retried every period and at the end, and one that still cannot + be resumed is reported as ``left_paused``.""" + + RESUME_TRIES_AT_END = 5 def __init__(self, pids: list[int], cap: float, period: float | None = None, suspend: Callable[[int], bool] | None = None, resume: Callable[[int], bool] | None = None): @@ -161,6 +165,7 @@ class DutyCap: self.paused_s = 0.0 self.cycles = 0 self.failed: set[int] = set() + self.owed: dict[int, int] = {} # pid -> suspends not yet undone self._stop = threading.Event() self._t: threading.Thread | None = None self._t0 = 0.0 @@ -173,17 +178,45 @@ class DutyCap: self._t.start() return self + @staticmethod + def _call(fn: Callable[[int], bool], pid: int) -> bool: + try: + return bool(fn(pid)) + except Exception: + return False + + def _resume(self, pid: int) -> None: + if self._call(self.resume, pid): + return + self.failed.add(pid) + self.owed[pid] = self.owed.get(pid, 0) + 1 + + def _settle_owed(self) -> None: + for pid, n in list(self.owed.items()): + while n and self._call(self.resume, pid): + n -= 1 + if n: + self.owed[pid] = n + else: + del self.owed[pid] + def _loop(self) -> None: while not self._stop.wait(self.period - self.pause): - held = [p for p in self.pids if self.suspend(p)] - self.failed.update(p for p in self.pids if p not in held) + self._settle_owed() + held = [] + for p in self.pids: + if p in self.owed: + continue # still paused from a resume that failed + if self._call(self.suspend, p): + held.append(p) + else: + self.failed.add(p) s0 = time.perf_counter() try: time.sleep(self.pause) finally: for p in held: - if not self.resume(p): - self.failed.add(p) + self._resume(p) self.paused_s += time.perf_counter() - s0 self.cycles += 1 @@ -191,6 +224,12 @@ class DutyCap: self._stop.set() if self._t: self._t.join(timeout=10) + for _ in range(self.RESUME_TRIES_AT_END): + if not self.owed: + break + self._settle_owed() + if self.owed: + time.sleep(0.05) self._t1 = time.perf_counter() def summary(self) -> dict: @@ -198,4 +237,4 @@ class DutyCap: return {"cap": self.cap, "period_s": self.period, "pause_s": round(self.pause, 4), "processes": len(self.pids), "cycles": self.cycles, "paused_share": round(self.paused_s / span, 3) if span > 0 else None, - "failed_processes": len(self.failed)} + "failed_processes": len(self.failed), "left_paused": len(self.owed)} diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 6a5e4d5..e8d87fb 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -514,9 +514,13 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels row["gpu_util"] = util.summary(cap) if cap: row["gpu_cap"] = duty.summary() - if row["gpu_cap"]["failed_processes"]: + if row["gpu_cap"]["left_paused"]: r.error(f"level {lv['key']} gpu cap", - f"{row['gpu_cap']['failed_processes']} process(es) could not be paused") + f"{row['gpu_cap']['left_paused']} process(es) are still paused") + elif row["gpu_cap"]["failed_processes"]: + r.error(f"level {lv['key']} gpu cap", + f"{row['gpu_cap']['failed_processes']} process(es) could not be " + "paused or resumed at once") for err in row["errors"]: r.error(f"level {lv['key']}", err) except Exception as e: diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index d11dd4f..9be61b9 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -1151,3 +1151,58 @@ def test_cap_without_the_engine_process_is_an_error_not_a_quiet_uncapped_row(tmp r, _, paused = _tune_with_cap(tmp_path, monkeypatch, fake_server, []) assert r["status"] == "partial" and not paused assert any("no engine process" in e["error"] for e in r["errors"]) + + +class _CountingProcs: + """NtSuspendProcess/NtResumeProcess on fake processes: each suspend adds one to the process's + suspend count and each successful resume takes one off; ``fail`` says when a resume fails.""" + + def __init__(self, fail): + self.count: dict[int, int] = {} + self.fail = fail + self.resumes = 0 + + def suspend(self, pid): + self.count[pid] = self.count.get(pid, 0) + 1 + return True + + def resume(self, pid): + self.resumes += 1 + if self.fail(pid, self.resumes): + return False + self.count[pid] -= 1 + return True + + +def test_a_resume_that_fails_once_is_retried_and_nothing_is_left_paused(): + import time as _time + procs = _CountingProcs(fail=lambda pid, n: pid == 1 and n == 1) + with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d: + _time.sleep(0.2) + s = d.summary() + assert all(v == 0 for v in procs.count.values()) + assert s["failed_processes"] == 1 and s["left_paused"] == 0 + + +def test_a_process_that_cannot_be_resumed_is_reported_as_left_paused(): + import time as _time + procs = _CountingProcs(fail=lambda pid, n: pid == 1) + with gpucap.DutyCap([1, 2], 0.9, period=0.02, suspend=procs.suspend, resume=procs.resume) as d: + _time.sleep(0.2) + assert procs.count[1] == 1 and procs.count[2] == 0 # paused once, never stacked + assert d.summary()["left_paused"] == 1 + + +def test_a_suspend_or_resume_that_raises_does_not_end_the_cap_with_processes_paused(): + import time as _time + procs = _CountingProcs(fail=lambda pid, n: False) + + def resume(pid): + if procs.resumes == 0: + procs.resumes += 1 + raise OSError("access denied") + return procs.resume(pid) + + with gpucap.DutyCap([1], 0.9, period=0.02, suspend=procs.suspend, resume=resume) as d: + _time.sleep(0.2) + assert procs.count[1] == 0 and d.summary()["left_paused"] == 0