From 290a4c9dcf0e9e2682324e199d9bbc393b326862 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Tue, 10 Nov 2026 19:16:14 +0900 Subject: [PATCH] fix(m0): record an STT load without timings as a gap, not a pass A worker that did not stop in time returned its earlier "loaded" note, was killed, and the placement still counted as ok with no transcription times. finish() now reports a timeout, and a worker that timed out, died, never started or measured no ten-speaker or single-clip timing is an error, so the stage is partial and the next run retries it. Co-Authored-By: Claude Opus 5.5 --- scripts/m0/m0lib/llm.py | 22 +++++++++++++++++++--- tests/test_m0_kit.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 96099bf..07b0230 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -295,7 +295,8 @@ class SttLoad: try: self.proc.wait(timeout=timeout) except subprocess.TimeoutExpired: - pass + # Whatever the file holds is the "loaded" note, not timings; close() kills the worker. + return {"status": "timeout", "errors": [f"stt worker did not stop within {timeout:.0f} s"]} return self._read() def close(self) -> None: @@ -306,6 +307,20 @@ class SttLoad: self._logf.close() +def stt_gaps(busy: dict) -> list[str]: + """Why an STT load next to the LLM did not measure what it should: its errors, a worker that + did not finish (timed out, died, never started), or no ten-speaker or single-clip timings.""" + gaps = list(busy.get("errors") or []) + if busy.get("status") != "done": + if not gaps: + gaps.append(f"stt worker ended as {busy.get('status') or 'nothing written'}") + return gaps + for key in ("batch10_s", "single_s"): + if not (busy.get(key) or {}).get("n"): + gaps.append(f"stt worker measured no {key}") + return gaps + + def port_in_use(port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.settimeout(2) @@ -430,8 +445,9 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels r.error(f"level {lv['key']}", e) if stt: busy = stt.finish() - row["stt"].update({k: busy.get(k) for k in ("batch10_s", "single_s", "errors", "seconds")}) - for err in busy.get("errors") or []: + row["stt"].update({k: busy.get(k) for k in ("status", "batch10_s", "single_s", "errors", + "seconds")}) + for err in stt_gaps(busy): r.error(f"level {lv['key']} stt", err) else: r.error(f"level {lv['key']}", "the placement did not start") diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index 5589069..f845433 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -661,7 +661,8 @@ STT_GPU_MIB = 2740 # M0 stt: the model 1300 MiB and 1440 MiB more for ten spea class _FakeStt: """Stands in for the STT worker process: on cuda its model holds VRAM while it is loaded.""" - def __init__(self, fail_device=None, broken_at_load=False): + def __init__(self, fail_device=None, broken_at_load=False, finish_result=None): + self.finish_result = finish_result self.events: list[str] = [] self.devices: list[str] = [] self.loaded: str | None = None @@ -690,6 +691,8 @@ class _FakeStt: def finish(self, timeout=120): fake.events.append("stt_finish") + if fake.finish_result is not None: + return fake.finish_result errs = ["RuntimeError: CUDA out of memory"] if device == fake.fail_device else [] return {"status": "done", "batch10_s": {"n": 4, "median": 0.9}, "single_s": {"n": 4, "median": 0.2}, "errors": errs} @@ -853,3 +856,30 @@ def test_stt_load_loop_does_not_run_when_stopped_before_go(tmp_path): stop.write_text("") res = stt.load_loop(lambda: 1 / 0, lambda: 1 / 0, tmp_path / "go", stop) assert res["status"] == "not_started" and res["errors"] == [] + + +@pytest.mark.parametrize("busy", [ + {"status": "timeout", "errors": []}, # the worker did not stop in time + {"status": "loaded"}, # it died before writing its timings + {"status": "not_started", "batch10_s": {"n": 0}, "single_s": {"n": 0}, "errors": []}, + {"status": "done", "batch10_s": {"n": 0}, "single_s": {"n": 3, "median": 0.2}, "errors": []}, +]) +def test_stt_load_without_timings_is_a_gap_not_a_pass(tmp_path, monkeypatch, fake_server, busy): + stt = _FakeStt(finish_result=busy) + r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together", + extra_opts=["stt_python=py"], stt=stt) + assert r["status"] == "partial" + assert sum(1 for e in r["errors"] if e["where"].endswith(" stt")) == len(llm.TOGETHER) + + +def test_stt_worker_that_ignores_stop_is_reported_as_timed_out_and_killed(tmp_path): + s = llm.SttLoad(sys.executable, tmp_path, "cpu", tmp_path / "stt.log") + s.cmd = [sys.executable, "-c", + "import json,sys,time; open(sys.argv[1],'w').write(json.dumps({'status':'loaded'})); time.sleep(60)", + str(s.out)] + assert s.start(timeout=30)["status"] == "loaded" + s.go() + res = s.finish(timeout=1) + assert res["status"] == "timeout" and res["errors"] + s.close() + assert s.proc.poll() is not None