fix(m0): record an STT load without timings as a gap, not a pass

A worker that did not stop in time returned its earlier "loaded" note, was
killed, and the placement still counted as ok with no transcription times.
finish() now reports a timeout, and a worker that timed out, died, never
started or measured no ten-speaker or single-clip timing is an error, so
the stage is partial and the next run retries it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-10 19:16:14 +09:00
parent ca5bb520b9
commit 290a4c9dcf
2 changed files with 50 additions and 4 deletions

View File

@@ -295,7 +295,8 @@ class SttLoad:
try:
self.proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
pass
# Whatever the file holds is the "loaded" note, not timings; close() kills the worker.
return {"status": "timeout", "errors": [f"stt worker did not stop within {timeout:.0f} s"]}
return self._read()
def close(self) -> None:
@@ -306,6 +307,20 @@ class SttLoad:
self._logf.close()
def stt_gaps(busy: dict) -> list[str]:
"""Why an STT load next to the LLM did not measure what it should: its errors, a worker that
did not finish (timed out, died, never started), or no ten-speaker or single-clip timings."""
gaps = list(busy.get("errors") or [])
if busy.get("status") != "done":
if not gaps:
gaps.append(f"stt worker ended as {busy.get('status') or 'nothing written'}")
return gaps
for key in ("batch10_s", "single_s"):
if not (busy.get(key) or {}).get("n"):
gaps.append(f"stt worker measured no {key}")
return gaps
def port_in_use(port: int) -> bool:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(2)
@@ -430,8 +445,9 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels
r.error(f"level {lv['key']}", e)
if stt:
busy = stt.finish()
row["stt"].update({k: busy.get(k) for k in ("batch10_s", "single_s", "errors", "seconds")})
for err in busy.get("errors") or []:
row["stt"].update({k: busy.get(k) for k in ("status", "batch10_s", "single_s", "errors",
"seconds")})
for err in stt_gaps(busy):
r.error(f"level {lv['key']} stt", err)
else:
r.error(f"level {lv['key']}", "the placement did not start")

View File

@@ -661,7 +661,8 @@ STT_GPU_MIB = 2740 # M0 stt: the model 1300 MiB and 1440 MiB more for ten spea
class _FakeStt:
"""Stands in for the STT worker process: on cuda its model holds VRAM while it is loaded."""
def __init__(self, fail_device=None, broken_at_load=False):
def __init__(self, fail_device=None, broken_at_load=False, finish_result=None):
self.finish_result = finish_result
self.events: list[str] = []
self.devices: list[str] = []
self.loaded: str | None = None
@@ -690,6 +691,8 @@ class _FakeStt:
def finish(self, timeout=120):
fake.events.append("stt_finish")
if fake.finish_result is not None:
return fake.finish_result
errs = ["RuntimeError: CUDA out of memory"] if device == fake.fail_device else []
return {"status": "done", "batch10_s": {"n": 4, "median": 0.9}, "single_s": {"n": 4, "median": 0.2},
"errors": errs}
@@ -853,3 +856,30 @@ def test_stt_load_loop_does_not_run_when_stopped_before_go(tmp_path):
stop.write_text("")
res = stt.load_loop(lambda: 1 / 0, lambda: 1 / 0, tmp_path / "go", stop)
assert res["status"] == "not_started" and res["errors"] == []
@pytest.mark.parametrize("busy", [
{"status": "timeout", "errors": []}, # the worker did not stop in time
{"status": "loaded"}, # it died before writing its timings
{"status": "not_started", "batch10_s": {"n": 0}, "single_s": {"n": 0}, "errors": []},
{"status": "done", "batch10_s": {"n": 0}, "single_s": {"n": 3, "median": 0.2}, "errors": []},
])
def test_stt_load_without_timings_is_a_gap_not_a_pass(tmp_path, monkeypatch, fake_server, busy):
stt = _FakeStt(finish_result=busy)
r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together",
extra_opts=["stt_python=py"], stt=stt)
assert r["status"] == "partial"
assert sum(1 for e in r["errors"] if e["where"].endswith(" stt")) == len(llm.TOGETHER)
def test_stt_worker_that_ignores_stop_is_reported_as_timed_out_and_killed(tmp_path):
s = llm.SttLoad(sys.executable, tmp_path, "cpu", tmp_path / "stt.log")
s.cmd = [sys.executable, "-c",
"import json,sys,time; open(sys.argv[1],'w').write(json.dumps({'status':'loaded'})); time.sleep(60)",
str(s.out)]
assert s.start(timeout=30)["status"] == "loaded"
s.go()
res = s.finish(timeout=1)
assert res["status"] == "timeout" and res["errors"]
s.close()
assert s.proc.poll() is not None