From 5762a22a670957d0d16210cf49aaba068e97ed26 Mon Sep 17 00:00:00 2001 From: javis-bot Date: Mon, 9 Nov 2026 15:53:40 +0900 Subject: [PATCH] fix(m0): retry failed LLM starts and never measure a foreign server - A Strata install or start that never reached a server is saved as "failed" (not "partial"), so the next run retries the stage. - A launch that exits non-zero is not ready, even if something answers on the port afterwards. - Before the cold start, the stage stops if another program already listens on port 8080 and tells the user to close it. Co-Authored-By: Claude Opus 5.5 --- scripts/m0/README.md | 3 ++- scripts/m0/m0lib/llm.py | 21 ++++++++++++++++++-- tests/test_m0_kit.py | 43 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scripts/m0/README.md b/scripts/m0/README.md index ebc74ec..b774433 100644 --- a/scripts/m0/README.md +++ b/scripts/m0/README.md @@ -13,7 +13,8 @@ v2 기본값을 정하려고 게임PC에서 한 번 실행하는 측정 스크 4. "이제 자리를 비우셔도 됩니다"가 나오면 그 뒤는 자동입니다. 모델 다운로드 때문에 몇 시간 걸릴 수 있습니다. **측정이 끝날 때까지 게임과 무거운 프로그램은 꺼 두세요.** 5. 끝나면 `jarvis_m0` 폴더에 `M0_results_날짜_시각.zip`이 생기고 탐색기에서 그 파일이 선택됩니다. 이 파일을 보내 주세요. -중간에 꺼져도 `run_m0.bat`을 다시 실행하면 끝난 단계는 건너뛰고 이어서 합니다. +중간에 꺼져도 `run_m0.bat`을 다시 실행하면 끝난 단계는 건너뛰고 이어서 합니다. 실패한 단계는 다시 시도합니다. +LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰고 있으면 그 단계는 실패로 기록되니, 그 프로그램을 끄고 다시 실행해 주세요. ## 직접 답하는 부분 diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 0fb4713..53e8a88 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -11,6 +11,7 @@ from __future__ import annotations import json import os import re +import socket import subprocess import sys import time @@ -121,7 +122,9 @@ class Strata: rc = run_logged(self.cmd("launch", *args), self.log, timeout=timeout, env=self.env, echo=True, echo_filter=progress) t_cmd = time.perf_counter() - t0 - ready = wait_ready(f"http://127.0.0.1:{PORT}", timeout=1800 if rc == 0 else 30) + # Only a successful launch counts: after a failed one, whatever still + # answers on the port is not the model that was asked for. + ready = wait_ready(f"http://127.0.0.1:{PORT}", timeout=1800) if rc == 0 else None return {"args": scrub(" ".join(args)), "dropped_flags": dropped, "exit": rc, "command_seconds": round(t_cmd, 1), "ready": ready is not None, "ready_seconds": round(time.perf_counter() - t0, 1) if ready is not None else None} @@ -139,6 +142,12 @@ class Strata: return scrub(p.stdout[-2000:]) +def port_in_use(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(2) + return sock.connect_ex(("127.0.0.1", port)) == 0 + + def wait_ready(root: str, timeout: float) -> float | None: t0 = time.perf_counter() while time.perf_counter() - t0 < timeout: @@ -185,6 +194,10 @@ def measure_strata(r: StageResult, work: Path, which: str, context: int) -> None # 1) Start (the first start downloads the model and engine). say(f" {preset['label']} 시작 중. 처음에는 모델을 내려받으므로 오래 걸립니다.") s.stop() + if port_in_use(PORT): + # Another program owns the port; measuring it would report the wrong model. + r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.") + return with GpuSampler(interval=2) as g: r["launch_cold"] = s.launch(preset, context, parallel=1) r["launch_cold"].update(g.summary()) @@ -237,7 +250,11 @@ def main() -> int: measure_strata(r, work, which, context) except Exception as e: r.error("llm", e) - r.save("ok" if not r["errors"] else "partial") + # No server reached means nothing was measured: "failed" so the next run retries it. + if "server" not in r.data: + r.save("failed") + else: + r.save("ok" if not r["errors"] else "partial") return 0 diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index 0c18500..7ae05d8 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -16,7 +16,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0")) -from m0lib import common # noqa: E402 +from m0lib import common, llm # noqa: E402 from m0lib.llm_client import ChatClient # noqa: E402 pytestmark = pytest.mark.unit @@ -100,3 +100,44 @@ def test_chat_client_assembles_streamed_tool_call_and_timings(fake_server): assert a["usage"]["completion_tokens"] == 9 # The first sentence ends at "요. ", before the whole answer is done. assert a["first_tool"] <= a["ttft"] <= a["first_sentence"] <= a["total"] + + +# --- LLM stage: a failed start must not count as measured ------------------- + + +def _run_llm_stage(tmp_path, monkeypatch) -> dict: + out = tmp_path / "results" / "llm_fast.json" + out.parent.mkdir(parents=True) + monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), "--opt", "model=fast"]) + llm.main() + return json.loads(out.read_text(encoding="utf-8")) + + +def test_llm_stage_is_failed_when_strata_install_fails(tmp_path, monkeypatch): + # A failed stage is retried on the next run; "partial" would be skipped. + monkeypatch.setattr(llm.Strata, "install", lambda self: False) + assert _run_llm_stage(tmp_path, monkeypatch)["status"] == "failed" + + +def test_llm_stage_does_not_measure_another_server_on_the_port(tmp_path, monkeypatch, fake_server): + port = int(fake_server.rsplit(":", 1)[1].split("/")[0]) + monkeypatch.setattr(llm, "PORT", port) + monkeypatch.setattr(llm.Strata, "install", lambda self: True) + monkeypatch.setattr(llm.Strata, "stop", lambda self: None) + monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {}) + monkeypatch.setattr(llm.Strata, "status", lambda self: {}) + launched = [] + monkeypatch.setattr(llm.Strata, "launch", lambda self, *a, **k: launched.append(a) or {"ready": True}) + r = _run_llm_stage(tmp_path, monkeypatch) + assert r["status"] == "failed" + assert launched == [] + assert "server" not in r + + +def test_strata_launch_that_fails_is_not_ready_even_if_a_server_answers(tmp_path, monkeypatch, fake_server): + port = int(fake_server.rsplit(":", 1)[1].split("/")[0]) + monkeypatch.setattr(llm, "PORT", port) + monkeypatch.setattr(llm, "run_logged", lambda *a, **k: 1) + s = llm.Strata(tmp_path, tmp_path / "strata.log") + monkeypatch.setattr(s, "supported_flags", lambda: set()) + assert s.launch(llm.PRESETS["fast"], 32768, parallel=1)["ready"] is False