diff --git a/scripts/m0/README.md b/scripts/m0/README.md index d65b7f3..b4f60d7 100644 --- a/scripts/m0/README.md +++ b/scripts/m0/README.md @@ -35,7 +35,8 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 | STT | faster-whisper large-v3-turbo의 전사 시간, 정확도, 말하는 중 부분 전사, 10명 동시 처리 | | LLM 빠름 / 권장 | Strata로 Qwen3.6-35B-A3B(약 23GB)와 Qwen3.8-Flash-Next Q2_0(약 66GB)을 받아서 첫 토큰 시간(맥락 길이별), 생성 속도, 도구 호출, 이미지 읽기(화질 단계별), 한국어 답변, 동시 요청, 게임 모드용 VRAM 비우기와 모델 다시 시작·전환 시간 | | PC 제어 도구 | agent-desktop으로 계산기 버튼을 눌러 12+3=15가 나오는지, 테스트용 브라우저 창(새 임시 프로필)의 입력칸에 글자를 넣고 다시 읽히는지, 디스코드·탐색기 화면을 읽는 시간, 계속 켜 두는 방식(MCP)과 매번 실행하는 방식의 명령 한 번 시간 | -| 게임 중 자동 조절 | 이미 받은 Qwen3.6-35B-A3B를 평소 배치와, 전문가 층 VRAM 몫을 6GB·4GB·2GB로 줄인 게임 중 배치(2GB는 CPU 작업자 절반으로도 한 번 더)로 차례로 다시 시작해서, 배치마다 전환 시간, 남는 VRAM, 출력 속도, 음성 첫 문장, 맥락 8천 토큰 첫 토큰 시간을 잽니다. 마지막에 평소 배치로 한 번 더 켜서 되돌리는 시간도 잽니다. 게임용 VRAM 8GB를 남기는 배치 중 가장 빠른 것을 결과에 적습니다 | +| 게임 중 자동 조절 | 이미 받은 Qwen3.6-35B-A3B를 평소 배치(동시 1개)와 게임 중 배치 3개(전문가 층 2GB 맥락 32K, 2GB 맥락 16K, 0GB 맥락 16K)로 차례로 다시 시작해서, 배치마다 전환 시간, 남는 VRAM, 출력 속도, 음성 첫 문장, 맥락 8천 토큰 첫 토큰 시간을 잽니다. 마지막에 평소 배치로 한 번 더 켜서 되돌리는 시간도 잽니다 | +| LLM과 음성 인식 함께 | 음성 인식(STT)을 먼저 켜 두고 LLM을 평소 배치(동시 1개, 2개)와 게임 중 배치(전문가 2GB·0GB, 맥락 16K)로 켭니다. LLM을 재는 동안 음성 인식은 10명 동시 전사를 계속 돌립니다. 게임 중 배치는 음성 인식을 GPU에 둔 경우와 CPU에 둔 경우를 둘 다 잽니다. 배치마다 남는 VRAM, LLM 속도, 10명 동시 전사 시간, 한 사람 전사 시간을 적습니다 | ## 개인정보 @@ -54,7 +55,7 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 .\run_m0.ps1 -Only stt,tts_melo -Redo # 특정 단계만 다시 ``` -단계 이름: `sysinfo`, `winchecks`, `vdisplay`, `tts_melo`, `tts_cosy2`, `tts_cosy3`, `stt`, `llm_fast`, `llm_recommended`, `tools_fast`, `tools_recommended`, `desktop`, `gamemode` +단계 이름: `sysinfo`, `winchecks`, `vdisplay`, `tts_melo`, `tts_cosy2`, `tts_cosy3`, `stt`, `llm_fast`, `llm_recommended`, `tools_fast`, `tools_recommended`, `desktop`, `gamemode`, `gamemode_stt`, `gamemode_live`(이 단계는 `-Only`로 고를 때만 돕니다) `tools_fast`, `tools_recommended`는 도구 호출만 다시 잽니다. 처음과 같은 방식(고치기 전)과, 자비스의 수정(지시문 규칙, 계획 단계, 건너뛴 도구 직접 실행)을 넣은 방식을 같은 모델로 비교합니다. 이미 받은 모델을 쓰므로 모델 하나에 몇 분이면 끝납니다. @@ -64,6 +65,8 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 `run_m0_desktop.bat`을 더블클릭하면 PC 제어 도구 단계(`desktop`)만 돕니다. agent-desktop(약 2MB)을 작업 폴더에 받아 쓰고, 1~2분이면 끝납니다. 계산기와 테스트용 브라우저 창이 잠깐 떴다가 닫히니 그동안 손대지 않으시면 됩니다. 입력칸이 값 넣기를 지원하지 않으면 키 입력으로 넣는데, 이때는 그 창이 잠깐 앞으로 나옵니다. -`run_m0_gamemode.bat`을 더블클릭하면 게임 중 자동 조절 단계(`gamemode`)만 돕니다. 같은 모델을 배치 7개(평소, 게임 중 후보 5개, 다시 평소)로 차례로 다시 켜서 잽니다. 게임 중 후보는 전문가 층 VRAM 몫(2GB, 1GB, 0GB)과 함께 대화 메모리 형식(q4_0), 맥락(16K, 8K), MTP 보조 층을 줄입니다. 모델은 이미 받아 둔 것을 쓰므로 새로 받지 않고, 10분 안팎 걸립니다. 그동안 게임이나 무거운 프로그램은 꺼 두시고 PC에 손대지 않으시면 됩니다. +`run_m0_gamemode.bat`을 더블클릭하면 게임 중 자동 조절 단계 두 개(`gamemode`, `gamemode_stt`)만 돕니다. 먼저 LLM만 배치 5개(평소, 게임 중 후보 3개, 다시 평소)로 재고, 이어서 음성 인식을 함께 켠 배치 6개를 잽니다. 모델은 이미 받아 둔 것을 쓰므로 새로 받지 않고(음성 인식 모델을 아직 안 받았으면 약 1.6GB를 받습니다), 20~30분 걸립니다. 그동안 게임이나 무거운 프로그램은 꺼 두시고 PC에 손대지 않으시면 됩니다. + +`run_m0_game_live.bat`을 더블클릭하면 실제 게임을 켠 상태로 잽니다(`gamemode_live`). 게임을 창 모드로 켜고 플레이 화면까지 들어간 뒤 Enter를 누르고 15초 안에 게임으로 돌아가면 됩니다. 게임 중 배치 4개(전문가 2GB·0GB, 음성 인식 GPU·CPU)를 차례로 재고 10~15분 걸리며, 끝나면 알림음이 울립니다. 게임 화면이나 게임 이름은 저장하지 않고 VRAM 사용량과 속도만 적습니다. 그래픽 메모리를 많이 쓰는 게임이면 배치가 켜지지 않을 수 있는데, 그것도 결과로 적습니다. 다 끝나고 필요 없으면 작업 폴더(`\JarvisM0`)를 통째로 지우면 됩니다. Python, 모델, Strata(엔진과 설정 포함)는 모두 그 안에 설치하도록 지정해 두었습니다. diff --git a/scripts/m0/m0.py b/scripts/m0/m0.py index 6c15d86..6d61e08 100644 --- a/scripts/m0/m0.py +++ b/scripts/m0/m0.py @@ -44,7 +44,17 @@ STAGES = [ ("desktop", "PC 제어 도구(agent-desktop) 확인", "", "desktop.py", [], 0), # Automatic tuning: the one model in each placement (normal, less VRAM for a game). ("gamemode", "게임 중 자동 조절: 모델 배치별 속도", "base", "llm.py", ["model=fast", "only=levels"], 0), + # The same placements with the STT loaded next to the LLM (on the GPU and on the CPU). + ("gamemode_stt", "게임 중 자동 조절: LLM과 음성 인식을 함께 켰을 때", "base", "llm.py", + ["model=fast", "only=together"], 0), + # With the user's game running. Needs someone at the PC, so only when asked for (--only). + ("gamemode_live", "게임을 켠 상태로 재기 (직접 게임을 켜 주셔야 합니다)", "base", "llm.py", + ["model=fast", "only=live"], 0), ] +# Stages left out of a full run: they need the user there after the unattended part. +ON_DEMAND = {"gamemode_live"} +# Stages that start the STT worker next to the LLM and need the stt venv's Python. +NEEDS_STT = {"gamemode_stt", "gamemode_live"} def free_gb(path: Path) -> float: @@ -61,6 +71,9 @@ def run_stage(key: str, venv: str, script: str, opts: list[str], work: Path, env if venv == "cosy": src = envs.ensure_cosy_source(work, log) opts = opts + [f"src={src}"] + if key in NEEDS_STT: + stt_py = envs.ensure_env("stt", work, work / "logs" / "env_stt.log") + opts = opts + [f"stt_python={stt_py}"] cmd = [py, str(KIT / "m0lib" / script), "--out", str(out), "--work", str(work)] for o in opts: cmd += ["--opt", o] @@ -128,7 +141,7 @@ def main() -> int: t0 = time.time() for key, title, venv, script, opts, need_gb in STAGES: - if (only and key not in only) or key in skip: + if (only and key not in only) or key in skip or (not only and key in ON_DEMAND): continue out = work / "results" / f"{key}.json" if out.exists() and not a.redo: diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 607977e..96099bf 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -7,6 +7,10 @@ Options (--opt key=value): only=tools only re-measure tool calling, before and after the Jarvis fix only=levels the fast model in each placement the app's automatic tuning switches between (normal, and less VRAM for a game) + only=together the same with the STT loaded next to it and transcribing ten + speakers at once, on the GPU and on the CPU (needs stt_python=) + only=live game placements and the STT while the user runs a real game + stt_python=PATH the Python of the stt venv (m0.py passes it) """ from __future__ import annotations @@ -22,8 +26,8 @@ import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, gpu_memory_mib, run_logged, say, # noqa: E402 - scrub, stage_args) +from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, ask, gpu_memory_mib, interactive, # noqa: E402 + read_json, run_logged, say, scrub, stage_args, wait_enter) from m0lib.llm_bench import (bench_single, quick_bench, test_decode, test_images, test_parallel, # noqa: E402 test_tools, test_vram_shrink, warmup) from m0lib.tool_fix import test_tools_fixed, tools_passed # noqa: E402 @@ -59,28 +63,40 @@ PORT = 8080 # routed experts; the rest stay in RAM) and a matching prompt chunk. A game # placement also reads pictures on the CPU, answers one request at a time and # leaves the game half the CPU (8 engine workers cost nothing on the 9950X3D). -# The first game PC run (2026-11-10) showed that the expert budget is not the -# floor: at 2 GB the model still held 10.8 GB and a 16 GB card kept only -# 4.3 GB free. The rest is 4.4 GiB of weights that never leave the card, the -# conversation memory (1.43 GiB per 32K in int8; q4_0 is half), the MTP -# drafter (about 1 GB) and buffers, so the smaller placements also shrink those. +# The game PC runs (2026-11-10) showed that the expert budget is not the floor: +# 4.4 GiB of weights never leave the card, the conversation memory scales with +# the context, and Strata 0.1.59's qwen36 engine takes no --kv and always loads +# the MTP block, so the smaller placements shrink the context instead. +# The normal placement answers one request at a time (faster on a 16 GB card); +# two only when a chat channel answers every message. # The last row starts the normal placement again: what returning after a game costs. GAME_RESERVE_MIB = 8000 _GAME = {"parallel": 1, "vision": "cpu"} _W8 = "pool-workers=8" +_EXP2 = ["expert-vram=2", "prefill=2048", _W8] +_EXP0 = ["expert-vram=0", "prefill=2048", _W8] LEVELS = [ - {"key": "normal", "parallel": 2, "vision": "gpu", "sets": []}, - {"key": "game_experts_2g", **_GAME, "sets": ["expert-vram=2", "prefill=2048", _W8]}, - {"key": "game_experts_1g_kv_q4_16k", **_GAME, "context": 16384, "kv": "q4_0", - "sets": ["expert-vram=1", "prefill=2048", _W8]}, - {"key": "game_experts_1g_kv_q4_16k_no_mtp", **_GAME, "context": 16384, "kv": "q4_0", "mtp": "off", - "sets": ["expert-vram=1", "prefill=2048", _W8]}, - {"key": "game_experts_0g_kv_q4_16k_no_mtp", **_GAME, "context": 16384, "kv": "q4_0", "mtp": "off", - "sets": ["expert-vram=0", "prefill=2048", _W8]}, - {"key": "game_experts_0g_kv_q4_8k_no_mtp", **_GAME, "context": 8192, "kv": "q4_0", "mtp": "off", - "sets": ["expert-vram=0", "prefill=2048", _W8]}, - {"key": "normal_again", "parallel": 2, "vision": "gpu", "sets": []}, + {"key": "normal", "parallel": 1, "vision": "gpu", "sets": []}, + {"key": "game_experts_2g", **_GAME, "sets": _EXP2}, + {"key": "game_experts_2g_16k", **_GAME, "context": 16384, "sets": _EXP2}, + {"key": "game_experts_0g_16k", **_GAME, "context": 16384, "sets": _EXP0}, + {"key": "normal_again", "parallel": 1, "vision": "gpu", "sets": []}, ] +# The same with the STT worker loaded first (as the app starts it) and transcribing +# ten speakers at once while the LLM is measured: the VRAM left is then the real +# game share. "stt" is where the STT runs; on the CPU it takes no VRAM but shares +# the cores with the LLM's CPU experts. +TOGETHER = [ + {"key": "normal_stt_gpu", "parallel": 1, "vision": "gpu", "sets": [], "stt": "cuda"}, + {"key": "normal_parallel2_stt_gpu", "parallel": 2, "vision": "gpu", "sets": [], "stt": "cuda"}, + {"key": "game_experts_2g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cuda"}, + {"key": "game_experts_2g_16k_stt_cpu", **_GAME, "context": 16384, "sets": _EXP2, "stt": "cpu"}, + {"key": "game_experts_0g_16k_stt_gpu", **_GAME, "context": 16384, "sets": _EXP0, "stt": "cuda"}, + {"key": "game_experts_0g_16k_stt_cpu", **_GAME, "context": 16384, "sets": _EXP0, "stt": "cpu"}, +] +# With a real game running: only game placements (the normal one would take the game's VRAM). +LIVE = [lv for lv in TOGETHER if lv["key"].startswith("game_")] +LEVEL_SETS = {"levels": LEVELS, "together": TOGETHER, "live": LIVE} INSTALL_URL = "https://github.com/Niko1221/Strata/releases/latest/download/install.ps1" @@ -236,6 +252,60 @@ class Strata: return scrub(p.stdout[-2000:]) +class SttLoad: + """The STT worker (stt.py mode=load in the stt venv) as a separate process next to the LLM.""" + + def __init__(self, python: str, work: Path, device: str, log: Path): + d = work / "cache" / f"stt_load_{device}" + shutil.rmtree(d, ignore_errors=True) + d.mkdir(parents=True) + self.out, self.go_file, self.stop_file = d / "result.json", d / "go", d / "stop" + self.log = log + self.cmd = [python, str(Path(__file__).resolve().parent / "stt.py"), "--out", str(self.out), + "--work", str(work), "--opt", "mode=load", "--opt", f"device={device}", + "--opt", f"go={self.go_file}", "--opt", f"stop={self.stop_file}"] + self.proc: subprocess.Popen | None = None + + def _read(self) -> dict: + try: + return read_json(self.out) + except (OSError, ValueError): + return {} + + def start(self, timeout: float = 900) -> dict: + """Load the model; returns the worker's result ("loaded", or "failed" with the reason).""" + self._logf = open(self.log, "a", encoding="utf-8") + self.proc = subprocess.Popen(self.cmd, stdout=self._logf, stderr=subprocess.STDOUT) + t0 = time.monotonic() + while time.monotonic() - t0 < timeout: + got = self._read() + if got.get("status") in ("loaded", "failed"): + return got + if self.proc.poll() is not None: + return got or {"status": "failed", "errors": [f"stt worker exited with {self.proc.returncode}"]} + time.sleep(0.5) + return {"status": "failed", "errors": ["stt model did not load in time"]} + + def go(self) -> None: + self.go_file.write_text("", encoding="utf-8") + + def finish(self, timeout: float = 300) -> dict: + """Stop the loop; the CPU can be in the middle of a long ten-speaker batch.""" + self.stop_file.write_text("", encoding="utf-8") + try: + self.proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass + return self._read() + + def close(self) -> None: + if self.proc and self.proc.poll() is None: + self.proc.kill() + self.proc.wait(timeout=30) + if getattr(self, "_logf", None): + self._logf.close() + + def port_in_use(port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.settimeout(2) @@ -313,8 +383,11 @@ def try_candidates(r: StageResult, which: str, default_summary: dict | None, mea return -def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels: list[dict] = LEVELS) -> None: - """Start the model in each placement and measure what the switch and the placement cost.""" +def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels: list[dict] = LEVELS, + stt_python: str = "", choose: bool = True) -> None: + """Start the model in each placement and measure what the switch and the placement cost. + A level with "stt" first loads the STT worker there and keeps it transcribing during the + measurement, so the VRAM left is what a game really gets.""" base = f"http://127.0.0.1:{PORT}/v1" rows: list[dict] = [] r["levels"] = rows @@ -327,26 +400,45 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels s.stop() idle = gpu_memory_mib() row["vram_idle_mib"] = idle[0] if idle else None + stt = None with GpuSampler(interval=1) as g: - row["launch"] = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"], - extra_sets=lv["sets"], kv=lv.get("kv"), mtp=lv.get("mtp"), timeout=3600) - ready = row["launch"].get("ready") - row["switch_s"] = round(time.perf_counter() - t0, 1) if ready else None - row["engine"] = s.engine_lines() - if ready: - try: - c = ChatClient(base) - if "server" not in r.data: - r["server"] = {"model": c.model, "info": server_info(base)} - c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time - row.update(quick_bench(c)) - for err in row["errors"]: - r.error(f"level {lv['key']}", err) - except Exception as e: - r.error(f"level {lv['key']}", e) - else: - r.error(f"level {lv['key']}", "the placement did not start") - now = gpu_memory_mib() if ready else None # the sampler can miss a short run + try: + if lv.get("stt"): + stt = SttLoad(stt_python, s.log.parent.parent, lv["stt"], s.log.with_name("stt_load.log")) + row["stt"] = {"device": lv["stt"], "load": stt.start()} + if row["stt"]["load"].get("status") != "loaded": + r.error(f"level {lv['key']}", "; ".join(row["stt"]["load"].get("errors") or []) + or "the STT did not load") + continue + row["launch"] = s.launch(preset, row["context"], parallel=lv["parallel"], vision=lv["vision"], + extra_sets=lv["sets"], kv=lv.get("kv"), mtp=lv.get("mtp"), timeout=3600) + ready = row["launch"].get("ready") + row["switch_s"] = round(time.perf_counter() - t0, 1) if ready else None + row["engine"] = s.engine_lines() + if ready: + try: + c = ChatClient(base) + if "server" not in r.data: + r["server"] = {"model": c.model, "info": server_info(base)} + c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time + if stt: + stt.go() + row.update(quick_bench(c)) + for err in row["errors"]: + r.error(f"level {lv['key']}", err) + except Exception as e: + r.error(f"level {lv['key']}", e) + if stt: + busy = stt.finish() + row["stt"].update({k: busy.get(k) for k in ("batch10_s", "single_s", "errors", "seconds")}) + for err in busy.get("errors") or []: + r.error(f"level {lv['key']} stt", err) + else: + r.error(f"level {lv['key']}", "the placement did not start") + now = gpu_memory_mib() if ready else None # the sampler can miss a short run + finally: + if stt: + stt.close() peak = max(x for x in (g.peak, now[0] if now else None) if x is not None) \ if g.peak is not None or now else None if now: @@ -354,10 +446,12 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels if peak is not None and idle: row["vram_peak_mib"] = peak row["vram_free_mib"] = idle[1] - peak - say(f" 전환 {row['switch_s']}s, 출력 {(row.get('decode_tok_s') or {}).get('median')} tok/s, " + say(f" 전환 {row.get('switch_s')}s, 출력 {(row.get('decode_tok_s') or {}).get('median')} tok/s, " f"남는 VRAM {row.get('vram_free_mib')} MiB") r.save() s.stop() + if not choose: + return r["game_choice"] = pick_game_level(rows) measured = [x for x in rows if x["key"].startswith("game_") and x.get("vram_free_mib") is not None] if r["game_choice"] is None and measured: @@ -366,6 +460,19 @@ def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels f"{most['key']}: {most['vram_free_mib']} MiB") +def ask_for_a_game() -> bool: + """The live stage needs someone to start a game first. False when nobody is there or they say no.""" + if not interactive(): + return False + say(" 게임을 켠 상태로 LLM과 음성 인식을 잽니다. 배치 4개를 재고 10~15분 걸립니다.") + if ask(" 지금 게임을 켜고 재시겠습니까?") != "y": + return False + wait_enter(" 게임을 창 모드(테두리 없는 창이면 가장 좋음)로 켜고 플레이 화면까지 들어간 뒤 Enter를 누르고, " + "15초 안에 게임으로 돌아가 주세요. 끝나면 알림음이 울립니다:") + time.sleep(15) + return True + + def pick_game_level(rows: list[dict], reserve_mib: int = GAME_RESERVE_MIB) -> dict | None: """The game placement the app would pick on this PC: the fastest one that leaves the game its reserve of VRAM. None when no game placement does.""" @@ -385,7 +492,7 @@ def measure_external(r: StageResult, url: str, context: int, only: str = "") -> if only == "tools": tools_only(r, c) return - if only == "levels": + if only in LEVEL_SETS: r.note("placements need Strata; not measured against an external server") return with GpuSampler() as g: @@ -394,7 +501,8 @@ def measure_external(r: StageResult, url: str, context: int, only: str = "") -> r["gpu"] = g.summary() -def measure_strata(r: StageResult, work: Path, which: str, context: int, only: str = "") -> None: +def measure_strata(r: StageResult, work: Path, which: str, context: int, only: str = "", + stt_python: str = "") -> None: preset = PRESETS[which] r["preset"] = dict(preset, key=which, context=context) s = Strata(work, work / "logs" / f"strata_{which}.log") @@ -403,12 +511,27 @@ def measure_strata(r: StageResult, work: Path, which: str, context: int, only: s return r["plan"] = s.plan(preset, context) r.save() - if only == "levels": + if only in LEVEL_SETS: s.stop() if port_in_use(PORT): r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.") return - measure_levels(r, s, preset, context) + if only != "levels" and not stt_python: + r.error("stt", "STT 환경을 찾지 못했습니다") + return + if only == "live": + if not ask_for_a_game(): + r["skipped"] = "no game started" + return + r.note("measured while the user's game was running") + measure_levels(r, s, preset, context, LEVEL_SETS[only], stt_python=stt_python, choose=only != "live") + if only == "live": + say("\a 게임 중 측정이 끝났습니다. 게임을 꺼도 됩니다.") + try: + import winsound + winsound.MessageBeep() + except Exception: + pass return # 1) Start (the first start downloads the model and engine). @@ -478,15 +601,19 @@ def main() -> int: which = opts.get("model", "fast") context = int(opts.get("context", "32768")) only = opts.get("only", "") - name = {"tools": f"tools_{which}", "levels": "gamemode"}.get(only, f"llm_{which}") + name = {"tools": f"tools_{which}", "levels": "gamemode", "together": "gamemode_stt", + "live": "gamemode_live"}.get(only, f"llm_{which}") r = StageResult(name, out) try: if opts.get("url"): measure_external(r, opts["url"], context, only) else: - measure_strata(r, work, which, context, only) + measure_strata(r, work, which, context, only, opts.get("stt_python", "")) except Exception as e: r.error("llm", e) + if r.get("skipped"): + r.save("skipped") + return 0 # No server reached means nothing was measured: "failed" so the next run retries it. if "server" not in r.data: r.save("failed") diff --git a/scripts/m0/m0lib/stt.py b/scripts/m0/m0lib/stt.py index 129e4f2..6025cc2 100644 --- a/scripts/m0/m0lib/stt.py +++ b/scripts/m0/m0lib/stt.py @@ -2,6 +2,11 @@ Runs inside its own venv (faster-whisper, nvidia-cublas-cu12, nvidia-cudnn-cu12). Audio: the synthetic Korean samples in assets/ko_samples (MeloTTS voice). + +With --opt mode=load it is instead the STT load the gamemode stages put next to +the LLM: load the model on device=cuda|cpu, write {"status": "loaded"} to --out, +wait for the file go=PATH, then transcribe ten speakers at once (and one final +clip) on a loop until the file stop=PATH appears, and write the timings. """ from __future__ import annotations @@ -14,7 +19,8 @@ import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from m0lib.common import GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats # noqa: E402 +from m0lib.common import (GpuSampler, StageResult, cer, gpu_memory_mib, say, stage_args, stats, # noqa: E402 + write_json) SR = 16000 ASSETS = Path(__file__).resolve().parent.parent / "assets" / "ko_samples" @@ -71,8 +77,60 @@ def batch_transcribe(model, audios: list, beam: int = 1) -> tuple[list[str], flo return texts, time.perf_counter() - t0 +def load_loop(batch, single, go: Path, stop: Path, wait_s: float = 3600, max_s: float = 600) -> dict: + """Wait for ``go``, then run ``batch()`` (ten speakers) and ``single()`` (one final clip) + in turn until ``stop`` appears. An error (out of memory) ends the loop and is kept.""" + t0 = time.monotonic() + while not go.exists(): + if stop.exists() or time.monotonic() - t0 > wait_s: + return {"status": "not_started", "batch10_s": stats([]), "single_s": stats([]), "errors": []} + time.sleep(0.2) + b, one, errors = [], [], [] + t1 = time.monotonic() + while not stop.exists() and time.monotonic() - t1 < max_s: + try: + for fn, into in ((batch, b), (single, one)): + s0 = time.perf_counter() + fn() + into.append(time.perf_counter() - s0) + except Exception as e: + errors.append(f"{type(e).__name__}: {str(e)[:300]}") + break + return {"status": "done", "batch10_s": stats(b), "single_s": stats(one), "errors": errors, + "seconds": round(time.monotonic() - t1, 1)} + + +def load_worker(out: Path, work: Path, opts: dict) -> int: + device = opts.get("device", "cuda") + ctype = opts.get("compute", "int8_float16" if device == "cuda" else "int8") + threads = int(opts.get("threads", "8")) + res: dict = {"device": device, "compute_type": ctype, "cpu_threads": threads if device == "cpu" else None} + try: + if device == "cuda": + add_cuda_dll_dirs() + from faster_whisper import WhisperModel + refs = json.loads((ASSETS / "refs.json").read_text(encoding="utf-8"))["samples"] + clips = [read_wav(ASSETS / refs[i % len(refs)]["file"]) for i in range(10)] + t0 = time.perf_counter() + model = WhisperModel(opts.get("model", "large-v3-turbo"), device=device, compute_type=ctype, + cpu_threads=threads if device == "cpu" else 0, + download_root=str(work / "models" / "stt")) + transcribe(model, clips[0], 1) # warm-up + res["load_s"] = round(time.perf_counter() - t0, 2) + except Exception as e: + write_json(out, dict(res, status="failed", errors=[f"load: {type(e).__name__}: {str(e)[:300]}"])) + return 0 + write_json(out, dict(res, status="loaded")) + res.update(load_loop(lambda: batch_transcribe(model, clips), lambda: transcribe(model, clips[0], 1), + Path(opts["go"]), Path(opts["stop"]))) + write_json(out, res) + return 0 + + def main() -> int: out, work, opts = stage_args() + if opts.get("mode") == "load": + return load_worker(out, work, opts) r = StageResult("stt", out) r["cuda_dll_dirs"] = len(add_cuda_dll_dirs()) os.environ.setdefault("HF_HOME", str(work / "models" / "hf")) diff --git a/scripts/m0/run_m0_game_live.bat b/scripts/m0/run_m0_game_live.bat new file mode 100644 index 0000000..aba1dd9 --- /dev/null +++ b/scripts/m0/run_m0_game_live.bat @@ -0,0 +1,5 @@ +@echo off +chcp 65001 >nul +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0run_m0.ps1" -Only gamemode_live -Redo %* +echo. +pause diff --git a/scripts/m0/run_m0_gamemode.bat b/scripts/m0/run_m0_gamemode.bat index 185b71f..6b08407 100644 --- a/scripts/m0/run_m0_gamemode.bat +++ b/scripts/m0/run_m0_gamemode.bat @@ -1,5 +1,5 @@ @echo off chcp 65001 >nul -powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0run_m0.ps1" -Only gamemode -Redo %* +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0run_m0.ps1" -Only gamemode,gamemode_stt -Redo %* echo. pause diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index d9e4fb7..5589069 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -454,19 +454,31 @@ def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path): # --- gamemode stage: the one model in each placement the app switches to ---- -def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True): +def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True, only="levels", + extra_opts=(), stt=None): """Run the stage with Strata replaced; ``gpu(running)`` is nvidia-smi's (used, total), where - ``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None.""" + ``running`` is the started placement ({"sets", "context", "kv", "mtp"}) or None. ``stt`` is a + _FakeStt whose loaded model adds to the used VRAM.""" port = int(fake_server.rsplit(":", 1)[1].split("/")[0]) monkeypatch.setattr(llm, "PORT", port) started = [] running = {"now": None} + if stt is not None: + stt.running = running + monkeypatch.setattr(llm, "SttLoad", stt.factory) + base_gpu = gpu + + def gpu(now): + used, total = base_gpu(now) + return used + stt.used_mib(), total def launch(self, preset, context, parallel, vision="on", extra_sets=None, restart=False, timeout=0, kv=None, mtp=None): spec = {"parallel": parallel, "vision": vision, "sets": list(extra_sets or []), "context": context, "kv": kv, "mtp": mtp} started.append(spec) + if stt is not None: + stt.events.append("llm_start") running["now"] = spec if ready(extra_sets or []) else None return {"ready": running["now"] is not None} @@ -482,8 +494,10 @@ def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["now"])) out = tmp_path / "results" / "gamemode.json" out.parent.mkdir(parents=True) - monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), - "--opt", "model=fast", "--opt", "only=levels"]) + opts = ["--opt", "model=fast", "--opt", f"only={only}"] + for o in extra_opts: + opts += ["--opt", o] + monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), *opts]) llm.main() return json.loads(out.read_text(encoding="utf-8")), started, running @@ -504,13 +518,18 @@ def _vram(running): return (2000 if running is None else 2000 + _model_mib(running)), 16303 +def _vram_20g(running): + """A 20 GB card, where the smaller game placements do leave the reserve.""" + return _vram(running)[0], 20480 + + def _vram_big_floor(running): """A card where even the smallest placement leaves the game less than its reserve.""" return (2000 if running is None else 2000 + _model_mib(running) + 3000), 16303 def test_gamemode_measures_every_placement_and_returns_to_normal(tmp_path, monkeypatch, fake_server): - r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram) + r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram_20g) assert r["stage"] == "gamemode" and r["status"] == "ok" assert [x["sets"] for x in started] == [lv["sets"] for lv in llm.LEVELS] # Each placement starts with its own context, conversation memory format and drafter. @@ -571,9 +590,9 @@ def test_placement_memory_format_and_drafter_reach_the_launch_command(tmp_path, def test_a_placement_that_does_not_start_is_recorded_and_the_rest_still_run(tmp_path, monkeypatch, fake_server): r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, - ready=lambda sets: "expert-vram=1" not in sets) + ready=lambda sets: "expert-vram=0" not in sets) assert len(started) == len(llm.LEVELS) - failed = [lv["key"] for lv in llm.LEVELS if "expert-vram=1" in lv["sets"]] + failed = [lv["key"] for lv in llm.LEVELS if "expert-vram=0" in lv["sets"]] row = next(x for x in r["levels"] if x["key"] == failed[0]) assert row["switch_s"] is None and "decode_tok_s" not in row # A placement that never started is a gap in the measurement, not a pass. @@ -619,3 +638,218 @@ def test_a_measurement_error_inside_a_placement_marks_the_stage_partial(tmp_path r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram) assert r["status"] == "partial" assert any(e["error"] == "decode: TimeoutError" for e in r["errors"]) + + +def test_gamemode_measures_the_2g_expert_placement_with_a_16k_context(tmp_path, monkeypatch, fake_server): + # The one placement between "2 GB, 32K" and "0 GB, 16K" that was never measured. + _, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram) + assert any("expert-vram=2" in x["sets"] and x["context"] == 16384 for x in started) + + +def test_normal_placement_answers_one_request_at_a_time(tmp_path, monkeypatch, fake_server): + # One request at a time is the default; two only when a chat channel answers every message. + _, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram) + assert started[0]["parallel"] == 1 and started[-1]["parallel"] == 1 + + +# --- gamemode_stt / gamemode_live: the LLM next to the STT that runs with it -- + + +STT_GPU_MIB = 2740 # M0 stt: the model 1300 MiB and 1440 MiB more for ten speakers at once + + +class _FakeStt: + """Stands in for the STT worker process: on cuda its model holds VRAM while it is loaded.""" + + def __init__(self, fail_device=None, broken_at_load=False): + self.events: list[str] = [] + self.devices: list[str] = [] + self.loaded: str | None = None + self.running: dict = {} + self.fail_device = fail_device + self.broken_at_load = broken_at_load + + def used_mib(self) -> int: + return STT_GPU_MIB if self.loaded == "cuda" else 0 + + def factory(self, python, work, device, log): + fake = self + fake.devices.append(device) + + class _One: + def start(self, timeout=900): + if fake.broken_at_load: + return {"status": "failed", "errors": ["load: RuntimeError: CUDA failed"]} + fake.loaded = device + fake.events.append("stt_loaded") + return {"status": "loaded", "load_s": 3.0} + + def go(self): + assert fake.running.get("now") is not None, "STT load must overlap a running LLM" + fake.events.append("stt_go") + + def finish(self, timeout=120): + fake.events.append("stt_finish") + errs = ["RuntimeError: CUDA out of memory"] if device == fake.fail_device else [] + return {"status": "done", "batch10_s": {"n": 4, "median": 0.9}, "single_s": {"n": 4, "median": 0.2}, + "errors": errs} + + def close(self): + fake.loaded = None + + return _One() + + +def test_together_stage_measures_the_llm_with_the_stt_loaded_and_busy(tmp_path, monkeypatch, fake_server): + stt = _FakeStt() + r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together", + extra_opts=["stt_python=py"], stt=stt) + assert r["stage"] == "gamemode_stt" and r["status"] == "ok" + assert len(started) == len(llm.TOGETHER) + # The STT model is in VRAM before the LLM picks its placement, and its ten-speaker loop runs + # while the LLM is measured (asserted by _FakeStt.go). + for i in range(len(llm.TOGETHER)): + assert stt.events[i * 4:(i + 1) * 4] == ["stt_loaded", "llm_start", "stt_go", "stt_finish"] + rows = {x["key"]: x for x in r["levels"]} + for lv in llm.TOGETHER: + row = rows[lv["key"]] + assert row["stt"]["device"] == lv["stt"] + assert row["stt"]["batch10_s"]["n"] > 0 and row["decode_tok_s"]["n"] > 0 + # What is left for the game already has the STT taken out when it runs on the GPU. + _, total = _vram(None) + assert row["vram_free_mib"] == total - row["vram_peak_mib"] + # Both places for the STT are measured in the same game placement. + game = [lv for lv in llm.TOGETHER if lv["key"].startswith("game_")] + assert {lv["stt"] for lv in game} == {"cuda", "cpu"} + pairs = [(rows[a["key"]], rows[b["key"]]) for a in game for b in game + if a["stt"] == "cuda" and b["stt"] == "cpu" and (a["sets"], a.get("context")) == (b["sets"], b.get("context"))] + assert pairs + for on_gpu, on_cpu in pairs: + assert on_cpu["vram_free_mib"] - on_gpu["vram_free_mib"] == STT_GPU_MIB + # A chat channel that answers every message runs two requests at once: measured with the STT too. + assert any(lv["parallel"] == 2 for lv in llm.TOGETHER if lv["key"].startswith("normal")) + assert stt.loaded is None and running["now"] is None # nothing is left running + + +def test_stt_error_next_to_the_llm_marks_the_together_stage_partial(tmp_path, monkeypatch, fake_server): + stt = _FakeStt(fail_device="cuda") + r, _, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together", + extra_opts=["stt_python=py"], stt=stt) + assert r["status"] == "partial" + assert any("out of memory" in e["error"] for e in r["errors"]) + + +def test_stt_that_does_not_load_skips_that_placement_and_the_rest_still_run(tmp_path, monkeypatch, fake_server): + stt = _FakeStt(broken_at_load=True) + r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together", + extra_opts=["stt_python=py"], stt=stt) + assert started == [] # an LLM measured without its STT would be the wrong number + assert len(stt.devices) == len(llm.TOGETHER) + assert r["status"] == "failed" + + +def test_together_stage_without_the_stt_environment_measures_nothing(tmp_path, monkeypatch, fake_server): + r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="together") + assert started == [] and r["status"] == "failed" + + +def test_live_stage_is_skipped_when_nobody_is_there_to_start_a_game(tmp_path, monkeypatch, fake_server): + monkeypatch.setattr(llm, "interactive", lambda: False) + stt = _FakeStt() + r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live", + extra_opts=["stt_python=py"], stt=stt) + assert r["status"] == "skipped" and started == [] + + +def test_live_stage_measures_game_placements_while_the_game_runs(tmp_path, monkeypatch, fake_server): + monkeypatch.setattr(llm, "interactive", lambda: True) + monkeypatch.setattr(llm, "ask", lambda *a, **k: "y") + monkeypatch.setattr(llm, "wait_enter", lambda *a, **k: "") + monkeypatch.setattr(llm.time, "sleep", lambda s: None) + stt = _FakeStt() + r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, only="live", + extra_opts=["stt_python=py"], stt=stt) + assert r["stage"] == "gamemode_live" and r["status"] == "ok" + # Only game placements: starting the normal one would take the game's VRAM. + assert len(started) == len(llm.LIVE) and all(lv["key"].startswith("game_") for lv in llm.LIVE) + assert {lv["stt"] for lv in llm.LIVE} == {"cuda", "cpu"} + assert all(x["vram_idle_mib"] is not None for x in r["levels"]) # Windows and the game + assert running["now"] is None and stt.loaded is None + + +def _fake_kit_run(tmp_path, monkeypatch, *args): + import m0 + ran = [] + monkeypatch.setattr(m0, "run_stage", lambda key, *a, **k: ran.append(key)) + monkeypatch.setattr(m0, "pack", lambda work: tmp_path / "x.zip") + monkeypatch.setattr(m0.subprocess, "run", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", ["m0.py", "--work", str(tmp_path / "w"), *args]) + m0.main() + return ran + + +def test_full_kit_run_leaves_out_the_stage_that_needs_a_running_game(tmp_path, monkeypatch): + ran = _fake_kit_run(tmp_path, monkeypatch) + assert "gamemode_stt" in ran and "gamemode_live" not in ran + assert _fake_kit_run(tmp_path, monkeypatch, "--only", "gamemode_live", "--redo") == ["gamemode_live"] + + +def test_stages_that_run_the_stt_next_to_the_llm_get_the_stt_python(tmp_path, monkeypatch): + import m0 + cmds = [] + monkeypatch.setattr(m0.envs, "ensure_env", lambda name, work, log: Path(f"/venvs/{name}/python")) + monkeypatch.setattr(m0.subprocess, "run", lambda cmd, **k: cmds.append(cmd) or type("P", (), {"returncode": 0})) + for key in ("gamemode_stt", "gamemode"): + m0.run_stage(key, "base", "llm.py", ["model=fast"], tmp_path, {}) + together, alone = cmds + assert f"stt_python={Path('/venvs/stt/python')}" in together + assert not any(str(x).startswith("stt_python=") for x in alone) + + +def test_one_click_runners_cover_the_new_gamemode_stages(): + import m0 + bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8") + assert "-Only gamemode,gamemode_stt" in bat + live = (Path(m0.KIT) / "run_m0_game_live.bat").read_text(encoding="utf-8") + assert "-Only gamemode_live" in live and "-Redo" in live + + +def test_stt_load_loop_waits_for_go_and_runs_until_stop(tmp_path): + from m0lib import stt + go, stop = tmp_path / "go", tmp_path / "stop" + calls = [] + + def batch(): + calls.append("b") + if not go.exists(): + raise AssertionError("ran before go") + + def single(): + calls.append("s") + if calls.count("s") == 3: + stop.write_text("") + + go.write_text("") + res = stt.load_loop(batch, single, go, stop) + assert res["status"] == "done" and res["batch10_s"]["n"] == 3 and res["single_s"]["n"] == 3 + assert calls == ["b", "s"] * 3 + + +def test_stt_load_loop_keeps_an_out_of_memory_and_stops(tmp_path): + from m0lib import stt + go, stop = tmp_path / "go", tmp_path / "stop" + go.write_text("") + + def batch(): + raise RuntimeError("CUDA failed with error out of memory") + + res = stt.load_loop(batch, lambda: None, go, stop) + assert res["batch10_s"]["n"] == 0 and "out of memory" in res["errors"][0] + + +def test_stt_load_loop_does_not_run_when_stopped_before_go(tmp_path): + from m0lib import stt + stop = tmp_path / "stop" + stop.write_text("") + res = stt.load_loop(lambda: 1 / 0, lambda: 1 / 0, tmp_path / "go", stop) + assert res["status"] == "not_started" and res["errors"] == []