diff --git a/scripts/m0/README.md b/scripts/m0/README.md index e3def28..7e0c971 100644 --- a/scripts/m0/README.md +++ b/scripts/m0/README.md @@ -35,6 +35,7 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 | STT | faster-whisper large-v3-turbo의 전사 시간, 정확도, 말하는 중 부분 전사, 10명 동시 처리 | | LLM 빠름 / 권장 | Strata로 Qwen3.6-35B-A3B(약 23GB)와 Qwen3.8-Flash-Next Q2_0(약 66GB)을 받아서 첫 토큰 시간(맥락 길이별), 생성 속도, 도구 호출, 이미지 읽기(화질 단계별), 한국어 답변, 동시 요청, 게임 모드용 VRAM 비우기와 모델 다시 시작·전환 시간 | | PC 제어 도구 | agent-desktop으로 계산기 버튼을 눌러 12+3=15가 나오는지, 테스트용 브라우저 창(새 임시 프로필)의 입력칸에 글자를 넣고 다시 읽히는지, 디스코드·탐색기 화면을 읽는 시간, 계속 켜 두는 방식(MCP)과 매번 실행하는 방식의 명령 한 번 시간 | +| 게임 중 자동 조절 | 이미 받은 Qwen3.6-35B-A3B를 평소 배치와, 전문가 층 VRAM 몫을 6GB·4GB·2GB로 줄인 게임 중 배치(2GB는 CPU 작업자 절반으로도 한 번 더)로 차례로 다시 시작해서, 배치마다 전환 시간, 남는 VRAM, 출력 속도, 음성 첫 문장, 맥락 8천 토큰 첫 토큰 시간을 잽니다. 마지막에 평소 배치로 한 번 더 켜서 되돌리는 시간도 잽니다. 게임용 VRAM 8GB를 남기는 배치 중 가장 빠른 것을 결과에 적습니다 | ## 개인정보 @@ -53,7 +54,7 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 .\run_m0.ps1 -Only stt,tts_melo -Redo # 특정 단계만 다시 ``` -단계 이름: `sysinfo`, `winchecks`, `vdisplay`, `tts_melo`, `tts_cosy2`, `tts_cosy3`, `stt`, `llm_fast`, `llm_recommended`, `tools_fast`, `tools_recommended`, `desktop` +단계 이름: `sysinfo`, `winchecks`, `vdisplay`, `tts_melo`, `tts_cosy2`, `tts_cosy3`, `stt`, `llm_fast`, `llm_recommended`, `tools_fast`, `tools_recommended`, `desktop`, `gamemode` `tools_fast`, `tools_recommended`는 도구 호출만 다시 잽니다. 처음과 같은 방식(고치기 전)과, 자비스의 수정(지시문 규칙, 계획 단계, 건너뛴 도구 직접 실행)을 넣은 방식을 같은 모델로 비교합니다. 이미 받은 모델을 쓰므로 모델 하나에 몇 분이면 끝납니다. @@ -63,4 +64,6 @@ LLM 단계는 8080 포트를 씁니다. 다른 프로그램이 이 포트를 쓰 `run_m0_desktop.bat`을 더블클릭하면 PC 제어 도구 단계(`desktop`)만 돕니다. agent-desktop(약 2MB)을 작업 폴더에 받아 쓰고, 1~2분이면 끝납니다. 계산기와 테스트용 브라우저 창이 잠깐 떴다가 닫히니 그동안 손대지 않으시면 됩니다. 입력칸이 값 넣기를 지원하지 않으면 키 입력으로 넣는데, 이때는 그 창이 잠깐 앞으로 나옵니다. +`run_m0_gamemode.bat`을 더블클릭하면 PC 제어 도구 단계(`desktop`)를 다시 재고, 이어서 게임 중 자동 조절 단계(`gamemode`)를 돕니다. 모델은 이미 받아 둔 것을 쓰므로 새로 받지 않고, 10~15분 걸립니다. 그동안 게임이나 무거운 프로그램은 꺼 두시고 PC에 손대지 않으시면 됩니다. + 다 끝나고 필요 없으면 작업 폴더(`\JarvisM0`)를 통째로 지우면 됩니다. Python, 모델, Strata(엔진과 설정 포함)는 모두 그 안에 설치하도록 지정해 두었습니다. diff --git a/scripts/m0/m0.py b/scripts/m0/m0.py index e1424fc..6c15d86 100644 --- a/scripts/m0/m0.py +++ b/scripts/m0/m0.py @@ -42,6 +42,8 @@ STAGES = [ ("tools_recommended", "도구 호출 다시 재기: 권장 모델", "base", "llm.py", ["model=recommended", "only=tools"], 0), # M7 PC control: agent-desktop (UI Automation) on calculator, a test page, Discord. ("desktop", "PC 제어 도구(agent-desktop) 확인", "", "desktop.py", [], 0), + # Automatic tuning: the one model in each placement (normal, less VRAM for a game). + ("gamemode", "게임 중 자동 조절: 모델 배치별 속도", "base", "llm.py", ["model=fast", "only=levels"], 0), ] @@ -141,7 +143,7 @@ def main() -> int: "errors": [{"where": "disk", "error": f"free {free_gb(work):.0f} GB < {need_gb} GB"}]}) continue stage_opts = list(opts) - if key.startswith(("llm_", "tools_")) and a.llm_url: + if key.startswith(("llm_", "tools_", "gamemode")) and a.llm_url: stage_opts.append(f"url={a.llm_url}") try: run_stage(key, venv, script, stage_opts, work, env) diff --git a/scripts/m0/m0lib/desktop.py b/scripts/m0/m0lib/desktop.py index e87a270..30655d3 100644 --- a/scripts/m0/m0lib/desktop.py +++ b/scripts/m0/m0lib/desktop.py @@ -55,6 +55,9 @@ INPUT_NAME = "jarvis-m0-input" INPUT_TEXT = "자비스 입력 테스트 15" READ_APPS = ["Discord", "explorer"] +# close-app only knows window-owning processes; a UWP app such as the +# calculator is owned by ApplicationFrameHost, so it is closed by image name. +FORCE_CLOSE = ["taskkill", "/F", "/IM"] REPEAT = 5 @@ -201,7 +204,23 @@ def check_calculator(desk: Desk, wait_s: float = 15) -> dict[str, Any]: out["error"] = "display does not show the sum" return out finally: - out["close_error"] = code(desk("close-app", app)) + close_launched(desk, app, out) + + +def close_launched(desk: Desk, app: str, out: dict[str, Any]) -> None: + """Close the calculator this stage started (never one the user had open).""" + out["close_error"] = code(desk("close-app", app)) + if out["close_error"] is not None: + try: + subprocess.run([*FORCE_CLOSE, app], capture_output=True, timeout=30) + except (OSError, subprocess.SubprocessError): + pass + out["closed"] = _calculator_app(desk, 0) is None + + +def _image_stem(name: str) -> str: + n = name.lower() + return n[:-4] if n.endswith(".exe") else n def _page(page_dir: Path) -> Path: @@ -258,15 +277,18 @@ def check_text_input(desk: Desk, open_page: Callable[[Path, str], Any], page_dir def check_apps(desk: Desk) -> dict[str, Any]: env = desk("list-apps") - running = {str(a.get("name") or "").lower() for a in ((env.get("data") or {}).get("apps") or [])} \ - if env.get("ok") else set() + # Windows lists process image names ("Discord.exe"); snapshot wants that exact name. + listed = [str(a.get("name") or "") for a in ((env.get("data") or {}).get("apps") or [])] \ + if env.get("ok") else [] + by_stem = {_image_stem(n): n for n in listed} out: dict[str, Any] = {} for app in READ_APPS: - if app.lower() not in running: + name = by_stem.get(_image_stem(app)) + if name is None: out[app] = {"running": False} continue t0 = time.perf_counter() - snap = desk("snapshot", "--app", app, "-i", "--compact", "--timeout-ms", "8000") + snap = desk("snapshot", "--app", name, "-i", "--compact", "--timeout-ms", "8000") d = snap.get("data") or {} if snap.get("ok") else {} out[app] = {"running": True, "ok": bool(snap.get("ok")), "error": code(snap), "ref_count": d.get("ref_count"), "complete": d.get("complete"), diff --git a/scripts/m0/m0lib/llm.py b/scripts/m0/m0lib/llm.py index 9f0cc63..aa10bde 100644 --- a/scripts/m0/m0lib/llm.py +++ b/scripts/m0/m0lib/llm.py @@ -5,6 +5,8 @@ Options (--opt key=value): url=http://.../v1 measure an already running server instead (no Strata management) context=32768 context size to start Strata with only=tools only re-measure tool calling, before and after the Jarvis fix + only=levels the fast model in each placement the app's automatic + tuning switches between (normal, and less VRAM for a game) """ from __future__ import annotations @@ -20,9 +22,10 @@ import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from m0lib.common import IS_WINDOWS, GpuSampler, StageResult, run_logged, say, scrub, stage_args # noqa: E402 -from m0lib.llm_bench import (bench_single, test_decode, test_images, test_parallel, test_tools, # noqa: E402 - test_vram_shrink, warmup) +from m0lib.common import (IS_WINDOWS, GpuSampler, StageResult, gpu_memory_mib, run_logged, say, # noqa: E402 + scrub, stage_args) +from m0lib.llm_bench import (bench_single, quick_bench, test_decode, test_images, test_parallel, # noqa: E402 + test_tools, test_vram_shrink, warmup) from m0lib.tool_fix import test_tools_fixed, tools_passed # noqa: E402 from m0lib.llm_client import ChatClient, http_json # noqa: E402 @@ -49,6 +52,24 @@ CANDIDATES = { ], } PORT = 8080 + +# The placements the app's automatic tuning moves the one model between. +# Strata's qwen36 engine cannot give VRAM back while it runs (POST /v1/vram +# answers 400), so each placement is a restart with --expert-vram (GiB for the +# routed experts; the rest stay in RAM) and a matching prompt chunk. A game +# placement also reads pictures on the CPU and answers one request at a time. +# The last row starts the normal placement again: what returning after a game costs. +GAME_RESERVE_MIB = 8000 +_GAME = {"parallel": 1, "vision": "cpu"} +LEVELS = [ + {"key": "normal", "parallel": 2, "vision": "gpu", "sets": []}, + {"key": "game_experts_6g", **_GAME, "sets": ["expert-vram=6", "prefill=2048"]}, + {"key": "game_experts_4g", **_GAME, "sets": ["expert-vram=4", "prefill=2048"]}, + {"key": "game_experts_2g", **_GAME, "sets": ["expert-vram=2", "prefill=2048"]}, + # A game also wants CPU cores: the same placement with half the engine's CPU workers. + {"key": "game_experts_2g_8_workers", **_GAME, "sets": ["expert-vram=2", "prefill=2048", "pool-workers=8"]}, + {"key": "normal_again", "parallel": 2, "vision": "gpu", "sets": []}, +] INSTALL_URL = "https://github.com/Niko1221/Strata/releases/latest/download/install.ps1" @@ -166,6 +187,14 @@ class Strata: dest = self.log.with_name(f"{self.log.stem}_engine_{src.name.removesuffix('.engine.log')}.log") dest.write_text(scrub(text), encoding="utf-8") + def engine_lines(self, prefix: str = "strata-q36:", limit: int = 20) -> list[str]: + """The newest engine log's placement lines (how much went to VRAM and RAM), scrubbed.""" + found = sorted(self.home.rglob("*.engine.log"), key=lambda p: p.stat().st_mtime) + if not found: + return [] + text = found[-1].read_text(encoding="utf-8", errors="replace")[-400_000:] + return [scrub(x.strip())[:400] for x in text.splitlines() if x.lstrip().startswith(prefix)][:limit] + def stop(self) -> None: run_logged(self.cmd("launch", "--stop"), self.log, timeout=300, env=self.env) time.sleep(3) @@ -256,12 +285,71 @@ def try_candidates(r: StageResult, which: str, default_summary: dict | None, mea return +def measure_levels(r: StageResult, s: Strata, preset: dict, context: int, levels: list[dict] = LEVELS) -> None: + """Start the model in each placement and measure what the switch and the placement cost.""" + base = f"http://127.0.0.1:{PORT}/v1" + rows: list[dict] = [] + r["levels"] = rows + for lv in levels: + row: dict = {"key": lv["key"], "parallel": lv["parallel"], "vision": lv["vision"], "sets": lv["sets"]} + rows.append(row) + say(f" - 배치 {lv['key']}: 다시 시작해서 재는 중") + t0 = time.perf_counter() + s.stop() + idle = gpu_memory_mib() + row["vram_idle_mib"] = idle[0] if idle else None + with GpuSampler(interval=1) as g: + row["launch"] = s.launch(preset, context, parallel=lv["parallel"], vision=lv["vision"], + extra_sets=lv["sets"], timeout=3600) + ready = row["launch"].get("ready") + row["switch_s"] = round(time.perf_counter() - t0, 1) if ready else None + row["engine"] = s.engine_lines() + if ready: + try: + c = ChatClient(base) + if "server" not in r.data: + r["server"] = {"model": c.model, "info": server_info(base)} + c.chat([{"role": "user", "content": "안녕"}], max_tokens=8) # loading is not answer time + row.update(quick_bench(c)) + except Exception as e: + r.error(f"level {lv['key']}", e) + now = gpu_memory_mib() if ready else None # the sampler can miss a short run + peak = max(x for x in (g.peak, now[0] if now else None) if x is not None) \ + if g.peak is not None or now else None + if now: + row["vram_after_mib"] = now[0] # settled; the peak (during the start) is what a running game must survive + if peak is not None and idle: + row["vram_peak_mib"] = peak + row["vram_free_mib"] = idle[1] - peak + say(f" 전환 {row['switch_s']}s, 출력 {(row.get('decode_tok_s') or {}).get('median')} tok/s, " + f"남는 VRAM {row.get('vram_free_mib')} MiB") + r.save() + s.stop() + r["game_choice"] = pick_game_level(rows) + + +def pick_game_level(rows: list[dict], reserve_mib: int = GAME_RESERVE_MIB) -> dict | None: + """The game placement the app would pick on this PC: the fastest one that + leaves the game its reserve of VRAM. None when no game placement does.""" + fits = [x for x in rows if x["key"].startswith("game_") and (x.get("vram_free_mib") or 0) >= reserve_mib + and (x.get("decode_tok_s") or {}).get("median")] + if not fits: + return None + best = max(fits, key=lambda x: x["decode_tok_s"]["median"]) + return {"key": best["key"], "reserve_mib": reserve_mib, "vram_free_mib": best["vram_free_mib"], + "decode_tok_s": best["decode_tok_s"]["median"], "switch_s": best.get("switch_s"), + "voice_first_sentence": (best.get("voice_first_sentence") or {}).get("median")} + + def measure_external(r: StageResult, url: str, context: int, only: str = "") -> None: c = ChatClient(url) r["server"] = {"url": url, "model": c.model, "info": server_info(c.base)} if only == "tools": tools_only(r, c) return + if only == "levels": + r.note("placements need Strata; not measured against an external server") + return with GpuSampler() as g: bench_single(r, c, context) test_parallel(r, c) @@ -277,6 +365,13 @@ def measure_strata(r: StageResult, work: Path, which: str, context: int, only: s return r["plan"] = s.plan(preset, context) r.save() + if only == "levels": + s.stop() + if port_in_use(PORT): + r.error("port", f"다른 프로그램이 {PORT} 포트를 쓰고 있습니다. 그 프로그램을 끄고 다시 실행해 주세요.") + return + measure_levels(r, s, preset, context) + return # 1) Start (the first start downloads the model and engine). say(f" {preset['label']} 시작 중. 처음에는 모델을 내려받으므로 오래 걸립니다.") @@ -345,7 +440,8 @@ def main() -> int: which = opts.get("model", "fast") context = int(opts.get("context", "32768")) only = opts.get("only", "") - r = StageResult(f"{'tools' if only == 'tools' else 'llm'}_{which}", out) + name = {"tools": f"tools_{which}", "levels": "gamemode"}.get(only, f"llm_{which}") + r = StageResult(name, out) try: if opts.get("url"): measure_external(r, opts["url"], context, only) diff --git a/scripts/m0/m0lib/llm_bench.py b/scripts/m0/m0lib/llm_bench.py index c37ae86..9ea87ba 100644 --- a/scripts/m0/m0lib/llm_bench.py +++ b/scripts/m0/m0lib/llm_bench.py @@ -364,6 +364,44 @@ def test_vram_shrink(r: StageResult, c: ChatClient, reserve_mib: int = 8000) -> r["vram_shrink"] = out +def quick_bench(c: ChatClient, context_tokens: int = 8192) -> dict[str, Any]: + """The numbers a placement change moves, in about half a minute: output + speed, the first spoken sentence and the first token after a long context. + Fail-open: an error is kept by name and the next measurement runs.""" + out: dict[str, Any] = {"errors": []} + runs = [] + for i in range(3): + try: + a = c.chat([{"role": "user", "content": f"({i}) 한국의 사계절을 각각 세 문장씩 자세히 설명해 줘."}], + max_tokens=320, extra=NO_THINK) + runs.append(a) + except Exception as e: + out["errors"].append(f"decode: {type(e).__name__}") + out["decode_tok_s"] = stats(x.get("decode_tok_s") for x in runs) + voice = [] + for kind, prompt in KOREAN_PROMPTS: + if kind != "voice": + continue + try: + a = c.chat([{"role": "system", "content": VOICE_SYSTEM}, {"role": "user", "content": prompt}], + max_tokens=120, extra=NO_THINK) + voice.append(a.get("first_sentence")) + except Exception as e: + out["errors"].append(f"voice: {type(e).__name__}") + out["voice_first_sentence"] = stats(voice) + seed = random.randint(0, 10**9) + doc = f"[문서 {seed}]\n" + filler(int(context_tokens * 1.5), seed) + try: + a = c.chat([{"role": "system", "content": VOICE_SYSTEM}, + {"role": "user", "content": doc + "\n\n위 글의 주제를 한 문장으로 말해 줘."}], + max_tokens=48, extra=NO_THINK) + out["context"] = {"prompt_tokens": (a.get("usage") or {}).get("prompt_tokens"), "ttft": a.get("ttft"), + "first_sentence": a.get("first_sentence")} + except Exception as e: + out["errors"].append(f"context: {type(e).__name__}") + return out + + def warmup(r: StageResult, c: ChatClient) -> None: """One throwaway request so model loading is not counted as answer time.""" try: diff --git a/scripts/m0/run_m0_gamemode.bat b/scripts/m0/run_m0_gamemode.bat new file mode 100644 index 0000000..54534fe --- /dev/null +++ b/scripts/m0/run_m0_gamemode.bat @@ -0,0 +1,5 @@ +@echo off +chcp 65001 >nul +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0run_m0.ps1" -Only desktop,gamemode -Redo %* +echo. +pause diff --git a/tests/test_m0_desktop.py b/tests/test_m0_desktop.py index af23140..ccceb16 100644 --- a/tests/test_m0_desktop.py +++ b/tests/test_m0_desktop.py @@ -55,11 +55,11 @@ FAKE_CLI = textwrap.dedent(r''' return args[args.index(name) + 1] if name in args else None def windows(s): - w = [{"id": "w-1", "app_name": "WINWORD", "title": SECRET + ".docx - Word", "accessible": True}] + w = [{"id": "w-1", "app_name": "WINWORD.EXE", "title": SECRET + ".docx - Word", "accessible": True}] if s.get("calc"): - w.append({"id": "w-200", "app_name": "CalculatorApp", "title": "계산기", "accessible": True}) + w.append({"id": "w-200", "app_name": "CalculatorApp.exe", "title": "계산기", "accessible": True}) if s.get("browser"): - w.append({"id": "w-100", "app_name": "chrome", "title": s["browser"] + " - Google Chrome", + w.append({"id": "w-100", "app_name": "chrome.exe", "title": s["browser"] + " - Google Chrome", "accessible": True}) return w @@ -96,7 +96,11 @@ FAKE_CLI = textwrap.dedent(r''' return ok(cmd, {"platform": "windows", "permissions": {"accessibility": {"state": "granted"}}, "supported_surfaces": ["window", "menu"]}) if cmd == "list-apps": - return ok(cmd, {"apps": [{"name": "Discord"}, {"name": "explorer"}, {"name": "WINWORD"}]}) + # Windows lists the process image names of window owners. A UWP + # app's windows belong to ApplicationFrameHost, so the calculator + # is not listed under its own name. + return ok(cmd, {"apps": [{"name": "ApplicationFrameHost.exe"}, {"name": "Discord.exe"}, + {"name": "explorer.exe"}, {"name": "WINWORD.EXE"}]}) if cmd == "list-windows": if MODE == "cli_fail": fail(cmd, "PLATFORM_ERROR", SECRET) @@ -108,13 +112,15 @@ FAKE_CLI = textwrap.dedent(r''' return ok(cmd, {"app": "calc", "pid": 1}) fail(cmd, "APP_NOT_FOUND", "no such app") if cmd == "close-app": + if a[1].lower() == "calculatorapp.exe": + fail(cmd, "APP_NOT_FOUND", "no running app named " + a[1]) s["calc"] = False save(s) return ok(cmd, {"app": a[1]}) if cmd == "find": if opt(a, "--native-id"): nid = opt(a, "--native-id") - if opt(a, "--app") != "CalculatorApp" or not s.get("calc"): + if opt(a, "--app") != "CalculatorApp.exe" or not s.get("calc"): return ok(cmd, {"matches": [], "total_matches": 0}) name = ("표시는 %s입니다" % s["display"]) if nid == "CalculatorResults" else nid return ok(cmd, {"matches": [{"ref_id": "@s1:" + nid, "name": name, "role": "button"}]}) @@ -144,9 +150,16 @@ FAKE_CLI = textwrap.dedent(r''' return ok(cmd, {"action": "set-value"}) if cmd == "get": return ok(cmd, {"property": "value", "ref": a[1], "value": s.get("value", "")}) + if cmd == "force-close": # stands in for taskkill /IM + if a[-1] == "CalculatorApp.exe": + s["calc"] = False + save(s) + return if cmd == "snapshot": app = opt(a, "--app") - if app == "explorer": + if app not in ("Discord.exe", "explorer.exe"): + fail(cmd, "APP_NOT_FOUND", "no running app named " + str(app)) + if app == "explorer.exe": fail(cmd, "AMBIGUOUS_TARGET", "windows: " + SECRET + ".docx") return ok(cmd, {"app": app, "ref_count": 120, "complete": True, "tree": {"role": "window", "name": SECRET + " DM"}}) @@ -165,6 +178,8 @@ def fake(tmp_path, monkeypatch): monkeypatch.setenv("FAKE_AD_SECRET", SECRET) monkeypatch.delenv("FAKE_AD_MODE", raising=False) + monkeypatch.setattr(desktop, "FORCE_CLOSE", [sys.executable, str(script), "force-close"]) + class Fake: cmd = [sys.executable, str(script)] log = tmp_path / "desktop.log" @@ -279,3 +294,19 @@ def test_an_already_open_calculator_is_left_alone(fake, tmp_path): assert result["ok"] is False and result["skipped"] s = json.loads(state.read_text(encoding="utf-8")) assert s["calc"] is True and s["display"] == "777" # not pressed, not closed + + +def test_running_apps_are_read_by_the_process_name_windows_lists(fake): + result = desktop.check_apps(fake.desk()) + assert result["Discord"]["running"] is True and result["Discord"]["ok"] is True + assert result["Discord"]["ref_count"] == 120 + assert result["explorer"]["running"] is True + assert result["explorer"]["error"] == "AMBIGUOUS_TARGET" + + +def test_the_launched_calculator_is_closed_when_close_app_cannot_find_it(fake, tmp_path): + result = desktop.check_calculator(fake.desk()) + assert result["ok"] is True + assert result["close_error"] == "APP_NOT_FOUND" + assert result["closed"] is True + assert json.loads((tmp_path / "state.json").read_text(encoding="utf-8"))["calc"] is False diff --git a/tests/test_m0_kit.py b/tests/test_m0_kit.py index f0e30bf..1979b3f 100644 --- a/tests/test_m0_kit.py +++ b/tests/test_m0_kit.py @@ -449,3 +449,91 @@ def test_a_lying_model_fails_the_stage_and_the_next_model_is_measured(tmp_path): llm.try_candidates(r, "fast", summary, lambda p: tried.append(p["model"]) or _summary(24, 24), free_gb=lambda: 999) assert tried == [llm.CANDIDATES["fast"][0]["model"]] assert r["tools_verdict"]["chosen"] == llm.CANDIDATES["fast"][0]["label"] + + +# --- gamemode stage: the one model in each placement the app switches to ---- + + +def _run_levels_stage(tmp_path, monkeypatch, fake_server, gpu, ready=lambda sets: True): + """Run the stage with Strata replaced; ``gpu(launched_sets)`` is nvidia-smi's (used, total).""" + port = int(fake_server.rsplit(":", 1)[1].split("/")[0]) + monkeypatch.setattr(llm, "PORT", port) + started = [] + running = {"sets": None} + + def launch(self, preset, context, parallel, vision="on", extra_sets=None, restart=False, timeout=0): + started.append({"parallel": parallel, "vision": vision, "sets": list(extra_sets or [])}) + ok = ready(extra_sets or []) + running["sets"] = list(extra_sets or []) if ok else None + return {"ready": ok} + + def stop(self): + running["sets"] = None + + monkeypatch.setattr(llm.Strata, "install", lambda self: True) + monkeypatch.setattr(llm.Strata, "plan", lambda self, preset, context: {}) + monkeypatch.setattr(llm.Strata, "launch", launch) + monkeypatch.setattr(llm.Strata, "stop", stop) + monkeypatch.setattr(llm, "port_in_use", lambda p: running["sets"] is not None) + monkeypatch.setattr(llm, "gpu_memory_mib", lambda: gpu(running["sets"])) + monkeypatch.setattr(common, "gpu_memory_mib", lambda: gpu(running["sets"])) + out = tmp_path / "results" / "gamemode.json" + out.parent.mkdir(parents=True) + monkeypatch.setattr(sys, "argv", ["llm.py", "--out", str(out), "--work", str(tmp_path), + "--opt", "model=fast", "--opt", "only=levels"]) + llm.main() + return json.loads(out.read_text(encoding="utf-8")), started, running + + +def _vram(sets): + """16 GB card, 2 GB used by Windows; the model takes less with a smaller expert budget.""" + if sets is None: + return 2000, 16303 + experts = next((int(s.split("=")[1]) for s in sets if s.startswith("expert-vram=")), None) + return (2000 + (13400 if experts is None else 1500 + experts * 1024)), 16303 + + +def test_gamemode_measures_every_placement_and_returns_to_normal(tmp_path, monkeypatch, fake_server): + r, started, running = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram) + assert r["stage"] == "gamemode" and r["status"] == "ok" + assert [x["sets"] for x in started] == [lv["sets"] for lv in llm.LEVELS] + assert started[-1] == started[0] # the last start is the normal placement again + for lv in llm.LEVELS: + row = next(x for x in r["levels"] if x["key"] == lv["key"]) + assert row["switch_s"] is not None + assert row["voice_first_sentence"]["n"] > 0 + rows = {x["key"]: x for x in r["levels"]} + assert rows["game_experts_2g"]["vram_free_mib"] > rows["game_experts_6g"]["vram_free_mib"] > rows["normal"]["vram_free_mib"] + assert running["sets"] is None # the model is not left running after the stage + + +def test_a_placement_that_does_not_start_is_recorded_and_the_rest_still_run(tmp_path, monkeypatch, fake_server): + r, started, _ = _run_levels_stage(tmp_path, monkeypatch, fake_server, _vram, + ready=lambda sets: "expert-vram=4" not in sets) + assert len(started) == len(llm.LEVELS) + row = next(x for x in r["levels"] if x["key"] == "game_experts_4g") + assert row["switch_s"] is None and "decode_tok_s" not in row + + +def _row(key, free, tok_s): + return {"key": key, "vram_free_mib": free, "decode_tok_s": {"median": tok_s}, "switch_s": 50.0} + + +def test_game_choice_is_the_fastest_placement_that_leaves_the_reserve_free(): + rows = [_row("normal", 900, 82.0), _row("game_experts_6g", 6000, 70.0), + _row("game_experts_4g", 8100, 60.0), _row("game_experts_2g", 10100, 45.0)] + choice = llm.pick_game_level(rows, reserve_mib=8000) + assert choice["key"] == "game_experts_4g" and choice["decode_tok_s"] == 60.0 + + +def test_no_game_choice_when_no_placement_leaves_the_reserve(): + rows = [_row("normal", 900, 82.0), _row("game_experts_2g", 7000, 45.0)] + assert llm.pick_game_level(rows, reserve_mib=8000) is None + + +def test_kit_has_a_one_click_runner_for_desktop_and_gamemode(): + import m0 + keys = [s[0] for s in m0.STAGES] + assert "gamemode" in keys + bat = (Path(m0.KIT) / "run_m0_gamemode.bat").read_text(encoding="utf-8") + assert "-Only desktop,gamemode" in bat and "-Redo" in bat