Files
javis_bot/scripts/m0/m0lib/desktop.py
javis-bot 13de978cf9 feat(m0): measure the one model's game placements and fix desktop app matching
Add a gamemode stage that restarts Qwen3.6-35B-A3B with a smaller routed
expert VRAM budget (6, 4, 2 GiB, and 2 GiB with half the CPU workers), then
the normal placement again, and records switch time, free VRAM, output speed,
first spoken sentence and 8K-context first token for each. It names the
fastest placement that leaves the game its 8 GB reserve. Strata 0.1.59's
qwen36 engine cannot shrink VRAM while running, so each placement is a restart.

The desktop stage now matches running apps by the process image name Windows
lists (Discord.exe) and closes the calculator it launched by image name when
close-app cannot find a UWP app.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 17:39:42 +09:00

478 lines
19 KiB
Python

"""Stage: agent-desktop (Windows UI Automation) as the M7 PC-control path.
agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0) reads a
program's accessibility tree and acts on its buttons and fields directly,
without moving the mouse or taking focus. This stage checks, on the gaming PC:
basics the CLI runs, what it reports about permissions and surfaces
calculator click 1, 2, +, 3, = in Windows Calculator and read 15 back
(buttons are found by automation id, so the UI language does
not matter)
text_input set a value in an input on a test page in a throwaway browser
profile and read it back; also how long the Chromium tree takes
to settle after launch
apps how long reading Discord and Explorer takes (counts only)
mcp a warm `agent-desktop mcp` server against one CLI process per
call, for the voice-latency choice between the two
Privacy: other programs' window titles, UI text and error messages are never
stored. Only counts, timings, error codes and the calculator's digits are.
"""
from __future__ import annotations
import hashlib
import io
import json
import os
import queue
import re
import shutil
import subprocess
import sys
import tarfile
import threading
import time
import urllib.request
from pathlib import Path
from typing import Any, Callable
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from m0lib.common import IS_WINDOWS, StageResult, say, stage_args, stats, step # noqa: E402
AD_VERSION = "1.2.0"
AD_URL = ("https://github.com/vercel-labs/agent-desktop/releases/download/"
f"v{AD_VERSION}/agent-desktop-v{AD_VERSION}-x86_64-pc-windows-msvc.tar.gz")
AD_SHA256 = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8"
# 12 + 3 = 15, pressed by automation id (language independent).
CALC_BUTTONS = ["clearButton", "num1Button", "num2Button", "plusButton", "num3Button", "equalButton"]
CALC_EXPECTED = "15"
CALC_RESULT_ID = "CalculatorResults"
PAGE_TITLE = "Jarvis 입력 테스트"
INPUT_NAME = "jarvis-m0-input"
INPUT_TEXT = "자비스 입력 테스트 15"
READ_APPS = ["Discord", "explorer"]
# close-app only knows window-owning processes; a UWP app such as the
# calculator is owned by ApplicationFrameHost, so it is closed by image name.
FORCE_CLOSE = ["taskkill", "/F", "/IM"]
REPEAT = 5
class Desk:
"""Runs agent-desktop commands and records how long each took."""
def __init__(self, cmd: list[str], log: Path, timeout: float = 60):
self.cmd = cmd
self.log = log
self.timeout = timeout
self.timings: dict[str, list[float]] = {}
def __call__(self, *args: str) -> dict[str, Any]:
t0 = time.perf_counter()
try:
proc = subprocess.run(self.cmd + list(args), capture_output=True, text=True, encoding="utf-8",
errors="replace", timeout=self.timeout, stdin=subprocess.DEVNULL)
out = proc.stdout.strip()
env = json.loads(out.splitlines()[-1]) if out else {}
except subprocess.TimeoutExpired:
env = {"ok": False, "error": {"code": "M0_TIMEOUT"}}
except (ValueError, IndexError):
env = {"ok": False, "error": {"code": "M0_NO_JSON"}}
elapsed = time.perf_counter() - t0
self.timings.setdefault(args[0], []).append(elapsed)
# Command names and codes only: arguments and output can carry UI text.
with open(self.log, "a", encoding="utf-8") as f:
f.write(f"{args[0]} ok={bool(env.get('ok'))} code={code(env)} {elapsed:.2f}s\n")
return env
def code(env: dict[str, Any]) -> str | None:
"""The error code of an envelope (never the message: it can name windows)."""
if env.get("ok"):
return None
err = env.get("error")
return str(err.get("code")) if isinstance(err, dict) else "M0_UNKNOWN"
def first_match(env: dict[str, Any]) -> dict[str, Any] | None:
matches = (env.get("data") or {}).get("matches") or [] if env.get("ok") else []
return matches[0] if matches else None
def windows(desk: Desk) -> list[dict[str, Any]]:
env = desk("list-windows")
data = env.get("data") if env.get("ok") else None
return data if isinstance(data, list) else []
# --------------------------------------------------------------------------
def install(work: Path, log: Path, url: str = AD_URL, sha256: str = AD_SHA256) -> Path:
"""Download the pinned Windows build into the work folder and verify it."""
dest = work / "tools" / f"agent-desktop-{AD_VERSION}"
exe = dest / "agent-desktop.exe"
if exe.exists():
return exe
say(f" agent-desktop {AD_VERSION} 받는 중")
with urllib.request.urlopen(url, timeout=300) as resp:
data = resp.read()
got = hashlib.sha256(data).hexdigest()
if got != sha256:
raise RuntimeError(f"agent-desktop archive hash mismatch: {got}")
tmp = dest.parent / (dest.name + ".tmp")
shutil.rmtree(tmp, ignore_errors=True)
tmp.mkdir(parents=True)
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
member = next(m for m in tar.getmembers() if m.isfile() and Path(m.name).name == "agent-desktop.exe")
(tmp / "agent-desktop.exe").write_bytes(tar.extractfile(member).read())
shutil.rmtree(dest, ignore_errors=True)
tmp.rename(dest)
with open(log, "a", encoding="utf-8") as f:
f.write(f"installed agent-desktop {AD_VERSION} sha256={got}\n")
return exe
def check_basics(desk: Desk) -> dict[str, Any]:
out: dict[str, Any] = {}
ver = desk("version")
out["version"] = (ver.get("data") or {}).get("version") if ver.get("ok") else None
st = desk("status")
if st.get("ok"):
d = st.get("data") or {}
out["platform"] = d.get("platform")
out["permissions"] = {k: (v or {}).get("state") for k, v in (d.get("permissions") or {}).items()}
out["surfaces"] = d.get("supported_surfaces")
else:
out["status_error"] = code(st)
wins = windows(desk)
out["window_count"] = len(wins)
out["accessible_windows"] = sum(1 for w in wins if w.get("accessible"))
out["ok"] = bool(out["version"]) and st.get("ok", False)
return out
def _calculator_app(desk: Desk, wait_s: float) -> str | None:
deadline = time.monotonic() + wait_s
while True:
for w in windows(desk):
name = str(w.get("app_name") or "")
if "calculator" in name.lower():
return name
if time.monotonic() > deadline:
return None
time.sleep(0.5)
def check_calculator(desk: Desk, wait_s: float = 15) -> dict[str, Any]:
out: dict[str, Any] = {"ok": False}
if _calculator_app(desk, 0) is not None:
# The user's own calculator: do not press its buttons or close it.
out["skipped"] = "calculator already open"
return out
t0 = time.perf_counter()
launched = desk("launch", "calc", "--timeout", "20000")
out["launch_error"] = code(launched)
if not launched.get("ok") and IS_WINDOWS:
subprocess.run(["cmd", "/c", "start", "", "calculator:"], capture_output=True, timeout=30)
out["launch_fallback"] = "calculator: URI"
app = _calculator_app(desk, wait_s)
out["window_s"] = round(time.perf_counter() - t0, 2)
if not app:
out["error"] = "calculator window not found"
return out
out["app_name"] = app
try:
for bid in CALC_BUTTONS:
m = first_match(desk("find", "--app", app, "--native-id", bid))
if not m:
if bid == "clearButton":
continue # optional: a fresh calculator already shows 0
out["error"] = f"button {bid} not found"
return out
clicked = desk("click", m["ref_id"])
if not clicked.get("ok"):
out["error"] = f"click {bid}: {code(clicked)}"
return out
m = first_match(desk("find", "--app", app, "--native-id", CALC_RESULT_ID))
digits = "".join(re.findall(r"\d", str((m or {}).get("name") or "")))
out["display_digits"] = digits
out["ok"] = digits.endswith(CALC_EXPECTED) and digits != ""
if not out["ok"]:
out["error"] = "display does not show the sum"
return out
finally:
close_launched(desk, app, out)
def close_launched(desk: Desk, app: str, out: dict[str, Any]) -> None:
"""Close the calculator this stage started (never one the user had open)."""
out["close_error"] = code(desk("close-app", app))
if out["close_error"] is not None:
try:
subprocess.run([*FORCE_CLOSE, app], capture_output=True, timeout=30)
except (OSError, subprocess.SubprocessError):
pass
out["closed"] = _calculator_app(desk, 0) is None
def _image_stem(name: str) -> str:
n = name.lower()
return n[:-4] if n.endswith(".exe") else n
def _page(page_dir: Path) -> Path:
page_dir.mkdir(parents=True, exist_ok=True)
page = page_dir / "jarvis_m0_input.html"
page.write_text(f"""<!doctype html><html><head><meta charset="utf-8"><title>{PAGE_TITLE}</title></head>
<body style="font:32px sans-serif;padding:40px"><p>M0 입력 테스트 (곧 닫힙니다)</p>
<input aria-label="{INPUT_NAME}" style="font:32px sans-serif;width:80%"></body></html>""", encoding="utf-8")
return page
def check_text_input(desk: Desk, open_page: Callable[[Path, str], Any], page_dir: Path,
wait_s: float = 40) -> dict[str, Any]:
out: dict[str, Any] = {"ok": False, "settle_s": None}
proc = open_page(_page(page_dir), PAGE_TITLE)
if proc is None:
out["error"] = "no Chrome or Edge found"
return out
try:
t0 = time.perf_counter()
ref = None
while time.perf_counter() - t0 < wait_s and not ref:
wid = next((w.get("id") for w in windows(desk) if PAGE_TITLE in str(w.get("title") or "")), None)
if wid:
m = first_match(desk("find", "--window-id", str(wid), "--role", "textfield", "--name", INPUT_NAME))
ref = (m or {}).get("ref_id")
if not ref:
time.sleep(1)
if not ref:
out["error"] = "test input not found"
return out
out["settle_s"] = round(time.perf_counter() - t0, 2)
# set-value is semantic (no focus, no keystrokes). When the field does not
# support it, fall back to keystroke typing, which takes the foreground.
set_env = desk("set-value", ref, INPUT_TEXT)
out["set_error"] = code(set_env)
out["input_method"] = "set-value"
if not set_env.get("ok"):
set_env = desk("type", ref, INPUT_TEXT, "--headed")
out["type_error"] = code(set_env)
out["input_method"] = "type --headed"
got = desk("get", ref, "--property", "value")
out["read_back"] = bool(got.get("ok")) and (got.get("data") or {}).get("value") == INPUT_TEXT
out["ok"] = bool(set_env.get("ok")) and out["read_back"]
if not out["ok"]:
out["error"] = "value not read back"
return out
finally:
try:
proc.terminate()
except Exception:
pass
def check_apps(desk: Desk) -> dict[str, Any]:
env = desk("list-apps")
# Windows lists process image names ("Discord.exe"); snapshot wants that exact name.
listed = [str(a.get("name") or "") for a in ((env.get("data") or {}).get("apps") or [])] \
if env.get("ok") else []
by_stem = {_image_stem(n): n for n in listed}
out: dict[str, Any] = {}
for app in READ_APPS:
name = by_stem.get(_image_stem(app))
if name is None:
out[app] = {"running": False}
continue
t0 = time.perf_counter()
snap = desk("snapshot", "--app", name, "-i", "--compact", "--timeout-ms", "8000")
d = snap.get("data") or {} if snap.get("ok") else {}
out[app] = {"running": True, "ok": bool(snap.get("ok")), "error": code(snap),
"ref_count": d.get("ref_count"), "complete": d.get("complete"),
"seconds": round(time.perf_counter() - t0, 2)}
return out
class _Mcp:
def __init__(self, cmd: list[str]):
self.proc = subprocess.Popen(cmd + ["mcp"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True, encoding="utf-8", bufsize=1)
self.lines: queue.Queue[str] = queue.Queue()
threading.Thread(target=self._read, daemon=True).start()
self.next_id = 0
def _read(self) -> None:
assert self.proc.stdout is not None
for line in self.proc.stdout:
self.lines.put(line)
def send(self, method: str, params: dict[str, Any] | None = None, notify: bool = False) -> Any:
msg: dict[str, Any] = {"jsonrpc": "2.0", "method": method, "params": params or {}}
if not notify:
self.next_id += 1
msg["id"] = self.next_id
assert self.proc.stdin is not None
self.proc.stdin.write(json.dumps(msg) + "\n")
self.proc.stdin.flush()
if notify:
return None
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
try:
resp = json.loads(self.lines.get(timeout=max(0.1, deadline - time.monotonic())))
except queue.Empty:
break
if resp.get("id") == msg["id"]:
if "error" in resp:
raise RuntimeError(f"mcp {method} error {resp['error'].get('code')}")
return resp.get("result")
raise RuntimeError(f"mcp {method} timed out")
def close(self) -> None:
try:
self.proc.kill()
except Exception:
pass
def _mcp_envelope(result: Any) -> dict[str, Any]:
"""The agent-desktop envelope inside an MCP tools/call result."""
result = result if isinstance(result, dict) else {}
env: dict[str, Any] = {}
for part in result.get("content") or []:
if isinstance(part, dict) and part.get("type") == "text":
try:
env = json.loads(part.get("text") or "")
except ValueError:
env = {}
break
if not isinstance(env, dict):
env = {}
if result.get("isError") and env.get("ok") is not False:
env = {"ok": False, "error": (env.get("error") if isinstance(env.get("error"), dict)
else {"code": "M0_MCP_IS_ERROR"})}
if "ok" not in env:
env = {"ok": False, "error": {"code": "M0_NO_JSON"}}
return env
def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]:
"""Speed of one call both ways. A failed call fails the check: an error
answers fast, so its timing says nothing about the working path."""
out: dict[str, Any] = {"ok": False}
cli = []
for _ in range(REPEAT):
t0 = time.perf_counter()
env = desk("list-windows")
cli.append(time.perf_counter() - t0)
if not env.get("ok"):
out["cli_error"] = code(env)
out["cli_call_s"] = stats(cli)
if out.get("cli_error"):
out["error"] = "cli call failed"
return out
server = _Mcp(cmd)
try:
t0 = time.perf_counter()
server.send("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
"clientInfo": {"name": "jarvis-m0", "version": "1"}})
server.send("notifications/initialized", notify=True)
out["start_s"] = round(time.perf_counter() - t0, 3)
tools = (server.send("tools/list") or {}).get("tools") or []
out["tool_count"] = len(tools)
name = next((t["name"] for t in tools if "list_windows" in t.get("name", "")), None)
if not name:
out["error"] = "no list-windows tool"
return out
warm = []
for _ in range(REPEAT):
t0 = time.perf_counter()
env = _mcp_envelope(server.send("tools/call", {"name": name, "arguments": {}}))
warm.append(time.perf_counter() - t0)
if not env.get("ok"):
out["mcp_error"] = code(env)
out["warm_call_s"] = stats(warm)
if out.get("mcp_error"):
out["error"] = "mcp call failed"
return out
out["ok"] = True
except Exception as e:
out["error"] = type(e).__name__ + ": " + str(e)[:200]
finally:
server.close()
return out
def open_browser_page(profile: Path) -> Callable[[Path, str], Any]:
"""Opens the test page in a throwaway Chrome/Edge profile (never the user's)."""
def _open(page: Path, title: str) -> Any:
cands = [os.path.expandvars(p) for p in (
r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe",
r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe")]
exe = next((p for p in cands if os.path.exists(p)), None)
if not exe:
return None
return subprocess.Popen([exe, f"--user-data-dir={profile}", "--no-first-run", "--no-default-browser-check",
"--new-window", page.as_uri()])
return _open
def measure(cmd: list[str], work: Path, out_path: Path, open_page: Callable[[Path, str], Any]) -> str:
r = StageResult("desktop", out_path)
log = work / "logs" / "desktop.log"
log.parent.mkdir(parents=True, exist_ok=True)
desk = Desk(cmd, log)
r["agent_desktop_version"] = AD_VERSION
checks = [("basics", "기본 동작", lambda: check_basics(desk)),
("calculator", "계산기 버튼 누르기", lambda: check_calculator(desk)),
("text_input", "브라우저 입력칸에 글자 넣기", lambda: check_text_input(
desk, open_page, work / "cache" / "desktop")),
("apps", "디스코드·탐색기 읽기 시간", lambda: check_apps(desk)),
("mcp", "계속 켜 두는 방식(MCP) 속도", lambda: check_mcp(cmd, desk))]
for key, title, fn in checks:
say(f" {title}")
try:
r[key] = fn()
except Exception as e:
r[key] = {"ok": False}
r.error(key, f"{type(e).__name__}")
if key == "basics" and not r[key].get("ok"):
r["timings"] = {k: stats(v) for k, v in desk.timings.items()}
r.save("failed")
return "failed"
r["timings"] = {k: stats(v) for k, v in desk.timings.items()}
core_ok = r["calculator"].get("ok") and r["text_input"].get("ok") and r["mcp"].get("ok")
status = "ok" if core_ok else "partial"
r.save(status)
return status
def main() -> int:
out, work, _opts = stage_args()
step("PC 제어 도구(agent-desktop) 확인")
if not IS_WINDOWS:
r = StageResult("desktop", out)
r.note("Windows only")
r.save("skipped")
return 0
say(" 계산기와 테스트용 브라우저 창이 잠깐 떴다가 닫힙니다. 손대지 않으셔도 됩니다.")
try:
exe = install(work, work / "logs" / "desktop.log")
except Exception as e:
r = StageResult("desktop", out)
r.error("install", e)
r.save("failed")
return 1
status = measure([str(exe)], work, out, open_browser_page(work / "cache" / "desktop" / "browser-profile"))
say(f" 결과: {status}")
return 0
if __name__ == "__main__":
sys.exit(main())