"""Stage: agent-desktop (Windows UI Automation) as the M7 PC-control path. agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0) reads a program's accessibility tree and acts on its buttons and fields directly, without moving the mouse or taking focus. This stage checks, on the gaming PC: basics the CLI runs, what it reports about permissions and surfaces calculator click 1, 2, +, 3, = in Windows Calculator and read 15 back (buttons are found by automation id, so the UI language does not matter) text_input set a value in an input on a test page in a throwaway browser profile and read it back; also how long the Chromium tree takes to settle after launch apps how long reading Discord and Explorer takes (counts only) mcp a warm `agent-desktop mcp` server against one CLI process per call, for the voice-latency choice between the two Privacy: other programs' window titles, UI text and error messages are never stored. Only counts, timings, error codes and the calculator's digits are. """ from __future__ import annotations import hashlib import io import json import os import queue import re import shutil import subprocess import sys import tarfile import threading import time import urllib.request from pathlib import Path from typing import Any, Callable sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from m0lib.common import IS_WINDOWS, StageResult, say, stage_args, stats, step # noqa: E402 AD_VERSION = "1.2.0" AD_URL = ("https://github.com/vercel-labs/agent-desktop/releases/download/" f"v{AD_VERSION}/agent-desktop-v{AD_VERSION}-x86_64-pc-windows-msvc.tar.gz") AD_SHA256 = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8" # 12 + 3 = 15, pressed by automation id (language independent). CALC_BUTTONS = ["clearButton", "num1Button", "num2Button", "plusButton", "num3Button", "equalButton"] CALC_EXPECTED = "15" CALC_RESULT_ID = "CalculatorResults" PAGE_TITLE = "Jarvis 입력 테스트" INPUT_NAME = "jarvis-m0-input" INPUT_TEXT = "자비스 입력 테스트 15" READ_APPS = ["Discord", "explorer"] REPEAT = 5 class Desk: """Runs agent-desktop commands and records how long each took.""" def __init__(self, cmd: list[str], log: Path, timeout: float = 60): self.cmd = cmd self.log = log self.timeout = timeout self.timings: dict[str, list[float]] = {} def __call__(self, *args: str) -> dict[str, Any]: t0 = time.perf_counter() try: proc = subprocess.run(self.cmd + list(args), capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=self.timeout, stdin=subprocess.DEVNULL) out = proc.stdout.strip() env = json.loads(out.splitlines()[-1]) if out else {} except subprocess.TimeoutExpired: env = {"ok": False, "error": {"code": "M0_TIMEOUT"}} except (ValueError, IndexError): env = {"ok": False, "error": {"code": "M0_NO_JSON"}} elapsed = time.perf_counter() - t0 self.timings.setdefault(args[0], []).append(elapsed) # Command names and codes only: arguments and output can carry UI text. with open(self.log, "a", encoding="utf-8") as f: f.write(f"{args[0]} ok={bool(env.get('ok'))} code={code(env)} {elapsed:.2f}s\n") return env def code(env: dict[str, Any]) -> str | None: """The error code of an envelope (never the message: it can name windows).""" if env.get("ok"): return None err = env.get("error") return str(err.get("code")) if isinstance(err, dict) else "M0_UNKNOWN" def first_match(env: dict[str, Any]) -> dict[str, Any] | None: matches = (env.get("data") or {}).get("matches") or [] if env.get("ok") else [] return matches[0] if matches else None def windows(desk: Desk) -> list[dict[str, Any]]: env = desk("list-windows") data = env.get("data") if env.get("ok") else None return data if isinstance(data, list) else [] # -------------------------------------------------------------------------- def install(work: Path, log: Path, url: str = AD_URL, sha256: str = AD_SHA256) -> Path: """Download the pinned Windows build into the work folder and verify it.""" dest = work / "tools" / f"agent-desktop-{AD_VERSION}" exe = dest / "agent-desktop.exe" if exe.exists(): return exe say(f" agent-desktop {AD_VERSION} 받는 중") with urllib.request.urlopen(url, timeout=300) as resp: data = resp.read() got = hashlib.sha256(data).hexdigest() if got != sha256: raise RuntimeError(f"agent-desktop archive hash mismatch: {got}") tmp = dest.parent / (dest.name + ".tmp") shutil.rmtree(tmp, ignore_errors=True) tmp.mkdir(parents=True) with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: member = next(m for m in tar.getmembers() if m.isfile() and Path(m.name).name == "agent-desktop.exe") (tmp / "agent-desktop.exe").write_bytes(tar.extractfile(member).read()) shutil.rmtree(dest, ignore_errors=True) tmp.rename(dest) with open(log, "a", encoding="utf-8") as f: f.write(f"installed agent-desktop {AD_VERSION} sha256={got}\n") return exe def check_basics(desk: Desk) -> dict[str, Any]: out: dict[str, Any] = {} ver = desk("version") out["version"] = (ver.get("data") or {}).get("version") if ver.get("ok") else None st = desk("status") if st.get("ok"): d = st.get("data") or {} out["platform"] = d.get("platform") out["permissions"] = {k: (v or {}).get("state") for k, v in (d.get("permissions") or {}).items()} out["surfaces"] = d.get("supported_surfaces") else: out["status_error"] = code(st) wins = windows(desk) out["window_count"] = len(wins) out["accessible_windows"] = sum(1 for w in wins if w.get("accessible")) out["ok"] = bool(out["version"]) and st.get("ok", False) return out def _calculator_app(desk: Desk, wait_s: float) -> str | None: deadline = time.monotonic() + wait_s while True: for w in windows(desk): name = str(w.get("app_name") or "") if "calculator" in name.lower(): return name if time.monotonic() > deadline: return None time.sleep(0.5) def check_calculator(desk: Desk, wait_s: float = 15) -> dict[str, Any]: out: dict[str, Any] = {"ok": False} if _calculator_app(desk, 0) is not None: # The user's own calculator: do not press its buttons or close it. out["skipped"] = "calculator already open" return out t0 = time.perf_counter() launched = desk("launch", "calc", "--timeout", "20000") out["launch_error"] = code(launched) if not launched.get("ok") and IS_WINDOWS: subprocess.run(["cmd", "/c", "start", "", "calculator:"], capture_output=True, timeout=30) out["launch_fallback"] = "calculator: URI" app = _calculator_app(desk, wait_s) out["window_s"] = round(time.perf_counter() - t0, 2) if not app: out["error"] = "calculator window not found" return out out["app_name"] = app try: for bid in CALC_BUTTONS: m = first_match(desk("find", "--app", app, "--native-id", bid)) if not m: if bid == "clearButton": continue # optional: a fresh calculator already shows 0 out["error"] = f"button {bid} not found" return out clicked = desk("click", m["ref_id"]) if not clicked.get("ok"): out["error"] = f"click {bid}: {code(clicked)}" return out m = first_match(desk("find", "--app", app, "--native-id", CALC_RESULT_ID)) digits = "".join(re.findall(r"\d", str((m or {}).get("name") or ""))) out["display_digits"] = digits out["ok"] = digits.endswith(CALC_EXPECTED) and digits != "" if not out["ok"]: out["error"] = "display does not show the sum" return out finally: out["close_error"] = code(desk("close-app", app)) def _page(page_dir: Path) -> Path: page_dir.mkdir(parents=True, exist_ok=True) page = page_dir / "jarvis_m0_input.html" page.write_text(f"""
M0 입력 테스트 (곧 닫힙니다)
""", encoding="utf-8") return page def check_text_input(desk: Desk, open_page: Callable[[Path, str], Any], page_dir: Path, wait_s: float = 40) -> dict[str, Any]: out: dict[str, Any] = {"ok": False, "settle_s": None} proc = open_page(_page(page_dir), PAGE_TITLE) if proc is None: out["error"] = "no Chrome or Edge found" return out try: t0 = time.perf_counter() ref = None while time.perf_counter() - t0 < wait_s and not ref: wid = next((w.get("id") for w in windows(desk) if PAGE_TITLE in str(w.get("title") or "")), None) if wid: m = first_match(desk("find", "--window-id", str(wid), "--role", "textfield", "--name", INPUT_NAME)) ref = (m or {}).get("ref_id") if not ref: time.sleep(1) if not ref: out["error"] = "test input not found" return out out["settle_s"] = round(time.perf_counter() - t0, 2) # set-value is semantic (no focus, no keystrokes). When the field does not # support it, fall back to keystroke typing, which takes the foreground. set_env = desk("set-value", ref, INPUT_TEXT) out["set_error"] = code(set_env) out["input_method"] = "set-value" if not set_env.get("ok"): set_env = desk("type", ref, INPUT_TEXT, "--headed") out["type_error"] = code(set_env) out["input_method"] = "type --headed" got = desk("get", ref, "--property", "value") out["read_back"] = bool(got.get("ok")) and (got.get("data") or {}).get("value") == INPUT_TEXT out["ok"] = bool(set_env.get("ok")) and out["read_back"] if not out["ok"]: out["error"] = "value not read back" return out finally: try: proc.terminate() except Exception: pass def check_apps(desk: Desk) -> dict[str, Any]: env = desk("list-apps") running = {str(a.get("name") or "").lower() for a in ((env.get("data") or {}).get("apps") or [])} \ if env.get("ok") else set() out: dict[str, Any] = {} for app in READ_APPS: if app.lower() not in running: out[app] = {"running": False} continue t0 = time.perf_counter() snap = desk("snapshot", "--app", app, "-i", "--compact", "--timeout-ms", "8000") d = snap.get("data") or {} if snap.get("ok") else {} out[app] = {"running": True, "ok": bool(snap.get("ok")), "error": code(snap), "ref_count": d.get("ref_count"), "complete": d.get("complete"), "seconds": round(time.perf_counter() - t0, 2)} return out class _Mcp: def __init__(self, cmd: list[str]): self.proc = subprocess.Popen(cmd + ["mcp"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8", bufsize=1) self.lines: queue.Queue[str] = queue.Queue() threading.Thread(target=self._read, daemon=True).start() self.next_id = 0 def _read(self) -> None: assert self.proc.stdout is not None for line in self.proc.stdout: self.lines.put(line) def send(self, method: str, params: dict[str, Any] | None = None, notify: bool = False) -> Any: msg: dict[str, Any] = {"jsonrpc": "2.0", "method": method, "params": params or {}} if not notify: self.next_id += 1 msg["id"] = self.next_id assert self.proc.stdin is not None self.proc.stdin.write(json.dumps(msg) + "\n") self.proc.stdin.flush() if notify: return None deadline = time.monotonic() + 30 while time.monotonic() < deadline: try: resp = json.loads(self.lines.get(timeout=max(0.1, deadline - time.monotonic()))) except queue.Empty: break if resp.get("id") == msg["id"]: if "error" in resp: raise RuntimeError(f"mcp {method} error {resp['error'].get('code')}") return resp.get("result") raise RuntimeError(f"mcp {method} timed out") def close(self) -> None: try: self.proc.kill() except Exception: pass def _mcp_envelope(result: Any) -> dict[str, Any]: """The agent-desktop envelope inside an MCP tools/call result.""" result = result if isinstance(result, dict) else {} env: dict[str, Any] = {} for part in result.get("content") or []: if isinstance(part, dict) and part.get("type") == "text": try: env = json.loads(part.get("text") or "") except ValueError: env = {} break if not isinstance(env, dict): env = {} if result.get("isError") and env.get("ok") is not False: env = {"ok": False, "error": (env.get("error") if isinstance(env.get("error"), dict) else {"code": "M0_MCP_IS_ERROR"})} if "ok" not in env: env = {"ok": False, "error": {"code": "M0_NO_JSON"}} return env def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]: """Speed of one call both ways. A failed call fails the check: an error answers fast, so its timing says nothing about the working path.""" out: dict[str, Any] = {"ok": False} cli = [] for _ in range(REPEAT): t0 = time.perf_counter() env = desk("list-windows") cli.append(time.perf_counter() - t0) if not env.get("ok"): out["cli_error"] = code(env) out["cli_call_s"] = stats(cli) if out.get("cli_error"): out["error"] = "cli call failed" return out server = _Mcp(cmd) try: t0 = time.perf_counter() server.send("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "jarvis-m0", "version": "1"}}) server.send("notifications/initialized", notify=True) out["start_s"] = round(time.perf_counter() - t0, 3) tools = (server.send("tools/list") or {}).get("tools") or [] out["tool_count"] = len(tools) name = next((t["name"] for t in tools if "list_windows" in t.get("name", "")), None) if not name: out["error"] = "no list-windows tool" return out warm = [] for _ in range(REPEAT): t0 = time.perf_counter() env = _mcp_envelope(server.send("tools/call", {"name": name, "arguments": {}})) warm.append(time.perf_counter() - t0) if not env.get("ok"): out["mcp_error"] = code(env) out["warm_call_s"] = stats(warm) if out.get("mcp_error"): out["error"] = "mcp call failed" return out out["ok"] = True except Exception as e: out["error"] = type(e).__name__ + ": " + str(e)[:200] finally: server.close() return out def open_browser_page(profile: Path) -> Callable[[Path, str], Any]: """Opens the test page in a throwaway Chrome/Edge profile (never the user's).""" def _open(page: Path, title: str) -> Any: cands = [os.path.expandvars(p) for p in ( r"%ProgramFiles%\Google\Chrome\Application\chrome.exe", r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe", r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe", r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe")] exe = next((p for p in cands if os.path.exists(p)), None) if not exe: return None return subprocess.Popen([exe, f"--user-data-dir={profile}", "--no-first-run", "--no-default-browser-check", "--new-window", page.as_uri()]) return _open def measure(cmd: list[str], work: Path, out_path: Path, open_page: Callable[[Path, str], Any]) -> str: r = StageResult("desktop", out_path) log = work / "logs" / "desktop.log" log.parent.mkdir(parents=True, exist_ok=True) desk = Desk(cmd, log) r["agent_desktop_version"] = AD_VERSION checks = [("basics", "기본 동작", lambda: check_basics(desk)), ("calculator", "계산기 버튼 누르기", lambda: check_calculator(desk)), ("text_input", "브라우저 입력칸에 글자 넣기", lambda: check_text_input( desk, open_page, work / "cache" / "desktop")), ("apps", "디스코드·탐색기 읽기 시간", lambda: check_apps(desk)), ("mcp", "계속 켜 두는 방식(MCP) 속도", lambda: check_mcp(cmd, desk))] for key, title, fn in checks: say(f" {title}") try: r[key] = fn() except Exception as e: r[key] = {"ok": False} r.error(key, f"{type(e).__name__}") if key == "basics" and not r[key].get("ok"): r["timings"] = {k: stats(v) for k, v in desk.timings.items()} r.save("failed") return "failed" r["timings"] = {k: stats(v) for k, v in desk.timings.items()} core_ok = r["calculator"].get("ok") and r["text_input"].get("ok") and r["mcp"].get("ok") status = "ok" if core_ok else "partial" r.save(status) return status def main() -> int: out, work, _opts = stage_args() step("PC 제어 도구(agent-desktop) 확인") if not IS_WINDOWS: r = StageResult("desktop", out) r.note("Windows only") r.save("skipped") return 0 say(" 계산기와 테스트용 브라우저 창이 잠깐 떴다가 닫힙니다. 손대지 않으셔도 됩니다.") try: exe = install(work, work / "logs" / "desktop.log") except Exception as e: r = StageResult("desktop", out) r.error("install", e) r.save("failed") return 1 status = measure([str(exe)], work, out, open_browser_page(work / "cache" / "desktop" / "browser-profile")) say(f" 결과: {status}") return 0 if __name__ == "__main__": sys.exit(main())