Add a 'desktop' stage to the M0 kit that installs the pinned agent-desktop 1.2.0 Windows build (SHA-256 checked) and checks, on the gaming PC: calculator buttons by automation id (12+3=15), set-value and read-back on a test page in a throwaway browser profile, Discord/Explorer read time, and a warm MCP server against one CLI process per call. Results keep only counts, timings, error codes and the calculator digits. Record agent-desktop (UI Automation) as the planned M7 control path in the v2 design docs, with raw Windows input only for screens without accessibility. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
423 lines
17 KiB
Python
423 lines
17 KiB
Python
"""Stage: agent-desktop (Windows UI Automation) as the M7 PC-control path.
|
|
|
|
agent-desktop (github.com/vercel-labs/agent-desktop, Apache-2.0) reads a
|
|
program's accessibility tree and acts on its buttons and fields directly,
|
|
without moving the mouse or taking focus. This stage checks, on the gaming PC:
|
|
|
|
basics the CLI runs, what it reports about permissions and surfaces
|
|
calculator click 1, 2, +, 3, = in Windows Calculator and read 15 back
|
|
(buttons are found by automation id, so the UI language does
|
|
not matter)
|
|
text_input set a value in an input on a test page in a throwaway browser
|
|
profile and read it back; also how long the Chromium tree takes
|
|
to settle after launch
|
|
apps how long reading Discord and Explorer takes (counts only)
|
|
mcp a warm `agent-desktop mcp` server against one CLI process per
|
|
call, for the voice-latency choice between the two
|
|
|
|
Privacy: other programs' window titles, UI text and error messages are never
|
|
stored. Only counts, timings, error codes and the calculator's digits are.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import queue
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from m0lib.common import IS_WINDOWS, StageResult, say, stage_args, stats, step # noqa: E402
|
|
|
|
AD_VERSION = "1.2.0"
|
|
AD_URL = ("https://github.com/vercel-labs/agent-desktop/releases/download/"
|
|
f"v{AD_VERSION}/agent-desktop-v{AD_VERSION}-x86_64-pc-windows-msvc.tar.gz")
|
|
AD_SHA256 = "a4199f741a441cedd349bf56c5eba01783b1261258b498efeb7ec9945220c3e8"
|
|
|
|
# 12 + 3 = 15, pressed by automation id (language independent).
|
|
CALC_BUTTONS = ["clearButton", "num1Button", "num2Button", "plusButton", "num3Button", "equalButton"]
|
|
CALC_EXPECTED = "15"
|
|
CALC_RESULT_ID = "CalculatorResults"
|
|
|
|
PAGE_TITLE = "Jarvis 입력 테스트"
|
|
INPUT_NAME = "jarvis-m0-input"
|
|
INPUT_TEXT = "자비스 입력 테스트 15"
|
|
|
|
READ_APPS = ["Discord", "explorer"]
|
|
REPEAT = 5
|
|
|
|
|
|
class Desk:
|
|
"""Runs agent-desktop commands and records how long each took."""
|
|
|
|
def __init__(self, cmd: list[str], log: Path, timeout: float = 60):
|
|
self.cmd = cmd
|
|
self.log = log
|
|
self.timeout = timeout
|
|
self.timings: dict[str, list[float]] = {}
|
|
|
|
def __call__(self, *args: str) -> dict[str, Any]:
|
|
t0 = time.perf_counter()
|
|
try:
|
|
proc = subprocess.run(self.cmd + list(args), capture_output=True, text=True, encoding="utf-8",
|
|
errors="replace", timeout=self.timeout, stdin=subprocess.DEVNULL)
|
|
out = proc.stdout.strip()
|
|
env = json.loads(out.splitlines()[-1]) if out else {}
|
|
except subprocess.TimeoutExpired:
|
|
env = {"ok": False, "error": {"code": "M0_TIMEOUT"}}
|
|
except (ValueError, IndexError):
|
|
env = {"ok": False, "error": {"code": "M0_NO_JSON"}}
|
|
elapsed = time.perf_counter() - t0
|
|
self.timings.setdefault(args[0], []).append(elapsed)
|
|
# Command names and codes only: arguments and output can carry UI text.
|
|
with open(self.log, "a", encoding="utf-8") as f:
|
|
f.write(f"{args[0]} ok={bool(env.get('ok'))} code={code(env)} {elapsed:.2f}s\n")
|
|
return env
|
|
|
|
|
|
def code(env: dict[str, Any]) -> str | None:
|
|
"""The error code of an envelope (never the message: it can name windows)."""
|
|
if env.get("ok"):
|
|
return None
|
|
err = env.get("error")
|
|
return str(err.get("code")) if isinstance(err, dict) else "M0_UNKNOWN"
|
|
|
|
|
|
def first_match(env: dict[str, Any]) -> dict[str, Any] | None:
|
|
matches = (env.get("data") or {}).get("matches") or [] if env.get("ok") else []
|
|
return matches[0] if matches else None
|
|
|
|
|
|
def windows(desk: Desk) -> list[dict[str, Any]]:
|
|
env = desk("list-windows")
|
|
data = env.get("data") if env.get("ok") else None
|
|
return data if isinstance(data, list) else []
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
def install(work: Path, log: Path, url: str = AD_URL, sha256: str = AD_SHA256) -> Path:
|
|
"""Download the pinned Windows build into the work folder and verify it."""
|
|
dest = work / "tools" / f"agent-desktop-{AD_VERSION}"
|
|
exe = dest / "agent-desktop.exe"
|
|
if exe.exists():
|
|
return exe
|
|
say(f" agent-desktop {AD_VERSION} 받는 중")
|
|
with urllib.request.urlopen(url, timeout=300) as resp:
|
|
data = resp.read()
|
|
got = hashlib.sha256(data).hexdigest()
|
|
if got != sha256:
|
|
raise RuntimeError(f"agent-desktop archive hash mismatch: {got}")
|
|
tmp = dest.parent / (dest.name + ".tmp")
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
tmp.mkdir(parents=True)
|
|
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
|
|
member = next(m for m in tar.getmembers() if m.isfile() and Path(m.name).name == "agent-desktop.exe")
|
|
(tmp / "agent-desktop.exe").write_bytes(tar.extractfile(member).read())
|
|
shutil.rmtree(dest, ignore_errors=True)
|
|
tmp.rename(dest)
|
|
with open(log, "a", encoding="utf-8") as f:
|
|
f.write(f"installed agent-desktop {AD_VERSION} sha256={got}\n")
|
|
return exe
|
|
|
|
|
|
def check_basics(desk: Desk) -> dict[str, Any]:
|
|
out: dict[str, Any] = {}
|
|
ver = desk("version")
|
|
out["version"] = (ver.get("data") or {}).get("version") if ver.get("ok") else None
|
|
st = desk("status")
|
|
if st.get("ok"):
|
|
d = st.get("data") or {}
|
|
out["platform"] = d.get("platform")
|
|
out["permissions"] = {k: (v or {}).get("state") for k, v in (d.get("permissions") or {}).items()}
|
|
out["surfaces"] = d.get("supported_surfaces")
|
|
else:
|
|
out["status_error"] = code(st)
|
|
wins = windows(desk)
|
|
out["window_count"] = len(wins)
|
|
out["accessible_windows"] = sum(1 for w in wins if w.get("accessible"))
|
|
out["ok"] = bool(out["version"]) and st.get("ok", False)
|
|
return out
|
|
|
|
|
|
def _calculator_app(desk: Desk, wait_s: float) -> str | None:
|
|
deadline = time.monotonic() + wait_s
|
|
while True:
|
|
for w in windows(desk):
|
|
name = str(w.get("app_name") or "")
|
|
if "calculator" in name.lower():
|
|
return name
|
|
if time.monotonic() > deadline:
|
|
return None
|
|
time.sleep(0.5)
|
|
|
|
|
|
def check_calculator(desk: Desk, wait_s: float = 15) -> dict[str, Any]:
|
|
out: dict[str, Any] = {"ok": False}
|
|
if _calculator_app(desk, 0) is not None:
|
|
# The user's own calculator: do not press its buttons or close it.
|
|
out["skipped"] = "calculator already open"
|
|
return out
|
|
t0 = time.perf_counter()
|
|
launched = desk("launch", "calc", "--timeout", "20000")
|
|
out["launch_error"] = code(launched)
|
|
if not launched.get("ok") and IS_WINDOWS:
|
|
subprocess.run(["cmd", "/c", "start", "", "calculator:"], capture_output=True, timeout=30)
|
|
out["launch_fallback"] = "calculator: URI"
|
|
app = _calculator_app(desk, wait_s)
|
|
out["window_s"] = round(time.perf_counter() - t0, 2)
|
|
if not app:
|
|
out["error"] = "calculator window not found"
|
|
return out
|
|
out["app_name"] = app
|
|
try:
|
|
for bid in CALC_BUTTONS:
|
|
m = first_match(desk("find", "--app", app, "--native-id", bid))
|
|
if not m:
|
|
if bid == "clearButton":
|
|
continue # optional: a fresh calculator already shows 0
|
|
out["error"] = f"button {bid} not found"
|
|
return out
|
|
clicked = desk("click", m["ref_id"])
|
|
if not clicked.get("ok"):
|
|
out["error"] = f"click {bid}: {code(clicked)}"
|
|
return out
|
|
m = first_match(desk("find", "--app", app, "--native-id", CALC_RESULT_ID))
|
|
digits = "".join(re.findall(r"\d", str((m or {}).get("name") or "")))
|
|
out["display_digits"] = digits
|
|
out["ok"] = digits.endswith(CALC_EXPECTED) and digits != ""
|
|
if not out["ok"]:
|
|
out["error"] = "display does not show the sum"
|
|
return out
|
|
finally:
|
|
out["close_error"] = code(desk("close-app", app))
|
|
|
|
|
|
def _page(page_dir: Path) -> Path:
|
|
page_dir.mkdir(parents=True, exist_ok=True)
|
|
page = page_dir / "jarvis_m0_input.html"
|
|
page.write_text(f"""<!doctype html><html><head><meta charset="utf-8"><title>{PAGE_TITLE}</title></head>
|
|
<body style="font:32px sans-serif;padding:40px"><p>M0 입력 테스트 (곧 닫힙니다)</p>
|
|
<input aria-label="{INPUT_NAME}" style="font:32px sans-serif;width:80%"></body></html>""", encoding="utf-8")
|
|
return page
|
|
|
|
|
|
def check_text_input(desk: Desk, open_page: Callable[[Path, str], Any], page_dir: Path,
|
|
wait_s: float = 40) -> dict[str, Any]:
|
|
out: dict[str, Any] = {"ok": False, "settle_s": None}
|
|
proc = open_page(_page(page_dir), PAGE_TITLE)
|
|
if proc is None:
|
|
out["error"] = "no Chrome or Edge found"
|
|
return out
|
|
try:
|
|
t0 = time.perf_counter()
|
|
ref = None
|
|
while time.perf_counter() - t0 < wait_s and not ref:
|
|
wid = next((w.get("id") for w in windows(desk) if PAGE_TITLE in str(w.get("title") or "")), None)
|
|
if wid:
|
|
m = first_match(desk("find", "--window-id", str(wid), "--role", "textfield", "--name", INPUT_NAME))
|
|
ref = (m or {}).get("ref_id")
|
|
if not ref:
|
|
time.sleep(1)
|
|
if not ref:
|
|
out["error"] = "test input not found"
|
|
return out
|
|
out["settle_s"] = round(time.perf_counter() - t0, 2)
|
|
# set-value is semantic (no focus, no keystrokes). When the field does not
|
|
# support it, fall back to keystroke typing, which takes the foreground.
|
|
set_env = desk("set-value", ref, INPUT_TEXT)
|
|
out["set_error"] = code(set_env)
|
|
out["input_method"] = "set-value"
|
|
if not set_env.get("ok"):
|
|
set_env = desk("type", ref, INPUT_TEXT, "--headed")
|
|
out["type_error"] = code(set_env)
|
|
out["input_method"] = "type --headed"
|
|
got = desk("get", ref, "--property", "value")
|
|
out["read_back"] = bool(got.get("ok")) and (got.get("data") or {}).get("value") == INPUT_TEXT
|
|
out["ok"] = bool(set_env.get("ok")) and out["read_back"]
|
|
if not out["ok"]:
|
|
out["error"] = "value not read back"
|
|
return out
|
|
finally:
|
|
try:
|
|
proc.terminate()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def check_apps(desk: Desk) -> dict[str, Any]:
|
|
env = desk("list-apps")
|
|
running = {str(a.get("name") or "").lower() for a in ((env.get("data") or {}).get("apps") or [])} \
|
|
if env.get("ok") else set()
|
|
out: dict[str, Any] = {}
|
|
for app in READ_APPS:
|
|
if app.lower() not in running:
|
|
out[app] = {"running": False}
|
|
continue
|
|
t0 = time.perf_counter()
|
|
snap = desk("snapshot", "--app", app, "-i", "--compact", "--timeout-ms", "8000")
|
|
d = snap.get("data") or {} if snap.get("ok") else {}
|
|
out[app] = {"running": True, "ok": bool(snap.get("ok")), "error": code(snap),
|
|
"ref_count": d.get("ref_count"), "complete": d.get("complete"),
|
|
"seconds": round(time.perf_counter() - t0, 2)}
|
|
return out
|
|
|
|
|
|
class _Mcp:
|
|
def __init__(self, cmd: list[str]):
|
|
self.proc = subprocess.Popen(cmd + ["mcp"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL, text=True, encoding="utf-8", bufsize=1)
|
|
self.lines: queue.Queue[str] = queue.Queue()
|
|
threading.Thread(target=self._read, daemon=True).start()
|
|
self.next_id = 0
|
|
|
|
def _read(self) -> None:
|
|
assert self.proc.stdout is not None
|
|
for line in self.proc.stdout:
|
|
self.lines.put(line)
|
|
|
|
def send(self, method: str, params: dict[str, Any] | None = None, notify: bool = False) -> Any:
|
|
msg: dict[str, Any] = {"jsonrpc": "2.0", "method": method, "params": params or {}}
|
|
if not notify:
|
|
self.next_id += 1
|
|
msg["id"] = self.next_id
|
|
assert self.proc.stdin is not None
|
|
self.proc.stdin.write(json.dumps(msg) + "\n")
|
|
self.proc.stdin.flush()
|
|
if notify:
|
|
return None
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
resp = json.loads(self.lines.get(timeout=max(0.1, deadline - time.monotonic())))
|
|
except queue.Empty:
|
|
break
|
|
if resp.get("id") == msg["id"]:
|
|
if "error" in resp:
|
|
raise RuntimeError(f"mcp {method} error {resp['error'].get('code')}")
|
|
return resp.get("result")
|
|
raise RuntimeError(f"mcp {method} timed out")
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
self.proc.kill()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]:
|
|
out: dict[str, Any] = {"ok": False}
|
|
cli = []
|
|
for _ in range(REPEAT):
|
|
t0 = time.perf_counter()
|
|
desk("list-windows")
|
|
cli.append(time.perf_counter() - t0)
|
|
out["cli_call_s"] = stats(cli)
|
|
server = _Mcp(cmd)
|
|
try:
|
|
t0 = time.perf_counter()
|
|
server.send("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
|
|
"clientInfo": {"name": "jarvis-m0", "version": "1"}})
|
|
server.send("notifications/initialized", notify=True)
|
|
out["start_s"] = round(time.perf_counter() - t0, 3)
|
|
tools = (server.send("tools/list") or {}).get("tools") or []
|
|
out["tool_count"] = len(tools)
|
|
name = next((t["name"] for t in tools if "list_windows" in t.get("name", "")), None)
|
|
if not name:
|
|
out["error"] = "no list-windows tool"
|
|
return out
|
|
warm = []
|
|
for _ in range(REPEAT):
|
|
t0 = time.perf_counter()
|
|
server.send("tools/call", {"name": name, "arguments": {}})
|
|
warm.append(time.perf_counter() - t0)
|
|
out["warm_call_s"] = stats(warm)
|
|
out["ok"] = True
|
|
except Exception as e:
|
|
out["error"] = type(e).__name__ + ": " + str(e)[:200]
|
|
finally:
|
|
server.close()
|
|
return out
|
|
|
|
|
|
def open_browser_page(profile: Path) -> Callable[[Path, str], Any]:
|
|
"""Opens the test page in a throwaway Chrome/Edge profile (never the user's)."""
|
|
def _open(page: Path, title: str) -> Any:
|
|
cands = [os.path.expandvars(p) for p in (
|
|
r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
|
|
r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe",
|
|
r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
|
|
r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe")]
|
|
exe = next((p for p in cands if os.path.exists(p)), None)
|
|
if not exe:
|
|
return None
|
|
return subprocess.Popen([exe, f"--user-data-dir={profile}", "--no-first-run", "--no-default-browser-check",
|
|
"--new-window", page.as_uri()])
|
|
return _open
|
|
|
|
|
|
def measure(cmd: list[str], work: Path, out_path: Path, open_page: Callable[[Path, str], Any]) -> str:
|
|
r = StageResult("desktop", out_path)
|
|
log = work / "logs" / "desktop.log"
|
|
log.parent.mkdir(parents=True, exist_ok=True)
|
|
desk = Desk(cmd, log)
|
|
r["agent_desktop_version"] = AD_VERSION
|
|
checks = [("basics", "기본 동작", lambda: check_basics(desk)),
|
|
("calculator", "계산기 버튼 누르기", lambda: check_calculator(desk)),
|
|
("text_input", "브라우저 입력칸에 글자 넣기", lambda: check_text_input(
|
|
desk, open_page, work / "cache" / "desktop")),
|
|
("apps", "디스코드·탐색기 읽기 시간", lambda: check_apps(desk)),
|
|
("mcp", "계속 켜 두는 방식(MCP) 속도", lambda: check_mcp(cmd, desk))]
|
|
for key, title, fn in checks:
|
|
say(f" {title}")
|
|
try:
|
|
r[key] = fn()
|
|
except Exception as e:
|
|
r[key] = {"ok": False}
|
|
r.error(key, f"{type(e).__name__}")
|
|
if key == "basics" and not r[key].get("ok"):
|
|
r["timings"] = {k: stats(v) for k, v in desk.timings.items()}
|
|
r.save("failed")
|
|
return "failed"
|
|
r["timings"] = {k: stats(v) for k, v in desk.timings.items()}
|
|
core_ok = r["calculator"].get("ok") and r["text_input"].get("ok") and r["mcp"].get("ok")
|
|
status = "ok" if core_ok else "partial"
|
|
r.save(status)
|
|
return status
|
|
|
|
|
|
def main() -> int:
|
|
out, work, _opts = stage_args()
|
|
step("PC 제어 도구(agent-desktop) 확인")
|
|
if not IS_WINDOWS:
|
|
r = StageResult("desktop", out)
|
|
r.note("Windows only")
|
|
r.save("skipped")
|
|
return 0
|
|
say(" 계산기와 테스트용 브라우저 창이 잠깐 떴다가 닫힙니다. 손대지 않으셔도 됩니다.")
|
|
try:
|
|
exe = install(work, work / "logs" / "desktop.log")
|
|
except Exception as e:
|
|
r = StageResult("desktop", out)
|
|
r.error("install", e)
|
|
r.save("failed")
|
|
return 1
|
|
status = measure([str(exe)], work, out, open_browser_page(work / "cache" / "desktop" / "browser-profile"))
|
|
say(f" 결과: {status}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|