check_mcp timed five CLI and five MCP list-windows calls but ignored their answers, so a server returning isError/ok=false still scored ok. Read the envelope of every call, record the error code (never the message), and fail the check on any failed call. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
282 lines
11 KiB
Python
282 lines
11 KiB
Python
"""Behaviour of the M0 ``desktop`` stage (agent-desktop on the gaming PC).
|
|
|
|
The stage drives a real CLI on Windows, so these tests run it against a fake
|
|
``agent-desktop`` that speaks the same JSON envelope: a calculator that adds,
|
|
a browser test page with one input, a private document window whose title
|
|
must never reach the results, and an MCP server over stdio.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tarfile
|
|
import textwrap
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts" / "m0"))
|
|
|
|
from m0lib import desktop # noqa: E402
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
SECRET = "비밀계획서"
|
|
|
|
FAKE_CLI = textwrap.dedent(r'''
|
|
import json, os, sys
|
|
STATE = os.environ["FAKE_AD_STATE"]
|
|
SECRET = os.environ["FAKE_AD_SECRET"]
|
|
MODE = os.environ.get("FAKE_AD_MODE", "")
|
|
|
|
def load():
|
|
try:
|
|
with open(STATE, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
except FileNotFoundError:
|
|
return {}
|
|
|
|
def save(s):
|
|
with open(STATE, "w", encoding="utf-8") as f:
|
|
json.dump(s, f)
|
|
|
|
def ok(cmd, data):
|
|
print(json.dumps({"version": "2.6", "ok": True, "command": cmd, "data": data}, ensure_ascii=False))
|
|
|
|
def fail(cmd, code, message):
|
|
print(json.dumps({"version": "2.6", "ok": False, "command": cmd,
|
|
"error": {"code": code, "message": message}}, ensure_ascii=False))
|
|
sys.exit(1)
|
|
|
|
def opt(args, name):
|
|
return args[args.index(name) + 1] if name in args else None
|
|
|
|
def windows(s):
|
|
w = [{"id": "w-1", "app_name": "WINWORD", "title": SECRET + ".docx - Word", "accessible": True}]
|
|
if s.get("calc"):
|
|
w.append({"id": "w-200", "app_name": "CalculatorApp", "title": "계산기", "accessible": True})
|
|
if s.get("browser"):
|
|
w.append({"id": "w-100", "app_name": "chrome", "title": s["browser"] + " - Google Chrome",
|
|
"accessible": True})
|
|
return w
|
|
|
|
def mcp():
|
|
tools = [{"name": "desktop_list_windows"}, {"name": "desktop_snapshot"}, {"name": "desktop_click"}]
|
|
for line in sys.stdin:
|
|
req = json.loads(line)
|
|
if "id" not in req:
|
|
continue
|
|
if req["method"] == "initialize":
|
|
result = {"protocolVersion": "2025-06-18", "capabilities": {"tools": {}}}
|
|
elif req["method"] == "tools/list":
|
|
result = {"tools": tools}
|
|
elif req["method"] == "tools/call" and MODE == "mcp_fail":
|
|
text = json.dumps({"ok": False, "error": {"code": "PLATFORM_ERROR", "message": SECRET}},
|
|
ensure_ascii=False)
|
|
result = {"isError": True, "content": [{"type": "text", "text": text}]}
|
|
elif req["method"] == "tools/call":
|
|
text = json.dumps({"ok": True, "data": windows(load())}, ensure_ascii=False)
|
|
result = {"content": [{"type": "text", "text": text}]}
|
|
else:
|
|
result = {}
|
|
print(json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": result}, ensure_ascii=False), flush=True)
|
|
|
|
def main():
|
|
a = sys.argv[1:]
|
|
cmd = a[0]
|
|
s = load()
|
|
if cmd == "mcp":
|
|
return mcp()
|
|
if cmd == "version":
|
|
return ok(cmd, {"os": "windows", "version": "1.2.0"})
|
|
if cmd == "status":
|
|
return ok(cmd, {"platform": "windows", "permissions": {"accessibility": {"state": "granted"}},
|
|
"supported_surfaces": ["window", "menu"]})
|
|
if cmd == "list-apps":
|
|
return ok(cmd, {"apps": [{"name": "Discord"}, {"name": "explorer"}, {"name": "WINWORD"}]})
|
|
if cmd == "list-windows":
|
|
if MODE == "cli_fail":
|
|
fail(cmd, "PLATFORM_ERROR", SECRET)
|
|
return ok(cmd, windows(s))
|
|
if cmd == "launch":
|
|
if a[1] == "calc":
|
|
s.update(calc=True, display="0", entry="", acc=None)
|
|
save(s)
|
|
return ok(cmd, {"app": "calc", "pid": 1})
|
|
fail(cmd, "APP_NOT_FOUND", "no such app")
|
|
if cmd == "close-app":
|
|
s["calc"] = False
|
|
save(s)
|
|
return ok(cmd, {"app": a[1]})
|
|
if cmd == "find":
|
|
if opt(a, "--native-id"):
|
|
nid = opt(a, "--native-id")
|
|
if opt(a, "--app") != "CalculatorApp" or not s.get("calc"):
|
|
return ok(cmd, {"matches": [], "total_matches": 0})
|
|
name = ("표시는 %s입니다" % s["display"]) if nid == "CalculatorResults" else nid
|
|
return ok(cmd, {"matches": [{"ref_id": "@s1:" + nid, "name": name, "role": "button"}]})
|
|
if opt(a, "--role") == "textfield":
|
|
if opt(a, "--window-id") == "w-100" and s.get("browser"):
|
|
return ok(cmd, {"matches": [{"ref_id": "@s2:e1", "name": opt(a, "--name"),
|
|
"role": "textfield"}]})
|
|
return ok(cmd, {"matches": [], "total_matches": 0})
|
|
if cmd == "click":
|
|
nid = a[1].split(":", 1)[1]
|
|
if MODE != "dead_clicks":
|
|
if nid.startswith("num"):
|
|
s["entry"] += nid[3]
|
|
s["display"] = s["entry"]
|
|
elif nid == "plusButton":
|
|
s["acc"], s["entry"] = int(s["entry"] or 0), ""
|
|
elif nid == "equalButton":
|
|
s["display"] = str((s["acc"] or 0) + int(s["entry"] or 0))
|
|
elif nid == "clearButton":
|
|
s.update(display="0", entry="", acc=None)
|
|
save(s)
|
|
return ok(cmd, {"action": "click"})
|
|
if cmd == "set-value":
|
|
if MODE != "lie":
|
|
s["value"] = a[2]
|
|
save(s)
|
|
return ok(cmd, {"action": "set-value"})
|
|
if cmd == "get":
|
|
return ok(cmd, {"property": "value", "ref": a[1], "value": s.get("value", "")})
|
|
if cmd == "snapshot":
|
|
app = opt(a, "--app")
|
|
if app == "explorer":
|
|
fail(cmd, "AMBIGUOUS_TARGET", "windows: " + SECRET + ".docx")
|
|
return ok(cmd, {"app": app, "ref_count": 120, "complete": True,
|
|
"tree": {"role": "window", "name": SECRET + " DM"}})
|
|
fail(cmd, "INVALID_ARGS", "unknown " + cmd)
|
|
|
|
main()
|
|
''')
|
|
|
|
|
|
@pytest.fixture
|
|
def fake(tmp_path, monkeypatch):
|
|
script = tmp_path / "fake_agent_desktop.py"
|
|
script.write_text(FAKE_CLI, encoding="utf-8")
|
|
state = tmp_path / "state.json"
|
|
monkeypatch.setenv("FAKE_AD_STATE", str(state))
|
|
monkeypatch.setenv("FAKE_AD_SECRET", SECRET)
|
|
monkeypatch.delenv("FAKE_AD_MODE", raising=False)
|
|
|
|
class Fake:
|
|
cmd = [sys.executable, str(script)]
|
|
log = tmp_path / "desktop.log"
|
|
|
|
def desk(self):
|
|
return desktop.Desk(self.cmd, self.log)
|
|
|
|
def open_page(self, page: Path, title: str):
|
|
# Stands in for the browser: the page's window appears in list-windows.
|
|
s = json.loads(state.read_text(encoding="utf-8")) if state.exists() else {}
|
|
s["browser"] = title
|
|
state.write_text(json.dumps(s), encoding="utf-8")
|
|
|
|
class Proc:
|
|
def terminate(self):
|
|
s2 = json.loads(state.read_text(encoding="utf-8"))
|
|
s2.pop("browser", None)
|
|
state.write_text(json.dumps(s2), encoding="utf-8")
|
|
return Proc()
|
|
|
|
return Fake()
|
|
|
|
|
|
def test_calculator_round_trip_passes_when_the_display_shows_the_sum(fake):
|
|
result = desktop.check_calculator(fake.desk())
|
|
assert result["ok"] is True
|
|
assert result["display_digits"] == desktop.CALC_EXPECTED
|
|
|
|
|
|
def test_calculator_fails_when_the_clicks_do_not_reach_the_app(fake, monkeypatch):
|
|
monkeypatch.setenv("FAKE_AD_MODE", "dead_clicks")
|
|
result = desktop.check_calculator(fake.desk())
|
|
assert result["ok"] is False
|
|
|
|
|
|
def test_text_input_reads_back_what_was_set(fake, tmp_path):
|
|
result = desktop.check_text_input(fake.desk(), fake.open_page, tmp_path)
|
|
assert result["ok"] is True
|
|
assert result["settle_s"] is not None
|
|
|
|
|
|
def test_text_input_fails_when_the_value_does_not_change(fake, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("FAKE_AD_MODE", "lie")
|
|
result = desktop.check_text_input(fake.desk(), fake.open_page, tmp_path)
|
|
assert result["ok"] is False
|
|
|
|
|
|
def test_warm_mcp_server_lists_tools_and_answers_repeated_calls(fake):
|
|
result = desktop.check_mcp(fake.cmd, fake.desk())
|
|
assert result["tool_count"] == 3
|
|
assert result["warm_call_s"]["n"] == desktop.REPEAT
|
|
assert result["cli_call_s"]["n"] == desktop.REPEAT
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["mcp_fail", "cli_fail"])
|
|
def test_failed_calls_fail_the_speed_check_with_their_code(fake, monkeypatch, mode):
|
|
monkeypatch.setenv("FAKE_AD_MODE", mode)
|
|
result = desktop.check_mcp(fake.cmd, fake.desk())
|
|
assert result["ok"] is False
|
|
assert "PLATFORM_ERROR" in json.dumps(result)
|
|
assert SECRET not in json.dumps(result, ensure_ascii=False)
|
|
|
|
|
|
def test_results_never_store_other_window_titles_or_ui_text(fake, tmp_path):
|
|
out = tmp_path / "results" / "desktop.json"
|
|
status = desktop.measure(fake.cmd, tmp_path, out, fake.open_page)
|
|
text = out.read_text(encoding="utf-8")
|
|
assert status == "ok"
|
|
data = json.loads(text)
|
|
assert data["calculator"]["ok"] and data["text_input"]["ok"]
|
|
assert SECRET not in text # private window title, snapshot names and error messages
|
|
|
|
|
|
def _archive(tmp_path: Path) -> tuple[str, str]:
|
|
buf = io.BytesIO()
|
|
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
|
payload = b"MZ fake exe"
|
|
info = tarfile.TarInfo("agent-desktop.exe")
|
|
info.size = len(payload)
|
|
tar.addfile(info, io.BytesIO(payload))
|
|
path = tmp_path / "ad.tar.gz"
|
|
path.write_bytes(buf.getvalue())
|
|
return path.as_uri(), hashlib.sha256(buf.getvalue()).hexdigest()
|
|
|
|
|
|
def test_install_unpacks_the_pinned_archive(tmp_path):
|
|
url, sha = _archive(tmp_path)
|
|
exe = desktop.install(tmp_path / "work", tmp_path / "log.txt", url=url, sha256=sha)
|
|
assert exe.exists() and exe.name == "agent-desktop.exe"
|
|
|
|
|
|
def test_install_rejects_an_archive_with_the_wrong_hash(tmp_path):
|
|
url, _ = _archive(tmp_path)
|
|
with pytest.raises(RuntimeError):
|
|
desktop.install(tmp_path / "work", tmp_path / "log.txt", url=url, sha256="0" * 64)
|
|
assert not list((tmp_path / "work").rglob("agent-desktop.exe"))
|
|
|
|
|
|
def test_kit_has_a_desktop_stage_and_a_one_click_runner():
|
|
import m0
|
|
keys = [s[0] for s in m0.STAGES]
|
|
assert "desktop" in keys
|
|
assert (Path(m0.KIT) / "m0lib" / m0.STAGES[keys.index("desktop")][3]).exists()
|
|
bat = (Path(m0.KIT) / "run_m0_desktop.bat").read_text(encoding="utf-8")
|
|
assert "-Only desktop" in bat
|
|
|
|
|
|
def test_an_already_open_calculator_is_left_alone(fake, tmp_path):
|
|
state = tmp_path / "state.json"
|
|
state.write_text(json.dumps({"calc": True, "display": "777", "entry": "777", "acc": None}), encoding="utf-8")
|
|
result = desktop.check_calculator(fake.desk())
|
|
assert result["ok"] is False and result["skipped"]
|
|
s = json.loads(state.read_text(encoding="utf-8"))
|
|
assert s["calc"] is True and s["display"] == "777" # not pressed, not closed
|