fix(m0): fail the agent-desktop speed check when its calls fail

check_mcp timed five CLI and five MCP list-windows calls but ignored their
answers, so a server returning isError/ok=false still scored ok. Read the
envelope of every call, record the error code (never the message), and
fail the check on any failed call.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
javis-bot
2026-11-10 17:19:33 +09:00
parent 66009d910d
commit c63a859ced
2 changed files with 50 additions and 2 deletions

View File

@@ -316,14 +316,42 @@ class _Mcp:
pass
def _mcp_envelope(result: Any) -> dict[str, Any]:
"""The agent-desktop envelope inside an MCP tools/call result."""
result = result if isinstance(result, dict) else {}
env: dict[str, Any] = {}
for part in result.get("content") or []:
if isinstance(part, dict) and part.get("type") == "text":
try:
env = json.loads(part.get("text") or "")
except ValueError:
env = {}
break
if not isinstance(env, dict):
env = {}
if result.get("isError") and env.get("ok") is not False:
env = {"ok": False, "error": (env.get("error") if isinstance(env.get("error"), dict)
else {"code": "M0_MCP_IS_ERROR"})}
if "ok" not in env:
env = {"ok": False, "error": {"code": "M0_NO_JSON"}}
return env
def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]:
"""Speed of one call both ways. A failed call fails the check: an error
answers fast, so its timing says nothing about the working path."""
out: dict[str, Any] = {"ok": False}
cli = []
for _ in range(REPEAT):
t0 = time.perf_counter()
desk("list-windows")
env = desk("list-windows")
cli.append(time.perf_counter() - t0)
if not env.get("ok"):
out["cli_error"] = code(env)
out["cli_call_s"] = stats(cli)
if out.get("cli_error"):
out["error"] = "cli call failed"
return out
server = _Mcp(cmd)
try:
t0 = time.perf_counter()
@@ -340,9 +368,14 @@ def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]:
warm = []
for _ in range(REPEAT):
t0 = time.perf_counter()
server.send("tools/call", {"name": name, "arguments": {}})
env = _mcp_envelope(server.send("tools/call", {"name": name, "arguments": {}}))
warm.append(time.perf_counter() - t0)
if not env.get("ok"):
out["mcp_error"] = code(env)
out["warm_call_s"] = stats(warm)
if out.get("mcp_error"):
out["error"] = "mcp call failed"
return out
out["ok"] = True
except Exception as e:
out["error"] = type(e).__name__ + ": " + str(e)[:200]

View File

@@ -73,6 +73,10 @@ FAKE_CLI = textwrap.dedent(r'''
result = {"protocolVersion": "2025-06-18", "capabilities": {"tools": {}}}
elif req["method"] == "tools/list":
result = {"tools": tools}
elif req["method"] == "tools/call" and MODE == "mcp_fail":
text = json.dumps({"ok": False, "error": {"code": "PLATFORM_ERROR", "message": SECRET}},
ensure_ascii=False)
result = {"isError": True, "content": [{"type": "text", "text": text}]}
elif req["method"] == "tools/call":
text = json.dumps({"ok": True, "data": windows(load())}, ensure_ascii=False)
result = {"content": [{"type": "text", "text": text}]}
@@ -94,6 +98,8 @@ FAKE_CLI = textwrap.dedent(r'''
if cmd == "list-apps":
return ok(cmd, {"apps": [{"name": "Discord"}, {"name": "explorer"}, {"name": "WINWORD"}]})
if cmd == "list-windows":
if MODE == "cli_fail":
fail(cmd, "PLATFORM_ERROR", SECRET)
return ok(cmd, windows(s))
if cmd == "launch":
if a[1] == "calc":
@@ -213,6 +219,15 @@ def test_warm_mcp_server_lists_tools_and_answers_repeated_calls(fake):
assert result["cli_call_s"]["n"] == desktop.REPEAT
@pytest.mark.parametrize("mode", ["mcp_fail", "cli_fail"])
def test_failed_calls_fail_the_speed_check_with_their_code(fake, monkeypatch, mode):
monkeypatch.setenv("FAKE_AD_MODE", mode)
result = desktop.check_mcp(fake.cmd, fake.desk())
assert result["ok"] is False
assert "PLATFORM_ERROR" in json.dumps(result)
assert SECRET not in json.dumps(result, ensure_ascii=False)
def test_results_never_store_other_window_titles_or_ui_text(fake, tmp_path):
out = tmp_path / "results" / "desktop.json"
status = desktop.measure(fake.cmd, tmp_path, out, fake.open_page)