fix(m0): fail the agent-desktop speed check when its calls fail
check_mcp timed five CLI and five MCP list-windows calls but ignored their answers, so a server returning isError/ok=false still scored ok. Read the envelope of every call, record the error code (never the message), and fail the check on any failed call. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -316,14 +316,42 @@ class _Mcp:
|
||||
pass
|
||||
|
||||
|
||||
def _mcp_envelope(result: Any) -> dict[str, Any]:
|
||||
"""The agent-desktop envelope inside an MCP tools/call result."""
|
||||
result = result if isinstance(result, dict) else {}
|
||||
env: dict[str, Any] = {}
|
||||
for part in result.get("content") or []:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
try:
|
||||
env = json.loads(part.get("text") or "")
|
||||
except ValueError:
|
||||
env = {}
|
||||
break
|
||||
if not isinstance(env, dict):
|
||||
env = {}
|
||||
if result.get("isError") and env.get("ok") is not False:
|
||||
env = {"ok": False, "error": (env.get("error") if isinstance(env.get("error"), dict)
|
||||
else {"code": "M0_MCP_IS_ERROR"})}
|
||||
if "ok" not in env:
|
||||
env = {"ok": False, "error": {"code": "M0_NO_JSON"}}
|
||||
return env
|
||||
|
||||
|
||||
def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]:
|
||||
"""Speed of one call both ways. A failed call fails the check: an error
|
||||
answers fast, so its timing says nothing about the working path."""
|
||||
out: dict[str, Any] = {"ok": False}
|
||||
cli = []
|
||||
for _ in range(REPEAT):
|
||||
t0 = time.perf_counter()
|
||||
desk("list-windows")
|
||||
env = desk("list-windows")
|
||||
cli.append(time.perf_counter() - t0)
|
||||
if not env.get("ok"):
|
||||
out["cli_error"] = code(env)
|
||||
out["cli_call_s"] = stats(cli)
|
||||
if out.get("cli_error"):
|
||||
out["error"] = "cli call failed"
|
||||
return out
|
||||
server = _Mcp(cmd)
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
@@ -340,9 +368,14 @@ def check_mcp(cmd: list[str], desk: Desk) -> dict[str, Any]:
|
||||
warm = []
|
||||
for _ in range(REPEAT):
|
||||
t0 = time.perf_counter()
|
||||
server.send("tools/call", {"name": name, "arguments": {}})
|
||||
env = _mcp_envelope(server.send("tools/call", {"name": name, "arguments": {}}))
|
||||
warm.append(time.perf_counter() - t0)
|
||||
if not env.get("ok"):
|
||||
out["mcp_error"] = code(env)
|
||||
out["warm_call_s"] = stats(warm)
|
||||
if out.get("mcp_error"):
|
||||
out["error"] = "mcp call failed"
|
||||
return out
|
||||
out["ok"] = True
|
||||
except Exception as e:
|
||||
out["error"] = type(e).__name__ + ": " + str(e)[:200]
|
||||
|
||||
@@ -73,6 +73,10 @@ FAKE_CLI = textwrap.dedent(r'''
|
||||
result = {"protocolVersion": "2025-06-18", "capabilities": {"tools": {}}}
|
||||
elif req["method"] == "tools/list":
|
||||
result = {"tools": tools}
|
||||
elif req["method"] == "tools/call" and MODE == "mcp_fail":
|
||||
text = json.dumps({"ok": False, "error": {"code": "PLATFORM_ERROR", "message": SECRET}},
|
||||
ensure_ascii=False)
|
||||
result = {"isError": True, "content": [{"type": "text", "text": text}]}
|
||||
elif req["method"] == "tools/call":
|
||||
text = json.dumps({"ok": True, "data": windows(load())}, ensure_ascii=False)
|
||||
result = {"content": [{"type": "text", "text": text}]}
|
||||
@@ -94,6 +98,8 @@ FAKE_CLI = textwrap.dedent(r'''
|
||||
if cmd == "list-apps":
|
||||
return ok(cmd, {"apps": [{"name": "Discord"}, {"name": "explorer"}, {"name": "WINWORD"}]})
|
||||
if cmd == "list-windows":
|
||||
if MODE == "cli_fail":
|
||||
fail(cmd, "PLATFORM_ERROR", SECRET)
|
||||
return ok(cmd, windows(s))
|
||||
if cmd == "launch":
|
||||
if a[1] == "calc":
|
||||
@@ -213,6 +219,15 @@ def test_warm_mcp_server_lists_tools_and_answers_repeated_calls(fake):
|
||||
assert result["cli_call_s"]["n"] == desktop.REPEAT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["mcp_fail", "cli_fail"])
|
||||
def test_failed_calls_fail_the_speed_check_with_their_code(fake, monkeypatch, mode):
|
||||
monkeypatch.setenv("FAKE_AD_MODE", mode)
|
||||
result = desktop.check_mcp(fake.cmd, fake.desk())
|
||||
assert result["ok"] is False
|
||||
assert "PLATFORM_ERROR" in json.dumps(result)
|
||||
assert SECRET not in json.dumps(result, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_results_never_store_other_window_titles_or_ui_text(fake, tmp_path):
|
||||
out = tmp_path / "results" / "desktop.json"
|
||||
status = desktop.measure(fake.cmd, tmp_path, out, fake.open_page)
|
||||
|
||||
Reference in New Issue
Block a user