Files
HermenixandClaude Opus 5 9708154eea test(mcp): count the unnarrowed resources against the listing, not a literal
`--toolsets` has to leave the resources alone, and the subtest that says so
asserted `5` and `3` -- so the stream's two resources turned a correct server
into a red check, which is the second literal in this file the surface outgrew
in one day.

Counted against `uris` and `templates` from the listing subtest instead. That
is the comparison the assertion was always about: not "six", but "as many as
this server lists when nothing is narrowed".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011ZeH65zd6pFTgBSu52W9PE
2026-10-11 23:08:13 +00:00

603 lines
28 KiB
Nix

# checks.<system>.vm-mcp -- the MCP server over stdio, in a guest, against a
# live broker and a real compositor.
#
# The same machine the other session checks use (`lib/session-machine.nix`),
# plus `services.hyprland-headless-stream.mcp.enable = true` -- which is also
# the first thing asserted, because "installing it" is the whole of that
# option. Then `hhs-mcp-call` (`lib/mcp_client.py`) spawns `hhstream-mcp` as the
# operator account and talks raw JSON-RPC to it: initialize, tools/list,
# tools/call.
#
# WHAT THIS PROVES, AND WHAT IT DOES NOT.
#
# It proves the *surface*, both halves of it: the handshake completes over a
# real pipe, all twenty-three tools and all ten resources are listed, a tool
# call reaches the broker and comes back with the broker's own protocol object,
# a resource read reaches it the same way, an image arrives as a blob with the
# geometry the module configured, `--toolsets` actually narrows what is
# offered, and a refusal arrives as a tool error (or, for a resource, a
# protocol error) carrying the frozen exit code rather than as a success.
#
# It does not prove the compositor behaviour the other checks own, and it must
# not be read as doing so. `vm-pointer` is where a click is shown to land in the
# window it was aimed at; `vm-keyboard` is where a typed symbol is shown to be
# decoded by a client from the keymap the driver uploaded. Here the victim is
# `foot`, which ignores `$HHSTREAM_APP_ID` and so matches no container rule
# (README.dev.md, known gap 4) -- so what the pointer and keyboard subtests
# below assert is that the lease was granted and the driver accepted the
# injection, which is the part the MCP layer is responsible for.
#
# The client is not the MCP SDK's, deliberately: the SDK on both ends would
# agree with itself about a framing neither side had to get right. Same choice
# as `tests/assert_stream.py` being a second implementation of the frame
# contract, and for the same reason.
#
# Habits copied from the other session checks: every wait is bounded, each
# subtest asserts once, and a failure dumps the compositor's diagnostics and
# the broker's journal before it propagates.
{
testers,
hhstream,
hhstream-mcp,
hhsModule,
testTools,
hyprland,
foot,
jq,
grim,
procps,
coreutils,
dejavu_fonts,
width ? 1280,
height ? 720,
sessionUser ? "hermes",
}:
testers.runNixOSTest (
{ pkgs, ... }:
let
tools = pkgs.callPackage ./lib/session-tools.nix { inherit hhstream; };
guest = import ./lib/session-machine.nix {
inherit
hhsModule
hhstream
hyprland
testTools
foot
jq
procps
grim
coreutils
dejavu_fonts
;
sessionTools = tools;
};
prelude = import ./lib/session-prelude.nix {
inherit hhstream coreutils sessionUser;
};
# The client, with a plain python3: it speaks JSON-RPC over a pipe and
# imports nothing but the standard library, which is what makes it a second
# implementation rather than the SDK talking to itself.
client = pkgs.runCommandLocal "hhs-mcp-call" { } ''
install -Dm755 ${./lib/mcp_client.py} $out/bin/hhs-mcp-call
sed -i '1s|.*|#!${pkgs.python3}/bin/python3|' $out/bin/hhs-mcp-call
$out/bin/hhs-mcp-call --help > /dev/null
# It must refuse to invent a request rather than printing nothing: a
# client that exits 0 having done nothing would make every subtest below
# vacuous.
if $out/bin/hhs-mcp-call --server ${coreutils}/bin/true; then
echo "hhs-mcp-call succeeded against a server that says nothing" >&2
exit 1
fi
'';
in
{
name = "hhstream-vm-mcp";
globalTimeout = 1800;
nodes.machine = {
imports = [
(guest {
inherit width height sessionUser;
pointer = true;
apps.foot = {
exec = "${foot}/bin/foot";
# The broker refuses unlisted arguments and this check passes none:
# `foot` with no argv opens a window, which is all the victim has
# to do.
argsAllow = [ ];
};
})
];
# The option under test. Installing the package IS the option, so the
# first subtest asserts the binary is on PATH and nothing else had to
# happen.
services.hyprland-headless-stream.mcp = {
enable = true;
package = hhstream-mcp;
};
# Enabled because two tools and two resources here are about streams,
# and with no daemon they could only be shown to *refuse*. What a stream
# actually serves is `vm-stream`'s subject -- it reads JPEGs off the
# port; what this check owns is that the URL (with its phrase) comes back
# through the MCP layer at all.
services.hyprland-headless-stream.sessions.stream = {
enable = true;
bind = "127.0.0.1";
port = 9988;
};
environment.systemPackages = [ client ];
};
testScript =
prelude
+ ''
import base64
import shlex
MCP = "${client}/bin/hhs-mcp-call"
WIDTH = ${toString width}
HEIGHT = ${toString height}
def mcp(argv, timeout=180):
"""`hhs-mcp-call …` as USER, parsed. Fails loudly if it printed no JSON."""
out = machine.succeed(as_user(f"{MCP} {argv}"), timeout=timeout)
try:
return json.loads(out)
except json.JSONDecodeError:
raise AssertionError(f"not JSON from `hhs-mcp-call {argv}`: {out!r}")
def call(tool, arguments=None, timeout=180):
"""One tools/call. Returns the CallToolResult as a dict."""
payload = json.dumps(arguments or {})
return mcp(f"{tool} {shlex.quote(payload)}", timeout=timeout)
def read(uri, timeout=180):
"""One resources/read that must succeed. Returns its contents list."""
return mcp(f"--read {shlex.quote(uri)}", timeout=timeout)["contents"]
def read_refused(uri, timeout=180):
"""One resources/read that must NOT succeed. Returns the error object.
A resource read has no `isError`, so a refusal is a protocol error
and the client exits 3 for it -- which is deliberately a different
status from the 2 it uses for "the server did not answer", so this
cannot pass because the server died.
"""
status, out = machine.execute(
as_user(f"{MCP} --read {shlex.quote(uri)}"), timeout=timeout
)
assert status == 3, f"expected a refusal (3) for {uri}, got {status}: {out}"
return json.loads(out)["error"]
def text_of(result):
"""The text part of a CallToolResult, which is where the CLI's own
output and (on a refusal) the remedy live."""
for block in result.get("content") or []:
if block.get("type") == "text":
return block["text"]
raise AssertionError(f"no text content in {result!r}")
def succeeded(tool, arguments=None, timeout=180):
"""A call that must not be an error, with the reason printed if it is."""
result = call(tool, arguments, timeout=timeout)
if result.get("isError"):
print(f"----- {tool} returned an error -----")
print(text_of(result))
assert not result.get("isError"), f"{tool} failed: {text_of(result)}"
return result
start_all()
bring_up("the compositor and broker")
with subtest("the as-user wrapper propagates exit status"):
# Every assertion below reads machine.succeed through it, so a
# wrapper that swallowed the status would make them all vacuous.
machine.succeed(as_user(TRUE), timeout=120)
machine.fail(as_user(FALSE), timeout=120)
with subtest("mcp.enable put the server on the system PATH"):
# The whole of the module option: no unit, no socket, no state dir.
path = machine.succeed(as_user("/run/current-system/sw/bin/sh -c 'command -v hhstream-mcp'")).strip()
print(f"hhstream-mcp resolves to {path}")
assert path.startswith("/run/current-system/sw/bin"), path
machine.fail("systemctl is-active hhstream-mcp.service")
with subtest("the stdio handshake completes and names this server"):
try:
handshake = mcp("--initialize")
print(json.dumps(handshake, indent=2)[:600])
assert handshake["serverInfo"]["name"] == "hhstream", handshake
assert handshake["capabilities"].get("tools") is not None, handshake
# The two rules, in the instructions a client is handed once.
instructions = handshake["instructions"]
assert "`session_touch` is the ONLY" in instructions
assert "POINTER lease" in instructions
except Exception:
dump("the mcp handshake")
raise
with subtest("tools/list carries every acting verb, with its annotations"):
listed = mcp("--list")["tools"]
names = sorted(tool["name"] for tool in listed)
print(f"{len(names)} tools: {', '.join(names)}")
assert len(names) == 23, names
for expected in (
"shot", "record", "run",
"session_start", "session_touch", "session_kill",
"session_stream_enable", "session_stream_disable",
"pointer_acquire", "pointer_click", "keyboard_type",
):
assert expected in names, f"{expected} is missing from the listing"
# The read-only verbs are resources now and must not be here.
for moved in ("status", "session_list", "session_status",
"session_logs", "pointer_state"):
assert moved not in names, f"{moved} is a resource, not a tool"
by_name = {tool["name"]: tool for tool in listed}
assert by_name["shot"]["annotations"]["readOnlyHint"] is True
assert by_name["session_kill"]["annotations"]["destructiveHint"] is True
# The rules have to be on the tool, not only in the instructions.
assert "session_touch" in by_name["session_gc"]["description"]
assert "POINTER lease" in by_name["keyboard_type"]["description"]
with subtest("resources/list and the templates carry the read-only half"):
both = mcp("--resources")
uris = sorted(r["uri"] for r in both["resources"])
templates = sorted(t["uriTemplate"] for t in both["resourceTemplates"])
print(f"resources: {', '.join(uris)}")
print(f"templates: {', '.join(templates)}")
assert uris == [
"hhstream://containers",
"hhstream://pointer",
"hhstream://screen",
"hhstream://sessions",
"hhstream://status",
"hhstream://streams",
], uris
assert templates == [
"hhstream://outputs/{name}",
"hhstream://sessions/{id}/logs{?lines}",
"hhstream://sessions/{id}/stream",
"hhstream://sessions/{id}{?events}",
], templates
by_uri = {r["uri"]: r for r in both["resources"]}
assert by_uri["hhstream://status"]["mimeType"] == "text/plain"
assert by_uri["hhstream://sessions"]["mimeType"] == "application/json"
# The idle-clock rule is on the reads it applies to.
assert "does NOT reset" in by_uri["hhstream://sessions"]["description"]
with subtest("a resource read reaches the broker and returns its object"):
try:
contents = read("hhstream://sessions")
assert len(contents) == 1, contents
assert contents[0]["mimeType"] == "application/json"
# The body is the CLI's own `--json` output, unmodified.
listing = json.loads(contents[0]["text"])
assert listing["ok"] is True, listing
assert "sessions" in listing, listing
except Exception:
dump("reading hhstream://sessions")
raise
with subtest("the status resource answers without the broker's help"):
# It reads the compositor's units and the frame socket, not the
# control socket -- which is why it is the thing to read first.
contents = read("hhstream://status")
assert contents[0]["mimeType"] == "text/plain"
report = contents[0]["text"]
print(report)
assert "hyprland-headless.service" in report
assert "frame:" in report
with subtest("a session started over mcp is the broker's own session"):
try:
started = succeeded("session_start", {"app": "foot"}, timeout=300)
session_id = started["structuredContent"]["id"]
print(f"started {session_id}")
assert text_of(started).startswith(f"session id: {session_id}")
# The broker is the authority on whether that happened, so it
# is asked directly rather than through the server under test.
listing = cli_json("list --json")
assert session_id in [s["id"] for s in listing["sessions"]], listing
except Exception:
dump("session_start over mcp")
raise
with subtest("reading a session does not reset its clock; touching it does"):
# The rule the whole surface repeats, measured rather than asserted
# about: two reads of the resource, then the tool, and the deadline
# has to move only for the tool.
try:
def expires_in():
body = json.loads(read(f"hhstream://sessions/{session_id}")[0]["text"])
return body["session"]["expires_in_s"]
first = expires_in()
second = expires_in()
assert second <= first, (
f"reading hhstream://sessions/{session_id} reset the idle "
f"clock: {first} -> {second}"
)
touched = succeeded("session_touch", {"id": session_id})
assert (
touched["structuredContent"]["session"]["expires_in_s"] >= second
), touched
except Exception:
dump("the idle clock over mcp", session_id)
raise
with subtest("a session's logs read through its template, with a query"):
try:
contents = read(f"hhstream://sessions/{session_id}/logs?lines=5")
body = json.loads(contents[0]["text"])
assert body["ok"] is True, body
assert "lines" in body, body
print(f"{len(body['lines'])} line(s) of {session_id}'s own output")
except Exception:
dump("reading a session's logs", session_id)
raise
with subtest("shot returns image content, sized as the publisher serves it"):
# MEASURED, and not what this check first assumed: the frame is
# 2560x720 on a guest configured for 1280x720. The publisher's
# default capture is `grim -t png -` with no `-o`, which takes the
# whole compositor layout -- and with containers on (the module's
# default) a session's own output sits beside the main one, so one
# session doubles the width. The geometry the module configures is
# the geometry of an *output*, which is what the next subtest
# captures by name.
try:
shot = succeeded("shot", timeout=300)
kinds = [block["type"] for block in shot["content"]]
assert kinds == ["text", "image"], kinds
image = shot["content"][1]
frame = base64.b64decode(image["data"])
print(f"{image['mimeType']}, {len(frame)} bytes: {text_of(shot)}")
assert image["mimeType"] in ("image/png", "image/jpeg")
# The media type is read off the bytes, never off the extension
# this server asked the CLI for -- so the two must agree.
magic = {"image/png": b"\x89PNG", "image/jpeg": b"\xff\xd8\xff"}
assert frame.startswith(magic[image["mimeType"]]), frame[:8]
reported = shot["structuredContent"]
assert reported["height"] == HEIGHT, reported
assert reported["width"] >= WIDTH, reported
assert reported["bytes"] == len(frame), reported
assert len(frame) > 1024, "a frame that small is not a desktop"
except Exception:
dump("shot over mcp", session_id)
raise
with subtest("the container is listed as a resource and reconciled as a tool"):
# The pair that has to stay split: the LIST only reads the record,
# so it is a resource; asking where the window is *now* re-reads the
# compositor and writes the answer back, so it is a tool. Both are
# asked here, and they have to agree about the output's name.
try:
listed = json.loads(read("hhstream://containers")[0]["text"])
assert listed["ok"] is True, listed
from_resource = [
c["output"] for c in (listed.get("containers") or [])
]
print(f"containers resource reports: {from_resource}")
assert f"hh-{session_id}" in from_resource, listed
reconciled = succeeded("session_container", {"id": session_id})
output = reconciled["structuredContent"]["container"]["output"]
assert output == f"hh-{session_id}", reconciled
except Exception:
dump("the container, both ways", session_id)
raise
with subtest("one output's frame reads as a blob, at exactly its own size"):
# The resource that is an image, and the one place the configured
# geometry is the geometry: `hhstream://screen` is the whole layout.
try:
output = f"hh-{session_id}"
contents = read(f"hhstream://outputs/{output}", timeout=300)
assert len(contents) == 1, contents
blob = contents[0]
assert blob["mimeType"] in ("image/png", "image/jpeg"), blob
assert "text" not in blob, "an image must not arrive as text"
frame = base64.b64decode(blob["blob"])
print(f"{output}: {blob['mimeType']}, {len(frame)} bytes")
assert frame[:4] in (b"\x89PNG", b"\xff\xd8\xff\xe0")
assert len(frame) > 1024, "a frame that small is not a desktop"
# And the tool agrees about the size, which is what makes the
# pointer's normalised coordinates meaningful.
framed = succeeded("shot", {"output": output}, timeout=300)
reported = framed["structuredContent"]
print(text_of(framed))
assert (reported["width"], reported["height"]) == (WIDTH, HEIGHT), reported
except Exception:
dump("reading an output's frame", session_id)
raise
with subtest("the pointer lease is granted, used and given back"):
try:
acquired = succeeded("pointer_acquire", {"id": session_id})
lease = acquired["structuredContent"]["lease_id"]
print(f"lease {lease}: {text_of(acquired).splitlines()[0]}")
assert text_of(acquired).startswith(f"lease_id: {lease}")
# Every gesture takes the lease as an explicit argument: the
# server caches nothing, which is what makes a transcript of
# tool calls readable.
succeeded("pointer_move", {"id": session_id, "lease": lease, "to": [0.5, 0.5]})
succeeded("pointer_click", {"id": session_id, "lease": lease})
succeeded("keyboard_type", {"id": session_id, "lease": lease, "text": "hello"})
typed = succeeded(
"keyboard_key", {"id": session_id, "lease": lease, "name": "Return"}
)
assert typed["structuredContent"]["ok"] is True, typed
released = succeeded("pointer_release", {"id": session_id, "lease": lease})
assert released["structuredContent"]["ok"] is True, released
# And the broker agrees that nobody holds it now.
assert cli_json("pointer state --json")["holder"] is None
except Exception:
dump("the pointer lease over mcp", session_id)
raise
with subtest("a gesture after the release is exit 9 and says to acquire"):
# The frozen NO_LEASE code, arriving as a tool error an agent can
# act on rather than as a success with a sad message in it.
refused = call("pointer_click", {"id": session_id, "lease": lease})
assert refused["isError"] is True, refused
assert refused["structuredContent"]["exit_code"] == 9, refused
said = text_of(refused)
print(said.splitlines()[0])
assert "exit 9 (NO_LEASE)" in said
assert "pointer_acquire" in said
with subtest("an unknown session is exit 3 from a tool, not a success"):
refused = call("session_touch", {"id": "no-such-session"})
assert refused["isError"] is True, refused
assert refused["structuredContent"]["exit_code"] == 3, refused
assert "hhstream://sessions" in text_of(refused)
with subtest("a refused read is a protocol error carrying the same remedy"):
# A resources/read has no `isError`, so this is the one place the
# refusal has to travel as an error -- with the exit code intact,
# or an agent cannot tell "no such session" from "broker down".
error = read_refused("hhstream://sessions/no-such-session")
print(error["message"].splitlines()[0])
assert error["data"]["exit_code"] == 3, error
assert error["data"]["error"] == "no_such_session", error
assert "exit 3 (NO_SUCH_SESSION)" in error["message"]
with subtest("a uri this server does not serve is refused without running anything"):
error = read_refused("hhstream://nonsense")
assert "no resource at" in error["message"], error
# And an undeclared query parameter, which the routing refuses for
# the reason every tool schema sets additionalProperties: false.
error = read_refused("hhstream://pointer?id=x")
assert "accepts: none" in error["message"], error
with subtest("an argument the schema refuses never reaches the broker"):
# No `lease`, so the server's own schema refuses before an argv list
# exists. The CLI is not run at all, which is the point.
refused = call("keyboard_type", {"id": session_id, "text": "x"})
assert refused["isError"] is True, refused
assert "lease" in text_of(refused), refused
with subtest("--toolsets narrows the tools and leaves the resources alone"):
# The flag exists because a tool listing is a budget. What it must
# not do is take the host's context away with it.
narrowed = json.loads(
machine.succeed(
as_user(f"{MCP} --server-arg=--toolsets=keyboard --list"),
timeout=180,
)
)["tools"]
names = sorted(tool["name"] for tool in narrowed)
print(f"--toolsets keyboard offers: {', '.join(names)}")
assert names == [
"keyboard_down", "keyboard_key", "keyboard_type", "keyboard_up",
], names
still_there = json.loads(
machine.succeed(
as_user(f"{MCP} --server-arg=--toolsets=keyboard --resources"),
timeout=180,
)
)
# Every one of them, unnarrowed: `--toolsets` is a tool budget and
# must not take the host's context away with it. Counted against
# the full listing above rather than against a literal, so the next
# resource added cannot make this pass by being forgotten.
assert len(still_there["resources"]) == len(uris), still_there
assert len(still_there["resourceTemplates"]) == len(templates), still_there
# And a tool outside the selection must be unreachable, not merely
# unlisted, or the flag is cosmetic.
refused = json.loads(
machine.succeed(
as_user(
f"{MCP} --server-arg=--toolsets=keyboard session_kill "
+ shlex.quote(json.dumps({"id": session_id}))
),
timeout=180,
)
)
assert refused["isError"] is True, refused
assert "without its toolset" in text_of(refused), refused
with subtest("kill over mcp ends the session the broker was tracking"):
try:
killed = succeeded("session_kill", {"id": session_id}, timeout=300)
print(text_of(killed))
assert killed["structuredContent"]["ok"] is True, killed
live = [s["id"] for s in cli_json("list --json")["sessions"]]
assert session_id not in live, live
except Exception:
dump("session_kill over mcp", session_id)
raise
with subtest("a stream asked for at start comes back as a url, twice"):
# The two places an agent can learn the phrase, and they have to
# agree: the `session_start` reply, and the resource for that
# session's stream. A URL is the daemon's to name and is never
# stored, so a reply built from the broker's record alone would
# carry a phrase and no URL -- which is how this went wrong once
# (see `SessionManager.start`).
streamed_id = None
try:
streamed = succeeded(
"session_start", {"app": "foot", "stream": True}, timeout=300
)
streamed_id = streamed["structuredContent"]["id"]
stream = streamed["structuredContent"]["stream"]
print(f"{streamed_id} -> {stream['url']}")
assert stream["url"].startswith("http://127.0.0.1:9988/"), stream
# The phrase is IN the URL and is the whole of its protection,
# so the text a model reads has to carry it too.
assert stream["url"] in text_of(streamed)
assert stream["phrase"] in stream["url"]
assert streamed["structuredContent"]["stream_error"] is None
body = json.loads(
read(f"hhstream://sessions/{streamed_id}/stream")[0]["text"]
)
assert body["stream"]["url"] == stream["url"], body
assert body["stream"]["attached"] is True, body
# And the listing resource names it, with the daemon reachable.
listed = json.loads(read("hhstream://streams")[0]["text"])
assert listed["available"] is True, listed
assert [row["id"] for row in listed["streams"]] == [streamed_id], listed
except Exception:
dump("session_start --stream over mcp", streamed_id)
raise
with subtest("disabling over mcp takes the stream off both resources"):
try:
succeeded("session_stream_disable", {"id": streamed_id}, timeout=180)
body = json.loads(
read(f"hhstream://sessions/{streamed_id}/stream")[0]["text"]
)
assert body["stream"] is None, body
assert json.loads(read("hhstream://streams")[0]["text"])["streams"] == []
finally:
succeeded("session_kill", {"id": streamed_id}, timeout=300)
'';
}
)