Skip to content

Commit e9786e2

Browse files
committed
ci(e2e): add MCP conformance coverage
Add a reusable MCP conformance workflow that runs upstream client scenarios through an OpenShell sandbox. Add a client image, wrapper, policy template, and expected-failures baseline for expanding MCP conformance coverage. Remove stale JSON-RPC e2e policy fields that are no longer accepted. Signed-off-by: Kris Hicks <khicks@nvidia.com>
1 parent c56760c commit e9786e2

8 files changed

Lines changed: 319 additions & 5 deletions

File tree

.github/workflows/branch-e2e.yml

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,16 @@ jobs:
111111
with:
112112
image-tag: ${{ github.sha }}
113113

114+
mcp-conformance:
115+
needs: [pr_metadata, build-gateway, build-supervisor]
116+
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
117+
permissions:
118+
contents: read
119+
packages: read
120+
uses: ./.github/workflows/mcp-conformance.yml
121+
with:
122+
image-tag: ${{ github.sha }}
123+
114124
kubernetes-ha-e2e:
115125
needs: [pr_metadata, build-gateway, build-supervisor]
116126
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
@@ -126,7 +136,7 @@ jobs:
126136

127137
core-e2e-result:
128138
name: Core E2E result
129-
needs: [pr_metadata, build-gateway, build-supervisor, e2e, kubernetes-e2e]
139+
needs: [pr_metadata, build-gateway, build-supervisor, e2e, kubernetes-e2e, mcp-conformance]
130140
if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
131141
runs-on: ubuntu-latest
132142
steps:
@@ -136,14 +146,16 @@ jobs:
136146
BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
137147
E2E_RESULT: ${{ needs.e2e.result }}
138148
KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }}
149+
MCP_CONFORMANCE_RESULT: ${{ needs.mcp-conformance.result }}
139150
run: |
140151
set -euo pipefail
141152
failed=0
142153
for item in \
143154
"build-gateway:$BUILD_GATEWAY_RESULT" \
144155
"build-supervisor:$BUILD_SUPERVISOR_RESULT" \
145156
"e2e:$E2E_RESULT" \
146-
"kubernetes-e2e:$KUBERNETES_E2E_RESULT"; do
157+
"kubernetes-e2e:$KUBERNETES_E2E_RESULT" \
158+
"mcp-conformance:$MCP_CONFORMANCE_RESULT"; do
147159
name="${item%%:*}"
148160
result="${item#*:}"
149161
if [ "$result" != "success" ]; then
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: MCP Conformance Test
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
image-tag:
7+
description: "Image tag to test (typically the commit SHA)"
8+
required: true
9+
type: string
10+
runner:
11+
description: "GitHub Actions runner label"
12+
required: false
13+
type: string
14+
default: "linux-amd64-cpu8"
15+
checkout-ref:
16+
description: "Git ref to check out for test inputs (defaults to the workflow SHA)"
17+
required: false
18+
type: string
19+
default: ""
20+
21+
permissions:
22+
contents: read
23+
packages: read
24+
25+
jobs:
26+
mcp-conformance:
27+
name: "MCP Conformance (${{ matrix.scenario }})"
28+
runs-on: ${{ inputs.runner }}
29+
timeout-minutes: 40
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
scenario:
34+
- initialize
35+
- tools-call
36+
container:
37+
image: ghcr.io/nvidia/openshell/ci:latest
38+
credentials:
39+
username: ${{ github.actor }}
40+
password: ${{ secrets.GITHUB_TOKEN }}
41+
options: --privileged
42+
volumes:
43+
- /var/run/docker.sock:/var/run/docker.sock
44+
- /home/runner/_work:/home/runner/_work
45+
env:
46+
MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
47+
IMAGE_TAG: ${{ inputs.image-tag }}
48+
OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
49+
OPENSHELL_REGISTRY_HOST: ghcr.io
50+
OPENSHELL_REGISTRY_NAMESPACE: nvidia/openshell
51+
OPENSHELL_REGISTRY_USERNAME: ${{ github.actor }}
52+
OPENSHELL_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
53+
OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }}
54+
OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE: openshell-mcp-conformance-client:${{ github.sha }}
55+
steps:
56+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
57+
with:
58+
ref: ${{ inputs['checkout-ref'] || github.sha }}
59+
60+
- name: Check out MCP conformance tests
61+
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
62+
with:
63+
repository: modelcontextprotocol/conformance
64+
ref: v0.1.11
65+
path: .cache/mcp-conformance
66+
67+
- name: Log in to GHCR with Docker
68+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
69+
70+
- name: Build OpenShell e2e binaries
71+
run: |
72+
cargo build -p openshell-server --bin openshell-gateway --features openshell-core/dev-settings
73+
cargo build -p openshell-cli --bin openshell --features openshell-core/dev-settings
74+
75+
- name: Build MCP conformance client image
76+
run: docker build --pull -f e2e/mcp-conformance/Dockerfile.client -t "${OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE}" .cache/mcp-conformance
77+
78+
- name: Run MCP conformance through OpenShell
79+
uses: modelcontextprotocol/conformance@v0.1.11
80+
with:
81+
mode: client
82+
scenario: ${{ matrix.scenario }}
83+
command: bash e2e/mcp-conformance/client-through-openshell.sh
84+
expected-failures: e2e/mcp-conformance/expected-failures.yml
85+
timeout: "900000"
86+
node-version: "22"
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM public.ecr.aws/docker/library/node:22-bookworm-slim
5+
6+
RUN apt-get update \
7+
&& apt-get install -y --no-install-recommends ca-certificates iproute2 \
8+
&& rm -rf /var/lib/apt/lists/*
9+
10+
# Match the sandbox user expected by OpenShell policies and supervisor setup.
11+
RUN groupadd -g 1000660000 sandbox \
12+
&& useradd -m -u 1000660000 -g sandbox sandbox
13+
14+
WORKDIR /opt/mcp-conformance
15+
16+
COPY . .
17+
RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi
18+
RUN chown -R sandbox:sandbox /opt/mcp-conformance /home/sandbox
19+
20+
USER sandbox
21+
CMD ["sleep", "infinity"]

e2e/mcp-conformance/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# MCP Conformance E2E
2+
3+
This directory contains the OpenShell wrapper for the upstream
4+
`modelcontextprotocol/conformance` GitHub Action.
5+
6+
The workflow uses conformance client mode. The upstream Action starts a real MCP
7+
test server, then invokes `client-through-openshell.sh` with that server URL.
8+
The wrapper starts the Docker-backed OpenShell e2e gateway and runs the upstream
9+
TypeScript `everything-client` inside an OpenShell sandbox, so the MCP traffic
10+
crosses the sandbox proxy.
11+
12+
The conformance server URL uses `localhost` from the GitHub Actions job
13+
container's perspective. Sandboxes run in separate Docker containers, so the
14+
wrapper rewrites local URLs to `host.openshell.internal`, the alias that
15+
`e2e/with-docker-gateway.sh` attaches to the job container on the e2e Docker
16+
network.
17+
18+
The generated policy allows valid JSON-RPC requests to the conformance server
19+
with `rpc_method: "*"`. That keeps OpenShell deny-by-default at the network
20+
boundary while allowing the upstream scenarios to exercise MCP behavior. The
21+
policy body lives in `policy-template.yaml`; the wrapper renders its host, port,
22+
and path placeholders from the upstream server URL.
23+
24+
When enabling broader upstream suites, add scenarios that OpenShell does not yet
25+
support through the JSON-RPC proxy to `expected-failures.yml`. The upstream
26+
runner treats listed failures as allowed and treats stale entries as failures.
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env bash
2+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
# Runs the upstream MCP conformance client through an OpenShell sandbox.
6+
#
7+
# The modelcontextprotocol/conformance GitHub Action starts a real MCP test
8+
# server in the GitHub Actions job container and invokes this script with that
9+
# server URL. This script starts the normal Docker-backed OpenShell e2e gateway,
10+
# creates a sandbox from the prebuilt conformance client image, and runs the
11+
# upstream TypeScript everything-client inside that sandbox. That keeps the MCP
12+
# client/server traffic in the OpenShell proxy data path.
13+
#
14+
# Conformance server URLs usually point at localhost in the job container.
15+
# Sandboxes are separate Docker containers, so localhost would point back at the
16+
# sandbox itself. The wrapper rewrites local URLs to host.openshell.internal,
17+
# which e2e/with-docker-gateway.sh attaches to the job container on the e2e
18+
# Docker network.
19+
20+
set -euo pipefail
21+
22+
usage() {
23+
echo "usage: $0 <conformance-server-url>" >&2
24+
}
25+
26+
if [ "$#" -ne 1 ]; then
27+
usage
28+
exit 2
29+
fi
30+
31+
# Parse the conformance runner's server URL and render the OpenShell policy.
32+
prepare_conformance_target() {
33+
local server_url=$1
34+
local policy_file=$2
35+
local policy_template=$3
36+
37+
python3 - "${server_url}" "${policy_file}" "${policy_template}" <<'PY'
38+
import json
39+
import string
40+
import sys
41+
from pathlib import Path
42+
from urllib.parse import urlparse, urlunparse
43+
44+
raw_url, policy_file, policy_template = sys.argv[1:4]
45+
parsed = urlparse(raw_url)
46+
47+
if parsed.scheme not in ("http", "https"):
48+
raise SystemExit(f"unsupported conformance server URL scheme: {parsed.scheme!r}")
49+
50+
host = parsed.hostname
51+
if not host:
52+
raise SystemExit(f"conformance server URL is missing a host: {raw_url}")
53+
54+
target_host = "host.openshell.internal" if host in {"localhost", "127.0.0.1", "::1"} else host
55+
port = parsed.port or (443 if parsed.scheme == "https" else 80)
56+
path = parsed.path or "/"
57+
netloc_host = f"[{target_host}]" if ":" in target_host and not target_host.startswith("[") else target_host
58+
netloc = f"{netloc_host}:{port}"
59+
rewritten = urlunparse((parsed.scheme, netloc, path, parsed.params, parsed.query, parsed.fragment))
60+
61+
template = string.Template(Path(policy_template).read_text(encoding="utf-8"))
62+
policy = template.substitute(
63+
host=json.dumps(target_host),
64+
port=str(port),
65+
path=json.dumps(path),
66+
)
67+
Path(policy_file).write_text(policy, encoding="utf-8")
68+
69+
print(rewritten)
70+
PY
71+
}
72+
73+
SERVER_URL="$1"
74+
CLIENT_IMAGE="${OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE:?set OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE to the prebuilt conformance client image}"
75+
ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
76+
POLICY_TEMPLATE="${ROOT}/e2e/mcp-conformance/policy-template.yaml"
77+
78+
POLICY_FILE="$(mktemp "${TMPDIR:-/tmp}/openshell-mcp-conformance-policy.XXXXXX.yaml")"
79+
trap 'rm -f "${POLICY_FILE}"' EXIT
80+
81+
CLIENT_SERVER_URL="$(prepare_conformance_target "${SERVER_URL}" "${POLICY_FILE}" "${POLICY_TEMPLATE}")"
82+
83+
ENV_ARGS=()
84+
# These environment variables are set by the upstream conformance test runner
85+
# before it invokes the configured client command. Forward them into the
86+
# sandbox because the sandboxed TypeScript client depends on them to select the
87+
# scenario and read scenario-specific context.
88+
for NAME in MCP_CONFORMANCE_SCENARIO MCP_CONFORMANCE_CONTEXT MCP_CONFORMANCE_PROTOCOL_VERSION; do
89+
if [ -n "${!NAME+x}" ]; then
90+
ENV_ARGS+=(--env "${NAME}=${!NAME}")
91+
fi
92+
done
93+
94+
# shellcheck source=e2e/support/gateway-common.sh disable=SC1091
95+
source "${ROOT}/e2e/support/gateway-common.sh"
96+
TARGET_DIR="$(e2e_cargo_target_dir "${ROOT}")"
97+
OPENSHELL_BIN="${OPENSHELL_BIN:-${TARGET_DIR}/debug/openshell}"
98+
export OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE:-${CLIENT_IMAGE}}"
99+
100+
# shellcheck disable=SC2016
101+
"${ROOT}/e2e/with-docker-gateway.sh" \
102+
"${OPENSHELL_BIN}" sandbox create \
103+
--from "${CLIENT_IMAGE}" \
104+
--policy "${POLICY_FILE}" \
105+
"${ENV_ARGS[@]}" \
106+
-- \
107+
sh -c 'cd /opt/mcp-conformance && exec ./node_modules/.bin/tsx examples/clients/typescript/everything-client.ts "$1"' \
108+
sh "${CLIENT_SERVER_URL}"
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# Add client scenarios here when enabling broader MCP conformance suites that
5+
# exercise features OpenShell does not yet support through the JSON-RPC proxy.
6+
client: []
7+
server: []
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
version: 1
5+
6+
filesystem_policy:
7+
include_workdir: true
8+
read_only:
9+
- /bin
10+
- /usr
11+
- /lib
12+
- /lib64
13+
- /proc
14+
- /sys
15+
- /dev/urandom
16+
- /etc
17+
- /opt
18+
- /var/log
19+
read_write:
20+
- /sandbox
21+
- /tmp
22+
- /dev/null
23+
- /home/sandbox
24+
25+
landlock:
26+
compatibility: best_effort
27+
28+
process:
29+
run_as_user: sandbox
30+
run_as_group: sandbox
31+
32+
network_policies:
33+
mcp_conformance:
34+
name: mcp_conformance
35+
endpoints:
36+
- host: ${host}
37+
port: ${port}
38+
path: ${path}
39+
protocol: json-rpc
40+
enforcement: enforce
41+
allowed_ips:
42+
- "10.0.0.0/8"
43+
- "172.0.0.0/8"
44+
- "192.168.0.0/16"
45+
- "fc00::/7"
46+
json_rpc:
47+
max_body_bytes: 131072
48+
rules:
49+
- allow:
50+
rpc_method: "*"
51+
binaries:
52+
- path: /bin/sh
53+
- path: /usr/bin/env
54+
- path: /usr/local/bin/node
55+
- path: /usr/bin/node
56+
- path: /opt/mcp-conformance/node_modules/.bin/*

e2e/rust/tests/forward_proxy_jsonrpc_l7.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,6 @@ network_policies:
102102
- "fc00::/7"
103103
json_rpc:
104104
max_body_bytes: 65536
105-
on_parse_error: deny
106-
batch_policy: deny_if_any_denied
107105
rules:
108106
- allow:
109107
rpc_method: initialize
@@ -318,7 +316,7 @@ results = {{
318316
{{"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {{"name": "blocked_action"}}}},
319317
]),
320318
321-
# forward proxy — invalid JSON body denied by on_parse_error: deny
319+
# forward proxy — invalid JSON body fails closed before generic rules apply
322320
"forward_invalid_json_denied": post_invalid_json(),
323321
324322
# CONNECT path — representative allowed and denied cases

0 commit comments

Comments
 (0)