From dff6e8424526857c6cf411b426cf30fd4d998570 Mon Sep 17 00:00:00 2001
From: Maciek Kisiel <mkisiel@google.com>
Date: Mon, 26 Jan 2026 15:59:59 +0000
Subject: [PATCH] conformance: prepare the conformance test suite

- move the conformance related files to a dedicated directory
- simplify parts of the conformance script
- create a scaffolding for a new GitHub action to run conformance tests
- use the new --output-dir flag from the conformance suite for saving
  results on demand
---
 .github/workflows/conformance.yml             | 39 +++++++++++++++++++
 conformance/baseline.yml                      |  4 ++
 .../everything-server}/main.go                |  0
 scripts/conformance.sh                        | 30 ++++++--------
 4 files changed, 55 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/conformance.yml
 create mode 100644 conformance/baseline.yml
 rename {examples/server/conformance => conformance/everything-server}/main.go (100%)

diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
new file mode 100644
index 00000000..021773a6
--- /dev/null
+++ b/.github/workflows/conformance.yml
@@ -0,0 +1,39 @@
+name: Conformance Tests
+
+on:
+  # TODO: enable automatic triggering when tested.
+  # push:
+  #     branches: [main]
+  # pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: conformance-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  server-conformance:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Check out code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
+        with:
+          go-version: "^1.25"
+      - name: Start everything-server
+        run: |
+          go run ./conformance/everything-server/main.go -http=":3001" &
+          # Wait for the server to be ready.
+          timeout 15 bash -c 'until curl -s http://localhost:3001/mcp; do sleep 0.5; done'
+      - name: "Run conformance tests" 
+        uses: modelcontextprotocol/conformance@c2f3fdaf781dcd5a862cb0d2f6454c1c210bf0f0 # v0.1.11
+        with:
+          mode: server
+          url: http://localhost:3001/mcp
+          suite: active
+          expected-failures: ./conformance/baseline.yml
diff --git a/conformance/baseline.yml b/conformance/baseline.yml
new file mode 100644
index 00000000..fd4e80a1
--- /dev/null
+++ b/conformance/baseline.yml
@@ -0,0 +1,4 @@
+server:
+  # All tests should pass.
+client:
+  # Tests are expected to fail. List will be populated after running the tests.
diff --git a/examples/server/conformance/main.go b/conformance/everything-server/main.go
similarity index 100%
rename from examples/server/conformance/main.go
rename to conformance/everything-server/main.go
diff --git a/scripts/conformance.sh b/scripts/conformance.sh
index 9e7d3007..aa43b843 100755
--- a/scripts/conformance.sh
+++ b/scripts/conformance.sh
@@ -50,11 +50,9 @@ done
 
 cleanup() {
     if [ -n "$SERVER_PID" ]; then
+        echo "Stopping server..."
         kill "$SERVER_PID" 2>/dev/null || true
-    fi
-    # Clean up the work directory unless --result_dir was specified.
-    if [ -z "$RESULT_DIR" ] && [ -n "$WORKDIR" ]; then
-        rm -rf "$WORKDIR"
+        wait "$SERVER_PID" 2>/dev/null || true
     fi
 }
 trap cleanup EXIT
@@ -68,7 +66,7 @@ else
 fi
 
 # Build the conformance server.
-go build -o "$WORKDIR/conformance-server" ./examples/server/conformance
+go build -o "$WORKDIR/conformance-server" ./conformance/everything-server
 
 # Start the server in the background
 echo "Starting conformance server on port $PORT..."
@@ -79,17 +77,10 @@ echo "Server pid is $SERVER_PID"
 
 # Wait for server to be ready
 echo "Waiting for server to be ready..."
-for i in {1..30}; do
-    if curl -s "http://localhost:$PORT" > /dev/null 2>&1; then
-        echo "Server is ready."
-        break
-    fi
-    if [ "$i" -eq 30 ]; then
-        echo "Server failed to start within 15 seconds."
-        exit 1
-    fi
-    sleep 0.5
-done
+if ! timeout 15 bash -c "until curl -s http://localhost:$PORT > /dev/null 2>&1; do sleep 0.5; done"; then
+    echo "Server failed to start within 15 seconds."
+    exit 1
+fi
 
 # Run conformance tests from the work directory to avoid writing results to the repo.
 echo "Running conformance tests..."
@@ -97,10 +88,13 @@ if [ -n "$CONFORMANCE_REPO" ]; then
     # Run from local checkout using npm run start.
     (cd "$WORKDIR" && \
         npm --prefix "$CONFORMANCE_REPO" run start -- \
-            server --url "http://localhost:$PORT")
+            server --url "http://localhost:$PORT" \
+            ${RESULT_DIR:+--output-dir "$RESULT_DIR"})
 else
     (cd "$WORKDIR" && \
-        npx @modelcontextprotocol/conformance@latest server --url "http://localhost:$PORT")
+        npx @modelcontextprotocol/conformance@latest \
+        server --url "http://localhost:$PORT" \
+        ${RESULT_DIR:+--output-dir "$RESULT_DIR"})
 fi
 
 echo ""