conformance: prepare the conformance test suite

.github/workflows/conformance.yml

@@ -0,0 +1,39 @@
+name: Conformance Tests
+
+on:
+  # TODO: enable automatic triggering when tested.
+  # push:
+  #     branches: [main]
+  # pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: conformance-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  server-conformance:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Check out code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
+        with:
+          go-version: "^1.25"
+      - name: Start everything-server
+        run: |
+          go run ./conformance/everything-server/main.go -http=":3001" &
+          # Wait for the server to be ready.
+          timeout 15 bash -c 'until curl -s http://localhost:3001/mcp; do sleep 0.5; done'
+      - name: "Run conformance tests" 
+        uses: modelcontextprotocol/conformance@c2f3fdaf781dcd5a862cb0d2f6454c1c210bf0f0 # v0.1.11
+        with:
+          mode: server
+          url: http://localhost:3001/mcp
+          suite: active
+          expected-failures: ./conformance/baseline.yml

conformance/baseline.yml

@@ -0,0 +1,4 @@
+server:
+  # All tests should pass.
+client:
+  # Tests are expected to fail. List will be populated after running the tests.

scripts/conformance.sh

@@ -50,11 +50,9 @@ done
 
 cleanup() {
     if [ -n "$SERVER_PID" ]; then
+        echo "Stopping server..."
         kill "$SERVER_PID" 2>/dev/null || true
-    fi
-    # Clean up the work directory unless --result_dir was specified.
-    if [ -z "$RESULT_DIR" ] && [ -n "$WORKDIR" ]; then
-        rm -rf "$WORKDIR"
+        wait "$SERVER_PID" 2>/dev/null || true
     fi
 }
 trap cleanup EXIT
@@ -68,7 +66,7 @@ else
 fi
 
 # Build the conformance server.
-go build -o "$WORKDIR/conformance-server" ./examples/server/conformance
+go build -o "$WORKDIR/conformance-server" ./conformance/everything-server
 
 # Start the server in the background
 echo "Starting conformance server on port $PORT..."
@@ -79,28 +77,24 @@ echo "Server pid is $SERVER_PID"
 
 # Wait for server to be ready
 echo "Waiting for server to be ready..."
-for i in {1..30}; do
-    if curl -s "http://localhost:$PORT" > /dev/null 2>&1; then
-        echo "Server is ready."
-        break
-    fi
-    if [ "$i" -eq 30 ]; then
-        echo "Server failed to start within 15 seconds."
-        exit 1
-    fi
-    sleep 0.5
-done
+if ! timeout 15 bash -c "until curl -s http://localhost:$PORT > /dev/null 2>&1; do sleep 0.5; done"; then
+    echo "Server failed to start within 15 seconds."
+    exit 1
+fi
 
 # Run conformance tests from the work directory to avoid writing results to the repo.
 echo "Running conformance tests..."
 if [ -n "$CONFORMANCE_REPO" ]; then
     # Run from local checkout using npm run start.
     (cd "$WORKDIR" && \
         npm --prefix "$CONFORMANCE_REPO" run start -- \
-            server --url "http://localhost:$PORT")
+            server --url "http://localhost:$PORT" \
+            ${RESULT_DIR:+--output-dir "$RESULT_DIR"})
 else
     (cd "$WORKDIR" && \
-        npx @modelcontextprotocol/conformance@latest server --url "http://localhost:$PORT")
+        npx @modelcontextprotocol/conformance@latest \
+        server --url "http://localhost:$PORT" \
+        ${RESULT_DIR:+--output-dir "$RESULT_DIR"})
 fi
 
 echo ""