diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..021773a6 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,39 @@ +name: Conformance Tests + +on: + # TODO: enable automatic triggering when tested. + # push: + # branches: [main] + # pull_request: + workflow_dispatch: + +concurrency: + group: conformance-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + server-conformance: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Check out code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5 + with: + go-version: "^1.25" + - name: Start everything-server + run: | + go run ./conformance/everything-server/main.go -http=":3001" & + # Wait for the server to be ready. + timeout 15 bash -c 'until curl -s http://localhost:3001/mcp; do sleep 0.5; done' + - name: "Run conformance tests" + uses: modelcontextprotocol/conformance@c2f3fdaf781dcd5a862cb0d2f6454c1c210bf0f0 # v0.1.11 + with: + mode: server + url: http://localhost:3001/mcp + suite: active + expected-failures: ./conformance/baseline.yml diff --git a/conformance/baseline.yml b/conformance/baseline.yml new file mode 100644 index 00000000..fd4e80a1 --- /dev/null +++ b/conformance/baseline.yml @@ -0,0 +1,4 @@ +server: + # All tests should pass. +client: + # Tests are expected to fail. List will be populated after running the tests. diff --git a/examples/server/conformance/main.go b/conformance/everything-server/main.go similarity index 100% rename from examples/server/conformance/main.go rename to conformance/everything-server/main.go diff --git a/scripts/conformance.sh b/scripts/conformance.sh index 9e7d3007..aa43b843 100755 --- a/scripts/conformance.sh +++ b/scripts/conformance.sh @@ -50,11 +50,9 @@ done cleanup() { if [ -n "$SERVER_PID" ]; then + echo "Stopping server..." kill "$SERVER_PID" 2>/dev/null || true - fi - # Clean up the work directory unless --result_dir was specified. - if [ -z "$RESULT_DIR" ] && [ -n "$WORKDIR" ]; then - rm -rf "$WORKDIR" + wait "$SERVER_PID" 2>/dev/null || true fi } trap cleanup EXIT @@ -68,7 +66,7 @@ else fi # Build the conformance server. -go build -o "$WORKDIR/conformance-server" ./examples/server/conformance +go build -o "$WORKDIR/conformance-server" ./conformance/everything-server # Start the server in the background echo "Starting conformance server on port $PORT..." @@ -79,17 +77,10 @@ echo "Server pid is $SERVER_PID" # Wait for server to be ready echo "Waiting for server to be ready..." -for i in {1..30}; do - if curl -s "http://localhost:$PORT" > /dev/null 2>&1; then - echo "Server is ready." - break - fi - if [ "$i" -eq 30 ]; then - echo "Server failed to start within 15 seconds." - exit 1 - fi - sleep 0.5 -done +if ! timeout 15 bash -c "until curl -s http://localhost:$PORT > /dev/null 2>&1; do sleep 0.5; done"; then + echo "Server failed to start within 15 seconds." + exit 1 +fi # Run conformance tests from the work directory to avoid writing results to the repo. echo "Running conformance tests..." @@ -97,10 +88,13 @@ if [ -n "$CONFORMANCE_REPO" ]; then # Run from local checkout using npm run start. (cd "$WORKDIR" && \ npm --prefix "$CONFORMANCE_REPO" run start -- \ - server --url "http://localhost:$PORT") + server --url "http://localhost:$PORT" \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) else (cd "$WORKDIR" && \ - npx @modelcontextprotocol/conformance@latest server --url "http://localhost:$PORT") + npx @modelcontextprotocol/conformance@latest \ + server --url "http://localhost:$PORT" \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) fi echo ""