test(conformance): pin conformance 0.2.0-alpha.1 and baseline the draft-spec suites (#2227)

Author: felixweinberger

.github/workflows/conformance.yml

@@ -16,7 +16,6 @@ permissions:
 jobs:
     client-conformance:
         runs-on: ubuntu-latest
-        continue-on-error: true
         steps:
             - uses: actions/checkout@v6
             - name: Install pnpm
@@ -34,7 +33,6 @@ jobs:
 
     server-conformance:
         runs-on: ubuntu-latest
-        continue-on-error: true
         steps:
             - uses: actions/checkout@v6
             - name: Install pnpm
@@ -49,3 +47,4 @@ jobs:
             - run: pnpm install
             - run: pnpm run build:all
             - run: pnpm run test:conformance:server
+            - run: pnpm run test:conformance:server:draft

package.json

@@ -40,6 +40,7 @@
         "test:conformance:client:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:all",
         "test:conformance:client:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:run",
         "test:conformance:server": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server",
+        "test:conformance:server:draft": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft",
         "test:conformance:server:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:all",
         "test:conformance:server:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:run",
         "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"

pnpm-lock.yaml

@@ -2,6 +2,8 @@
 
 This directory contains conformance test implementations for the TypeScript MCP SDK.
 
+> `@modelcontextprotocol/conformance` is pinned to an exact version (no `^` range) in `package.json`. New conformance releases are adopted by deliberately bumping the pin and updating `expected-failures.yaml` for any new scenarios/checks in the same change.
+
 ## Client Conformance Tests
 
 Tests the SDK's client implementation against a conformance test server.

test/conformance/README.md

@@ -1,4 +1,93 @@
 # Conformance scenarios not yet implemented in the v2 TypeScript SDK.
 # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
+#
+# Baseline established against @modelcontextprotocol/conformance 0.2.0-alpha.1,
+# which adds the draft-spec scenario suite (SEP-2575, SEP-2322, SEP-2243, SEP-2549,
+# SEP-2468, ...) plus new checks on existing scenarios (SEP-837 application_type
+# during DCR).
+#
+# Entries are grouped by SEP. As each SEP/milestone is implemented in the SDK the
+# corresponding scenarios start passing and MUST be removed from this list (the
+# runner fails on stale entries), so the baseline burns down per milestone.
 
-client: []
+client:
+    # --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) ---
+    # SEP-2575 (request metadata / _meta envelope): client does not populate the
+    # _meta envelope or the MCP-Protocol-Version header semantics yet.
+    - request-metadata
+    # SEP-2322 (multi-round-trip requests): client does not echo requestState /
+    # handle IncompleteResult yet.
+    - sep-2322-client-request-state
+    # SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
+    - http-custom-headers
+    - http-invalid-tool-headers
+    # SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs.
+    - json-schema-ref-no-deref
+    # SEP-2468 (authorization response iss parameter): not implemented in the client.
+    - auth/iss-supported
+    - auth/iss-not-advertised
+    - auth/iss-supported-missing
+    - auth/iss-wrong-issuer
+    - auth/iss-unexpected
+    - auth/iss-normalized
+    - auth/metadata-issuer-mismatch
+    # SEP-2352 (authorization server migration): client does not re-register when
+    # PRM authorization_servers changes.
+    - auth/authorization-server-migration
+    # SEP-2207 (offline_access scope) scenario, currently failing only on the new
+    # SEP-837 application_type check (see the SEP-837 group below).
+    - auth/offline-access-not-supported
+
+    # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 ---
+    # SEP-837: client MUST send application_type during Dynamic Client Registration.
+    # Single new check; everything else in these scenarios passes.
+    - auth/metadata-default
+    - auth/metadata-var1
+    - auth/metadata-var2
+    - auth/metadata-var3
+    - auth/scope-from-www-authenticate
+    - auth/scope-from-scopes-supported
+    - auth/scope-omitted-when-undefined
+    - auth/scope-step-up
+    - auth/scope-retry-limit
+    - auth/token-endpoint-auth-basic
+    - auth/token-endpoint-auth-post
+    - auth/token-endpoint-auth-none
+    - auth/2025-03-26-oauth-metadata-backcompat
+    # SEP-990 (enterprise-managed authorization extension): no fixture handler /
+    # client support for the token-exchange + JWT bearer flow.
+    - auth/enterprise-managed-authorization
+
+server:
+    # --- Draft-spec scenarios (in `--suite draft`; the default `active` suite is green) ---
+    # SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode,
+    # _meta-derived capabilities, error-code mappings, or server/discover yet.
+    - server-stateless
+    # SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented;
+    # most scenarios currently fail early with "Session ID required" because the
+    # fixture only runs in stateful mode.
+    - input-required-result-basic-elicitation
+    - input-required-result-basic-sampling
+    - input-required-result-basic-list-roots
+    - input-required-result-request-state
+    - input-required-result-multiple-input-requests
+    - input-required-result-multi-round
+    - input-required-result-non-tool-request
+    - input-required-result-result-type
+    - input-required-result-tampered-state
+    - input-required-result-capability-check
+    # SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the
+    # stateful-mode "Session ID required" error.
+    - caching
+    # SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and
+    # case-insensitive/whitespace-trimmed header validation not implemented.
+    - http-header-validation
+    - http-custom-header-server-validation
+    # WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level
+    # WARNINGs, but the expected-failures evaluator counts WARNINGs as failures.
+    # SEP-2164: server returns -32002 without the requested URI in error.data.
+    - sep-2164-resource-not-found
+    # SEP-2322 SHOULD-level behaviours (re-request missing inputResponses, ignore
+    # unrecognized inputResponses keys).
+    - input-required-result-missing-input-response
+    - input-required-result-ignore-extra-params

test/conformance/expected-failures.yaml

@@ -32,12 +32,13 @@
         "test:conformance:client:all": "conformance client --command 'npx tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml",
         "test:conformance:client:run": "npx tsx ./src/everythingClient.ts",
         "test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml",
+        "test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml",
         "test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml",
         "test:conformance:server:run": "npx tsx ./src/everythingServer.ts",
         "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"
     },
     "devDependencies": {
-        "@modelcontextprotocol/conformance": "0.1.15",
+        "@modelcontextprotocol/conformance": "0.2.0-alpha.1",
         "@modelcontextprotocol/client": "workspace:^",
         "@modelcontextprotocol/server": "workspace:^",
         "@modelcontextprotocol/core": "workspace:^",