|
1 | 1 | # Conformance scenarios not yet implemented in the v2 TypeScript SDK. |
2 | 2 | # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. |
| 3 | +# |
| 4 | +# Baseline established against @modelcontextprotocol/conformance 0.2.0-alpha.1, |
| 5 | +# which adds the draft-spec scenario suite (SEP-2575, SEP-2322, SEP-2243, SEP-2549, |
| 6 | +# SEP-2468, ...) plus new checks on existing scenarios (SEP-837 application_type |
| 7 | +# during DCR). |
| 8 | +# |
| 9 | +# Entries are grouped by SEP. As each SEP/milestone is implemented in the SDK the |
| 10 | +# corresponding scenarios start passing and MUST be removed from this list (the |
| 11 | +# runner fails on stale entries), so the baseline burns down per milestone. |
3 | 12 |
|
4 | | -client: [] |
| 13 | +client: |
| 14 | + # --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) --- |
| 15 | + # SEP-2575 (request metadata / _meta envelope): client does not populate the |
| 16 | + # _meta envelope or the MCP-Protocol-Version header semantics yet. |
| 17 | + - request-metadata |
| 18 | + # SEP-2322 (multi-round-trip requests): client does not echo requestState / |
| 19 | + # handle IncompleteResult yet. |
| 20 | + - sep-2322-client-request-state |
| 21 | + # SEP-2243 (HTTP standardization): no fixture handler / client header support yet. |
| 22 | + - http-custom-headers |
| 23 | + - http-invalid-tool-headers |
| 24 | + # SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs. |
| 25 | + - json-schema-ref-no-deref |
| 26 | + # SEP-2468 (authorization response iss parameter): not implemented in the client. |
| 27 | + - auth/iss-supported |
| 28 | + - auth/iss-not-advertised |
| 29 | + - auth/iss-supported-missing |
| 30 | + - auth/iss-wrong-issuer |
| 31 | + - auth/iss-unexpected |
| 32 | + - auth/iss-normalized |
| 33 | + - auth/metadata-issuer-mismatch |
| 34 | + # SEP-2352 (authorization server migration): client does not re-register when |
| 35 | + # PRM authorization_servers changes. |
| 36 | + - auth/authorization-server-migration |
| 37 | + # SEP-2207 (offline_access scope) scenario, currently failing only on the new |
| 38 | + # SEP-837 application_type check (see the SEP-837 group below). |
| 39 | + - auth/offline-access-not-supported |
| 40 | + |
| 41 | + # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 --- |
| 42 | + # SEP-837: client MUST send application_type during Dynamic Client Registration. |
| 43 | + # Single new check; everything else in these scenarios passes. |
| 44 | + - auth/metadata-default |
| 45 | + - auth/metadata-var1 |
| 46 | + - auth/metadata-var2 |
| 47 | + - auth/metadata-var3 |
| 48 | + - auth/scope-from-www-authenticate |
| 49 | + - auth/scope-from-scopes-supported |
| 50 | + - auth/scope-omitted-when-undefined |
| 51 | + - auth/scope-step-up |
| 52 | + - auth/scope-retry-limit |
| 53 | + - auth/token-endpoint-auth-basic |
| 54 | + - auth/token-endpoint-auth-post |
| 55 | + - auth/token-endpoint-auth-none |
| 56 | + - auth/2025-03-26-oauth-metadata-backcompat |
| 57 | + # SEP-990 (enterprise-managed authorization extension): no fixture handler / |
| 58 | + # client support for the token-exchange + JWT bearer flow. |
| 59 | + - auth/enterprise-managed-authorization |
| 60 | + |
| 61 | +server: |
| 62 | + # --- Draft-spec scenarios (in `--suite draft`; the default `active` suite is green) --- |
| 63 | + # SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode, |
| 64 | + # _meta-derived capabilities, error-code mappings, or server/discover yet. |
| 65 | + - server-stateless |
| 66 | + # SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented; |
| 67 | + # most scenarios currently fail early with "Session ID required" because the |
| 68 | + # fixture only runs in stateful mode. |
| 69 | + - input-required-result-basic-elicitation |
| 70 | + - input-required-result-basic-sampling |
| 71 | + - input-required-result-basic-list-roots |
| 72 | + - input-required-result-request-state |
| 73 | + - input-required-result-multiple-input-requests |
| 74 | + - input-required-result-multi-round |
| 75 | + - input-required-result-non-tool-request |
| 76 | + - input-required-result-result-type |
| 77 | + - input-required-result-tampered-state |
| 78 | + - input-required-result-capability-check |
| 79 | + # SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the |
| 80 | + # stateful-mode "Session ID required" error. |
| 81 | + - caching |
| 82 | + # SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and |
| 83 | + # case-insensitive/whitespace-trimmed header validation not implemented. |
| 84 | + - http-header-validation |
| 85 | + - http-custom-header-server-validation |
| 86 | + # WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level |
| 87 | + # WARNINGs, but the expected-failures evaluator counts WARNINGs as failures. |
| 88 | + # SEP-2164: server returns -32002 without the requested URI in error.data. |
| 89 | + - sep-2164-resource-not-found |
| 90 | + # SEP-2322 SHOULD-level behaviours (re-request missing inputResponses, ignore |
| 91 | + # unrecognized inputResponses keys). |
| 92 | + - input-required-result-missing-input-response |
| 93 | + - input-required-result-ignore-extra-params |
0 commit comments