From 27255c49446ecb8ff184ccf5d2e850324135f3e6 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 4 Sep 2026 13:28:27 +0100 Subject: [PATCH 1/3] ci: isolate dataplane conformance Signed-off-by: lucarlig --- .github/workflows/conformance.yml | 4 ++-- Makefile | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9821850..02edce0 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -62,8 +62,8 @@ jobs: with: tool: cargo-binstall - - name: Install cf-integration 0.2.1 binary - run: cargo binstall cf-integration@0.2.1 --no-confirm + - name: Install cf-integration 0.3.1 binary + run: cargo binstall cf-integration@0.3.1 --no-confirm - name: Prepare the exact data-plane image env: diff --git a/Makefile b/Makefile index 240dc62..24a8251 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,8 @@ conformance: ## Run strict modern MCP conformance against the committed data-pla "$(CF_INTEGRATION)" conformance run \ --client-era modern \ --server-era modern \ - --lane external-data-plane \ + --lane external \ + --standalone \ --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ --output-dir "$(CF_INTEGRATION_DIR)/reports" @@ -63,7 +64,8 @@ conformance-bless: ## Run strict modern conformance and atomically update its ba "$(CF_INTEGRATION)" conformance run \ --client-era modern \ --server-era modern \ - --lane external-data-plane \ + --lane external \ + --standalone \ --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ --output-dir "$(CF_INTEGRATION_DIR)/reports" \ --bless From 754a7136a0b60d0dda8936d36abb2c676b9824c5 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 4 Sep 2026 14:41:40 +0100 Subject: [PATCH 2/3] test: baseline standalone dataplane conformance Signed-off-by: lucarlig --- .../2026-07-28/modern/external-data-plane.yml | 158 +++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml index 2cb8db3..00d45c5 100644 --- a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml @@ -1 +1,157 @@ -findings: [] +findings: +- scenario: caching + check: sep-2549-cache-scope-valid + name: CacheScopeValid + status: FAILURE +- scenario: caching + check: sep-2549-prompts-list-caching-hints + name: PromptsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-list-caching-hints + name: ResourcesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-resources-templates-list-caching-hints + name: ResourcesTemplatesListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-tools-list-caching-hints + name: ToolsListCachingHints + status: FAILURE +- scenario: caching + check: sep-2549-ttl-non-negative + name: TtlNonNegative + status: FAILURE +- scenario: completion-complete + check: completion-complete + name: CompletionComplete + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-custom-setup + name: HttpCustomHeaderServerValidationSetup + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-decode-base64 + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-invalid-param-chars + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-reject-param-mismatch + name: NotObserved + status: FAILURE +- scenario: http-custom-header-server-validation + check: sep-2243-server-validate-param-match + name: NotObserved + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-accepts-whitespace-header-value + name: ServerAcceptsWhitespaceHeaderValue + status: FAILURE +- scenario: http-header-validation + check: sep-2243-server-standard-setup + name: HttpHeaderValidationSetup + status: FAILURE +- scenario: input-required-result-basic-elicitation + check: sep-2322-elicitation-incomplete + name: InputRequiredResultElicitationIncomplete + status: FAILURE +- scenario: input-required-result-basic-list-roots + check: sep-2322-list-roots-incomplete + name: InputRequiredResultListRootsIncomplete + status: FAILURE +- scenario: input-required-result-basic-sampling + check: sep-2322-sampling-incomplete + name: InputRequiredResultSamplingIncomplete + status: FAILURE +- scenario: input-required-result-capability-check + check: sep-2322-respect-client-capabilities + name: RespectClientCapabilities + status: FAILURE +- scenario: input-required-result-ignore-extra-params + check: sep-2322-ignore-unexpected-params + name: IgnoreUnexpectedParams + status: WARNING +- scenario: input-required-result-missing-input-response + check: sep-2322-missing-response-rerequests + name: InputRequiredResultMissingResponseRerequests + status: WARNING +- scenario: input-required-result-multi-round + check: sep-2322-multi-round-r1 + name: InputRequiredResultMultiRoundR1 + status: FAILURE +- scenario: input-required-result-multiple-input-requests + check: sep-2322-multiple-inputs-incomplete + name: InputRequiredResultMultipleInputsIncomplete + status: FAILURE +- scenario: input-required-result-non-tool-request + check: sep-2322-non-tool-incomplete + name: InputRequiredResultNonToolIncomplete + status: FAILURE +- scenario: input-required-result-request-state + check: sep-2322-request-state-incomplete + name: InputRequiredResultRequestStateIncomplete + status: FAILURE +- scenario: input-required-result-result-type + check: sep-2322-result-type-included + name: ResultTypeIncluded + status: FAILURE +- scenario: input-required-result-tampered-state + check: sep-2322-reject-tampered-state + name: RejectTamperedState + status: FAILURE +- scenario: json-schema-2020-12 + check: json-schema-2020-12-error + name: JsonSchema2020_12Error + status: FAILURE +- scenario: prompts-list + check: prompts-list + name: PromptsList + status: FAILURE +- scenario: resources-list + check: resources-list + name: ResourcesList + status: FAILURE +- scenario: resources-templates-read + check: resources-templates-read + name: ResourcesTemplateRead + status: FAILURE +- scenario: sep-2164-resource-not-found + check: sep-2164-data-uri + name: ResourcesNotFoundDataUri + status: WARNING +- scenario: server-stateless + check: sep-2575-discover-capabilities-match-handlers + name: DiscoverCapabilitiesMatchHandlers + status: FAILURE +- scenario: server-stateless + check: sep-2575-http-server-no-independent-requests-on-stream + name: HttpServerNoIndependentRequestsOnStream + status: FAILURE +- scenario: server-stateless + check: sep-2575-missing-capability-http-400 + name: MissingCapabilityHttp400 + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-declares-prompts-in-discover + name: ServerDeclaresPromptsInDiscover + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-no-log-without-loglevel + name: ServerNoLogWithoutLogLevel + status: FAILURE +- scenario: server-stateless + check: sep-2575-server-rejects-undeclared-capability + name: ServerRejectsUndeclaredCapability + status: FAILURE +- scenario: tools-call-with-progress + check: tools-call-with-progress + name: ToolsCallWithProgress + status: FAILURE +- scenario: tools-list + check: tools-list + name: ToolsList + status: FAILURE From 4721e7be8a962195f9d3918f3b0597e35c933e6c Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 4 Sep 2026 21:26:52 +0100 Subject: [PATCH 3/3] test: refresh conformance baseline from discovered fixture Signed-off-by: lucarlig --- _context/wiki/testing.md | 21 ++++++++++++++++--- .../2026-07-28/modern/external-data-plane.yml | 16 -------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index d3cc32b..87f68ed 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -76,11 +76,15 @@ Comment exactly `/conformance` on a pull request to run the **Conformance** Actions workflow. Only repository owners, members, and collaborators can start it. The workflow acknowledges the command, tests the pull request merge commit, and reports the final result back to the pull request. It runs the modern client -and modern server eras through the external dataplane. Selecting that lane also -runs the fixture-direct server leg and the scoped external-dataplane client leg: +and modern server eras through the external dataplane in standalone mode. This +starts Redis, the dataplane, nginx, and the official fixture without the control +plane. The harness discovers the fixture's tools, resources, templates, and +prompts and publishes their routes and actual tool schemas through the +dataplane serializer. Selecting that lane also runs the fixture-direct server +leg and the scoped external-dataplane client leg: ```bash -cargo binstall cf-integration@0.1.0 --no-confirm +cargo binstall cf-integration@0.3.1 --no-confirm make conformance ``` @@ -106,3 +110,14 @@ use `fixture-direct.yml` and `external-data-plane.yml`; scoped client findings use `client/external-data-plane.yml`. Operational failures are always failures and cannot be blessed. Runtime checkouts, logs, results, and reports remain beneath `.integration/`. + +A strict baseline match means the observed findings have not changed; it does +not mean complete protocol coverage. Inspect the report's individual checks +before blessing changes. In particular, the external dataplane deliberately +rejects catalog list methods owned by the control plane. Scenarios that need +`tools/list` for setup cannot exercise the subsequent schema or parameter-header +checks through this lane; `NotObserved` findings record missing coverage, not +successful validation. The pinned fixture also lacks an `x-mcp-header`-annotated +tool for the custom-header server scenario. Parameter-header validation remains +covered by the active in-repo tests described above. Keep these setup gaps +distinct from observed protocol failures when interpreting conformance results. diff --git a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml index 00d45c5..84efe29 100644 --- a/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml +++ b/tests/conformance/baselines/2026-07-28/modern/external-data-plane.yml @@ -71,10 +71,6 @@ findings: check: sep-2322-respect-client-capabilities name: RespectClientCapabilities status: FAILURE -- scenario: input-required-result-ignore-extra-params - check: sep-2322-ignore-unexpected-params - name: IgnoreUnexpectedParams - status: WARNING - scenario: input-required-result-missing-input-response check: sep-2322-missing-response-rerequests name: InputRequiredResultMissingResponseRerequests @@ -131,22 +127,10 @@ findings: check: sep-2575-http-server-no-independent-requests-on-stream name: HttpServerNoIndependentRequestsOnStream status: FAILURE -- scenario: server-stateless - check: sep-2575-missing-capability-http-400 - name: MissingCapabilityHttp400 - status: FAILURE - scenario: server-stateless check: sep-2575-server-declares-prompts-in-discover name: ServerDeclaresPromptsInDiscover status: FAILURE -- scenario: server-stateless - check: sep-2575-server-no-log-without-loglevel - name: ServerNoLogWithoutLogLevel - status: FAILURE -- scenario: server-stateless - check: sep-2575-server-rejects-undeclared-capability - name: ServerRejectsUndeclaredCapability - status: FAILURE - scenario: tools-call-with-progress check: tools-call-with-progress name: ToolsCallWithProgress