From 450e69af16f87aee810401b774b08906ed7ea4aa Mon Sep 17 00:00:00 2001 From: root Date: Wed, 2 Sep 2026 08:57:25 +0000 Subject: [PATCH 01/14] Trigger workflow scan From 90f1f3dffb9a70292dbc56f03b28ebb386f8255f Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 15 Sep 2026 02:39:41 +0000 Subject: [PATCH 02/14] test(mcp): establish 2026-07-28 conformance baseline --- .github/workflows/ci.yml | 38 ++- docs/MCP_CONFORMANCE.md | 124 +++++++++ docs/TESTING.md | 8 + scripts/mcp_conformance.sh | 182 +++++++++++++ scripts/mcp_conformance_report.py | 273 +++++++++++++++++++ scripts/tests/test_mcp_conformance_report.py | 167 ++++++++++++ src/mcp_tests.rs | 2 + src/mcp_tests/artifact_export.rs | 3 + src/mcp_tests/conformance.rs | 61 +++++ src/mcp_tests/project_connector.rs | 6 + tests/fixtures/mcp/conformance/baseline.json | 26 ++ 11 files changed, 887 insertions(+), 3 deletions(-) create mode 100644 docs/MCP_CONFORMANCE.md create mode 100644 scripts/mcp_conformance.sh create mode 100644 scripts/mcp_conformance_report.py create mode 100644 scripts/tests/test_mcp_conformance_report.py create mode 100644 src/mcp_tests/conformance.rs create mode 100644 tests/fixtures/mcp/conformance/baseline.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15b9f3415..907b3ccf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,37 @@ jobs: explicit_resume_advanced_compatibility_schema_and_metadata_are_retained \ http_project_connector_lists_and_dispatches_only_canonical_capabilities + mcp-conformance: + # Dated MCP wire evidence is independent from ordinary unit/integration tests. + # The script pins the external referee and rejects missing or silently skipped + # coverage; known failures require exact check-level classification. + needs: contract + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: linux-ci + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Check conformance report gate + run: python3 scripts/tests/test_mcp_conformance_report.py + - name: Run pinned MCP dated-revision baseline + run: bash scripts/mcp_conformance.sh + - name: Upload MCP conformance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: mcp-conformance-${{ github.sha }} + path: target/mcp-conformance/reports + if-no-files-found: ignore + retention-days: 7 + test-linux-rust: # Complete Linux Rust coverage is mandatory before merge, including owner PRs. # Start it immediately in parallel with contract/frontend work; the stable @@ -760,7 +791,7 @@ jobs: require_lane desktop "$NEEDS_DESKTOP" "$DESKTOP_RESULT" test-native: - needs: [contract, changes, test-docker-server, test-macos, test-windows] + needs: [contract, changes, mcp-conformance, test-docker-server, test-macos, test-windows] # Stable branch-protection context: daily native aggregates and the path-aware # Server image smoke must resolve according to classifier policy. if: always() @@ -772,13 +803,14 @@ jobs: CONTRACT_RESULT: ${{ needs.contract.result }} CHANGES_RESULT: ${{ needs.changes.result }} NEEDS_DOCKER: ${{ needs.changes.outputs.needs_docker }} + MCP_CONFORMANCE_RESULT: ${{ needs.mcp-conformance.result }} DOCKER_RESULT: ${{ needs.test-docker-server.result }} MACOS_RESULT: ${{ needs.test-macos.result }} WINDOWS_RESULT: ${{ needs.test-windows.result }} run: | set -euo pipefail - echo "changes=$CHANGES_RESULT contract=$CONTRACT_RESULT docker=$DOCKER_RESULT macos=$MACOS_RESULT windows=$WINDOWS_RESULT needs_docker=$NEEDS_DOCKER" - if [ "$CHANGES_RESULT" != success ] || [ "$CONTRACT_RESULT" != success ]; then + echo "changes=$CHANGES_RESULT contract=$CONTRACT_RESULT mcp_conformance=$MCP_CONFORMANCE_RESULT docker=$DOCKER_RESULT macos=$MACOS_RESULT windows=$WINDOWS_RESULT needs_docker=$NEEDS_DOCKER" + if [ "$CHANGES_RESULT" != success ] || [ "$CONTRACT_RESULT" != success ] || [ "$MCP_CONFORMANCE_RESULT" != success ]; then echo "CI policy or contract classification did not succeed" >&2 exit 1 fi diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md new file mode 100644 index 000000000..ee073b18c --- /dev/null +++ b/docs/MCP_CONFORMANCE.md @@ -0,0 +1,124 @@ +# MCP conformance baseline + +WebCodex tracks MCP wire compatibility with the upstream +[`modelcontextprotocol/conformance`](https://github.com/modelcontextprotocol/conformance) +referee in addition to its focused Rust tests. The baseline is evidence about the +checks that actually ran; it is not a blanket "MCP compliant" badge. + +## Pinned referee and profiles + +`scripts/mcp_conformance.sh` pins the upstream referee to the immutable commit in +`tests/fixtures/mcp/conformance/baseline.json`. The script rejects any checkout at +a different commit and installs dependencies from that checkout's lockfile before +building it. Updating the pin is therefore a reviewed test-contract change, not a +floating dependency update. + +The ordinary baseline runs two dated server profiles independently: + +- `2026-07-28`, using the stateless per-request protocol and that revision's + frozen requirements; +- `2025-11-25`, using the stateful lifecycle and its reconstructed frozen + requirements. + +Focused repository tests continue to cover `2025-06-18`; the conformance harness +is not used to invent a frozen requirements set that it does not publish. + +Run the same baseline locally with: + +```bash +bash scripts/mcp_conformance.sh +``` + +To reuse an already checked-out referee without changing the pin: + +```bash +WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR=/path/to/conformance \ + bash scripts/mcp_conformance.sh 2026-07-28 +``` + +The supplied checkout must resolve to the exact pinned commit. Reports are written +to `target/mcp-conformance/reports` by default. CI uploads that directory even +when the gate fails so the raw evidence remains inspectable. + +## What is exercised + +The script starts one ignored, test-only Rust fixture on `127.0.0.1:0`. The +fixture composes the same Salvo `AuthMiddleware` and `/mcp` handlers used by the +production HTTP surface, with temporary database/runtime state and anonymous test +authority. The referee therefore reaches real WebCodex parsing, dispatch and +rendering without production credentials or an external deployment. + +No conformance-only tool is added to the production registry. Some upstream +scenarios intentionally expect named reference-fixture tools. When WebCodex does +not expose such a tool, that result is a coverage gap until a test-only adapter is +provided; it must not be relabeled as a product protocol failure merely to make a +summary green. + +Authenticated Runtime/Connector behavior remains covered by focused synthetic- +credential integration tests. The upstream server CLI does not provide a generic +way to inject WebCodex authorization headers, so an authentication-blocked +scenario is inconclusive rather than a pass. + +## Coverage-aware report gate + +The upstream runner saves one `checks.json` file per scenario. After each profile, +`scripts/mcp_conformance_report.py` verifies the raw reports against the exact +required server-scenario list and the reviewed baseline. + +Every `FAILURE`, `WARNING`, or `SKIPPED` check must have an exact +`, ` classification in +`tests/fixtures/mcp/conformance/baseline.json`. Whole-scenario wildcards are +rejected because they can hide a newly failing check behind an unrelated known +failure. Supported classifications are: + +| Classification | Meaning | +|---|---| +| `genuine_protocol_failure` | The intended WebCodex behavior was reached and a protocol requirement is known to be violated. | +| `missing_harness_fixture` | The upstream scenario depends on reference-fixture behavior that the product endpoint does not expose. The product behavior remains untested by that check. | +| `optional_capability_not_implemented` | A capability is genuinely optional and is not advertised/implemented. The entry must cite capability/spec evidence. | +| `harness_limitation_pending` | The pinned referee itself marks or demonstrates a limitation/pending check. | +| `inconclusive_infrastructure` | Authentication blocking, timeout, harness exception, or other infrastructure failure. This is recorded but **never satisfies the CI gate**. | + +The gate also fails when: + +- no applicable `SUCCESS`/`FAILURE`/`WARNING` checks were emitted; +- a required scenario report is missing, duplicated, or unexpectedly present; +- a new non-success check has no exact classification; +- a classified check is no longer emitted or now passes, making its baseline + entry stale; +- an inconclusive infrastructure result is present. + +This is intentionally stricter than trusting the referee process exit code. A +zero exit can coexist with skipped/unscored coverage, and an expected-failure run +can intentionally tolerate known protocol defects. + +The report gate's own regression suite is dependency-free: + +```bash +python3 scripts/tests/test_mcp_conformance_report.py +``` + +It covers all-skipped output, missing required scenarios, harness errors, new +failures, stale classifications, and rejection of broad masks. + +## Raw evidence and review policy + +Each profile retains: + +- the referee's raw per-scenario `checks.json` files; +- the referee stdout/stderr log; +- metadata containing the WebCodex source SHA, referee SHA, profile, upstream + referee exit code, and exact required scenario list; +- the coverage-aware WebCodex summary. + +Baseline updates should be narrow. Add a classification only after reproducing +and understanding the exact check. Remove it as soon as that check passes. Do not +replace multiple check IDs with a scenario-wide exception. + +This baseline deliberately records two already-demonstrated 2026 result-shape findings +without fixing production behavior: streamed artifact `resources/read` omits the +cache hints emitted by ordinary resource reads, and a completed Connector Task +embeds its original tool result without a nested `resultType`. Their focused Rust +observations and planned follow-up are recorded in the baseline fixture. Fixes +belong in a separate protocol-behavior change so this baseline remains an +independent measuring instrument. diff --git a/docs/TESTING.md b/docs/TESTING.md index 4d354f171..1e6980682 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -33,6 +33,7 @@ waits, and the cost of each test lane. | real-process job reconciliation harness | Boot a real server plus a WebSocket runner that advertises `job_state_reconciliation`. Scenario A keeps a raw async Job running across a SERVER-only restart and asserts the SAME runner instance, original `job_id`, preserved ownership/project/session, non-regressing sequence/log cursors, `recovered_after_server_restart`, original-process stop, and one side-effect set. Scenario B lets a Job complete while the Server is offline and reconciles the terminal result without duplicate logs or execution. Scenario C forces `run_process` past its synchronous grace window, then proves the handed-off structured Job survives a Server restart and an old Server-epoch observation token refreshes immediately for the same `job_id`. Scenario D uses a delayed Cargo fixture to force a real `cargo_check` validation handoff past its sync window, then proves the same restart/token-refresh/stop contract with the validation command started exactly once. Ordinary Runner-owned Jobs keep the Runner process alive for these scenarios; `run_detached_process` restart survival is a separate supervisor-ownership contract covered by its focused Runner suites and production dogfood. | Local processes, temp dirs/ports/tokens, and a temp project; no production services or QUIC certs. Scenario D intentionally takes roughly the validation sync window plus restart time. | `bash scripts/e2e_job_reconciliation_ws.sh` | | real-process job recovery failure/non-reconciliation harness | Cover the failure and non-reconciliation paths the happy-path reconciliation harness omits, using `WEBCODEX_JOB_RECOVERY_GRACE_SECS=10` (clamped, above the 5s floor) so the deadline is bounded without waiting the 120s default. Scenario C: kill the runner only (server stays up), let the job enter `recovering`, and assert the non-request-triggered recovery-timeout sweep transitions it to `lost` with `runner_recovery_deadline_exceeded`, `ended_at` set once, one list record, stop-on-lost stable, and the command never re-executes. Scenario D: instance B replaces instance A (same client_id, new `agent_instance_id`); A's job becomes `lost` with `runner_instance_replaced`, B starts its own new job, A's late update is rejected, first `ended_at`/reason preserved. Scenario E: a generation-2 Runner registered with `WEBCODEX_RUNNER_DISABLE_JOB_STATE_RECONCILIATION=1` (no capability, no inventory) dispatches a job and, on disconnect, deterministically fences it to `lost` with `runner_disconnected_without_reconciliation` (never `recovering`); after a server restart its public lost receipt remains observable within retention, and a same-client new no-reconciliation instance cannot revive execution. Scenario F: a long job across three server restarts keeps the same `job_id`, runs the command once, keeps `last_update_seq`/log cursors non-regressing and markers non-duplicating, and reaches a terminal `stopped` that survives a third restart with `ended_at` unchanged by terminal inventory replay. | Local processes, temp dirs/ports/tokens, and a temp project; no production services or QUIC certs. | `bash scripts/e2e_job_recovery_failures_ws.sh` | | security auth matrix | Cover OAuth, scope policy, shared-key behavior, token classes, read-only session guards, and denied mutations. | No external identity provider by default; use local fixtures and synthetic tokens. | `cargo test -p webcodex --lib oauth -- --nocapture`; `cargo test -p webcodex --lib scope -- --nocapture`; `cargo test -p webcodex --lib metadata -- --nocapture` | +| MCP conformance baseline | Exercise the real Salvo `/mcp` handlers through a disposable loopback fixture against a pinned upstream referee, then gate on check-level coverage/classification rather than process exit alone. | Loopback product fixture; immutable external referee source/dependencies fetched only by the explicit script/CI lane; no production credentials. | `bash scripts/mcp_conformance.sh`; see [`MCP_CONFORMANCE.md`](MCP_CONFORMANCE.md). | The MCP Apps' DOM/message-order regression tests run without browser or npm dependencies: `node --test src/mcp_tests/*.test.mjs`. They execute the embedded @@ -95,6 +96,13 @@ The lanes above define test semantics; workflows decide when to run them. `test-windows`, and `test-native` aggregates always resolve and verify each child lane is `success` when required or `skipped` when not required, avoiding a skipped required-check context that could leave branch protection pending. +- MCP dated-revision evidence has its own bounded `mcp-conformance` lane. It pins + the upstream referee, runs the `2026-07-28` and `2025-11-25` server requirements + against a test-only loopback WebCodex endpoint, validates the report gate itself, + and uploads raw reports on success or failure. `test-native` requires this lane + to succeed, so missing coverage, stale classifications, and unclassified new + failures are merge-blocking even when the upstream referee exits zero. See + [`MCP_CONFORMANCE.md`](MCP_CONFORMANCE.md) for baseline semantics. - Linux Rust execution remains package-sharded: the server package `webcodex`, the Runner/LSP packages, and the remaining workspace crates run in parallel. The Runner/LSP shard compiles with `--features runner-real-process-tests` to prevent diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh new file mode 100644 index 000000000..f875305c9 --- /dev/null +++ b/scripts/mcp_conformance.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The conformance referee is intentionally immutable. Updating it is a reviewed +# baseline change because scenario/check semantics can change between commits. +BASELINE="tests/fixtures/mcp/conformance/baseline.json" +HARNESS_COMMIT="$(python3 - "$BASELINE" <<'PY' +import json +import pathlib +import re +import sys +value = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +commit = value.get("harness_commit", "") +if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise SystemExit("baseline harness_commit must be a full lowercase Git SHA") +print(commit) +PY +)" +WORK_ROOT="${WEBCODEX_MCP_CONFORMANCE_WORK_ROOT:-target/mcp-conformance}" +HARNESS_DIR="${WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR:-$WORK_ROOT/harness}" +REPORT_ROOT="${WEBCODEX_MCP_CONFORMANCE_REPORT_ROOT:-$WORK_ROOT/reports}" +PROFILES=("$@") + +if [ "$#" -eq 0 ]; then + PROFILES=("2026-07-28" "2025-11-25") +fi +for profile in "${PROFILES[@]}"; do + case "$profile" in + 2026-07-28|2025-11-25) ;; + *) echo "unsupported MCP conformance profile: $profile" >&2; exit 2 ;; + esac +done + +mkdir -p "$WORK_ROOT" "$REPORT_ROOT" + +prepare_harness() { + if [ ! -d "$HARNESS_DIR/.git" ]; then + rm -rf "$HARNESS_DIR" + mkdir -p "$HARNESS_DIR" + git -C "$HARNESS_DIR" init -q + git -C "$HARNESS_DIR" remote add origin https://github.com/modelcontextprotocol/conformance.git + GIT_TERMINAL_PROMPT=0 git -C "$HARNESS_DIR" fetch -q --depth 1 origin "$HARNESS_COMMIT" + git -C "$HARNESS_DIR" checkout -q --detach FETCH_HEAD + fi + actual="$(git -C "$HARNESS_DIR" rev-parse HEAD)" + if [ "$actual" != "$HARNESS_COMMIT" ]; then + echo "MCP conformance harness must be exactly $HARNESS_COMMIT, got $actual" >&2 + exit 2 + fi + if [ ! -f "$HARNESS_DIR/package-lock.json" ]; then + echo "pinned harness has no package-lock.json" >&2 + exit 2 + fi + if [ ! -f "$HARNESS_DIR/dist/index.js" ]; then + npm --prefix "$HARNESS_DIR" ci --ignore-scripts --no-audit --no-fund + npm --prefix "$HARNESS_DIR" run build + fi +} + +prepare_harness +python3 scripts/tests/test_mcp_conformance_report.py + +# Compile before starting the bounded readiness clock. A cold WebCodex test build +# can take several minutes on small builders; readiness should measure server +# startup, not Rust compilation time. +cargo test --locked -p webcodex --lib mcp_conformance_fixture_server --no-run + +fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/webcodex-mcp-conformance.XXXXXX")" +url_file="$fixture_dir/url" +stop_file="$fixture_dir/stop" +fixture_log="$REPORT_ROOT/fixture.log" +fixture_pid="" +cleanup() { + touch "$stop_file" 2>/dev/null || true + if [ -n "$fixture_pid" ]; then + for _ in $(seq 1 50); do + if ! kill -0 "$fixture_pid" 2>/dev/null; then break; fi + sleep 0.1 + done + if kill -0 "$fixture_pid" 2>/dev/null; then + kill "$fixture_pid" 2>/dev/null || true + fi + wait "$fixture_pid" 2>/dev/null || true + fi + rm -rf "$fixture_dir" +} +trap cleanup EXIT INT TERM + +rm -rf "$REPORT_ROOT" +mkdir -p "$REPORT_ROOT" +WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ +WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ + cargo test --locked -p webcodex --lib mcp_conformance_fixture_server -- --ignored --nocapture \ + >"$fixture_log" 2>&1 & +fixture_pid=$! + +ready_deadline=$((SECONDS + 60)) +while [ ! -s "$url_file" ]; do + if ! kill -0 "$fixture_pid" 2>/dev/null; then + echo "MCP conformance fixture exited before publishing its URL" >&2 + tail -80 "$fixture_log" >&2 || true + exit 1 + fi + if [ "$SECONDS" -ge "$ready_deadline" ]; then + echo "timed out waiting for MCP conformance fixture URL" >&2 + tail -80 "$fixture_log" >&2 || true + exit 1 + fi + sleep 0.1 +done +fixture_url="$(tr -d '\r\n' < "$url_file")" +case "$fixture_url" in + http://127.0.0.1:*'/mcp') ;; + *) echo "fixture published unexpected URL: $fixture_url" >&2; exit 1 ;; +esac + +server_sha="$(git rev-parse HEAD)" +overall=0 +for profile in "${PROFILES[@]}"; do + profile_root="$REPORT_ROOT/$profile" + raw="$profile_root/raw" + mkdir -p "$raw" + log="$profile_root/harness.log" + set +e + node "$HARNESS_DIR/dist/index.js" server \ + --url "$fixture_url" \ + --requirements "$profile" \ + --output-dir "$raw" \ + --timeout 10000 \ + >"$log" 2>&1 + harness_exit=$? + set -e + + metadata="$profile_root/metadata.json" + python3 - "$HARNESS_DIR/requirements/$profile.yaml" "$metadata" "$profile" \ + "$server_sha" "$HARNESS_COMMIT" "$harness_exit" <<'PY' +import json +import pathlib +import sys + +requirements_path, output_path, profile, server_sha, harness_sha, harness_exit = sys.argv[1:] +server = [] +in_server = False +for raw in pathlib.Path(requirements_path).read_text(encoding="utf-8").splitlines(): + if raw == "server:": + in_server = True + continue + if in_server and raw and not raw.startswith(" "): + break + if in_server and raw.startswith(" - "): + server.append(raw[4:].strip()) +if not server: + raise SystemExit(f"no scored server scenarios found in {requirements_path}") +pathlib.Path(output_path).write_text( + json.dumps( + { + "schema_version": 1, + "profile": profile, + "server_sha": server_sha, + "harness_sha": harness_sha, + "harness_exit_code": int(harness_exit), + "required_scenarios": server, + }, + indent=2, + sort_keys=True, + ) + "\n", + encoding="utf-8", +) +PY + + summary="$profile_root/summary.json" + if ! python3 scripts/mcp_conformance_report.py \ + --reports "$raw" \ + --metadata "$metadata" \ + --baseline "$BASELINE" \ + --profile "$profile" \ + --summary "$summary"; then + overall=1 + fi +done + +exit "$overall" diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py new file mode 100644 index 000000000..2aa3c8ec1 --- /dev/null +++ b/scripts/mcp_conformance_report.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Coverage-aware gate for pinned MCP conformance reports. + +The upstream conformance runner intentionally answers a different question from +this gate: it executes scenarios and reports their checks. WebCodex keeps the +raw upstream output, then this script verifies that the expected scenario set was +actually exercised and that every non-success result has a narrow, reviewed +classification. A zero upstream process exit is never treated as proof of +coverage or compliance. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +CLASSIFICATIONS = { + "genuine_protocol_failure", + "missing_harness_fixture", + "optional_capability_not_implemented", + "harness_limitation_pending", + "inconclusive_infrastructure", +} +NON_SUCCESS_STATUSES = {"FAILURE", "WARNING", "SKIPPED"} +APPLICABLE_STATUSES = {"SUCCESS", "FAILURE", "WARNING"} +SCENARIO_DIR_RE = re.compile(r"^server-(.+)-\d{4}-\d{2}-\d{2}T.*Z$") + + +class GateError(RuntimeError): + """Raised when report or baseline input is malformed.""" + + +@dataclass(frozen=True) +class Classification: + scenario: str + check_id: str + classification: str + reason: str + evidence: str + + @property + def key(self) -> tuple[str, str]: + return (self.scenario, self.check_id) + + +def _load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise GateError(f"cannot read JSON {path}: {exc}") from exc + + +def _scenario_from_result_dir(path: Path) -> str: + match = SCENARIO_DIR_RE.match(path.parent.name) + if not match: + raise GateError( + f"unexpected upstream result directory {path.parent.name!r}; " + "expected server--" + ) + return match.group(1) + + +def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: + reports: dict[str, list[dict[str, Any]]] = {} + for checks_path in sorted(report_root.glob("server-*/checks.json")): + scenario = _scenario_from_result_dir(checks_path) + if scenario in reports: + raise GateError( + f"multiple raw reports found for scenario {scenario!r}; clean the output directory" + ) + value = _load_json(checks_path) + if not isinstance(value, list): + raise GateError(f"{checks_path} must contain a JSON array") + checks: list[dict[str, Any]] = [] + for index, check in enumerate(value): + if not isinstance(check, dict): + raise GateError(f"{checks_path}[{index}] is not an object") + check_id = check.get("id") + status = check.get("status") + if not isinstance(check_id, str) or not check_id: + raise GateError(f"{checks_path}[{index}] has no non-empty check id") + if not isinstance(status, str) or not status: + raise GateError(f"{checks_path}[{index}] has no non-empty status") + checks.append(check) + reports[scenario] = checks + return reports + + +def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, str], Classification]: + baseline = _load_json(baseline_path) + if not isinstance(baseline, dict) or baseline.get("schema_version") != 1: + raise GateError("baseline must be an object with schema_version=1") + profiles = baseline.get("profiles") + if not isinstance(profiles, dict) or profile not in profiles: + raise GateError(f"baseline has no profile {profile!r}") + profile_config = profiles[profile] + if not isinstance(profile_config, dict): + raise GateError(f"baseline profile {profile!r} must be an object") + raw_entries = profile_config.get("classifications", []) + if not isinstance(raw_entries, list): + raise GateError(f"baseline profile {profile!r} classifications must be an array") + + entries: dict[tuple[str, str], Classification] = {} + for index, raw in enumerate(raw_entries): + if not isinstance(raw, dict): + raise GateError(f"classification #{index + 1} must be an object") + scenario = raw.get("scenario") + check_id = raw.get("check_id") + kind = raw.get("classification") + reason = raw.get("reason") + evidence = raw.get("evidence") + if not isinstance(scenario, str) or not scenario: + raise GateError(f"classification #{index + 1} needs a non-empty scenario") + # Whole-scenario masks are deliberately unsupported. They can hide a newly + # failing check when an older, unrelated check is already expected to fail. + if not isinstance(check_id, str) or not check_id or check_id == "*": + raise GateError( + f"classification {scenario!r} must name one exact check_id; broad scenario masks are forbidden" + ) + if kind not in CLASSIFICATIONS: + raise GateError( + f"classification {scenario}:{check_id} has unsupported kind {kind!r}" + ) + if not isinstance(reason, str) or not reason.strip(): + raise GateError(f"classification {scenario}:{check_id} needs a reason") + if not isinstance(evidence, str) or not evidence.strip(): + raise GateError(f"classification {scenario}:{check_id} needs evidence") + entry = Classification(scenario, check_id, kind, reason.strip(), evidence.strip()) + if entry.key in entries: + raise GateError(f"duplicate classification for {scenario}:{check_id}") + entries[entry.key] = entry + return entries + + +def evaluate_reports( + report_root: Path, + metadata_path: Path, + baseline_path: Path, + profile: str, +) -> dict[str, Any]: + metadata = _load_json(metadata_path) + if not isinstance(metadata, dict): + raise GateError("metadata must be a JSON object") + if metadata.get("profile") != profile: + raise GateError( + f"metadata profile {metadata.get('profile')!r} does not match requested {profile!r}" + ) + required = metadata.get("required_scenarios") + if not isinstance(required, list) or not required or not all(isinstance(item, str) and item for item in required): + raise GateError("metadata required_scenarios must be a non-empty string array") + if len(required) != len(set(required)): + raise GateError("metadata required_scenarios contains duplicates") + + reports = load_reports(report_root) + classifications = load_classifications(baseline_path, profile) + problems: list[str] = [] + missing_scenarios = sorted(set(required) - set(reports)) + unexpected_scenarios = sorted(set(reports) - set(required)) + if missing_scenarios: + problems.append("missing required scenario reports: " + ", ".join(missing_scenarios)) + if unexpected_scenarios: + problems.append("unexpected scenario reports: " + ", ".join(unexpected_scenarios)) + + statuses = Counter() + emitted: dict[tuple[str, str], set[str]] = {} + applicable_checks = 0 + unclassified: list[str] = [] + expected: list[dict[str, str]] = [] + inconclusive: list[str] = [] + + for scenario, checks in sorted(reports.items()): + for check in checks: + check_id = check["id"] + status = check["status"] + statuses[status] += 1 + key = (scenario, check_id) + emitted.setdefault(key, set()).add(status) + if status in APPLICABLE_STATUSES: + applicable_checks += 1 + if status in NON_SUCCESS_STATUSES: + entry = classifications.get(key) + label = f"{scenario}:{check_id} [{status}]" + if entry is None: + unclassified.append(label) + continue + expected.append( + { + "scenario": scenario, + "check_id": check_id, + "status": status, + "classification": entry.classification, + "reason": entry.reason, + "evidence": entry.evidence, + } + ) + if entry.classification == "inconclusive_infrastructure": + inconclusive.append(label) + + if applicable_checks == 0: + problems.append("zero applicable SUCCESS/FAILURE/WARNING checks were emitted") + if unclassified: + problems.append("unclassified non-success checks: " + ", ".join(sorted(unclassified))) + if inconclusive: + problems.append( + "inconclusive infrastructure results cannot satisfy the gate: " + + ", ".join(sorted(inconclusive)) + ) + + stale: list[str] = [] + for key, entry in sorted(classifications.items()): + states = emitted.get(key) + label = f"{entry.scenario}:{entry.check_id}" + if not states: + stale.append(label + " (check not emitted)") + elif states <= {"INFO"}: + stale.append(label + " (no verdict emitted)") + elif states <= {"SUCCESS", "INFO"} and "SUCCESS" in states: + stale.append(label + " (now passing)") + if stale: + problems.append("stale or uncovered classifications: " + ", ".join(stale)) + + return { + "schema_version": 1, + "profile": profile, + "server_sha": metadata.get("server_sha"), + "harness_sha": metadata.get("harness_sha"), + "harness_exit_code": metadata.get("harness_exit_code"), + "required_scenario_count": len(required), + "reported_scenario_count": len(reports), + "applicable_check_count": applicable_checks, + "status_counts": dict(sorted(statuses.items())), + "classified_non_success": expected, + "missing_scenarios": missing_scenarios, + "unexpected_scenarios": unexpected_scenarios, + "problems": problems, + "gate_passed": not problems, + } + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reports", required=True, type=Path) + parser.add_argument("--metadata", required=True, type=Path) + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--profile", required=True) + parser.add_argument("--summary", type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + try: + summary = evaluate_reports(args.reports, args.metadata, args.baseline, args.profile) + except GateError as exc: + print(f"MCP conformance report gate input error: {exc}", file=sys.stderr) + return 2 + + rendered = json.dumps(summary, indent=2, sort_keys=True) + "\n" + if args.summary: + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if summary["gate_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py new file mode 100644 index 000000000..47cae5934 --- /dev/null +++ b/scripts/tests/test_mcp_conformance_report.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "mcp_conformance_report.py" +SPEC = importlib.util.spec_from_file_location("mcp_conformance_report", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +report_gate = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = report_gate +SPEC.loader.exec_module(report_gate) + + +class ReportGateTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.raw = self.root / "raw" + self.raw.mkdir() + self.metadata = self.root / "metadata.json" + self.baseline = self.root / "baseline.json" + self.profile = "2026-07-28" + + def tearDown(self) -> None: + self.tmp.cleanup() + + def write_metadata(self, scenarios: list[str]) -> None: + self.metadata.write_text( + json.dumps( + { + "profile": self.profile, + "server_sha": "a" * 40, + "harness_sha": "b" * 40, + "harness_exit_code": 0, + "required_scenarios": scenarios, + } + ), + encoding="utf-8", + ) + + def write_baseline(self, entries: list[dict[str, str]]) -> None: + self.baseline.write_text( + json.dumps( + { + "schema_version": 1, + "profiles": {self.profile: {"classifications": entries}}, + } + ), + encoding="utf-8", + ) + + def write_checks(self, scenario: str, checks: list[dict[str, str]]) -> None: + result = self.raw / f"server-{scenario}-2026-09-14T00-00-00-000Z" + result.mkdir() + (result / "checks.json").write_text(json.dumps(checks), encoding="utf-8") + + @staticmethod + def check(check_id: str, status: str) -> dict[str, str]: + return { + "id": check_id, + "name": check_id, + "description": check_id, + "status": status, + "timestamp": "2026-09-14T00:00:00Z", + } + + @staticmethod + def classification(check_id: str, kind: str = "genuine_protocol_failure") -> dict[str, str]: + return { + "scenario": "scenario-a", + "check_id": check_id, + "classification": kind, + "reason": "reviewed baseline reason", + "evidence": "MCP 2026-07-28 section/example", + } + + def evaluate(self): + return report_gate.evaluate_reports(self.raw, self.metadata, self.baseline, self.profile) + + def test_successful_nonempty_coverage_passes(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("wire-shape", "SUCCESS")]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + self.assertEqual(summary["applicable_check_count"], 1) + + def test_all_skipped_is_not_mistaken_for_coverage(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline( + [self.classification("not-applicable", "inconclusive_infrastructure")] + ) + self.write_checks("scenario-a", [self.check("not-applicable", "SKIPPED")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("zero applicable" in problem for problem in summary["problems"])) + + def test_missing_required_scenario_fails(self) -> None: + self.write_metadata(["scenario-a", "scenario-b"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertEqual(summary["missing_scenarios"], ["scenario-b"]) + + def test_classified_harness_error_remains_inconclusive(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline( + [self.classification("scenario-a", "inconclusive_infrastructure")] + ) + self.write_checks("scenario-a", [self.check("scenario-a", "FAILURE")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("inconclusive infrastructure" in p for p in summary["problems"])) + + def test_new_failure_requires_exact_classification(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("new-regression", "FAILURE")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("new-regression" in p for p in summary["problems"])) + + def test_passing_expected_failure_is_stale(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([self.classification("fixed-check")]) + self.write_checks("scenario-a", [self.check("fixed-check", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("now passing" in p for p in summary["problems"])) + + def test_info_only_expected_failure_is_uncovered(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([self.classification("no-verdict")]) + self.write_checks( + "scenario-a", + [self.check("no-verdict", "INFO"), self.check("ok", "SUCCESS")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("no verdict emitted" in p for p in summary["problems"])) + + def test_broad_expected_failure_masks_are_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline( + [ + { + "scenario": "scenario-a", + "check_id": "*", + "classification": "genuine_protocol_failure", + "reason": "too broad", + "evidence": "none", + } + ] + ) + self.write_checks("scenario-a", [self.check("failure", "FAILURE")]) + with self.assertRaisesRegex(report_gate.GateError, "broad scenario masks"): + self.evaluate() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/mcp_tests.rs b/src/mcp_tests.rs index 666472c30..2920fca2f 100644 --- a/src/mcp_tests.rs +++ b/src/mcp_tests.rs @@ -214,6 +214,8 @@ mod agent_continuation_app; mod artifact_export; #[path = "mcp_tests/computer_app.rs"] mod computer_app; +#[path = "mcp_tests/conformance.rs"] +mod conformance; #[path = "mcp_tests/file_import.rs"] mod file_import; #[path = "mcp_tests/goal_plan_app.rs"] diff --git a/src/mcp_tests/artifact_export.rs b/src/mcp_tests/artifact_export.rs index 327d901a1..7745db234 100644 --- a/src/mcp_tests/artifact_export.rs +++ b/src/mcp_tests/artifact_export.rs @@ -992,6 +992,9 @@ async fn http_mcp_artifact_export_resources_read_streams_valid_json_blob() { assert_eq!(body["jsonrpc"], "2.0"); assert_eq!(body["id"], 3190); assert_eq!(body["result"]["resultType"], "complete"); + // MCP 2026 conformance baseline finding: the optimized streaming path has + // not yet been aligned with the cache hints emitted by ordinary resources/read. + // This baseline records the current wire shape only; a focused follow-up fixes it. assert!(body["result"].get("ttlMs").is_none()); assert!(body["result"].get("cacheScope").is_none()); assert_eq!(body["result"]["contents"][0]["uri"], uri); diff --git a/src/mcp_tests/conformance.rs b/src/mcp_tests/conformance.rs new file mode 100644 index 000000000..2964499e1 --- /dev/null +++ b/src/mcp_tests/conformance.rs @@ -0,0 +1,61 @@ +use super::*; +use salvo::conn::{Acceptor, Listener, TcpListener}; +use std::path::PathBuf; +use std::time::Duration; +use tokio::time::Instant; + +const CONFORMANCE_URL_FILE_ENV: &str = "WEBCODEX_MCP_CONFORMANCE_URL_FILE"; +const CONFORMANCE_STOP_FILE_ENV: &str = "WEBCODEX_MCP_CONFORMANCE_STOP_FILE"; +const CONFORMANCE_FIXTURE_LIFETIME: Duration = Duration::from_secs(15 * 60); + +/// External MCP conformance uses a real loopback socket, but the endpoint is +/// still the same production Salvo/AuthMiddleware/mcp_post composition used by +/// the in-process HTTP tests. It is ignored because it deliberately waits for an +/// external referee process; `scripts/mcp_conformance.sh` owns its lifecycle. +#[tokio::test] +#[ignore = "started only by scripts/mcp_conformance.sh with the pinned external MCP referee"] +async fn mcp_conformance_fixture_server() { + let url_file = PathBuf::from( + std::env::var_os(CONFORMANCE_URL_FILE_ENV) + .expect("conformance fixture requires WEBCODEX_MCP_CONFORMANCE_URL_FILE"), + ); + let stop_file = PathBuf::from( + std::env::var_os(CONFORMANCE_STOP_FILE_ENV) + .expect("conformance fixture requires WEBCODEX_MCP_CONFORMANCE_STOP_FILE"), + ); + + let config = test_config(None); + let (_db_tmp, db) = test_db(); + let runtime = Arc::new(test_runtime()); + let router = build_test_router(config, db, runtime); + let acceptor = TcpListener::new("127.0.0.1:0").bind().await; + let addr = acceptor.holdings()[0] + .local_addr + .clone() + .into_std() + .expect("MCP conformance listener must be TCP"); + let server_task = tokio::spawn(async move { + Server::new(acceptor).serve(router).await; + }); + + let temporary_url_file = url_file.with_extension("tmp"); + std::fs::write(&temporary_url_file, format!("http://{addr}/mcp\n")) + .expect("write MCP conformance fixture URL"); + std::fs::rename(&temporary_url_file, &url_file) + .expect("publish MCP conformance fixture URL atomically"); + + let deadline = Instant::now() + CONFORMANCE_FIXTURE_LIFETIME; + loop { + if stop_file.exists() { + break; + } + assert!( + Instant::now() < deadline, + "external MCP conformance fixture exceeded its bounded lifetime" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + + server_task.abort(); + let _ = server_task.await; +} diff --git a/src/mcp_tests/project_connector.rs b/src/mcp_tests/project_connector.rs index 042c83f87..0ec1f24d4 100644 --- a/src/mcp_tests/project_connector.rs +++ b/src/mcp_tests/project_connector.rs @@ -907,6 +907,12 @@ async fn http_project_connector_2026_tasks_poll_durable_execution_across_reopen( let completed_body: Value = completed.take_json().await.unwrap(); assert_eq!(completed_body["result"]["status"], "completed"); assert_eq!(completed_body["result"]["taskId"], task_id); + // MCP 2026 conformance baseline finding: the completed Task is finalized, + // but the nested original tool result is not yet discriminated as complete. + // Keep the observation explicit until the focused result-shape follow-up. + assert!(completed_body["result"]["result"] + .get("resultType") + .is_none()); assert_eq!( completed_body["result"]["result"]["structuredContent"]["data"]["execution"] ["execution_status"], diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json new file mode 100644 index 000000000..a8d206088 --- /dev/null +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "harness_commit": "7169291ec0b68eb370fddcd9947313ab0d5e4156", + "profiles": { + "2025-11-25": { + "classifications": [] + }, + "2026-07-28": { + "classifications": [] + } + }, + "supplemental_findings": [ + { + "id": "2026-streamed-resource-cache-hints", + "classification": "genuine_protocol_failure", + "evidence": "src/mcp_tests/artifact_export.rs::http_mcp_artifact_export_resources_read_streams_valid_json_blob", + "summary": "The streamed artifact resources/read result currently omits ttlMs and cacheScope. This baseline records the finding but does not change production framing." + }, + { + "id": "2026-completed-task-nested-result-type", + "classification": "genuine_protocol_failure", + "evidence": "src/mcp_tests/project_connector.rs::http_project_connector_2026_tasks_poll_durable_execution_across_reopen", + "summary": "A completed Connector Task currently embeds its original tool result without resultType. This baseline records the finding but does not change production framing." + } + ] +} From 180d4d62923d314f97c4b2d7dafe56ff368f0a1f Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 15 Sep 2026 03:21:46 +0000 Subject: [PATCH 03/14] test(mcp): classify first conformance evidence --- docs/MCP_CONFORMANCE.md | 22 +- scripts/mcp_conformance.sh | 48 +- scripts/mcp_conformance_report.py | 64 ++- scripts/tests/test_mcp_conformance_report.py | 31 +- tests/fixtures/mcp/conformance/baseline.json | 503 ++++++++++++++++++- 5 files changed, 648 insertions(+), 20 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index ee073b18c..c2792cd84 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -65,11 +65,14 @@ The upstream runner saves one `checks.json` file per scenario. After each profil `scripts/mcp_conformance_report.py` verifies the raw reports against the exact required server-scenario list and the reviewed baseline. -Every `FAILURE`, `WARNING`, or `SKIPPED` check must have an exact -`, ` classification in +Every `FAILURE`, `WARNING`, or `SKIPPED` check from a **scored required** +scenario must have an exact `, ` classification in `tests/fixtures/mcp/conformance/baseline.json`. Whole-scenario wildcards are rejected because they can hide a newly failing check behind an unrelated known -failure. Supported classifications are: +failure. The referee's `not_scored` server scenarios are still required to run +and are retained as informational evidence, but their verdicts do not affect the +gate and do not consume expected-failure classifications. Supported scored +classifications are: | Classification | Meaning | |---|---| @@ -82,8 +85,9 @@ failure. Supported classifications are: The gate also fails when: - no applicable `SUCCESS`/`FAILURE`/`WARNING` checks were emitted; -- a required scenario report is missing, duplicated, or unexpectedly present; -- a new non-success check has no exact classification; +- a scored or `not_scored` scenario report expected by the frozen requirements is + missing, duplicated, or an unknown scenario is unexpectedly present; +- a new non-success check in a scored required scenario has no exact classification; - a classified check is no longer emitted or now passes, making its baseline entry stale; - an inconclusive infrastructure result is present. @@ -98,8 +102,9 @@ The report gate's own regression suite is dependency-free: python3 scripts/tests/test_mcp_conformance_report.py ``` -It covers all-skipped output, missing required scenarios, harness errors, new -failures, stale classifications, and rejection of broad masks. +It covers all-skipped scored output, missing scored/informational scenarios, +harness errors, new scored failures, informational non-success reporting, stale +classifications, and rejection of broad masks. ## Raw evidence and review policy @@ -108,7 +113,8 @@ Each profile retains: - the referee's raw per-scenario `checks.json` files; - the referee stdout/stderr log; - metadata containing the WebCodex source SHA, referee SHA, profile, upstream - referee exit code, and exact required scenario list; + referee exit code, exact scored scenario list, and frozen `not_scored` server + scenarios with their upstream reasons; - the coverage-aware WebCodex summary. Baseline updates should be narrow. Add a classification only after reproducing diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index f875305c9..4802bdc91 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -140,17 +140,52 @@ import sys requirements_path, output_path, profile, server_sha, harness_sha, harness_exit = sys.argv[1:] server = [] -in_server = False +not_scored = [] +section = None +current_not_scored = None + +def retain_not_scored(entry): + if not entry or entry.get("leg") != "server": + return + scenario = entry.get("scenario") + reason = entry.get("reason") + if not scenario or not reason: + raise SystemExit(f"malformed server not_scored entry in {requirements_path}: {entry}") + not_scored.append({"scenario": scenario, "reason": reason}) + for raw in pathlib.Path(requirements_path).read_text(encoding="utf-8").splitlines(): - if raw == "server:": - in_server = True + if raw in {"server:", "client:", "not_scored:"}: + retain_not_scored(current_not_scored) + current_not_scored = None + section = raw[:-1] + continue + if raw and not raw.startswith(" "): + retain_not_scored(current_not_scored) + current_not_scored = None + section = None continue - if in_server and raw and not raw.startswith(" "): - break - if in_server and raw.startswith(" - "): + if section == "server" and raw.startswith(" - "): server.append(raw[4:].strip()) + elif section == "not_scored": + if raw.startswith(" - scenario: "): + retain_not_scored(current_not_scored) + current_not_scored = {"scenario": raw[len(" - scenario: "):].strip()} + elif current_not_scored is not None and raw.startswith(" leg: "): + current_not_scored["leg"] = raw[len(" leg: "):].strip() + elif current_not_scored is not None and raw.startswith(" reason: "): + current_not_scored["reason"] = raw[len(" reason: "):].strip() +retain_not_scored(current_not_scored) + if not server: raise SystemExit(f"no scored server scenarios found in {requirements_path}") +if len(server) != len(set(server)): + raise SystemExit(f"duplicate scored server scenarios in {requirements_path}") +not_scored_names = [entry["scenario"] for entry in not_scored] +if len(not_scored_names) != len(set(not_scored_names)): + raise SystemExit(f"duplicate server not_scored scenarios in {requirements_path}") +if set(server) & set(not_scored_names): + raise SystemExit(f"server scenario is both scored and not_scored in {requirements_path}") + pathlib.Path(output_path).write_text( json.dumps( { @@ -160,6 +195,7 @@ pathlib.Path(output_path).write_text( "harness_sha": harness_sha, "harness_exit_code": int(harness_exit), "required_scenarios": server, + "not_scored_scenarios": not_scored, }, indent=2, sort_keys=True, diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index 2aa3c8ec1..32e4eac32 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -157,29 +157,81 @@ def evaluate_reports( if len(required) != len(set(required)): raise GateError("metadata required_scenarios contains duplicates") + raw_not_scored = metadata.get("not_scored_scenarios", []) + if not isinstance(raw_not_scored, list): + raise GateError("metadata not_scored_scenarios must be an array") + not_scored: dict[str, str] = {} + for index, raw in enumerate(raw_not_scored): + if not isinstance(raw, dict): + raise GateError(f"metadata not_scored_scenarios[{index}] must be an object") + scenario = raw.get("scenario") + reason = raw.get("reason") + if not isinstance(scenario, str) or not scenario or not isinstance(reason, str) or not reason: + raise GateError( + f"metadata not_scored_scenarios[{index}] requires non-empty scenario and reason" + ) + if scenario in not_scored: + raise GateError(f"metadata not_scored_scenarios duplicates {scenario!r}") + not_scored[scenario] = reason + + required_set = set(required) + if required_set & set(not_scored): + raise GateError("metadata scenario cannot be both required and not_scored") + reports = load_reports(report_root) classifications = load_classifications(baseline_path, profile) problems: list[str] = [] - missing_scenarios = sorted(set(required) - set(reports)) - unexpected_scenarios = sorted(set(reports) - set(required)) + missing_scenarios = sorted(required_set - set(reports)) + missing_not_scored_scenarios = sorted(set(not_scored) - set(reports)) + expected_scenarios = required_set | set(not_scored) + unexpected_scenarios = sorted(set(reports) - expected_scenarios) if missing_scenarios: problems.append("missing required scenario reports: " + ", ".join(missing_scenarios)) + if missing_not_scored_scenarios: + problems.append( + "missing not-scored scenario reports: " + ", ".join(missing_not_scored_scenarios) + ) if unexpected_scenarios: problems.append("unexpected scenario reports: " + ", ".join(unexpected_scenarios)) + invalid_classification_scenarios = sorted( + {entry.scenario for entry in classifications.values() if entry.scenario not in required_set} + ) + if invalid_classification_scenarios: + problems.append( + "classifications may target scored required scenarios only: " + + ", ".join(invalid_classification_scenarios) + ) + statuses = Counter() + informational_statuses = Counter() emitted: dict[tuple[str, str], set[str]] = {} applicable_checks = 0 unclassified: list[str] = [] expected: list[dict[str, str]] = [] + informational_non_success: list[dict[str, str]] = [] inconclusive: list[str] = [] for scenario, checks in sorted(reports.items()): + scored = scenario in required_set for check in checks: check_id = check["id"] status = check["status"] - statuses[status] += 1 key = (scenario, check_id) + if not scored: + informational_statuses[status] += 1 + if scenario in not_scored and status in NON_SUCCESS_STATUSES: + informational_non_success.append( + { + "scenario": scenario, + "check_id": check_id, + "status": status, + "reason": not_scored[scenario], + } + ) + continue + + statuses[status] += 1 emitted.setdefault(key, set()).add(status) if status in APPLICABLE_STATUSES: applicable_checks += 1 @@ -214,6 +266,8 @@ def evaluate_reports( stale: list[str] = [] for key, entry in sorted(classifications.items()): + if entry.scenario not in required_set: + continue states = emitted.get(key) label = f"{entry.scenario}:{entry.check_id}" if not states: @@ -232,11 +286,15 @@ def evaluate_reports( "harness_sha": metadata.get("harness_sha"), "harness_exit_code": metadata.get("harness_exit_code"), "required_scenario_count": len(required), + "not_scored_scenario_count": len(not_scored), "reported_scenario_count": len(reports), "applicable_check_count": applicable_checks, "status_counts": dict(sorted(statuses.items())), + "informational_status_counts": dict(sorted(informational_statuses.items())), "classified_non_success": expected, + "informational_non_success": informational_non_success, "missing_scenarios": missing_scenarios, + "missing_not_scored_scenarios": missing_not_scored_scenarios, "unexpected_scenarios": unexpected_scenarios, "problems": problems, "gate_passed": not problems, diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 47cae5934..820f1e141 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -29,7 +29,11 @@ def setUp(self) -> None: def tearDown(self) -> None: self.tmp.cleanup() - def write_metadata(self, scenarios: list[str]) -> None: + def write_metadata( + self, + scenarios: list[str], + not_scored: list[dict[str, str]] | None = None, + ) -> None: self.metadata.write_text( json.dumps( { @@ -38,6 +42,7 @@ def write_metadata(self, scenarios: list[str]) -> None: "harness_sha": "b" * 40, "harness_exit_code": 0, "required_scenarios": scenarios, + "not_scored_scenarios": not_scored or [], } ), encoding="utf-8", @@ -108,6 +113,30 @@ def test_missing_required_scenario_fails(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertEqual(summary["missing_scenarios"], ["scenario-b"]) + def test_not_scored_failure_is_visible_but_does_not_require_classification(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "extension"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("extension-a", [self.check("pending", "FAILURE")]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + self.assertEqual(summary["informational_status_counts"], {"FAILURE": 1}) + self.assertEqual(summary["informational_non_success"][0]["reason"], "extension") + + def test_missing_not_scored_scenario_fails_coverage_integrity(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "pending-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertEqual(summary["missing_not_scored_scenarios"], ["pending-a"]) + def test_classified_harness_error_remains_inconclusive(self) -> None: self.write_metadata(["scenario-a"]) self.write_baseline( diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index a8d206088..4c327a4e7 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -3,10 +3,509 @@ "harness_commit": "7169291ec0b68eb370fddcd9947313ab0d5e4156", "profiles": { "2025-11-25": { - "classifications": [] + "classifications": [ + { + "scenario": "completion-complete", + "check_id": "completion-complete", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "logging-set-level", + "check_id": "logging-set-level", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-embedded-resource", + "check_id": "prompts-get-embedded-resource", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-simple", + "check_id": "prompts-get-simple", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-with-args", + "check_id": "prompts-get-with-args", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-with-image", + "check_id": "prompts-get-with-image", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-list", + "check_id": "prompts-list", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-list", + "check_id": "resources-list", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-subscribe", + "check_id": "resources-subscribe", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-unsubscribe", + "check_id": "resources-unsubscribe", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-with-logging", + "check_id": "tools-call-with-logging", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "dns-rebinding-protection", + "check_id": "localhost-host-rebinding-rejected", + "classification": "genuine_protocol_failure", + "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", + "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "elicitation-sep1034-defaults", + "check_id": "elicitation-sep1034-general", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "elicitation-sep1330-enums", + "check_id": "elicitation-sep1330-general", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-read-binary", + "check_id": "resources-read-binary", + "classification": "optional_capability_not_implemented", + "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-read-text", + "check_id": "resources-read-text", + "classification": "optional_capability_not_implemented", + "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-templates-read", + "check_id": "resources-templates-read", + "classification": "optional_capability_not_implemented", + "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-audio", + "check_id": "tools-call-audio", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-elicitation", + "check_id": "tools-call-elicitation", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-embedded-resource", + "check_id": "tools-call-embedded-resource", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-error", + "check_id": "tools-call-error", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-image", + "check_id": "tools-call-image", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-mixed-content", + "check_id": "tools-call-mixed-content", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-sampling", + "check_id": "tools-call-sampling", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-simple-text", + "check_id": "tools-call-simple-text", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-with-progress", + "check_id": "tools-call-with-progress", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-list", + "check_id": "wire-schema-valid", + "classification": "genuine_protocol_failure", + "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", + "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report" + } + ] }, "2026-07-28": { - "classifications": [] + "classifications": [ + { + "scenario": "caching", + "check_id": "sep-2549-prompts-list-caching-hints", + "classification": "optional_capability_not_implemented", + "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", + "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "caching", + "check_id": "sep-2549-resources-templates-list-caching-hints", + "classification": "optional_capability_not_implemented", + "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", + "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "caching", + "check_id": "sep-2549-resources-read-caching-hints", + "classification": "missing_harness_fixture", + "reason": "resources/read caching could not be exercised because the fixture exposed no reference resource to read.", + "evidence": "SEP-2549 caching scenario result from first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "completion-complete", + "check_id": "completion-complete", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-embedded-resource", + "check_id": "prompts-get-embedded-resource", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-simple", + "check_id": "prompts-get-simple", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-with-args", + "check_id": "prompts-get-with-args", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-get-with-image", + "check_id": "prompts-get-with-image", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "prompts-list", + "check_id": "prompts-list", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "dns-rebinding-protection", + "check_id": "localhost-host-rebinding-rejected", + "classification": "genuine_protocol_failure", + "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", + "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-basic-elicitation", + "check_id": "sep-2322-elicitation-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-basic-list-roots", + "check_id": "sep-2322-list-roots-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-basic-sampling", + "check_id": "sep-2322-sampling-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-capability-check", + "check_id": "sep-2322-respect-client-capabilities", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-ignore-extra-params", + "check_id": "sep-2322-ignore-unexpected-params", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-missing-input-response", + "check_id": "sep-2322-missing-response-rerequests", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-multi-round", + "check_id": "sep-2322-multi-round-r1", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-multiple-input-requests", + "check_id": "sep-2322-multiple-inputs-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-request-state", + "check_id": "sep-2322-request-state-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-result-type", + "check_id": "sep-2322-result-type-included", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-tampered-state", + "check_id": "sep-2322-reject-tampered-state", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "input-required-result-non-tool-request", + "check_id": "sep-2322-non-tool-incomplete", + "classification": "optional_capability_not_implemented", + "reason": "This MRTR probe uses prompts/get, but the current WebCodex surface does not implement the optional prompts capability.", + "evidence": "SEP-2322 non-tool request scenario plus server capability model; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-read-binary", + "check_id": "resources-read-binary", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-read-text", + "check_id": "resources-read-text", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "resources-templates-read", + "check_id": "resources-templates-read", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "sep-2164-resource-not-found", + "check_id": "sep-2164-data-uri", + "classification": "genuine_protocol_failure", + "reason": "The resource-not-found error omits the recommended data.uri echo.", + "evidence": "SEP-2164 resource-not-found error data; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-http-server-no-independent-requests-on-stream", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-missing-capability-http-400", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-no-log-without-loglevel", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-rejects-undeclared-capability", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-request-meta-invalid-missing-meta", + "classification": "genuine_protocol_failure", + "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", + "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-request-meta-invalid-missing-protocol-version", + "classification": "genuine_protocol_failure", + "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", + "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-honors-notification-filter", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-sends-prompts-list-changed-on-subscription", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-sends-subscription-ack", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-sends-tools-list-changed-on-subscription", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-tags-subscription-id", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-audio", + "check_id": "tools-call-audio", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-embedded-resource", + "check_id": "tools-call-embedded-resource", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-error", + "check_id": "tools-call-error", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-image", + "check_id": "tools-call-image", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-mixed-content", + "check_id": "tools-call-mixed-content", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-simple-text", + "check_id": "tools-call-simple-text", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + }, + { + "scenario": "tools-call-with-progress", + "check_id": "tools-call-with-progress", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + } + ] } }, "supplemental_findings": [ From b0f148f33c7ffee083e8b497afe17869b7a35c77 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:05:41 +0000 Subject: [PATCH 04/14] test(mcp): harden conformance baseline gate --- docs/MCP_CONFORMANCE.md | 70 +++-- docs/TESTING.md | 13 +- scripts/mcp_conformance.sh | 162 ++++++++--- scripts/mcp_conformance_report.py | 148 +++++++++- scripts/tests/test_mcp_conformance_report.py | 191 +++++++++++-- tests/fixtures/mcp/conformance/baseline.json | 286 ++++++++++++++----- 6 files changed, 714 insertions(+), 156 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index c2792cd84..f1bd0da1a 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -8,10 +8,12 @@ checks that actually ran; it is not a blanket "MCP compliant" badge. ## Pinned referee and profiles `scripts/mcp_conformance.sh` pins the upstream referee to the immutable commit in -`tests/fixtures/mcp/conformance/baseline.json`. The script rejects any checkout at -a different commit and installs dependencies from that checkout's lockfile before -building it. Updating the pin is therefore a reviewed test-contract change, not a -floating dependency update. +`tests/fixtures/mcp/conformance/baseline.json`. The script rejects an external +checkout at a different commit, archives the pinned Git object into a fresh +script-owned build directory, and runs `npm ci` plus the referee build from that +archive every time. Existing `dist/` output and working-tree modifications are +therefore never trusted as pinned evidence. Updating the pin is a reviewed +test-contract change, not a floating dependency update. The ordinary baseline runs two dated server profiles independently: @@ -36,9 +38,14 @@ WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR=/path/to/conformance \ bash scripts/mcp_conformance.sh 2026-07-28 ``` -The supplied checkout must resolve to the exact pinned commit. Reports are written -to `target/mcp-conformance/reports` by default. CI uploads that directory even -when the gate fails so the raw evidence remains inspectable. +The supplied checkout must resolve to the exact pinned commit. A checkout supplied +through `WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR` is read-only from this script's +perspective: invalid or mismatched checkouts are rejected rather than deleted or +rewritten, and local working-tree changes are ignored by the pinned `git archive`. +Reports are written to `target/mcp-conformance/reports` by default. An explicitly +supplied report directory must be empty or carry the WebCodex ownership marker +before old profile output is removed. CI uploads the report directory even when +the gate fails so the raw evidence remains inspectable. ## What is exercised @@ -67,11 +74,17 @@ required server-scenario list and the reviewed baseline. Every `FAILURE`, `WARNING`, or `SKIPPED` check from a **scored required** scenario must have an exact `, ` classification in -`tests/fixtures/mcp/conformance/baseline.json`. Whole-scenario wildcards are -rejected because they can hide a newly failing check behind an unrelated known -failure. The referee's `not_scored` server scenarios are still required to run -and are retained as informational evidence, but their verdicts do not affect the -gate and do not consume expected-failure classifications. Supported scored +`tests/fixtures/mcp/conformance/baseline.json`. Each classification also pins the +observed status and a SHA-256 fingerprint of the stable failure evidence +(`errorMessage`, then structured details/metadata, falling back to the check's +name/description). If the same check ID starts failing for a different reason or +with a different severity, the gate requires review instead of reusing the old +classification. Whole-scenario wildcards are rejected. + +The referee's `not_scored` server scenarios are still required to run and are +retained as informational protocol evidence. Their protocol verdicts do not +consume expected-failure classifications, but infrastructure failures such as a +scenario timeout or connection failure remain gate-blocking. Supported scored classifications are: | Classification | Meaning | @@ -85,16 +98,22 @@ classifications are: The gate also fails when: - no applicable `SUCCESS`/`FAILURE`/`WARNING` checks were emitted; +- an individual required scenario emits no verdict (for example an empty or + INFO-only report); - a scored or `not_scored` scenario report expected by the frozen requirements is missing, duplicated, or an unknown scenario is unexpectedly present; +- a check contains an unknown status; - a new non-success check in a scored required scenario has no exact classification; -- a classified check is no longer emitted or now passes, making its baseline - entry stale; -- an inconclusive infrastructure result is present. +- a classified check changes status/evidence, disappears, or now passes; +- the referee exits abnormally, its exit code disagrees with the scored FAILURE + set, or any scored/not-scored scenario reports a timeout, connection failure, + or other recognized infrastructure error; +- an explicitly classified inconclusive infrastructure result is present. -This is intentionally stricter than trusting the referee process exit code. A -zero exit can coexist with skipped/unscored coverage, and an expected-failure run -can intentionally tolerate known protocol defects. +For the pinned referee, exit `0` means no scored FAILURE and exit `1` means at +least one scored FAILURE; other exit codes are treated as runner/infrastructure +failures. This is stricter than trusting the process exit code alone because the +raw reports still need complete coverage and reviewed classifications. The report gate's own regression suite is dependency-free: @@ -102,9 +121,10 @@ The report gate's own regression suite is dependency-free: python3 scripts/tests/test_mcp_conformance_report.py ``` -It covers all-skipped scored output, missing scored/informational scenarios, -harness errors, new scored failures, informational non-success reporting, stale -classifications, and rejection of broad masks. +It covers all-skipped output, missing/empty/INFO-only scenarios, unknown statuses, +abnormal referee exits, scored and not-scored infrastructure failures, changed +status/evidence for an existing check ID, new scored failures, informational +non-success reporting, stale classifications, and rejection of broad masks. ## Raw evidence and review policy @@ -118,8 +138,12 @@ Each profile retains: - the coverage-aware WebCodex summary. Baseline updates should be narrow. Add a classification only after reproducing -and understanding the exact check. Remove it as soon as that check passes. Do not -replace multiple check IDs with a scenario-wide exception. +and understanding the exact check, then record its observed status and evidence +fingerprint. A changed fingerprint is a review prompt, not something to refresh +mechanically. Remove the entry as soon as the check passes. Do not replace +multiple check IDs with a scenario-wide exception. The SEP-2164 `data.uri` +finding is kept as an advisory SHOULD/WARNING rather than described as a MUST +violation. This baseline deliberately records two already-demonstrated 2026 result-shape findings without fixing production behavior: streamed artifact `resources/read` omits the diff --git a/docs/TESTING.md b/docs/TESTING.md index 1e6980682..b625c8406 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -97,12 +97,13 @@ The lanes above define test semantics; workflows decide when to run them. lane is `success` when required or `skipped` when not required, avoiding a skipped required-check context that could leave branch protection pending. - MCP dated-revision evidence has its own bounded `mcp-conformance` lane. It pins - the upstream referee, runs the `2026-07-28` and `2025-11-25` server requirements - against a test-only loopback WebCodex endpoint, validates the report gate itself, - and uploads raw reports on success or failure. `test-native` requires this lane - to succeed, so missing coverage, stale classifications, and unclassified new - failures are merge-blocking even when the upstream referee exits zero. See - [`MCP_CONFORMANCE.md`](MCP_CONFORMANCE.md) for baseline semantics. + and freshly builds the upstream referee, runs the `2026-07-28` and `2025-11-25` + server requirements against a test-only loopback WebCodex endpoint, validates + the report gate itself, and uploads raw reports on success or failure. + `test-native` requires this lane to succeed, so missing per-scenario verdicts, + abnormal/infrastructure runs, stale or changed classifications, and unclassified + new failures are merge-blocking. See [`MCP_CONFORMANCE.md`](MCP_CONFORMANCE.md) + for baseline semantics. - Linux Rust execution remains package-sharded: the server package `webcodex`, the Runner/LSP packages, and the remaining workspace crates run in parallel. The Runner/LSP shard compiles with `--features runner-real-process-tests` to prevent diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index 4802bdc91..c172671fe 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -17,9 +17,27 @@ print(commit) PY )" WORK_ROOT="${WEBCODEX_MCP_CONFORMANCE_WORK_ROOT:-target/mcp-conformance}" -HARNESS_DIR="${WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR:-$WORK_ROOT/harness}" -REPORT_ROOT="${WEBCODEX_MCP_CONFORMANCE_REPORT_ROOT:-$WORK_ROOT/reports}" +HARNESS_DIR_EXTERNAL=0 +if [ "${WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR+x}" = x ]; then + HARNESS_DIR="${WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR}" + HARNESS_DIR_EXTERNAL=1 +else + HARNESS_DIR="$WORK_ROOT/harness" +fi +REPORT_ROOT_EXTERNAL=0 +if [ "${WEBCODEX_MCP_CONFORMANCE_REPORT_ROOT+x}" = x ]; then + REPORT_ROOT="${WEBCODEX_MCP_CONFORMANCE_REPORT_ROOT}" + REPORT_ROOT_EXTERNAL=1 +else + REPORT_ROOT="$WORK_ROOT/reports" +fi PROFILES=("$@") +HARNESS_OWNER_MARKER=".webcodex-mcp-conformance-harness" +REPORT_OWNER_MARKER=".webcodex-mcp-conformance-reports" +harness_build_dir="" +fixture_dir="" +fixture_pid="" +cleanup_started=0 if [ "$#" -eq 0 ]; then PROFILES=("2026-07-28" "2025-11-25") @@ -31,33 +49,130 @@ for profile in "${PROFILES[@]}"; do esac done -mkdir -p "$WORK_ROOT" "$REPORT_ROOT" +if [ -z "$WORK_ROOT" ] || [ -z "$HARNESS_DIR" ] || [ -z "$REPORT_ROOT" ]; then + echo "MCP conformance work, harness, and report paths must be non-empty" >&2 + exit 2 +fi +mkdir -p "$WORK_ROOT" + +stop_fixture() { + [ -n "$fixture_pid" ] || return 0 + touch "$fixture_dir/stop" 2>/dev/null || true + for _ in $(seq 1 50); do + if ! kill -0 "$fixture_pid" 2>/dev/null; then break; fi + sleep 0.1 + done + if kill -0 "$fixture_pid" 2>/dev/null; then + kill "$fixture_pid" 2>/dev/null || true + fi + for _ in $(seq 1 50); do + if ! kill -0 "$fixture_pid" 2>/dev/null; then break; fi + sleep 0.1 + done + if kill -0 "$fixture_pid" 2>/dev/null; then + kill -KILL "$fixture_pid" 2>/dev/null || true + fi + wait "$fixture_pid" 2>/dev/null || true + fixture_pid="" +} + +cleanup() { + [ "$cleanup_started" -eq 0 ] || return 0 + cleanup_started=1 + stop_fixture + if [ -n "$fixture_dir" ]; then + rm -rf "$fixture_dir" + fi + if [ -n "$harness_build_dir" ]; then + rm -rf "$harness_build_dir" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM -prepare_harness() { - if [ ! -d "$HARNESS_DIR/.git" ]; then - rm -rf "$HARNESS_DIR" +is_git_worktree() { + git -C "$1" rev-parse --is-inside-work-tree >/dev/null 2>&1 +} + +prepare_harness_source() { + if [ "$HARNESS_DIR_EXTERNAL" -eq 1 ]; then + if ! is_git_worktree "$HARNESS_DIR"; then + echo "WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR must name an existing Git worktree; refusing to modify it: $HARNESS_DIR" >&2 + exit 2 + fi + elif [ ! -e "$HARNESS_DIR" ]; then mkdir -p "$HARNESS_DIR" + touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" git -C "$HARNESS_DIR" init -q git -C "$HARNESS_DIR" remote add origin https://github.com/modelcontextprotocol/conformance.git + elif ! is_git_worktree "$HARNESS_DIR"; then + echo "default harness path exists but is not a Git worktree; refusing to remove it: $HARNESS_DIR" >&2 + exit 2 + elif [ ! -f "$HARNESS_DIR/$HARNESS_OWNER_MARKER" ]; then + origin_url="$(git -C "$HARNESS_DIR" remote get-url origin 2>/dev/null || true)" + case "$origin_url" in + https://github.com/modelcontextprotocol/conformance.git|https://github.com/modelcontextprotocol/conformance) + touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" + ;; + *) + echo "default harness checkout is not WebCodex-owned and has an unexpected origin; refusing to modify it" >&2 + exit 2 + ;; + esac + fi + + actual="$(git -C "$HARNESS_DIR" rev-parse HEAD 2>/dev/null || true)" + if [ "$actual" != "$HARNESS_COMMIT" ]; then + if [ "$HARNESS_DIR_EXTERNAL" -eq 1 ]; then + echo "MCP conformance harness must be exactly $HARNESS_COMMIT, got ${actual:-unresolved}; external checkout is never modified" >&2 + exit 2 + fi GIT_TERMINAL_PROMPT=0 git -C "$HARNESS_DIR" fetch -q --depth 1 origin "$HARNESS_COMMIT" git -C "$HARNESS_DIR" checkout -q --detach FETCH_HEAD + actual="$(git -C "$HARNESS_DIR" rev-parse HEAD)" fi - actual="$(git -C "$HARNESS_DIR" rev-parse HEAD)" if [ "$actual" != "$HARNESS_COMMIT" ]; then echo "MCP conformance harness must be exactly $HARNESS_COMMIT, got $actual" >&2 exit 2 fi - if [ ! -f "$HARNESS_DIR/package-lock.json" ]; then +} + +prepare_report_root() { + if [ -e "$REPORT_ROOT" ] && [ ! -d "$REPORT_ROOT" ]; then + echo "MCP conformance report root is not a directory: $REPORT_ROOT" >&2 + exit 2 + fi + mkdir -p "$REPORT_ROOT" + if [ "$REPORT_ROOT_EXTERNAL" -eq 1 ] && [ ! -f "$REPORT_ROOT/$REPORT_OWNER_MARKER" ]; then + if find "$REPORT_ROOT" -mindepth 1 -maxdepth 1 -print -quit | grep -q .; then + echo "external MCP report root is non-empty and not WebCodex-owned; refusing to delete its contents: $REPORT_ROOT" >&2 + exit 2 + fi + fi + touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" + rm -rf "$REPORT_ROOT/2026-07-28" "$REPORT_ROOT/2025-11-25" + rm -f "$REPORT_ROOT/fixture.log" +} + +prepare_harness_build() { + harness_build_dir="$(mktemp -d "$WORK_ROOT/harness-build.XXXXXX")" + git -C "$HARNESS_DIR" archive "$HARNESS_COMMIT" | tar -x -C "$harness_build_dir" + if [ ! -f "$harness_build_dir/package-lock.json" ]; then echo "pinned harness has no package-lock.json" >&2 exit 2 fi - if [ ! -f "$HARNESS_DIR/dist/index.js" ]; then - npm --prefix "$HARNESS_DIR" ci --ignore-scripts --no-audit --no-fund - npm --prefix "$HARNESS_DIR" run build + npm --prefix "$harness_build_dir" ci --ignore-scripts --no-audit --no-fund + npm --prefix "$harness_build_dir" run build + if [ ! -f "$harness_build_dir/dist/index.js" ]; then + echo "pinned harness build did not produce dist/index.js" >&2 + exit 2 fi } -prepare_harness +prepare_harness_source +prepare_report_root +prepare_harness_build python3 scripts/tests/test_mcp_conformance_report.py # Compile before starting the bounded readiness clock. A cold WebCodex test build @@ -69,25 +184,6 @@ fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/webcodex-mcp-conformance.XXXXXX")" url_file="$fixture_dir/url" stop_file="$fixture_dir/stop" fixture_log="$REPORT_ROOT/fixture.log" -fixture_pid="" -cleanup() { - touch "$stop_file" 2>/dev/null || true - if [ -n "$fixture_pid" ]; then - for _ in $(seq 1 50); do - if ! kill -0 "$fixture_pid" 2>/dev/null; then break; fi - sleep 0.1 - done - if kill -0 "$fixture_pid" 2>/dev/null; then - kill "$fixture_pid" 2>/dev/null || true - fi - wait "$fixture_pid" 2>/dev/null || true - fi - rm -rf "$fixture_dir" -} -trap cleanup EXIT INT TERM - -rm -rf "$REPORT_ROOT" -mkdir -p "$REPORT_ROOT" WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ cargo test --locked -p webcodex --lib mcp_conformance_fixture_server -- --ignored --nocapture \ @@ -122,7 +218,7 @@ for profile in "${PROFILES[@]}"; do mkdir -p "$raw" log="$profile_root/harness.log" set +e - node "$HARNESS_DIR/dist/index.js" server \ + node "$harness_build_dir/dist/index.js" server \ --url "$fixture_url" \ --requirements "$profile" \ --output-dir "$raw" \ @@ -132,7 +228,7 @@ for profile in "${PROFILES[@]}"; do set -e metadata="$profile_root/metadata.json" - python3 - "$HARNESS_DIR/requirements/$profile.yaml" "$metadata" "$profile" \ + python3 - "$harness_build_dir/requirements/$profile.yaml" "$metadata" "$profile" \ "$server_sha" "$HARNESS_COMMIT" "$harness_exit" <<'PY' import json import pathlib diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index 32e4eac32..a1ef37119 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -12,6 +12,7 @@ from __future__ import annotations import argparse +import hashlib import json import re import sys @@ -27,9 +28,26 @@ "harness_limitation_pending", "inconclusive_infrastructure", } +VALID_STATUSES = {"SUCCESS", "FAILURE", "WARNING", "SKIPPED", "INFO"} NON_SUCCESS_STATUSES = {"FAILURE", "WARNING", "SKIPPED"} APPLICABLE_STATUSES = {"SUCCESS", "FAILURE", "WARNING"} +VERDICT_STATUSES = {"SUCCESS", "FAILURE", "WARNING", "SKIPPED"} SCENARIO_DIR_RE = re.compile(r"^server-(.+)-\d{4}-\d{2}-\d{2}T.*Z$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +INFRASTRUCTURE_ERROR_MARKERS = ( + "econnrefused", + "connection refused", + "econnreset", + "connection reset", + "enotfound", + "fetch failed", + "failed to fetch", + "network error", + "socket hang up", + "did not complete within", + "timed out", + "connect timeout", +) class GateError(RuntimeError): @@ -40,6 +58,8 @@ class GateError(RuntimeError): class Classification: scenario: str check_id: str + expected_status: str + evidence_sha256: str classification: str reason: str evidence: str @@ -49,6 +69,44 @@ def key(self) -> tuple[str, str]: return (self.scenario, self.check_id) +def check_evidence_sha256(check: dict[str, Any]) -> str: + """Hash stable failure evidence while excluding timestamps and verbose logs.""" + + error_message = check.get("errorMessage") + if isinstance(error_message, str) and error_message: + payload: dict[str, Any] = {"errorMessage": error_message} + elif "details" in check: + payload = {"details": check.get("details")} + elif "metadata" in check: + payload = {"metadata": check.get("metadata")} + else: + payload = { + "name": check.get("name"), + "description": check.get("description"), + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _infrastructure_reason(check: dict[str, Any]) -> str | None: + if check.get("id") == "scenario-timeout": + return "referee scenario timeout" + if check.get("description") == "Failed to run scenario": + return "referee failed to run scenario" + error_message = check.get("errorMessage") + if isinstance(error_message, str): + lowered = error_message.lower() + for marker in INFRASTRUCTURE_ERROR_MARKERS: + if marker in lowered: + return f"network/timeout error marker {marker!r}" + return None + + def _load_json(path: Path) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) @@ -87,6 +145,8 @@ def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: raise GateError(f"{checks_path}[{index}] has no non-empty check id") if not isinstance(status, str) or not status: raise GateError(f"{checks_path}[{index}] has no non-empty status") + if status not in VALID_STATUSES: + raise GateError(f"{checks_path}[{index}] has unknown status {status!r}") checks.append(check) reports[scenario] = checks return reports @@ -94,8 +154,8 @@ def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, str], Classification]: baseline = _load_json(baseline_path) - if not isinstance(baseline, dict) or baseline.get("schema_version") != 1: - raise GateError("baseline must be an object with schema_version=1") + if not isinstance(baseline, dict) or baseline.get("schema_version") != 2: + raise GateError("baseline must be an object with schema_version=2") profiles = baseline.get("profiles") if not isinstance(profiles, dict) or profile not in profiles: raise GateError(f"baseline has no profile {profile!r}") @@ -112,6 +172,8 @@ def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, s raise GateError(f"classification #{index + 1} must be an object") scenario = raw.get("scenario") check_id = raw.get("check_id") + expected_status = raw.get("expected_status") + evidence_sha256 = raw.get("evidence_sha256") kind = raw.get("classification") reason = raw.get("reason") evidence = raw.get("evidence") @@ -123,6 +185,15 @@ def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, s raise GateError( f"classification {scenario!r} must name one exact check_id; broad scenario masks are forbidden" ) + if expected_status not in NON_SUCCESS_STATUSES: + raise GateError( + f"classification {scenario}:{check_id} needs expected_status in " + f"{sorted(NON_SUCCESS_STATUSES)}, got {expected_status!r}" + ) + if not isinstance(evidence_sha256, str) or not SHA256_RE.fullmatch(evidence_sha256): + raise GateError( + f"classification {scenario}:{check_id} needs a lowercase 64-hex evidence_sha256" + ) if kind not in CLASSIFICATIONS: raise GateError( f"classification {scenario}:{check_id} has unsupported kind {kind!r}" @@ -131,7 +202,15 @@ def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, s raise GateError(f"classification {scenario}:{check_id} needs a reason") if not isinstance(evidence, str) or not evidence.strip(): raise GateError(f"classification {scenario}:{check_id} needs evidence") - entry = Classification(scenario, check_id, kind, reason.strip(), evidence.strip()) + entry = Classification( + scenario, + check_id, + expected_status, + evidence_sha256, + kind, + reason.strip(), + evidence.strip(), + ) if entry.key in entries: raise GateError(f"duplicate classification for {scenario}:{check_id}") entries[entry.key] = entry @@ -151,6 +230,9 @@ def evaluate_reports( raise GateError( f"metadata profile {metadata.get('profile')!r} does not match requested {profile!r}" ) + harness_exit_code = metadata.get("harness_exit_code") + if isinstance(harness_exit_code, bool) or not isinstance(harness_exit_code, int): + raise GateError("metadata harness_exit_code must be an integer") required = metadata.get("required_scenarios") if not isinstance(required, list) or not required or not all(isinstance(item, str) and item for item in required): raise GateError("metadata required_scenarios must be a non-empty string array") @@ -211,13 +293,25 @@ def evaluate_reports( expected: list[dict[str, str]] = [] informational_non_success: list[dict[str, str]] = [] inconclusive: list[str] = [] + evidence_mismatches: list[str] = [] + infrastructure_failures: list[str] = [] + baseline_observations: list[dict[str, str]] = [] for scenario, checks in sorted(reports.items()): scored = scenario in required_set + if scored and not any(check["status"] in VERDICT_STATUSES for check in checks): + problems.append(f"required scenario {scenario!r} emitted no verdict checks") + if not scored and not checks: + problems.append(f"not-scored scenario {scenario!r} emitted an empty report") for check in checks: check_id = check["id"] status = check["status"] key = (scenario, check_id) + infra_reason = _infrastructure_reason(check) + if infra_reason is not None and status in NON_SUCCESS_STATUSES: + infrastructure_failures.append( + f"{scenario}:{check_id} [{status}] ({infra_reason})" + ) if not scored: informational_statuses[status] += 1 if scenario in not_scored and status in NON_SUCCESS_STATUSES: @@ -238,14 +332,35 @@ def evaluate_reports( if status in NON_SUCCESS_STATUSES: entry = classifications.get(key) label = f"{scenario}:{check_id} [{status}]" + observed_evidence = check_evidence_sha256(check) + baseline_observations.append( + { + "scenario": scenario, + "check_id": check_id, + "expected_status": status, + "evidence_sha256": observed_evidence, + } + ) if entry is None: unclassified.append(label) continue + if entry.expected_status != status: + evidence_mismatches.append( + f"{label} expected status {entry.expected_status}" + ) + continue + if entry.evidence_sha256 != observed_evidence: + evidence_mismatches.append( + f"{label} evidence changed: expected {entry.evidence_sha256}, " + f"observed {observed_evidence}" + ) + continue expected.append( { "scenario": scenario, "check_id": check_id, "status": status, + "evidence_sha256": observed_evidence, "classification": entry.classification, "reason": entry.reason, "evidence": entry.evidence, @@ -256,8 +371,29 @@ def evaluate_reports( if applicable_checks == 0: problems.append("zero applicable SUCCESS/FAILURE/WARNING checks were emitted") + if harness_exit_code not in {0, 1}: + problems.append( + f"conformance referee exited abnormally with code {harness_exit_code}; only 0/1 are normal" + ) + scored_failures = statuses.get("FAILURE", 0) + if harness_exit_code == 0 and scored_failures > 0: + problems.append( + f"referee exit code 0 disagrees with {scored_failures} scored FAILURE check(s)" + ) + if harness_exit_code == 1 and scored_failures == 0: + problems.append("referee exit code 1 had no scored FAILURE checks") + if infrastructure_failures: + problems.append( + "infrastructure failures cannot satisfy the gate: " + + ", ".join(sorted(infrastructure_failures)) + ) if unclassified: problems.append("unclassified non-success checks: " + ", ".join(sorted(unclassified))) + if evidence_mismatches: + problems.append( + "classified check status/evidence changed and requires review: " + + ", ".join(sorted(evidence_mismatches)) + ) if inconclusive: problems.append( "inconclusive infrastructure results cannot satisfy the gate: " @@ -280,11 +416,11 @@ def evaluate_reports( problems.append("stale or uncovered classifications: " + ", ".join(stale)) return { - "schema_version": 1, + "schema_version": 2, "profile": profile, "server_sha": metadata.get("server_sha"), "harness_sha": metadata.get("harness_sha"), - "harness_exit_code": metadata.get("harness_exit_code"), + "harness_exit_code": harness_exit_code, "required_scenario_count": len(required), "not_scored_scenario_count": len(not_scored), "reported_scenario_count": len(reports), @@ -293,6 +429,8 @@ def evaluate_reports( "informational_status_counts": dict(sorted(informational_statuses.items())), "classified_non_success": expected, "informational_non_success": informational_non_success, + "baseline_observations": baseline_observations, + "infrastructure_failures": infrastructure_failures, "missing_scenarios": missing_scenarios, "missing_not_scored_scenarios": missing_not_scored_scenarios, "unexpected_scenarios": unexpected_scenarios, diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 820f1e141..85c3eca40 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -33,6 +33,7 @@ def write_metadata( self, scenarios: list[str], not_scored: list[dict[str, str]] | None = None, + harness_exit_code: int = 0, ) -> None: self.metadata.write_text( json.dumps( @@ -40,7 +41,7 @@ def write_metadata( "profile": self.profile, "server_sha": "a" * 40, "harness_sha": "b" * 40, - "harness_exit_code": 0, + "harness_exit_code": harness_exit_code, "required_scenarios": scenarios, "not_scored_scenarios": not_scored or [], } @@ -52,33 +53,49 @@ def write_baseline(self, entries: list[dict[str, str]]) -> None: self.baseline.write_text( json.dumps( { - "schema_version": 1, + "schema_version": 2, "profiles": {self.profile: {"classifications": entries}}, } ), encoding="utf-8", ) - def write_checks(self, scenario: str, checks: list[dict[str, str]]) -> None: + def write_checks(self, scenario: str, checks: list[dict[str, object]]) -> None: result = self.raw / f"server-{scenario}-2026-09-14T00-00-00-000Z" result.mkdir() (result / "checks.json").write_text(json.dumps(checks), encoding="utf-8") @staticmethod - def check(check_id: str, status: str) -> dict[str, str]: - return { + def check( + check_id: str, + status: str, + *, + error_message: str | None = None, + description: str | None = None, + ) -> dict[str, object]: + result: dict[str, object] = { "id": check_id, "name": check_id, - "description": check_id, + "description": description or check_id, "status": status, "timestamp": "2026-09-14T00:00:00Z", } + if error_message is not None: + result["errorMessage"] = error_message + return result @staticmethod - def classification(check_id: str, kind: str = "genuine_protocol_failure") -> dict[str, str]: + def classification( + check: dict[str, object], + kind: str = "genuine_protocol_failure", + *, + scenario: str = "scenario-a", + ) -> dict[str, str]: return { - "scenario": "scenario-a", - "check_id": check_id, + "scenario": scenario, + "check_id": str(check["id"]), + "expected_status": str(check["status"]), + "evidence_sha256": report_gate.check_evidence_sha256(check), "classification": kind, "reason": "reviewed baseline reason", "evidence": "MCP 2026-07-28 section/example", @@ -95,12 +112,21 @@ def test_successful_nonempty_coverage_passes(self) -> None: self.assertTrue(summary["gate_passed"]) self.assertEqual(summary["applicable_check_count"], 1) + def test_exact_classified_failure_passes(self) -> None: + failed = self.check("known-gap", "FAILURE", error_message="stable protocol mismatch") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(failed)]) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + def test_all_skipped_is_not_mistaken_for_coverage(self) -> None: + skipped = self.check("not-applicable", "SKIPPED") self.write_metadata(["scenario-a"]) self.write_baseline( - [self.classification("not-applicable", "inconclusive_infrastructure")] + [self.classification(skipped, "inconclusive_infrastructure")] ) - self.write_checks("scenario-a", [self.check("not-applicable", "SKIPPED")]) + self.write_checks("scenario-a", [skipped]) summary = self.evaluate() self.assertFalse(summary["gate_passed"]) self.assertTrue(any("zero applicable" in problem for problem in summary["problems"])) @@ -113,6 +139,31 @@ def test_missing_required_scenario_fails(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertEqual(summary["missing_scenarios"], ["scenario-b"]) + def test_required_empty_report_fails_even_with_other_coverage(self) -> None: + self.write_metadata(["scenario-a", "scenario-b"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("scenario-b", []) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("scenario-b" in p and "no verdict" in p for p in summary["problems"])) + + def test_required_info_only_report_fails_even_with_other_coverage(self) -> None: + self.write_metadata(["scenario-a", "scenario-b"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("scenario-b", [self.check("info", "INFO")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("scenario-b" in p and "no verdict" in p for p in summary["problems"])) + + def test_unknown_status_is_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("broken", "ERROR")]) + with self.assertRaisesRegex(report_gate.GateError, "unknown status"): + self.evaluate() + def test_not_scored_failure_is_visible_but_does_not_require_classification(self) -> None: self.write_metadata( ["scenario-a"], @@ -126,6 +177,27 @@ def test_not_scored_failure_is_visible_but_does_not_require_classification(self) self.assertEqual(summary["informational_status_counts"], {"FAILURE": 1}) self.assertEqual(summary["informational_non_success"][0]["reason"], "extension") + def test_not_scored_timeout_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "extension"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "extension-a", + [ + self.check( + "scenario-timeout", + "FAILURE", + error_message="Scenario 'extension-a' did not complete within 10000ms.", + ) + ], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + def test_missing_not_scored_scenario_fails_coverage_integrity(self) -> None: self.write_metadata( ["scenario-a"], @@ -137,35 +209,118 @@ def test_missing_not_scored_scenario_fails_coverage_integrity(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertEqual(summary["missing_not_scored_scenarios"], ["pending-a"]) + def test_empty_not_scored_report_fails_coverage_integrity(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "pending-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("pending-a", []) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("not-scored scenario" in p and "empty" in p for p in summary["problems"])) + def test_classified_harness_error_remains_inconclusive(self) -> None: - self.write_metadata(["scenario-a"]) + failed = self.check("scenario-a", "FAILURE", error_message="unreviewable harness result") + self.write_metadata(["scenario-a"], harness_exit_code=1) self.write_baseline( - [self.classification("scenario-a", "inconclusive_infrastructure")] + [self.classification(failed, "inconclusive_infrastructure")] ) - self.write_checks("scenario-a", [self.check("scenario-a", "FAILURE")]) + self.write_checks("scenario-a", [failed]) summary = self.evaluate() self.assertFalse(summary["gate_passed"]) self.assertTrue(any("inconclusive infrastructure" in p for p in summary["problems"])) + def test_abnormal_harness_exit_is_infrastructure_failure(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=137) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("exited abnormally" in p for p in summary["problems"])) + + def test_exit_one_without_scored_failure_is_rejected(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("exit code 1 had no scored FAILURE" in p for p in summary["problems"])) + def test_new_failure_requires_exact_classification(self) -> None: - self.write_metadata(["scenario-a"]) + self.write_metadata(["scenario-a"], harness_exit_code=1) self.write_baseline([]) self.write_checks("scenario-a", [self.check("new-regression", "FAILURE")]) summary = self.evaluate() self.assertFalse(summary["gate_passed"]) self.assertTrue(any("new-regression" in p for p in summary["problems"])) + def test_changed_failure_evidence_requires_review(self) -> None: + expected = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="Unknown tool: test_simple_text", + ) + actual = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="Tool returned a different protocol error", + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_connection_failure_cannot_hide_behind_fixture_classification(self) -> None: + expected = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="Unknown tool: test_simple_text", + ) + actual = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="connect ECONNREFUSED 127.0.0.1:12345", + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_status_change_requires_review(self) -> None: + expected = self.check("subscription-ack", "SKIPPED") + actual = self.check( + "subscription-ack", + "FAILURE", + error_message="subscription protocol assertion failed", + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline( + [self.classification(expected, "optional_capability_not_implemented")] + ) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("expected status SKIPPED" in p for p in summary["problems"])) + def test_passing_expected_failure_is_stale(self) -> None: + expected = self.check("fixed-check", "FAILURE", error_message="known bug") self.write_metadata(["scenario-a"]) - self.write_baseline([self.classification("fixed-check")]) + self.write_baseline([self.classification(expected)]) self.write_checks("scenario-a", [self.check("fixed-check", "SUCCESS")]) summary = self.evaluate() self.assertFalse(summary["gate_passed"]) self.assertTrue(any("now passing" in p for p in summary["problems"])) def test_info_only_expected_failure_is_uncovered(self) -> None: + expected = self.check("no-verdict", "FAILURE", error_message="known bug") self.write_metadata(["scenario-a"]) - self.write_baseline([self.classification("no-verdict")]) + self.write_baseline([self.classification(expected)]) self.write_checks( "scenario-a", [self.check("no-verdict", "INFO"), self.check("ok", "SUCCESS")], @@ -181,6 +336,8 @@ def test_broad_expected_failure_masks_are_rejected(self) -> None: { "scenario": "scenario-a", "check_id": "*", + "expected_status": "FAILURE", + "evidence_sha256": "0" * 64, "classification": "genuine_protocol_failure", "reason": "too broad", "evidence": "none", diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index 4c327a4e7..0bf81ee6b 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -1,5 +1,5 @@ { - "schema_version": 1, + "schema_version": 2, "harness_commit": "7169291ec0b68eb370fddcd9947313ab0d5e4156", "profiles": { "2025-11-25": { @@ -9,189 +9,243 @@ "check_id": "completion-complete", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "logging-set-level", "check_id": "logging-set-level", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-embedded-resource", "check_id": "prompts-get-embedded-resource", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-simple", "check_id": "prompts-get-simple", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-with-args", "check_id": "prompts-get-with-args", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-with-image", "check_id": "prompts-get-with-image", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-list", "check_id": "prompts-list", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-list", "check_id": "resources-list", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-subscribe", "check_id": "resources-subscribe", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-unsubscribe", "check_id": "resources-unsubscribe", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-with-logging", "check_id": "tools-call-with-logging", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "dns-rebinding-protection", "check_id": "localhost-host-rebinding-rejected", "classification": "genuine_protocol_failure", "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", - "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report" + "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "elicitation-sep1034-defaults", "check_id": "elicitation-sep1034-general", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "elicitation-sep1330-enums", "check_id": "elicitation-sep1330-general", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-read-binary", "check_id": "resources-read-binary", "classification": "optional_capability_not_implemented", "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", - "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report" + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-read-text", "check_id": "resources-read-text", "classification": "optional_capability_not_implemented", "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", - "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report" + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-templates-read", "check_id": "resources-templates-read", "classification": "optional_capability_not_implemented", "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", - "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report" + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-audio", "check_id": "tools-call-audio", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-elicitation", "check_id": "tools-call-elicitation", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-embedded-resource", "check_id": "tools-call-embedded-resource", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-error", "check_id": "tools-call-error", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-image", "check_id": "tools-call-image", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-mixed-content", "check_id": "tools-call-mixed-content", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-sampling", "check_id": "tools-call-sampling", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-simple-text", "check_id": "tools-call-simple-text", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-with-progress", "check_id": "tools-call-with-progress", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-list", "check_id": "wire-schema-valid", "classification": "genuine_protocol_failure", "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", - "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report" + "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" } ] }, @@ -202,308 +256,396 @@ "check_id": "sep-2549-prompts-list-caching-hints", "classification": "optional_capability_not_implemented", "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", - "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "caching", "check_id": "sep-2549-resources-templates-list-caching-hints", "classification": "optional_capability_not_implemented", "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", - "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "caching", "check_id": "sep-2549-resources-read-caching-hints", "classification": "missing_harness_fixture", "reason": "resources/read caching could not be exercised because the fixture exposed no reference resource to read.", - "evidence": "SEP-2549 caching scenario result from first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2549 caching scenario result from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "completion-complete", "check_id": "completion-complete", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-embedded-resource", "check_id": "prompts-get-embedded-resource", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-simple", "check_id": "prompts-get-simple", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-with-args", "check_id": "prompts-get-with-args", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-get-with-image", "check_id": "prompts-get-with-image", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "prompts-list", "check_id": "prompts-list", "classification": "optional_capability_not_implemented", "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", - "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report" + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "dns-rebinding-protection", "check_id": "localhost-host-rebinding-rejected", "classification": "genuine_protocol_failure", "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", - "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report" + "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-basic-elicitation", "check_id": "sep-2322-elicitation-incomplete", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-basic-list-roots", "check_id": "sep-2322-list-roots-incomplete", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-basic-sampling", "check_id": "sep-2322-sampling-incomplete", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-capability-check", "check_id": "sep-2322-respect-client-capabilities", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-ignore-extra-params", "check_id": "sep-2322-ignore-unexpected-params", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "WARNING", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-missing-input-response", "check_id": "sep-2322-missing-response-rerequests", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "WARNING", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-multi-round", "check_id": "sep-2322-multi-round-r1", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-multiple-input-requests", "check_id": "sep-2322-multiple-inputs-incomplete", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-request-state", "check_id": "sep-2322-request-state-incomplete", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-result-type", "check_id": "sep-2322-result-type-included", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-tampered-state", "check_id": "sep-2322-reject-tampered-state", "classification": "missing_harness_fixture", "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", - "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "input-required-result-non-tool-request", "check_id": "sep-2322-non-tool-incomplete", "classification": "optional_capability_not_implemented", "reason": "This MRTR probe uses prompts/get, but the current WebCodex surface does not implement the optional prompts capability.", - "evidence": "SEP-2322 non-tool request scenario plus server capability model; first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2322 non-tool request scenario plus server capability model; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-read-binary", "check_id": "resources-read-binary", "classification": "missing_harness_fixture", "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", - "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-read-text", "check_id": "resources-read-text", "classification": "missing_harness_fixture", "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", - "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "resources-templates-read", "check_id": "resources-templates-read", "classification": "missing_harness_fixture", "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", - "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "sep-2164-resource-not-found", "check_id": "sep-2164-data-uri", "classification": "genuine_protocol_failure", "reason": "The resource-not-found error omits the recommended data.uri echo.", - "evidence": "SEP-2164 resource-not-found error data; first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2164 resource-not-found error data; first GitHub Actions conformance baseline raw report", + "expected_status": "WARNING", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-http-server-no-independent-requests-on-stream", "classification": "missing_harness_fixture", "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", - "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-missing-capability-http-400", "classification": "missing_harness_fixture", "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", - "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-no-log-without-loglevel", "classification": "missing_harness_fixture", "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", - "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-rejects-undeclared-capability", "classification": "missing_harness_fixture", "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", - "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-request-meta-invalid-missing-meta", "classification": "genuine_protocol_failure", "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", - "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-request-meta-invalid-missing-protocol-version", "classification": "genuine_protocol_failure", "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", - "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-honors-notification-filter", "classification": "optional_capability_not_implemented", "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", - "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-sends-prompts-list-changed-on-subscription", "classification": "optional_capability_not_implemented", "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", - "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-sends-subscription-ack", "classification": "optional_capability_not_implemented", "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", - "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-sends-tools-list-changed-on-subscription", "classification": "optional_capability_not_implemented", "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", - "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "server-stateless", "check_id": "sep-2575-server-tags-subscription-id", "classification": "optional_capability_not_implemented", "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", - "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report" + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-audio", "check_id": "tools-call-audio", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-embedded-resource", "check_id": "tools-call-embedded-resource", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-error", "check_id": "tools-call-error", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-image", "check_id": "tools-call-image", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-mixed-content", "check_id": "tools-call-mixed-content", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-simple-text", "check_id": "tools-call-simple-text", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, { "scenario": "tools-call-with-progress", "check_id": "tools-call-with-progress", "classification": "missing_harness_fixture", "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", - "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report" + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" } ] } From 3c72b1c55908dcc7a40c81465cfce71770e0eaa2 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:27:17 +0000 Subject: [PATCH 05/14] test(mcp): bind conformance baseline evidence --- docs/MCP_CONFORMANCE.md | 24 ++- scripts/mcp_conformance.sh | 131 +++++++++++++++- scripts/mcp_conformance_report.py | 85 +++++++--- scripts/tests/test_mcp_conformance_report.py | 81 +++++++++- tests/fixtures/mcp/conformance/baseline.json | 156 ++++++++++--------- 5 files changed, 371 insertions(+), 106 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index f1bd0da1a..644b4726f 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -81,6 +81,14 @@ name/description). If the same check ID starts failing for a different reason or with a different severity, the gate requires review instead of reusing the old classification. Whole-scenario wildcards are rejected. +Classifications that rely on an optional capability also bind to the fixture's +actual capability advertisement. Before the referee run, the script probes the +same HTTP endpoint with `server/discover` for `2026-07-28` and `initialize` for +`2025-11-25`, keeps the relevant tools/resources/prompts/logging/completions +capabilities, and compares them with the profile snapshot in the baseline. If the +server starts advertising a previously absent capability, those optional +classifications require review even when the old check ID still fails. + The referee's `not_scored` server scenarios are still required to run and are retained as informational protocol evidence. Their protocol verdicts do not consume expected-failure classifications, but infrastructure failures such as a @@ -106,8 +114,10 @@ The gate also fails when: - a new non-success check in a scored required scenario has no exact classification; - a classified check changes status/evidence, disappears, or now passes; - the referee exits abnormally, its exit code disagrees with the scored FAILURE - set, or any scored/not-scored scenario reports a timeout, connection failure, - or other recognized infrastructure error; + set, or any scored/not-scored scenario emits one of the pinned referee's + structural infrastructure failures (`scenario-timeout` or `Failed to run scenario`); +- the server capability advertisement changes while optional-capability baseline + entries depend on the previous advertisement; - an explicitly classified inconclusive infrastructure result is present. For the pinned referee, exit `0` means no scored FAILURE and exit `1` means at @@ -123,8 +133,10 @@ python3 scripts/tests/test_mcp_conformance_report.py It covers all-skipped output, missing/empty/INFO-only scenarios, unknown statuses, abnormal referee exits, scored and not-scored infrastructure failures, changed -status/evidence for an existing check ID, new scored failures, informational -non-success reporting, stale classifications, and rejection of broad masks. +status/evidence for an existing check ID, capability-advertisement drift, +duplicate check IDs, harness-pin mismatch, placeholder evidence, new scored +failures, informational non-success reporting, stale classifications, and +rejection of broad masks. ## Raw evidence and review policy @@ -133,8 +145,8 @@ Each profile retains: - the referee's raw per-scenario `checks.json` files; - the referee stdout/stderr log; - metadata containing the WebCodex source SHA, referee SHA, profile, upstream - referee exit code, exact scored scenario list, and frozen `not_scored` server - scenarios with their upstream reasons; + referee exit code, observed server capability projection, exact scored scenario + list, and frozen `not_scored` server scenarios with their upstream reasons; - the coverage-aware WebCodex summary. Baseline updates should be narrow. Add a classification only after reproducing diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index c172671fe..4f2d4bf3f 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -152,7 +152,7 @@ prepare_report_root() { fi touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" rm -rf "$REPORT_ROOT/2026-07-28" "$REPORT_ROOT/2025-11-25" - rm -f "$REPORT_ROOT/fixture.log" + rm -f "$REPORT_ROOT/fixture.log" "$REPORT_ROOT/server-capabilities.json" } prepare_harness_build() { @@ -210,6 +210,118 @@ case "$fixture_url" in *) echo "fixture published unexpected URL: $fixture_url" >&2; exit 1 ;; esac +capabilities_file="$REPORT_ROOT/server-capabilities.json" +python3 - "$fixture_url" "$capabilities_file" <<'PY' +import json +import pathlib +import sys +import urllib.error +import urllib.request + +server_url, output_path = sys.argv[1:] + +def rpc_request(payload, headers=None): + encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") + request_headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + if headers: + request_headers.update(headers) + request = urllib.request.Request( + server_url, + data=encoded, + headers=request_headers, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + content_type = response.headers.get("Content-Type", "") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise SystemExit(f"capability probe HTTP {exc.code}: {body[:500]}") from exc + except OSError as exc: + raise SystemExit(f"capability probe transport error: {exc}") from exc + + messages = [] + if "text/event-stream" in content_type: + for line in raw.splitlines(): + if not line.startswith("data:"): + continue + candidate = line[len("data:"):].strip() + if candidate: + messages.append(json.loads(candidate)) + else: + messages.append(json.loads(raw)) + request_id = payload["id"] + message = next((item for item in messages if item.get("id") == request_id), None) + if not isinstance(message, dict): + raise SystemExit("capability probe did not receive the matching JSON-RPC response") + if "error" in message: + raise SystemExit(f"capability probe returned JSON-RPC error: {message['error']}") + result = message.get("result") + if not isinstance(result, dict): + raise SystemExit("capability probe response has no result object") + capabilities = result.get("capabilities") + if not isinstance(capabilities, dict): + raise SystemExit("capability probe result has no capabilities object") + return capabilities + +modern = rpc_request( + { + "jsonrpc": "2.0", + "id": "webcodex-capabilities-2026", + "method": "server/discover", + "params": { + "_meta": { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": {}, + "io.modelcontextprotocol/clientInfo": { + "name": "webcodex-conformance-capability-probe", + "version": "1.0", + }, + } + }, + }, + { + "MCP-Protocol-Version": "2026-07-28", + "Mcp-Method": "server/discover", + }, +) +legacy = rpc_request( + { + "jsonrpc": "2.0", + "id": "webcodex-capabilities-2025", + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { + "name": "webcodex-conformance-capability-probe", + "version": "1.0", + }, + }, + } +) + +relevant_keys = ("completions", "logging", "prompts", "resources", "tools") +def project(capabilities): + return {key: capabilities[key] for key in relevant_keys if key in capabilities} + +pathlib.Path(output_path).write_text( + json.dumps( + { + "2026-07-28": project(modern), + "2025-11-25": project(legacy), + }, + indent=2, + sort_keys=True, + ) + "\n", + encoding="utf-8", +) +PY + server_sha="$(git rev-parse HEAD)" overall=0 for profile in "${PROFILES[@]}"; do @@ -229,12 +341,24 @@ for profile in "${PROFILES[@]}"; do metadata="$profile_root/metadata.json" python3 - "$harness_build_dir/requirements/$profile.yaml" "$metadata" "$profile" \ - "$server_sha" "$HARNESS_COMMIT" "$harness_exit" <<'PY' + "$server_sha" "$HARNESS_COMMIT" "$harness_exit" "$capabilities_file" <<'PY' import json import pathlib import sys -requirements_path, output_path, profile, server_sha, harness_sha, harness_exit = sys.argv[1:] +( + requirements_path, + output_path, + profile, + server_sha, + harness_sha, + harness_exit, + capabilities_path, +) = sys.argv[1:] +capability_map = json.loads(pathlib.Path(capabilities_path).read_text(encoding="utf-8")) +server_capabilities = capability_map.get(profile) +if not isinstance(server_capabilities, dict): + raise SystemExit(f"missing capability probe result for {profile}") server = [] not_scored = [] section = None @@ -290,6 +414,7 @@ pathlib.Path(output_path).write_text( "server_sha": server_sha, "harness_sha": harness_sha, "harness_exit_code": int(harness_exit), + "server_capabilities": server_capabilities, "required_scenarios": server, "not_scored_scenarios": not_scored, }, diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index a1ef37119..64856643d 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -34,20 +34,7 @@ VERDICT_STATUSES = {"SUCCESS", "FAILURE", "WARNING", "SKIPPED"} SCENARIO_DIR_RE = re.compile(r"^server-(.+)-\d{4}-\d{2}-\d{2}T.*Z$") SHA256_RE = re.compile(r"^[0-9a-f]{64}$") -INFRASTRUCTURE_ERROR_MARKERS = ( - "econnrefused", - "connection refused", - "econnreset", - "connection reset", - "enotfound", - "fetch failed", - "failed to fetch", - "network error", - "socket hang up", - "did not complete within", - "timed out", - "connect timeout", -) +GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") class GateError(RuntimeError): @@ -94,16 +81,14 @@ def check_evidence_sha256(check: dict[str, Any]) -> str: def _infrastructure_reason(check: dict[str, Any]) -> str | None: + # These are structural failure shapes emitted by the pinned referee itself. + # Do not scan arbitrary assertion text for words such as "timeout": a valid + # protocol/schema assertion can mention those words and must remain a scored + # protocol result rather than being promoted to infrastructure failure. if check.get("id") == "scenario-timeout": return "referee scenario timeout" if check.get("description") == "Failed to run scenario": return "referee failed to run scenario" - error_message = check.get("errorMessage") - if isinstance(error_message, str): - lowered = error_message.lower() - for marker in INFRASTRUCTURE_ERROR_MARKERS: - if marker in lowered: - return f"network/timeout error marker {marker!r}" return None @@ -136,6 +121,7 @@ def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: if not isinstance(value, list): raise GateError(f"{checks_path} must contain a JSON array") checks: list[dict[str, Any]] = [] + check_ids: set[str] = set() for index, check in enumerate(value): if not isinstance(check, dict): raise GateError(f"{checks_path}[{index}] is not an object") @@ -143,6 +129,9 @@ def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: status = check.get("status") if not isinstance(check_id, str) or not check_id: raise GateError(f"{checks_path}[{index}] has no non-empty check id") + if check_id in check_ids: + raise GateError(f"{checks_path} contains duplicate check id {check_id!r}") + check_ids.add(check_id) if not isinstance(status, str) or not status: raise GateError(f"{checks_path}[{index}] has no non-empty status") if status not in VALID_STATUSES: @@ -194,6 +183,10 @@ def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, s raise GateError( f"classification {scenario}:{check_id} needs a lowercase 64-hex evidence_sha256" ) + if evidence_sha256 == "0" * 64: + raise GateError( + f"classification {scenario}:{check_id} has placeholder evidence_sha256" + ) if kind not in CLASSIFICATIONS: raise GateError( f"classification {scenario}:{check_id} has unsupported kind {kind!r}" @@ -226,10 +219,28 @@ def evaluate_reports( metadata = _load_json(metadata_path) if not isinstance(metadata, dict): raise GateError("metadata must be a JSON object") + if metadata.get("schema_version") != 1: + raise GateError("metadata must have schema_version=1") if metadata.get("profile") != profile: raise GateError( f"metadata profile {metadata.get('profile')!r} does not match requested {profile!r}" ) + server_sha = metadata.get("server_sha") + harness_sha = metadata.get("harness_sha") + if not isinstance(server_sha, str) or not GIT_SHA_RE.fullmatch(server_sha): + raise GateError("metadata server_sha must be a full lowercase Git SHA") + if not isinstance(harness_sha, str) or not GIT_SHA_RE.fullmatch(harness_sha): + raise GateError("metadata harness_sha must be a full lowercase Git SHA") + baseline_metadata = _load_json(baseline_path) + if not isinstance(baseline_metadata, dict) or baseline_metadata.get("schema_version") != 2: + raise GateError("baseline must be an object with schema_version=2") + pinned_harness = baseline_metadata.get("harness_commit") + if not isinstance(pinned_harness, str) or not GIT_SHA_RE.fullmatch(pinned_harness): + raise GateError("baseline harness_commit must be a full lowercase Git SHA") + if harness_sha != pinned_harness: + raise GateError( + f"metadata harness_sha {harness_sha} does not match pinned baseline {pinned_harness}" + ) harness_exit_code = metadata.get("harness_exit_code") if isinstance(harness_exit_code, bool) or not isinstance(harness_exit_code, int): raise GateError("metadata harness_exit_code must be an integer") @@ -263,6 +274,35 @@ def evaluate_reports( reports = load_reports(report_root) classifications = load_classifications(baseline_path, profile) problems: list[str] = [] + optional_classifications = [ + entry + for entry in classifications.values() + if entry.classification == "optional_capability_not_implemented" + ] + server_capabilities = metadata.get("server_capabilities") + if optional_classifications: + if not isinstance(server_capabilities, dict): + raise GateError( + "metadata server_capabilities must be an object when optional-capability classifications exist" + ) + profiles = baseline_metadata.get("profiles") + profile_config = profiles.get(profile) if isinstance(profiles, dict) else None + expected_capabilities = ( + profile_config.get("expected_server_capabilities") + if isinstance(profile_config, dict) + else None + ) + if not isinstance(expected_capabilities, dict): + raise GateError( + f"baseline profile {profile!r} needs expected_server_capabilities for optional classifications" + ) + if server_capabilities != expected_capabilities: + problems.append( + "server capability advertisement changed; optional-capability classifications " + "require review" + ) + elif server_capabilities is not None and not isinstance(server_capabilities, dict): + raise GateError("metadata server_capabilities must be an object when present") missing_scenarios = sorted(required_set - set(reports)) missing_not_scored_scenarios = sorted(set(not_scored) - set(reports)) expected_scenarios = required_set | set(not_scored) @@ -418,9 +458,10 @@ def evaluate_reports( return { "schema_version": 2, "profile": profile, - "server_sha": metadata.get("server_sha"), - "harness_sha": metadata.get("harness_sha"), + "server_sha": server_sha, + "harness_sha": harness_sha, "harness_exit_code": harness_exit_code, + "server_capabilities": server_capabilities, "required_scenario_count": len(required), "not_scored_scenario_count": len(not_scored), "reported_scenario_count": len(reports), diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 85c3eca40..0cf90cb4a 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -34,14 +34,19 @@ def write_metadata( scenarios: list[str], not_scored: list[dict[str, str]] | None = None, harness_exit_code: int = 0, + server_capabilities: dict[str, object] | None = None, ) -> None: self.metadata.write_text( json.dumps( { "profile": self.profile, + "schema_version": 1, "server_sha": "a" * 40, "harness_sha": "b" * 40, "harness_exit_code": harness_exit_code, + "server_capabilities": server_capabilities + if server_capabilities is not None + else {"tools": {"listChanged": False}}, "required_scenarios": scenarios, "not_scored_scenarios": not_scored or [], } @@ -49,12 +54,24 @@ def write_metadata( encoding="utf-8", ) - def write_baseline(self, entries: list[dict[str, str]]) -> None: + def write_baseline( + self, + entries: list[dict[str, str]], + expected_server_capabilities: dict[str, object] | None = None, + ) -> None: self.baseline.write_text( json.dumps( { "schema_version": 2, - "profiles": {self.profile: {"classifications": entries}}, + "harness_commit": "b" * 40, + "profiles": { + self.profile: { + "expected_server_capabilities": expected_server_capabilities + if expected_server_capabilities is not None + else {"tools": {"listChanged": False}}, + "classifications": entries, + } + }, } ), encoding="utf-8", @@ -164,6 +181,26 @@ def test_unknown_status_is_rejected(self) -> None: with self.assertRaisesRegex(report_gate.GateError, "unknown status"): self.evaluate() + def test_duplicate_check_id_is_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks( + "scenario-a", + [self.check("duplicate", "SUCCESS"), self.check("duplicate", "SUCCESS")], + ) + with self.assertRaisesRegex(report_gate.GateError, "duplicate check id"): + self.evaluate() + + def test_metadata_harness_sha_must_match_baseline_pin(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + metadata = json.loads(self.metadata.read_text(encoding="utf-8")) + metadata["harness_sha"] = "c" * 40 + self.metadata.write_text(json.dumps(metadata), encoding="utf-8") + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + with self.assertRaisesRegex(report_gate.GateError, "does not match pinned baseline"): + self.evaluate() + def test_not_scored_failure_is_visible_but_does_not_require_classification(self) -> None: self.write_metadata( ["scenario-a"], @@ -284,6 +321,7 @@ def test_connection_failure_cannot_hide_behind_fixture_classification(self) -> N "tools-call-simple-text", "FAILURE", error_message="connect ECONNREFUSED 127.0.0.1:12345", + description="Failed to run scenario", ) self.write_metadata(["scenario-a"], harness_exit_code=1) self.write_baseline([self.classification(expected, "missing_harness_fixture")]) @@ -308,6 +346,26 @@ def test_status_change_requires_review(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("expected status SKIPPED" in p for p in summary["problems"])) + def test_optional_classification_requires_same_capability_advertisement(self) -> None: + failed = self.check( + "prompts-list", + "FAILURE", + error_message="Failed: method not found", + ) + self.write_metadata( + ["scenario-a"], + harness_exit_code=1, + server_capabilities={"tools": {"listChanged": False}, "prompts": {}}, + ) + self.write_baseline( + [self.classification(failed, "optional_capability_not_implemented")], + expected_server_capabilities={"tools": {"listChanged": False}}, + ) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("capability advertisement changed" in p for p in summary["problems"])) + def test_passing_expected_failure_is_stale(self) -> None: expected = self.check("fixed-check", "FAILURE", error_message="known bug") self.write_metadata(["scenario-a"]) @@ -329,6 +387,25 @@ def test_info_only_expected_failure_is_uncovered(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("no verdict emitted" in p for p in summary["problems"])) + def test_placeholder_evidence_hash_is_rejected(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline( + [ + { + "scenario": "scenario-a", + "check_id": "failure", + "expected_status": "FAILURE", + "evidence_sha256": "0" * 64, + "classification": "genuine_protocol_failure", + "reason": "not reviewed", + "evidence": "placeholder", + } + ] + ) + self.write_checks("scenario-a", [self.check("failure", "FAILURE")]) + with self.assertRaisesRegex(report_gate.GateError, "placeholder evidence_sha256"): + self.evaluate() + def test_broad_expected_failure_masks_are_rejected(self) -> None: self.write_metadata(["scenario-a"]) self.write_baseline( diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index 0bf81ee6b..c00cc4495 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -11,7 +11,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "3fb31b87a162a1318b40f9d1ddee34b892156f5d42e865bb31a78e9cbb2fa5e3" }, { "scenario": "logging-set-level", @@ -20,7 +20,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "2eea842958ca1c21934dbf7f31c526278313834d87d01c77b24d28e2ea1d7f17" }, { "scenario": "prompts-get-embedded-resource", @@ -29,7 +29,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" }, { "scenario": "prompts-get-simple", @@ -38,7 +38,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" }, { "scenario": "prompts-get-with-args", @@ -47,7 +47,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" }, { "scenario": "prompts-get-with-image", @@ -56,7 +56,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" }, { "scenario": "prompts-list", @@ -65,7 +65,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "500284cbe8b04997354d450cc6004d54c245c404c89baa1d45e8f657bda7ff6b" }, { "scenario": "resources-list", @@ -74,7 +74,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "940374420a5847d2786704f3964d1f6a9b6f93d45d5caca7c867109789721091" }, { "scenario": "resources-subscribe", @@ -83,7 +83,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "6fa45e84678894951bba5b600f36e0eeb6fa9f4760bc4e54aa7aa4fd03caa465" }, { "scenario": "resources-unsubscribe", @@ -92,7 +92,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "6fa45e84678894951bba5b600f36e0eeb6fa9f4760bc4e54aa7aa4fd03caa465" }, { "scenario": "tools-call-with-logging", @@ -101,7 +101,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "2eea842958ca1c21934dbf7f31c526278313834d87d01c77b24d28e2ea1d7f17" }, { "scenario": "dns-rebinding-protection", @@ -110,7 +110,7 @@ "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9cb48cfaaa1b12cefb5bb3466b2df926a4b379bd230b00e51a539ef9f0b1a182" }, { "scenario": "elicitation-sep1034-defaults", @@ -119,7 +119,7 @@ "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "83d6fd31cb9a2b72c67b9770fb8061dbffa3bce28ee8b3010cddc583ff678321" }, { "scenario": "elicitation-sep1330-enums", @@ -128,7 +128,7 @@ "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "26a3bf8993b0c55a57bb73ec0464c55409ab344b298c1247c272e037de824b35" }, { "scenario": "resources-read-binary", @@ -137,7 +137,7 @@ "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "5ec605e03cf3cb3c48dae40d35358da3caffebcba8576ed4050cc206d0a12327" }, { "scenario": "resources-read-text", @@ -146,7 +146,7 @@ "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "5ec605e03cf3cb3c48dae40d35358da3caffebcba8576ed4050cc206d0a12327" }, { "scenario": "resources-templates-read", @@ -155,7 +155,7 @@ "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "5ec605e03cf3cb3c48dae40d35358da3caffebcba8576ed4050cc206d0a12327" }, { "scenario": "tools-call-audio", @@ -164,7 +164,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "70365f79b9a6aa9469dbc6ab94bf8f24af5a0c801f76d7b34f5a81f0f4f1ae29" }, { "scenario": "tools-call-elicitation", @@ -173,7 +173,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "1e7da23eb35ac8b41db36c90d516268f47bb337b0db0840e5c175a6ab4ce7aa7" }, { "scenario": "tools-call-embedded-resource", @@ -182,7 +182,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "45387ed5643a97cb78461bdf8c85072032feb6dc1cbbc8ada48497d57827f8e9" }, { "scenario": "tools-call-error", @@ -191,7 +191,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "c0a78aef84e5353a71161e750cf1688e0fdaaa937d5592fedb7547ecfd73a64c" }, { "scenario": "tools-call-image", @@ -200,7 +200,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "d8b841aa282c1ff888eac2d3103399bab5d049971d628dc2225b49603fe3da99" }, { "scenario": "tools-call-mixed-content", @@ -209,7 +209,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "a24ee9f4b00fa5402e4ab02d0fe49836b2292f0f3bf45760aa86050219719890" }, { "scenario": "tools-call-sampling", @@ -218,7 +218,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9a95b1e3dd93a5c2ca1203d2f406135f08b19739f7a3ff29fbc2c6f3ea71d010" }, { "scenario": "tools-call-simple-text", @@ -227,7 +227,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "85a43c02a38ca393c7b6fd7740cdc6a204b62b051066a18f41019b9f23d6327c" }, { "scenario": "tools-call-with-progress", @@ -236,7 +236,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "42e6639388cb452a7b6e336bf057634a1a0b2c6c9fd05fdbf566d2d6153f3c6d" }, { "scenario": "tools-list", @@ -245,9 +245,14 @@ "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "853ee32a673acee6464bf043eaed3098e3da523045d6acd96b21592706fe5842" } - ] + ], + "expected_server_capabilities": { + "tools": { + "listChanged": false + } + } }, "2026-07-28": { "classifications": [ @@ -258,7 +263,7 @@ "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "bb452629335c11ead82c495e5f5e338e3242ffa46057da1c7c4f08a028e71497" }, { "scenario": "caching", @@ -267,7 +272,7 @@ "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "1d8f9b5b01db46426f7a6c6e730c4575938939d3e6a28009f08c798864413b28" }, { "scenario": "caching", @@ -276,7 +281,7 @@ "reason": "resources/read caching could not be exercised because the fixture exposed no reference resource to read.", "evidence": "SEP-2549 caching scenario result from first GitHub Actions conformance baseline raw report", "expected_status": "SKIPPED", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "e8e86cdaa5319e85265044f35b33d006da90dbab5ee709fd55269c6e9892d0d0" }, { "scenario": "completion-complete", @@ -285,7 +290,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "662966f5250ef4190d1d1dfa6f5b05348cf61d995e15b8a435240872b3273ae1" }, { "scenario": "prompts-get-embedded-resource", @@ -294,7 +299,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" }, { "scenario": "prompts-get-simple", @@ -303,7 +308,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" }, { "scenario": "prompts-get-with-args", @@ -312,7 +317,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" }, { "scenario": "prompts-get-with-image", @@ -321,7 +326,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" }, { "scenario": "prompts-list", @@ -330,7 +335,7 @@ "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "96a058e400b47135503f462d35b6865092c786aee626904bcfd88ffa4f51aa02" }, { "scenario": "dns-rebinding-protection", @@ -339,7 +344,7 @@ "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9cb48cfaaa1b12cefb5bb3466b2df926a4b379bd230b00e51a539ef9f0b1a182" }, { "scenario": "input-required-result-basic-elicitation", @@ -348,7 +353,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "a45ff6b1c8bdf199012aa9981b96ebc919728dbcba11feca2e13bdb691ff8143" }, { "scenario": "input-required-result-basic-list-roots", @@ -357,7 +362,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "2bc53d620969f34d0981d7d5fe6acf718a3a3d8e5726fb56ce76fcb9d13d9ce0" }, { "scenario": "input-required-result-basic-sampling", @@ -366,7 +371,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "1d5d13f6588d6ea704a0ff01b0c49acfb3c1188b088210aed7cb1c9661d251b9" }, { "scenario": "input-required-result-capability-check", @@ -375,7 +380,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "8ca1123a297295ee6687823184c59c4ef98e7181fbd0bdbac892ccd767ef5900" }, { "scenario": "input-required-result-ignore-extra-params", @@ -384,7 +389,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "242650bdb1be45878f0a4a8236da3a578c3b30cdf5d1d33c00239a616d31dfcc" }, { "scenario": "input-required-result-missing-input-response", @@ -393,7 +398,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "511d84da640d8dcd5b41923a2e1b97f0c8849ca2e224b5b7565ec062cb6dcf2a" }, { "scenario": "input-required-result-multi-round", @@ -402,7 +407,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "3521f1986753263f2678691a266a3b3f7073b7772fd7d441963c57a0c586ecd7" }, { "scenario": "input-required-result-multiple-input-requests", @@ -411,7 +416,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "5d9f35a56d677d40ae9c848dc8ad342aeedb7cefbd0f81593f2732d213cf31e4" }, { "scenario": "input-required-result-request-state", @@ -420,7 +425,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "787cc0f014e8ccd28230db81eab394b0f47392cf65210a102608e60c2143c656" }, { "scenario": "input-required-result-result-type", @@ -429,7 +434,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "a45ff6b1c8bdf199012aa9981b96ebc919728dbcba11feca2e13bdb691ff8143" }, { "scenario": "input-required-result-tampered-state", @@ -438,7 +443,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "ae62de054b2a479f9f52a8d06482d068dac60854ce75e4f0fb84580995a826d1" }, { "scenario": "input-required-result-non-tool-request", @@ -447,7 +452,7 @@ "reason": "This MRTR probe uses prompts/get, but the current WebCodex surface does not implement the optional prompts capability.", "evidence": "SEP-2322 non-tool request scenario plus server capability model; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "c1c0268e795faa2ed47e9ac10ab235bbc1b0a29ed4f73187459aab0257226bc1" }, { "scenario": "resources-read-binary", @@ -456,7 +461,7 @@ "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "7efc3ca5577c0ceb01a87bc6dca89d81cb0dd380440005a6438951c2ed0ecaa2" }, { "scenario": "resources-read-text", @@ -465,7 +470,7 @@ "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "7efc3ca5577c0ceb01a87bc6dca89d81cb0dd380440005a6438951c2ed0ecaa2" }, { "scenario": "resources-templates-read", @@ -474,7 +479,7 @@ "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "7efc3ca5577c0ceb01a87bc6dca89d81cb0dd380440005a6438951c2ed0ecaa2" }, { "scenario": "sep-2164-resource-not-found", @@ -483,7 +488,7 @@ "reason": "The resource-not-found error omits the recommended data.uri echo.", "evidence": "SEP-2164 resource-not-found error data; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "1eac24b0b70f00d528822678810a5ebf3a13986bef525f6dc175ac2991f257ad" }, { "scenario": "server-stateless", @@ -492,7 +497,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "18cbc6f45d27b694f190edbe99b354f0a0088aa9068c683511902583f07e1ee3" }, { "scenario": "server-stateless", @@ -501,7 +506,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "6a886aa9a507ed141f1d9ae77bbe156dcdfb85204481ca63179db175435319f8" }, { "scenario": "server-stateless", @@ -510,7 +515,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "01a7e2476184fdb6cbaba03327cfd84443f9a1edc1e1695bdfff7672f430c7fc" }, { "scenario": "server-stateless", @@ -519,7 +524,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "04d4a4bea3a86334565b22e03d9b5bb34679eedb46b8b1feaa0e56b43a3e0e37" }, { "scenario": "server-stateless", @@ -528,7 +533,7 @@ "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "f36b764f2c0df40406923e05c785784c2df3fc7c52985ae1447bb0c84f8bc5f7" }, { "scenario": "server-stateless", @@ -537,7 +542,7 @@ "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "f36b764f2c0df40406923e05c785784c2df3fc7c52985ae1447bb0c84f8bc5f7" }, { "scenario": "server-stateless", @@ -546,7 +551,7 @@ "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", "expected_status": "SKIPPED", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "3526c6b7099730a5aa95fcdb13c9a9256c97a831c6adf4a6fcea428948faedc1" }, { "scenario": "server-stateless", @@ -555,7 +560,7 @@ "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", "expected_status": "SKIPPED", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "bc64f7ce41051860e4ea6a058fa1fc33182b920635312562fa174807df371b2c" }, { "scenario": "server-stateless", @@ -564,7 +569,7 @@ "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", "expected_status": "SKIPPED", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "3526c6b7099730a5aa95fcdb13c9a9256c97a831c6adf4a6fcea428948faedc1" }, { "scenario": "server-stateless", @@ -573,7 +578,7 @@ "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", "expected_status": "SKIPPED", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "9b3210383e2285dfff82e29bfce618a0124128dac675e6d498f11495e20af5f6" }, { "scenario": "server-stateless", @@ -582,7 +587,7 @@ "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", "expected_status": "SKIPPED", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "3526c6b7099730a5aa95fcdb13c9a9256c97a831c6adf4a6fcea428948faedc1" }, { "scenario": "tools-call-audio", @@ -591,7 +596,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "6f8af85c68030374f0172b2a48e00df300146d7e14f11a37050da19e52bd661b" }, { "scenario": "tools-call-embedded-resource", @@ -600,7 +605,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "847e62c211f21d7b531c72225355fd80ff2b99175ee64e59cc7a5cefa3263cbb" }, { "scenario": "tools-call-error", @@ -609,7 +614,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "0fa33b84480756ee34f1cb755233ab4fb6de29371e34d623d218da344b0272aa" }, { "scenario": "tools-call-image", @@ -618,7 +623,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "eed3a72319705ab76276f68b868a9bfd7c32146fbf5de1fb5e207137f7df4764" }, { "scenario": "tools-call-mixed-content", @@ -627,7 +632,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "e7ba2d298b0f72354f1c2b8e3c01ae103ce371208df94b245c8779e5cd934b0e" }, { "scenario": "tools-call-simple-text", @@ -636,7 +641,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "7e56ee1105a41b3e15bc7f1231ba28c9b69d8c905d7fa3a224b8f20766d1a5fc" }, { "scenario": "tools-call-with-progress", @@ -645,9 +650,14 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + "evidence_sha256": "fb6394b7f3c96a61167a0a89ea934533a154f990bfbcdf757be026706d8f20a7" + } + ], + "expected_server_capabilities": { + "tools": { + "listChanged": false } - ] + } } }, "supplemental_findings": [ From 21882823b7567b8fbf4cd4cf632608c1dc7db75f Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:48:54 +0000 Subject: [PATCH 06/14] test(mcp): close conformance gate review gaps --- docs/MCP_CONFORMANCE.md | 2 +- scripts/mcp_conformance.sh | 53 +++++++++++++------- scripts/mcp_conformance_report.py | 33 +++++++----- scripts/tests/test_mcp_conformance_report.py | 51 +++++++++++++++++-- tests/fixtures/mcp/conformance/baseline.json | 46 ++++++++--------- 5 files changed, 128 insertions(+), 57 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index 644b4726f..3579ba01d 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -134,7 +134,7 @@ python3 scripts/tests/test_mcp_conformance_report.py It covers all-skipped output, missing/empty/INFO-only scenarios, unknown statuses, abnormal referee exits, scored and not-scored infrastructure failures, changed status/evidence for an existing check ID, capability-advertisement drift, -duplicate check IDs, harness-pin mismatch, placeholder evidence, new scored +ambiguous duplicate non-success check IDs, harness-pin mismatch, placeholder evidence, new scored failures, informational non-success reporting, stale classifications, and rejection of broad masks. diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index 4f2d4bf3f..c60ccb2bc 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -55,22 +55,26 @@ if [ -z "$WORK_ROOT" ] || [ -z "$HARNESS_DIR" ] || [ -z "$REPORT_ROOT" ]; then fi mkdir -p "$WORK_ROOT" +fixture_group_alive() { + [ -n "$fixture_pid" ] && kill -0 -- "-$fixture_pid" 2>/dev/null +} + stop_fixture() { [ -n "$fixture_pid" ] || return 0 touch "$fixture_dir/stop" 2>/dev/null || true for _ in $(seq 1 50); do - if ! kill -0 "$fixture_pid" 2>/dev/null; then break; fi + if ! fixture_group_alive; then break; fi sleep 0.1 done - if kill -0 "$fixture_pid" 2>/dev/null; then - kill "$fixture_pid" 2>/dev/null || true + if fixture_group_alive; then + kill -TERM -- "-$fixture_pid" 2>/dev/null || true fi for _ in $(seq 1 50); do - if ! kill -0 "$fixture_pid" 2>/dev/null; then break; fi + if ! fixture_group_alive; then break; fi sleep 0.1 done - if kill -0 "$fixture_pid" 2>/dev/null; then - kill -KILL "$fixture_pid" 2>/dev/null || true + if fixture_group_alive; then + kill -KILL -- "-$fixture_pid" 2>/dev/null || true fi wait "$fixture_pid" 2>/dev/null || true fixture_pid="" @@ -109,17 +113,23 @@ prepare_harness_source() { elif ! is_git_worktree "$HARNESS_DIR"; then echo "default harness path exists but is not a Git worktree; refusing to remove it: $HARNESS_DIR" >&2 exit 2 - elif [ ! -f "$HARNESS_DIR/$HARNESS_OWNER_MARKER" ]; then + fi + + if [ "$HARNESS_DIR_EXTERNAL" -eq 0 ]; then origin_url="$(git -C "$HARNESS_DIR" remote get-url origin 2>/dev/null || true)" case "$origin_url" in - https://github.com/modelcontextprotocol/conformance.git|https://github.com/modelcontextprotocol/conformance) - touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" - ;; + https://github.com/modelcontextprotocol/conformance.git|https://github.com/modelcontextprotocol/conformance) ;; *) - echo "default harness checkout is not WebCodex-owned and has an unexpected origin; refusing to modify it" >&2 + echo "default harness checkout has an unexpected origin; refusing to modify it: $HARNESS_DIR" >&2 exit 2 ;; esac + # Older versions of this script created the same default checkout before the + # ownership marker existed. Adopt only that exact known-origin checkout; an + # externally supplied worktree remains read-only and is never marked/modified. + if [ ! -f "$HARNESS_DIR/$HARNESS_OWNER_MARKER" ]; then + touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" + fi fi actual="$(git -C "$HARNESS_DIR" rev-parse HEAD 2>/dev/null || true)" @@ -143,14 +153,16 @@ prepare_report_root() { echo "MCP conformance report root is not a directory: $REPORT_ROOT" >&2 exit 2 fi - mkdir -p "$REPORT_ROOT" - if [ "$REPORT_ROOT_EXTERNAL" -eq 1 ] && [ ! -f "$REPORT_ROOT/$REPORT_OWNER_MARKER" ]; then + if [ ! -e "$REPORT_ROOT" ]; then + mkdir -p "$REPORT_ROOT" + touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" + elif [ ! -f "$REPORT_ROOT/$REPORT_OWNER_MARKER" ]; then if find "$REPORT_ROOT" -mindepth 1 -maxdepth 1 -print -quit | grep -q .; then - echo "external MCP report root is non-empty and not WebCodex-owned; refusing to delete its contents: $REPORT_ROOT" >&2 + echo "MCP report root is non-empty and lacks the WebCodex ownership marker; refusing to delete its contents: $REPORT_ROOT" >&2 exit 2 fi + touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" fi - touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" rm -rf "$REPORT_ROOT/2026-07-28" "$REPORT_ROOT/2025-11-25" rm -f "$REPORT_ROOT/fixture.log" "$REPORT_ROOT/server-capabilities.json" } @@ -184,15 +196,20 @@ fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/webcodex-mcp-conformance.XXXXXX")" url_file="$fixture_dir/url" stop_file="$fixture_dir/stop" fixture_log="$REPORT_ROOT/fixture.log" -WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ -WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ +if ! command -v setsid >/dev/null 2>&1; then + echo "MCP conformance fixture supervision requires setsid" >&2 + exit 2 +fi +setsid env \ + WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ + WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ cargo test --locked -p webcodex --lib mcp_conformance_fixture_server -- --ignored --nocapture \ >"$fixture_log" 2>&1 & fixture_pid=$! ready_deadline=$((SECONDS + 60)) while [ ! -s "$url_file" ]; do - if ! kill -0 "$fixture_pid" 2>/dev/null; then + if ! fixture_group_alive; then echo "MCP conformance fixture exited before publishing its URL" >&2 tail -80 "$fixture_log" >&2 || true exit 1 diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index 64856643d..d3979821c 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -59,14 +59,11 @@ def key(self) -> tuple[str, str]: def check_evidence_sha256(check: dict[str, Any]) -> str: """Hash stable failure evidence while excluding timestamps and verbose logs.""" - error_message = check.get("errorMessage") - if isinstance(error_message, str) and error_message: - payload: dict[str, Any] = {"errorMessage": error_message} - elif "details" in check: - payload = {"details": check.get("details")} - elif "metadata" in check: - payload = {"metadata": check.get("metadata")} - else: + payload: dict[str, Any] = {} + for field in ("errorMessage", "details", "metadata"): + if field in check: + payload[field] = check.get(field) + if not payload: payload = { "name": check.get("name"), "description": check.get("description"), @@ -121,7 +118,6 @@ def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: if not isinstance(value, list): raise GateError(f"{checks_path} must contain a JSON array") checks: list[dict[str, Any]] = [] - check_ids: set[str] = set() for index, check in enumerate(value): if not isinstance(check, dict): raise GateError(f"{checks_path}[{index}] is not an object") @@ -129,9 +125,6 @@ def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: status = check.get("status") if not isinstance(check_id, str) or not check_id: raise GateError(f"{checks_path}[{index}] has no non-empty check id") - if check_id in check_ids: - raise GateError(f"{checks_path} contains duplicate check id {check_id!r}") - check_ids.add(check_id) if not isinstance(status, str) or not status: raise GateError(f"{checks_path}[{index}] has no non-empty status") if status not in VALID_STATUSES: @@ -339,6 +332,22 @@ def evaluate_reports( for scenario, checks in sorted(reports.items()): scored = scenario in required_set + if scored: + checks_by_id: dict[str, list[dict[str, Any]]] = {} + for check in checks: + checks_by_id.setdefault(check["id"], []).append(check) + ambiguous_duplicates = sorted( + check_id + for check_id, repeated in checks_by_id.items() + if len(repeated) > 1 + and any(item["status"] in NON_SUCCESS_STATUSES for item in repeated) + ) + if ambiguous_duplicates: + problems.append( + f"required scenario {scenario!r} emitted duplicate non-success check IDs " + "that cannot be matched to one baseline entry: " + + ", ".join(ambiguous_duplicates) + ) if scored and not any(check["status"] in VERDICT_STATUSES for check in checks): problems.append(f"required scenario {scenario!r} emitted no verdict checks") if not scored and not checks: diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 0cf90cb4a..99f5fc28e 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -181,15 +181,25 @@ def test_unknown_status_is_rejected(self) -> None: with self.assertRaisesRegex(report_gate.GateError, "unknown status"): self.evaluate() - def test_duplicate_check_id_is_rejected(self) -> None: + def test_duplicate_success_check_id_is_allowed(self) -> None: self.write_metadata(["scenario-a"]) self.write_baseline([]) self.write_checks( "scenario-a", [self.check("duplicate", "SUCCESS"), self.check("duplicate", "SUCCESS")], ) - with self.assertRaisesRegex(report_gate.GateError, "duplicate check id"): - self.evaluate() + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_duplicate_non_success_check_id_fails_closed(self) -> None: + first = self.check("duplicate", "FAILURE", error_message="first failure") + second = self.check("duplicate", "FAILURE", error_message="second failure") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(first)]) + self.write_checks("scenario-a", [first, second]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("duplicate non-success check IDs" in p for p in summary["problems"])) def test_metadata_harness_sha_must_match_baseline_pin(self) -> None: self.write_metadata(["scenario-a"]) @@ -235,6 +245,21 @@ def test_not_scored_timeout_is_infrastructure_failure(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + def test_not_scored_duplicate_check_ids_remain_informational(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "extension-a", + [self.check("repeated", "FAILURE"), self.check("repeated", "FAILURE")], + ) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + self.assertEqual(summary["informational_status_counts"], {"FAILURE": 2}) + def test_missing_not_scored_scenario_fails_coverage_integrity(self) -> None: self.write_metadata( ["scenario-a"], @@ -311,6 +336,26 @@ def test_changed_failure_evidence_requires_review(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_changed_failure_details_with_same_message_requires_review(self) -> None: + expected = self.check( + "stable-message", + "FAILURE", + error_message="protocol assertion failed", + ) + expected["details"] = {"actual": 200, "expected": 400} + actual = self.check( + "stable-message", + "FAILURE", + error_message="protocol assertion failed", + ) + actual["details"] = {"actual": 500, "expected": 400} + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_connection_failure_cannot_hide_behind_fixture_classification(self) -> None: expected = self.check( "tools-call-simple-text", diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index c00cc4495..c6609544b 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -110,7 +110,7 @@ "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "9cb48cfaaa1b12cefb5bb3466b2df926a4b379bd230b00e51a539ef9f0b1a182" + "evidence_sha256": "a9ce6a3e09ef612a093679f8168ec4fb3fa2aca1297a5768620a6bc32f284b09" }, { "scenario": "elicitation-sep1034-defaults", @@ -245,7 +245,7 @@ "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "853ee32a673acee6464bf043eaed3098e3da523045d6acd96b21592706fe5842" + "evidence_sha256": "ececf4d2bd6e6309d77a640908419f8c8124c93a0dcc41701c6f361b6771273c" } ], "expected_server_capabilities": { @@ -263,7 +263,7 @@ "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "bb452629335c11ead82c495e5f5e338e3242ffa46057da1c7c4f08a028e71497" + "evidence_sha256": "7af406127f9e0edb58db00d3c7a974c4a64e9dc2eb966779a6de918741a9f5f8" }, { "scenario": "caching", @@ -272,7 +272,7 @@ "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "1d8f9b5b01db46426f7a6c6e730c4575938939d3e6a28009f08c798864413b28" + "evidence_sha256": "e3fb42ce657d615df87789d7f92cc5b379ed29c676d48cd6f60f27515347ec75" }, { "scenario": "caching", @@ -344,7 +344,7 @@ "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "9cb48cfaaa1b12cefb5bb3466b2df926a4b379bd230b00e51a539ef9f0b1a182" + "evidence_sha256": "318016600ed6861ea36888a269d82ca3105673a7d3d7e540c15b2ea493571b2c" }, { "scenario": "input-required-result-basic-elicitation", @@ -353,7 +353,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "a45ff6b1c8bdf199012aa9981b96ebc919728dbcba11feca2e13bdb691ff8143" + "evidence_sha256": "18bdbb4b49ac7b1a73e056914b6627a1b3ffe9c36cd1c438b72c09a4be638f1b" }, { "scenario": "input-required-result-basic-list-roots", @@ -362,7 +362,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "2bc53d620969f34d0981d7d5fe6acf718a3a3d8e5726fb56ce76fcb9d13d9ce0" + "evidence_sha256": "09733701c42953223f9761ea327f831d5eef286a08e3807e2f2687ffafc45625" }, { "scenario": "input-required-result-basic-sampling", @@ -371,7 +371,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "1d5d13f6588d6ea704a0ff01b0c49acfb3c1188b088210aed7cb1c9661d251b9" + "evidence_sha256": "a220d559b1a7050e97a4634952062d2c2eab238542450d87980f0c0aece9733e" }, { "scenario": "input-required-result-capability-check", @@ -380,7 +380,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "8ca1123a297295ee6687823184c59c4ef98e7181fbd0bdbac892ccd767ef5900" + "evidence_sha256": "253ff0b64d1e6b62721dcd10c4d48f9df0dcd12648f406e7e96a15f19824d3e7" }, { "scenario": "input-required-result-ignore-extra-params", @@ -389,7 +389,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "242650bdb1be45878f0a4a8236da3a578c3b30cdf5d1d33c00239a616d31dfcc" + "evidence_sha256": "52a17049b7fa7e0e6e9d2654c9d42125aa6b79edde9ee4b37751e60f39bca83f" }, { "scenario": "input-required-result-missing-input-response", @@ -398,7 +398,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "511d84da640d8dcd5b41923a2e1b97f0c8849ca2e224b5b7565ec062cb6dcf2a" + "evidence_sha256": "e6fd1be79feba098d507c9f3d21453d995503479cbcc91d09c33fbf331721e66" }, { "scenario": "input-required-result-multi-round", @@ -407,7 +407,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "3521f1986753263f2678691a266a3b3f7073b7772fd7d441963c57a0c586ecd7" + "evidence_sha256": "3afcf29210ae83e39892a28ac859e41baac1ab6838e5ad869bd193f08cfd505b" }, { "scenario": "input-required-result-multiple-input-requests", @@ -416,7 +416,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "5d9f35a56d677d40ae9c848dc8ad342aeedb7cefbd0f81593f2732d213cf31e4" + "evidence_sha256": "cba19b857b74e5a744f92234cd251678d0cf1ec67feb1563d4b3732a6536e4e3" }, { "scenario": "input-required-result-request-state", @@ -425,7 +425,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "787cc0f014e8ccd28230db81eab394b0f47392cf65210a102608e60c2143c656" + "evidence_sha256": "bb56f96f2a4741573281dd15c3af52bb3dce1dcf68d966b5eb36cff46aec2ae1" }, { "scenario": "input-required-result-result-type", @@ -434,7 +434,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "a45ff6b1c8bdf199012aa9981b96ebc919728dbcba11feca2e13bdb691ff8143" + "evidence_sha256": "18bdbb4b49ac7b1a73e056914b6627a1b3ffe9c36cd1c438b72c09a4be638f1b" }, { "scenario": "input-required-result-tampered-state", @@ -452,7 +452,7 @@ "reason": "This MRTR probe uses prompts/get, but the current WebCodex surface does not implement the optional prompts capability.", "evidence": "SEP-2322 non-tool request scenario plus server capability model; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "c1c0268e795faa2ed47e9ac10ab235bbc1b0a29ed4f73187459aab0257226bc1" + "evidence_sha256": "19847c48286d6012d963b8274997698a9c1a9cdfb53fc7f9980729303d5c62bc" }, { "scenario": "resources-read-binary", @@ -488,7 +488,7 @@ "reason": "The resource-not-found error omits the recommended data.uri echo.", "evidence": "SEP-2164 resource-not-found error data; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "1eac24b0b70f00d528822678810a5ebf3a13986bef525f6dc175ac2991f257ad" + "evidence_sha256": "03e182c64ad2607c2749b90609cbf22dc87eda95ae0951f9c7a4c2f1227aabe0" }, { "scenario": "server-stateless", @@ -497,7 +497,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "18cbc6f45d27b694f190edbe99b354f0a0088aa9068c683511902583f07e1ee3" + "evidence_sha256": "b76faa90c503520992faac5ee3e81c11fa174de69d97e1bfdc977759e91daf0f" }, { "scenario": "server-stateless", @@ -506,7 +506,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "6a886aa9a507ed141f1d9ae77bbe156dcdfb85204481ca63179db175435319f8" + "evidence_sha256": "00318fb6a220abec56b408ee72fa031253a43ec35e93d0247bd7f49f3f0ce1e3" }, { "scenario": "server-stateless", @@ -515,7 +515,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "01a7e2476184fdb6cbaba03327cfd84443f9a1edc1e1695bdfff7672f430c7fc" + "evidence_sha256": "d54d3cc76357dd0289e936526cdced33478df35af58bfbe516084dfd8a478207" }, { "scenario": "server-stateless", @@ -524,7 +524,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "04d4a4bea3a86334565b22e03d9b5bb34679eedb46b8b1feaa0e56b43a3e0e37" + "evidence_sha256": "9b03051fab076a4266977ed4b87c04f57ac9ea8077e40474f6c4d72971a3ffd9" }, { "scenario": "server-stateless", @@ -533,7 +533,7 @@ "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "f36b764f2c0df40406923e05c785784c2df3fc7c52985ae1447bb0c84f8bc5f7" + "evidence_sha256": "f72e87a4dcd615f71164cc3d3a94bc2e9a8db5fd4da6364c1379115262c62a15" }, { "scenario": "server-stateless", @@ -542,7 +542,7 @@ "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "f36b764f2c0df40406923e05c785784c2df3fc7c52985ae1447bb0c84f8bc5f7" + "evidence_sha256": "453717c56bda626408c987bb9e6c8dd5c2582e1523481557197a85f3169fc463" }, { "scenario": "server-stateless", From 2ca40473f33929638d8d59d9f35d7c9e03cc353b Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:50:04 +0000 Subject: [PATCH 07/14] docs(mcp): match conformance evidence hashing --- docs/MCP_CONFORMANCE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index 3579ba01d..ac4718fb6 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -76,8 +76,8 @@ Every `FAILURE`, `WARNING`, or `SKIPPED` check from a **scored required** scenario must have an exact `, ` classification in `tests/fixtures/mcp/conformance/baseline.json`. Each classification also pins the observed status and a SHA-256 fingerprint of the stable failure evidence -(`errorMessage`, then structured details/metadata, falling back to the check's -name/description). If the same check ID starts failing for a different reason or +(`errorMessage` plus structured details/metadata when present, falling back to +the check's name/description). If the same check ID starts failing for a different reason or with a different severity, the gate requires review instead of reusing the old classification. Whole-scenario wildcards are rejected. From 59684b68c3ca64f1e4365c62aa665d816fa81022 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:55:32 +0000 Subject: [PATCH 08/14] test(mcp): tighten conformance harness ownership --- scripts/mcp_conformance.sh | 12 ++++++++++-- scripts/tests/test_mcp_conformance_report.py | 9 +++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index c60ccb2bc..617ba5ae4 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -125,9 +125,17 @@ prepare_harness_source() { ;; esac # Older versions of this script created the same default checkout before the - # ownership marker existed. Adopt only that exact known-origin checkout; an - # externally supplied worktree remains read-only and is never marked/modified. + # ownership marker existed. Adopt it only when it is already at the pinned + # commit and has no tracked edits; otherwise ownership is ambiguous and we + # refuse to mutate it. Externally supplied worktrees are always read-only. if [ ! -f "$HARNESS_DIR/$HARNESS_OWNER_MARKER" ]; then + legacy_actual="$(git -C "$HARNESS_DIR" rev-parse HEAD 2>/dev/null || true)" + if [ "$legacy_actual" != "$HARNESS_COMMIT" ] || \ + ! git -C "$HARNESS_DIR" diff --quiet --no-ext-diff || \ + ! git -C "$HARNESS_DIR" diff --cached --quiet --no-ext-diff; then + echo "default harness checkout is unowned and not an exact clean pinned checkout; refusing to modify it: $HARNESS_DIR" >&2 + exit 2 + fi touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" fi fi diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 99f5fc28e..61ca82d56 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -310,6 +310,15 @@ def test_exit_one_without_scored_failure_is_rejected(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("exit code 1 had no scored FAILURE" in p for p in summary["problems"])) + def test_exit_zero_with_scored_failure_is_rejected(self) -> None: + failed = self.check("unexpected-failure", "FAILURE", error_message="protocol mismatch") + self.write_metadata(["scenario-a"], harness_exit_code=0) + self.write_baseline([self.classification(failed)]) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("exit code 0" in p and "scored FAILURE" in p for p in summary["problems"])) + def test_new_failure_requires_exact_classification(self) -> None: self.write_metadata(["scenario-a"], harness_exit_code=1) self.write_baseline([]) From 480d88d5c4bed81ab40d69ccca8d8df377a08ae3 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 01:06:57 +0000 Subject: [PATCH 09/14] test(mcp): restore reliable fixture supervision --- scripts/mcp_conformance.sh | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index 617ba5ae4..750fbc84d 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -55,26 +55,26 @@ if [ -z "$WORK_ROOT" ] || [ -z "$HARNESS_DIR" ] || [ -z "$REPORT_ROOT" ]; then fi mkdir -p "$WORK_ROOT" -fixture_group_alive() { - [ -n "$fixture_pid" ] && kill -0 -- "-$fixture_pid" 2>/dev/null +fixture_alive() { + [ -n "$fixture_pid" ] && kill -0 "$fixture_pid" 2>/dev/null } stop_fixture() { [ -n "$fixture_pid" ] || return 0 touch "$fixture_dir/stop" 2>/dev/null || true for _ in $(seq 1 50); do - if ! fixture_group_alive; then break; fi + if ! fixture_alive; then break; fi sleep 0.1 done - if fixture_group_alive; then - kill -TERM -- "-$fixture_pid" 2>/dev/null || true + if fixture_alive; then + kill -TERM "$fixture_pid" 2>/dev/null || true fi for _ in $(seq 1 50); do - if ! fixture_group_alive; then break; fi + if ! fixture_alive; then break; fi sleep 0.1 done - if fixture_group_alive; then - kill -KILL -- "-$fixture_pid" 2>/dev/null || true + if fixture_alive; then + kill -KILL "$fixture_pid" 2>/dev/null || true fi wait "$fixture_pid" 2>/dev/null || true fixture_pid="" @@ -204,20 +204,15 @@ fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/webcodex-mcp-conformance.XXXXXX")" url_file="$fixture_dir/url" stop_file="$fixture_dir/stop" fixture_log="$REPORT_ROOT/fixture.log" -if ! command -v setsid >/dev/null 2>&1; then - echo "MCP conformance fixture supervision requires setsid" >&2 - exit 2 -fi -setsid env \ - WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ - WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ +WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ +WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ cargo test --locked -p webcodex --lib mcp_conformance_fixture_server -- --ignored --nocapture \ >"$fixture_log" 2>&1 & fixture_pid=$! ready_deadline=$((SECONDS + 60)) while [ ! -s "$url_file" ]; do - if ! fixture_group_alive; then + if ! fixture_alive; then echo "MCP conformance fixture exited before publishing its URL" >&2 tail -80 "$fixture_log" >&2 || true exit 1 From 50a3486321f78b8c099ecb681a52829c43e63202 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 01:34:57 +0000 Subject: [PATCH 10/14] test(mcp): stabilize conformance evidence fingerprints --- docs/MCP_CONFORMANCE.md | 11 +- scripts/mcp_conformance_report.py | 42 +++++++- scripts/tests/test_mcp_conformance_report.py | 101 +++++++++++++++++++ tests/fixtures/mcp/conformance/baseline.json | 6 +- 4 files changed, 152 insertions(+), 8 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index ac4718fb6..0250804ae 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -135,8 +135,8 @@ It covers all-skipped output, missing/empty/INFO-only scenarios, unknown statuse abnormal referee exits, scored and not-scored infrastructure failures, changed status/evidence for an existing check ID, capability-advertisement drift, ambiguous duplicate non-success check IDs, harness-pin mismatch, placeholder evidence, new scored -failures, informational non-success reporting, stale classifications, and -rejection of broad masks. +failures, informational non-success reporting, stale classifications, volatile +JSON-RPC request IDs, wire-schema payload drift, and rejection of broad masks. ## Raw evidence and review policy @@ -151,8 +151,11 @@ Each profile retains: Baseline updates should be narrow. Add a classification only after reproducing and understanding the exact check, then record its observed status and evidence -fingerprint. A changed fingerprint is a review prompt, not something to refresh -mechanically. Remove the entry as soon as the check passes. Do not replace +fingerprint. Fingerprints retain semantic diagnostics but normalize two known +referee noise sources: millisecond-epoch JSON-RPC probe IDs and the complete +offending message duplicated inside wire-schema violations (the violation context +and errors remain hashed). A changed fingerprint is a review prompt, not something +to refresh mechanically. Remove the entry as soon as the check passes. Do not replace multiple check IDs with a scenario-wide exception. The SEP-2164 `data.uri` finding is kept as an advisory SHOULD/WARNING rather than described as a MUST violation. diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index d3979821c..5ff8e1ae5 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -56,13 +56,53 @@ def key(self) -> tuple[str, str]: return (self.scenario, self.check_id) +def _normalize_evidence_value(value: Any) -> Any: + """Remove only known referee noise while preserving diagnostic semantics.""" + + if isinstance(value, list): + return [_normalize_evidence_value(item) for item in value] + if not isinstance(value, dict): + return value + + # Wire-schema violations include the complete offending JSON-RPC message. + # That payload can change whenever an unrelated tool description/schema + # changes even though the actual schema violation is identical. Keep the + # violation context/errors/spec version, but omit the redundant full message. + wire_schema_violation = ( + isinstance(value.get("origin"), str) + and isinstance(value.get("context"), str) + and isinstance(value.get("errors"), list) + and isinstance(value.get("specVersion"), str) + and isinstance(value.get("message"), dict) + ) + + normalized: dict[str, Any] = {} + for field, item in value.items(): + if wire_schema_violation and field == "message": + continue + # The pinned stateless referee uses Date.now() as request IDs for some + # diagnostic probes. Those millisecond-epoch IDs are run-specific and do + # not describe the protocol failure. Fixed/small JSON-RPC IDs remain part + # of the fingerprint so meaningful response-shape changes still surface. + if ( + field == "id" + and value.get("jsonrpc") == "2.0" + and isinstance(item, int) + and not isinstance(item, bool) + and abs(item) >= 1_000_000_000_000 + ): + continue + normalized[field] = _normalize_evidence_value(item) + return normalized + + def check_evidence_sha256(check: dict[str, Any]) -> str: """Hash stable failure evidence while excluding timestamps and verbose logs.""" payload: dict[str, Any] = {} for field in ("errorMessage", "details", "metadata"): if field in check: - payload[field] = check.get(field) + payload[field] = _normalize_evidence_value(check.get(field)) if not payload: payload = { "name": check.get("name"), diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 61ca82d56..06997bc2b 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -365,6 +365,107 @@ def test_changed_failure_details_with_same_message_requires_review(self) -> None self.assertFalse(summary["gate_passed"]) self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_timestamp_jsonrpc_id_does_not_change_evidence(self) -> None: + expected = self.check( + "stream-probe", + "FAILURE", + error_message="diagnostic tool is unavailable", + ) + expected["details"] = { + "untestable": True, + "response": { + "jsonrpc": "2.0", + "id": 1_789_000_000_001, + "error": {"code": -32602, "message": "tool unavailable"}, + }, + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = 1_789_000_000_999 + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_fixed_jsonrpc_id_change_still_requires_review(self) -> None: + expected = self.check( + "fixed-id-probe", + "FAILURE", + error_message="protocol assertion failed", + ) + expected["details"] = { + "response": {"jsonrpc": "2.0", "id": 1, "error": {"code": -32602}} + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = 2 + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_wire_schema_full_message_drift_does_not_change_evidence(self) -> None: + expected = self.check( + "wire-schema-valid", + "FAILURE", + error_message="Schema validation failed", + ) + expected["details"] = { + "messagesValidated": 5, + "violations": [ + { + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["outputSchema: must have required property 'type'"], + "message": { + "jsonrpc": "2.0", + "id": 1, + "result": {"tools": [{"name": "old-tool"}]}, + }, + } + ], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["message"]["result"] = { + "tools": [{"name": "new-unrelated-tool", "description": "changed"}] + } + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_wire_schema_error_change_still_requires_review(self) -> None: + expected = self.check( + "wire-schema-valid", + "FAILURE", + error_message="Schema validation failed", + ) + expected["details"] = { + "messagesValidated": 5, + "violations": [ + { + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["outputSchema: must have required property 'type'"], + "message": {"jsonrpc": "2.0", "id": 1, "result": {}}, + } + ], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["errors"] = [ + "inputSchema: must have required property 'type'" + ] + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_connection_failure_cannot_hide_behind_fixture_classification(self) -> None: expected = self.check( "tools-call-simple-text", diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index c6609544b..58a0d1089 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -245,7 +245,7 @@ "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "ececf4d2bd6e6309d77a640908419f8c8124c93a0dcc41701c6f361b6771273c" + "evidence_sha256": "633e7603d65e71cb89b4dc77d388e25e3306160b001dc2b2411c4a9709562006" } ], "expected_server_capabilities": { @@ -497,7 +497,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "b76faa90c503520992faac5ee3e81c11fa174de69d97e1bfdc977759e91daf0f" + "evidence_sha256": "a8ac03871432cc0afcb1c24d0df149cb1c162fa989f0dd806294f7cd069b8bd4" }, { "scenario": "server-stateless", @@ -515,7 +515,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "d54d3cc76357dd0289e936526cdced33478df35af58bfbe516084dfd8a478207" + "evidence_sha256": "8bda1e39d0e9d45f5c39f16903533b068a7a7b73ba83590b2b769275246209f4" }, { "scenario": "server-stateless", From 42b51c325580a8f0357d2f0b0dc62db8acb75356 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:16:57 +0000 Subject: [PATCH 11/14] test(mcp): close conformance gate fail-open paths --- docs/MCP_CONFORMANCE.md | 29 ++- scripts/mcp_conformance_report.py | 177 ++++++++++++--- scripts/tests/test_mcp_conformance_report.py | 226 ++++++++++++++++++- tests/fixtures/mcp/conformance/baseline.json | 6 +- 4 files changed, 386 insertions(+), 52 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index 0250804ae..77c4245db 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -51,8 +51,8 @@ the gate fails so the raw evidence remains inspectable. The script starts one ignored, test-only Rust fixture on `127.0.0.1:0`. The fixture composes the same Salvo `AuthMiddleware` and `/mcp` handlers used by the -production HTTP surface, with temporary database/runtime state and anonymous test -authority. The referee therefore reaches real WebCodex parsing, dispatch and +production HTTP surface, with temporary database/runtime state and authentication +disabled in the isolated test config. The referee therefore reaches real WebCodex parsing, dispatch and rendering without production credentials or an external deployment. No conformance-only tool is added to the production registry. Some upstream @@ -114,8 +114,10 @@ The gate also fails when: - a new non-success check in a scored required scenario has no exact classification; - a classified check changes status/evidence, disappears, or now passes; - the referee exits abnormally, its exit code disagrees with the scored FAILURE - set, or any scored/not-scored scenario emits one of the pinned referee's - structural infrastructure failures (`scenario-timeout` or `Failed to run scenario`); + set, or any scored/not-scored scenario emits `scenario-timeout`, + `wire-schema-harness-error`, one of the pinned referee's generic top-level + scenario-error checks, the outer `Failed to run scenario` shape, or an exact + transport-exception form emitted by the pinned Node runner; - the server capability advertisement changes while optional-capability baseline entries depend on the previous advertisement; - an explicitly classified inconclusive infrastructure result is present. @@ -132,11 +134,12 @@ python3 scripts/tests/test_mcp_conformance_report.py ``` It covers all-skipped output, missing/empty/INFO-only scenarios, unknown statuses, -abnormal referee exits, scored and not-scored infrastructure failures, changed -status/evidence for an existing check ID, capability-advertisement drift, +abnormal referee exits, scored and not-scored infrastructure/transport failures, +changed status/evidence for an existing check ID, capability-advertisement drift, ambiguous duplicate non-success check IDs, harness-pin mismatch, placeholder evidence, new scored -failures, informational non-success reporting, stale classifications, volatile -JSON-RPC request IDs, wire-schema payload drift, and rejection of broad masks. +failures, informational non-success reporting, stale classifications, exact-path +JSON-RPC request-ID normalization, wire-schema payload drift, ordinary lookalike +objects, empty structured evidence, and rejection of broad masks. ## Raw evidence and review policy @@ -151,10 +154,12 @@ Each profile retains: Baseline updates should be narrow. Add a classification only after reproducing and understanding the exact check, then record its observed status and evidence -fingerprint. Fingerprints retain semantic diagnostics but normalize two known -referee noise sources: millisecond-epoch JSON-RPC probe IDs and the complete -offending message duplicated inside wire-schema violations (the violation context -and errors remain hashed). A changed fingerprint is a review prompt, not something +fingerprint. Fingerprints retain semantic diagnostics but normalize two narrowly +scoped referee noise sources: the numeric value (not presence/type) of the two +known SEP-2575 millisecond-epoch response IDs, and the duplicated offending +message only inside the structured `wire-schema-valid` violation list. Wire-schema +origin/context/errors/specVersion remain hashed, and ordinary objects with similar +field names are not normalized. A changed fingerprint is a review prompt, not something to refresh mechanically. Remove the entry as soon as the check passes. Do not replace multiple check IDs with a scenario-wide exception. The SEP-2164 `data.uri` finding is kept as an advisory SHOULD/WARNING rather than described as a MUST diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index 5ff8e1ae5..0619a52da 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -36,6 +36,38 @@ SHA256_RE = re.compile(r"^[0-9a-f]{64}$") GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +# The pinned referee uses Date.now() only for these two stateless diagnostic +# probes. Keep this deliberately narrow: a large integer in any other check is +# semantic evidence, not generic timestamp noise. +DYNAMIC_JSONRPC_ID_CHECKS = { + "sep-2575-http-server-no-independent-requests-on-stream", + "sep-2575-server-no-log-without-loglevel", +} +DYNAMIC_JSONRPC_ID_PATH = ("details", "response", "id") +DYNAMIC_JSONRPC_ID_SENTINEL = { + "$webcodexEvidence": "dynamic-millisecond-jsonrpc-id", + "sourceType": "integer", +} +WIRE_SCHEMA_CHECK_ID = "wire-schema-valid" +WIRE_SCHEMA_MESSAGE_SENTINEL = "" + +# Structural or top-level scenario failures emitted by the immutable referee pin. +# These IDs mean the scenario/harness aborted outside its intended protocol +# assertions. Per-assertion task failures and setup checks are deliberately not +# blanket-listed; transport exceptions from those paths are recognized separately. +PINNED_REFEREE_INFRA_CHECK_IDS = { + "scenario-timeout", + "wire-schema-harness-error", + "json-schema-2020-12-error", + "server-session-lifecycle-error", + "server-sse-multiple-streams-error", + "server-sse-polling-error", +} +PINNED_TRANSPORT_ERROR_RE = re.compile( + r"^(?:(?:Failed(?::| to initialize:| to set up tests:)|Error:)\s*)?" + r"(?:fetch failed|connect E(?:CONNREFUSED|CONNRESET|HOSTUNREACH|NETUNREACH)\b.*)$" +) + class GateError(RuntimeError): """Raised when report or baseline input is malformed.""" @@ -56,54 +88,124 @@ def key(self) -> tuple[str, str]: return (self.scenario, self.check_id) -def _normalize_evidence_value(value: Any) -> Any: - """Remove only known referee noise while preserving diagnostic semantics.""" +def _normalize_evidence_value( + value: Any, + *, + path: tuple[str, ...] = (), + normalize_dynamic_probe_id: bool = False, +) -> Any: + """Normalize only pin-specific noise at an exact known evidence path.""" if isinstance(value, list): - return [_normalize_evidence_value(item) for item in value] + return [ + _normalize_evidence_value( + item, + path=path, + normalize_dynamic_probe_id=normalize_dynamic_probe_id, + ) + for item in value + ] if not isinstance(value, dict): return value - # Wire-schema violations include the complete offending JSON-RPC message. - # That payload can change whenever an unrelated tool description/schema - # changes even though the actual schema violation is identical. Keep the - # violation context/errors/spec version, but omit the redundant full message. - wire_schema_violation = ( - isinstance(value.get("origin"), str) - and isinstance(value.get("context"), str) - and isinstance(value.get("errors"), list) - and isinstance(value.get("specVersion"), str) - and isinstance(value.get("message"), dict) - ) - normalized: dict[str, Any] = {} for field, item in value.items(): - if wire_schema_violation and field == "message": - continue - # The pinned stateless referee uses Date.now() as request IDs for some - # diagnostic probes. Those millisecond-epoch IDs are run-specific and do - # not describe the protocol failure. Fixed/small JSON-RPC IDs remain part - # of the fingerprint so meaningful response-shape changes still surface. + item_path = path + (field,) if ( - field == "id" + normalize_dynamic_probe_id + and item_path == DYNAMIC_JSONRPC_ID_PATH and value.get("jsonrpc") == "2.0" and isinstance(item, int) and not isinstance(item, bool) and abs(item) >= 1_000_000_000_000 ): + # Preserve field presence/type semantics while ignoring only the + # Date.now()-style numeric value used by the pinned probe. + normalized[field] = DYNAMIC_JSONRPC_ID_SENTINEL continue - normalized[field] = _normalize_evidence_value(item) + normalized[field] = _normalize_evidence_value( + item, + path=item_path, + normalize_dynamic_probe_id=normalize_dynamic_probe_id, + ) return normalized +def _wire_schema_semantic_details(check: dict[str, Any]) -> Any | None: + """Return structured wire-schema evidence with only the duplicated message normalized.""" + + if check.get("id") != WIRE_SCHEMA_CHECK_ID: + return None + details = check.get("details") + if not isinstance(details, dict) or not isinstance(details.get("messagesValidated"), int): + return None + violations = details.get("violations") + if not isinstance(violations, list) or not violations: + return None + + normalized = _normalize_evidence_value(details, path=("details",)) + normalized_violations = normalized.get("violations") + if not isinstance(normalized_violations, list): + return None + + recognized = False + for index, violation in enumerate(violations): + if not isinstance(violation, dict): + continue + if ( + violation.get("origin") not in {"harness", "implementation"} + or not isinstance(violation.get("context"), str) + or not isinstance(violation.get("errors"), list) + or not isinstance(violation.get("specVersion"), str) + or "message" not in violation + ): + continue + normalized_violation = normalized_violations[index] + if not isinstance(normalized_violation, dict): + continue + # The same complete JSON-RPC object is also rendered into the referee's + # errorMessage. Retain message *presence* with a sentinel and hash the + # authoritative origin/context/errors/specVersion fields instead. + normalized_violation["message"] = WIRE_SCHEMA_MESSAGE_SENTINEL + recognized = True + return normalized if recognized else None + + +def _has_substantive_evidence(value: Any) -> bool: + if value is None or value == "": + return False + if isinstance(value, dict): + return any(_has_substantive_evidence(item) for item in value.values()) + if isinstance(value, list): + return any(_has_substantive_evidence(item) for item in value) + return True + + def check_evidence_sha256(check: dict[str, Any]) -> str: - """Hash stable failure evidence while excluding timestamps and verbose logs.""" + """Hash stable failure evidence while excluding only proven referee noise.""" payload: dict[str, Any] = {} - for field in ("errorMessage", "details", "metadata"): - if field in check: - payload[field] = _normalize_evidence_value(check.get(field)) - if not payload: + wire_schema_details = _wire_schema_semantic_details(check) + if wire_schema_details is not None: + # The structured violations are the authoritative evidence. The pinned + # referee's errorMessage redundantly serializes the entire offending + # message, so including it would reintroduce unrelated tool/schema drift. + payload["details"] = wire_schema_details + if "metadata" in check: + payload["metadata"] = _normalize_evidence_value( + check.get("metadata"), path=("metadata",) + ) + else: + normalize_dynamic_probe_id = check.get("id") in DYNAMIC_JSONRPC_ID_CHECKS + for field in ("errorMessage", "details", "metadata"): + if field in check: + payload[field] = _normalize_evidence_value( + check.get(field), + path=(field,), + normalize_dynamic_probe_id=normalize_dynamic_probe_id, + ) + + if not payload or not any(_has_substantive_evidence(value) for value in payload.values()): payload = { "name": check.get("name"), "description": check.get("description"), @@ -118,14 +220,21 @@ def check_evidence_sha256(check: dict[str, Any]) -> str: def _infrastructure_reason(check: dict[str, Any]) -> str | None: - # These are structural failure shapes emitted by the pinned referee itself. - # Do not scan arbitrary assertion text for words such as "timeout": a valid - # protocol/schema assertion can mention those words and must remain a scored - # protocol result rather than being promoted to infrastructure failure. - if check.get("id") == "scenario-timeout": - return "referee scenario timeout" + # These are structural/top-level failure shapes emitted by the immutable referee + # pin. Do not scan arbitrary assertion text for words such as "timeout": a + # valid protocol assertion may contain those words. + check_id = check.get("id") + if check_id in PINNED_REFEREE_INFRA_CHECK_IDS: + if check_id == "scenario-timeout": + return "referee scenario timeout" + if check_id == "wire-schema-harness-error": + return "referee wire-schema harness error" + return f"referee top-level scenario failure ({check_id})" if check.get("description") == "Failed to run scenario": return "referee failed to run scenario" + error_message = check.get("errorMessage") + if isinstance(error_message, str) and PINNED_TRANSPORT_ERROR_RE.fullmatch(error_message.strip()): + return "referee transport exception" return None diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 06997bc2b..8218494ca 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -156,6 +156,37 @@ def test_missing_required_scenario_fails(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertEqual(summary["missing_scenarios"], ["scenario-b"]) + def test_unexpected_scenario_fails(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("scenario-extra", [self.check("extra", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertEqual(summary["unexpected_scenarios"], ["scenario-extra"]) + + def test_duplicate_scenario_reports_are_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + duplicate = self.raw / "server-scenario-a-2026-09-14T00-00-01-000Z" + duplicate.mkdir() + (duplicate / "checks.json").write_text( + json.dumps([self.check("ok-again", "SUCCESS")]), encoding="utf-8" + ) + with self.assertRaisesRegex(report_gate.GateError, "multiple raw reports"): + self.evaluate() + + def test_required_and_not_scored_overlap_is_rejected(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "scenario-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + with self.assertRaisesRegex(report_gate.GateError, "both required and not_scored"): + self.evaluate() + def test_required_empty_report_fails_even_with_other_coverage(self) -> None: self.write_metadata(["scenario-a", "scenario-b"]) self.write_baseline([]) @@ -367,7 +398,7 @@ def test_changed_failure_details_with_same_message_requires_review(self) -> None def test_timestamp_jsonrpc_id_does_not_change_evidence(self) -> None: expected = self.check( - "stream-probe", + "sep-2575-http-server-no-independent-requests-on-stream", "FAILURE", error_message="diagnostic tool is unavailable", ) @@ -387,9 +418,49 @@ def test_timestamp_jsonrpc_id_does_not_change_evidence(self) -> None: summary = self.evaluate() self.assertTrue(summary["gate_passed"]) + def test_missing_timestamp_jsonrpc_id_requires_review(self) -> None: + expected = self.check( + "sep-2575-http-server-no-independent-requests-on-stream", + "FAILURE", + error_message="diagnostic tool is unavailable", + ) + expected["details"] = { + "response": { + "jsonrpc": "2.0", + "id": 1_789_000_000_001, + "error": {"code": -32602, "message": "tool unavailable"}, + } + } + actual = json.loads(json.dumps(expected)) + del actual["details"]["response"]["id"] + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_timestamp_jsonrpc_id_type_change_requires_review(self) -> None: + expected = self.check( + "sep-2575-http-server-no-independent-requests-on-stream", + "FAILURE", + error_message="diagnostic tool is unavailable", + ) + expected["details"] = { + "response": {"jsonrpc": "2.0", "id": 1_789_000_000_001, "error": {"code": -32602}} + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = "" + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_fixed_jsonrpc_id_change_still_requires_review(self) -> None: expected = self.check( - "fixed-id-probe", + "sep-2575-http-server-no-independent-requests-on-stream", "FAILURE", error_message="protocol assertion failed", ) @@ -405,11 +476,25 @@ def test_fixed_jsonrpc_id_change_still_requires_review(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_timestamp_id_on_unrelated_check_is_not_normalized(self) -> None: + expected = self.check("other-check", "FAILURE", error_message="protocol assertion failed") + expected["details"] = { + "response": {"jsonrpc": "2.0", "id": 1_789_000_000_001, "error": {"code": -32602}} + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = 1_789_000_000_999 + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_wire_schema_full_message_drift_does_not_change_evidence(self) -> None: expected = self.check( "wire-schema-valid", "FAILURE", - error_message="Schema validation failed", + error_message='Schema validation failed; offending message: {"tools":[{"name":"old-tool"}]}', ) expected["details"] = { "messagesValidated": 5, @@ -431,6 +516,10 @@ def test_wire_schema_full_message_drift_does_not_change_evidence(self) -> None: actual["details"]["violations"][0]["message"]["result"] = { "tools": [{"name": "new-unrelated-tool", "description": "changed"}] } + actual["errorMessage"] = ( + 'Schema validation failed; offending message: ' + '{"tools":[{"name":"new-unrelated-tool","description":"changed"}]}' + ) self.write_metadata(["scenario-a"], harness_exit_code=1) self.write_baseline([self.classification(expected)]) self.write_checks("scenario-a", [actual]) @@ -466,6 +555,127 @@ def test_wire_schema_error_change_still_requires_review(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + def test_business_message_change_is_not_wire_schema_noise(self) -> None: + expected = self.check( + "business-error", + "FAILURE", + error_message="tool failed", + ) + expected["details"] = { + "origin": "application", + "context": "tool-call", + "errors": [], + "specVersion": "business-v1", + "message": {"code": "FIXTURE_MISSING"}, + } + actual = json.loads(json.dumps(expected)) + actual["details"]["message"]["code"] = "AUTH_DENIED" + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_empty_structured_evidence_falls_back_to_description(self) -> None: + expected = self.check("empty-details", "FAILURE", description="first diagnostic") + expected["details"] = {} + actual = self.check("empty-details", "FAILURE", description="changed diagnostic") + actual["details"] = {} + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_not_scored_wire_schema_harness_error_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "extension-a", + [self.check("wire-schema-harness-error", "FAILURE", error_message="harness emitted invalid JSON-RPC")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_not_scored_scenario_catch_error_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "json-schema-2020-12", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "json-schema-2020-12", + [self.check("json-schema-2020-12-error", "FAILURE", error_message="Failed: fetch failed")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_not_scored_top_level_scenario_error_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "server-sse-polling", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "server-sse-polling", + [ + self.check( + "server-sse-polling-error", + "FAILURE", + error_message="Error: unexpected scenario exception", + ) + ], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("top-level scenario failure" in p for p in summary["problems"])) + + def test_not_scored_transport_exception_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "tasks-mrtr-composition", "reason": "extension"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "tasks-mrtr-composition", + [ + self.check( + "sep-2663-mrtr-synchronous-before-task-creation", + "FAILURE", + error_message="fetch failed", + ) + ], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("transport exception" in p for p in summary["problems"])) + + def test_not_scored_setup_failure_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "http-header-validation", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "http-header-validation", + [self.check("sep-2243-server-standard-setup", "FAILURE", error_message="Failed to set up tests: fetch failed")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + def test_connection_failure_cannot_hide_behind_fixture_classification(self) -> None: expected = self.check( "tools-call-simple-text", @@ -501,6 +711,16 @@ def test_status_change_requires_review(self) -> None: self.assertFalse(summary["gate_passed"]) self.assertTrue(any("expected status SKIPPED" in p for p in summary["problems"])) + def test_warning_to_failure_requires_review(self) -> None: + expected = self.check("advisory", "WARNING", error_message="SHOULD mismatch") + actual = self.check("advisory", "FAILURE", error_message="SHOULD mismatch") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("expected status WARNING" in p for p in summary["problems"])) + def test_optional_classification_requires_same_capability_advertisement(self) -> None: failed = self.check( "prompts-list", diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index 58a0d1089..78ad31143 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -245,7 +245,7 @@ "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "633e7603d65e71cb89b4dc77d388e25e3306160b001dc2b2411c4a9709562006" + "evidence_sha256": "6b9808d684345474a82659c7ddbbcbf8d249325ee5abc1f4a1813ed283cb45d7" } ], "expected_server_capabilities": { @@ -497,7 +497,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "a8ac03871432cc0afcb1c24d0df149cb1c162fa989f0dd806294f7cd069b8bd4" + "evidence_sha256": "e7955d92d110e3187d92d1a199891c9e6a2bda2a880ba499f49e8bc8d1934c32" }, { "scenario": "server-stateless", @@ -515,7 +515,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "8bda1e39d0e9d45f5c39f16903533b068a7a7b73ba83590b2b769275246209f4" + "evidence_sha256": "c5d626bc1ba209485e7961f24e1e9a1dbf1fcb30b900565bf45b99fb8af1da28" }, { "scenario": "server-stateless", From 594b8c060b493922c16026ad65b323d7ae1c7948 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:44:07 +0000 Subject: [PATCH 12/14] test(mcp): stabilize wire finding identity --- docs/MCP_CONFORMANCE.md | 10 +++-- scripts/mcp_conformance_report.py | 39 ++++++++++++++++++ scripts/tests/test_mcp_conformance_report.py | 43 ++++++++++++++++++++ tests/fixtures/mcp/conformance/baseline.json | 2 +- 4 files changed, 90 insertions(+), 4 deletions(-) diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md index 77c4245db..4c5005700 100644 --- a/docs/MCP_CONFORMANCE.md +++ b/docs/MCP_CONFORMANCE.md @@ -157,9 +157,13 @@ and understanding the exact check, then record its observed status and evidence fingerprint. Fingerprints retain semantic diagnostics but normalize two narrowly scoped referee noise sources: the numeric value (not presence/type) of the two known SEP-2575 millisecond-epoch response IDs, and the duplicated offending -message only inside the structured `wire-schema-valid` violation list. Wire-schema -origin/context/errors/specVersion remain hashed, and ordinary objects with similar -field names are not normalized. A changed fingerprint is a review prompt, not something +message only inside the structured `wire-schema-valid` violation list. For a +`ListToolsResult/tools//...` diagnostic, that volatile array ordinal is +resolved through the offending response to the referenced tool name before hashing, +so unrelated tool insertion/reordering does not rename the finding; moving the defect +to a different tool still changes the fingerprint. Wire-schema origin/context/errors/ +specVersion remain hashed, and ordinary objects with similar field names are not +normalized. A changed fingerprint is a review prompt, not something to refresh mechanically. Remove the entry as soon as the check passes. Do not replace multiple check IDs with a scenario-wide exception. The SEP-2164 `data.uri` finding is kept as an advisory SHOULD/WARNING rather than described as a MUST diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py index 0619a52da..d44a073ab 100644 --- a/scripts/mcp_conformance_report.py +++ b/scripts/mcp_conformance_report.py @@ -50,6 +50,7 @@ } WIRE_SCHEMA_CHECK_ID = "wire-schema-valid" WIRE_SCHEMA_MESSAGE_SENTINEL = "" +WIRE_SCHEMA_TOOL_INDEX_RE = re.compile(r"ListToolsResult/tools/(?P\d+)") # Structural or top-level scenario failures emitted by the immutable referee pin. # These IDs mean the scenario/harness aborted outside its intended protocol @@ -131,6 +132,36 @@ def _normalize_evidence_value( return normalized +def _wire_schema_errors_with_tool_names(errors: Any, message: Any) -> Any: + """Replace volatile tools/list array ordinals with the referenced tool name.""" + + if not isinstance(errors, list) or not isinstance(message, dict): + return errors + result = message.get("result") + tools = result.get("tools") if isinstance(result, dict) else None + if not isinstance(tools, list): + return errors + + def replace_tool_index(match: re.Match[str]) -> str: + tool_index = int(match.group("index")) + if tool_index >= len(tools): + return match.group(0) + tool = tools[tool_index] + if not isinstance(tool, dict): + return match.group(0) + tool_name = tool.get("name") + if not isinstance(tool_name, str) or not tool_name: + return match.group(0) + return "ListToolsResult/tools[" + json.dumps(tool_name, ensure_ascii=False) + "]" + + return [ + WIRE_SCHEMA_TOOL_INDEX_RE.sub(replace_tool_index, error) + if isinstance(error, str) + else error + for error in errors + ] + + def _wire_schema_semantic_details(check: dict[str, Any]) -> Any | None: """Return structured wire-schema evidence with only the duplicated message normalized.""" @@ -163,6 +194,14 @@ def _wire_schema_semantic_details(check: dict[str, Any]) -> Any | None: normalized_violation = normalized_violations[index] if not isinstance(normalized_violation, dict): continue + # Tool insertion/reordering changes the numeric array position in the + # referee diagnostic even when the same named tool has the same schema + # defect. Resolve that ordinal through the offending response before the + # complete message is replaced. If resolution fails, keep the raw index + # so the gate fails closed. + normalized_violation["errors"] = _wire_schema_errors_with_tool_names( + normalized_violation.get("errors"), violation.get("message") + ) # The same complete JSON-RPC object is also rendered into the referee's # errorMessage. Retain message *presence* with a sentinel and hash the # authoritative origin/context/errors/specVersion fields instead. diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py index 8218494ca..c28e348e2 100644 --- a/scripts/tests/test_mcp_conformance_report.py +++ b/scripts/tests/test_mcp_conformance_report.py @@ -526,6 +526,49 @@ def test_wire_schema_full_message_drift_does_not_change_evidence(self) -> None: summary = self.evaluate() self.assertTrue(summary["gate_passed"]) + def test_wire_schema_tool_index_shift_uses_tool_identity(self) -> None: + expected = self.check("wire-schema-valid", "FAILURE", error_message="Schema validation failed") + expected["details"] = { + "messagesValidated": 5, + "violations": [{ + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["ListToolsResult/tools/1/outputSchema: must have required property 'type' (result of 'tools/list')"], + "message": {"jsonrpc": "2.0", "id": 1, "result": {"tools": [{"name": "other"}, {"name": "ssh_resource"}]}}, + }], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["errors"] = [ + "ListToolsResult/tools/2/outputSchema: must have required property 'type' (result of 'tools/list')" + ] + actual["details"]["violations"][0]["message"]["result"]["tools"] = [ + {"name": "inserted"}, {"name": "other"}, {"name": "ssh_resource"} + ] + self.assertEqual( + report_gate.check_evidence_sha256(expected), + report_gate.check_evidence_sha256(actual), + ) + + def test_wire_schema_tool_identity_change_requires_review(self) -> None: + expected = self.check("wire-schema-valid", "FAILURE", error_message="Schema validation failed") + expected["details"] = { + "messagesValidated": 5, + "violations": [{ + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["ListToolsResult/tools/0/outputSchema: must have required property 'type' (result of 'tools/list')"], + "message": {"jsonrpc": "2.0", "id": 1, "result": {"tools": [{"name": "ssh_resource"}]}}, + }], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["message"]["result"]["tools"][0]["name"] = "different_tool" + self.assertNotEqual( + report_gate.check_evidence_sha256(expected), + report_gate.check_evidence_sha256(actual), + ) + def test_wire_schema_error_change_still_requires_review(self) -> None: expected = self.check( "wire-schema-valid", diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index 78ad31143..92c13f3c8 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -245,7 +245,7 @@ "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "6b9808d684345474a82659c7ddbbcbf8d249325ee5abc1f4a1813ed283cb45d7" + "evidence_sha256": "04ac9b7699a61e4970c6efdd1296dc61971f8097fa21e90873d6992f2e1d40ca" } ], "expected_server_capabilities": { From d578abb0c4608a9bfce960d158d5344fcf0dc4c2 Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Wed, 16 Sep 2026 13:49:18 +0800 Subject: [PATCH 13/14] test(mcp): harden conformance fixture execution --- scripts/mcp_conformance.sh | 19 ++++++++++++++++--- src/mcp_tests/conformance.rs | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh index 750fbc84d..527a8b176 100644 --- a/scripts/mcp_conformance.sh +++ b/scripts/mcp_conformance.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + # The conformance referee is intentionally immutable. Updating it is a reviewed # baseline change because scenario/check semantics can change between commits. BASELINE="tests/fixtures/mcp/conformance/baseline.json" @@ -56,7 +60,13 @@ fi mkdir -p "$WORK_ROOT" fixture_alive() { - [ -n "$fixture_pid" ] && kill -0 "$fixture_pid" 2>/dev/null + [ -n "$fixture_pid" ] || return 1 + kill -0 -- "-$fixture_pid" 2>/dev/null || kill -0 "$fixture_pid" 2>/dev/null +} + +signal_fixture() { + local signal="$1" + kill "-$signal" -- "-$fixture_pid" 2>/dev/null || kill "-$signal" "$fixture_pid" 2>/dev/null || true } stop_fixture() { @@ -67,14 +77,14 @@ stop_fixture() { sleep 0.1 done if fixture_alive; then - kill -TERM "$fixture_pid" 2>/dev/null || true + signal_fixture TERM fi for _ in $(seq 1 50); do if ! fixture_alive; then break; fi sleep 0.1 done if fixture_alive; then - kill -KILL "$fixture_pid" 2>/dev/null || true + signal_fixture KILL fi wait "$fixture_pid" 2>/dev/null || true fixture_pid="" @@ -204,8 +214,11 @@ fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/webcodex-mcp-conformance.XXXXXX")" url_file="$fixture_dir/url" stop_file="$fixture_dir/stop" fixture_log="$REPORT_ROOT/fixture.log" +# Keep the entire Cargo + test-binary fixture tree in one private process group so +# fallback teardown cannot leave the loopback server alive if Cargo exits first. WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ + python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ cargo test --locked -p webcodex --lib mcp_conformance_fixture_server -- --ignored --nocapture \ >"$fixture_log" 2>&1 & fixture_pid=$! diff --git a/src/mcp_tests/conformance.rs b/src/mcp_tests/conformance.rs index 2964499e1..e2db650c4 100644 --- a/src/mcp_tests/conformance.rs +++ b/src/mcp_tests/conformance.rs @@ -26,7 +26,7 @@ async fn mcp_conformance_fixture_server() { let config = test_config(None); let (_db_tmp, db) = test_db(); - let runtime = Arc::new(test_runtime()); + let runtime = Arc::new(test_runtime_from_model_surface_env(None)); let router = build_test_router(config, db, runtime); let acceptor = TcpListener::new("127.0.0.1:0").bind().await; let addr = acceptor.holdings()[0] From 67c06f675b633c2bf755296c09bac9773a9bf039 Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Wed, 16 Sep 2026 14:08:30 +0800 Subject: [PATCH 14/14] test(mcp): refresh adaptive conformance baseline --- tests/fixtures/mcp/conformance/baseline.json | 81 +++++++++----------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json index 92c13f3c8..921527d4b 100644 --- a/tests/fixtures/mcp/conformance/baseline.json +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -110,7 +110,7 @@ "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "a9ce6a3e09ef612a093679f8168ec4fb3fa2aca1297a5768620a6bc32f284b09" + "evidence_sha256": "55fef6d01afbb12e2235724cc78cef53e13ad7b61fb8b387d4b260c19ce74b89" }, { "scenario": "elicitation-sep1034-defaults", @@ -119,7 +119,7 @@ "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "83d6fd31cb9a2b72c67b9770fb8061dbffa3bce28ee8b3010cddc583ff678321" + "evidence_sha256": "5ab028bbd3467c6dc5fd5c8bd1c19af5732391da6e7c6543737d21962039db54" }, { "scenario": "elicitation-sep1330-enums", @@ -128,7 +128,7 @@ "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "26a3bf8993b0c55a57bb73ec0464c55409ab344b298c1247c272e037de824b35" + "evidence_sha256": "6043343fce6e7d657c351276e50c1092274ab5679572e2cfbac15397db729633" }, { "scenario": "resources-read-binary", @@ -164,7 +164,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "70365f79b9a6aa9469dbc6ab94bf8f24af5a0c801f76d7b34f5a81f0f4f1ae29" + "evidence_sha256": "f52ab456f88f932ae8fbda66972454a1eb01e97650550de960ce5f78d251eafe" }, { "scenario": "tools-call-elicitation", @@ -173,7 +173,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "1e7da23eb35ac8b41db36c90d516268f47bb337b0db0840e5c175a6ab4ce7aa7" + "evidence_sha256": "2e5b0ff0322968f3f71159940edc711a62a8570ff5b807d17060695f40b921e2" }, { "scenario": "tools-call-embedded-resource", @@ -182,7 +182,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "45387ed5643a97cb78461bdf8c85072032feb6dc1cbbc8ada48497d57827f8e9" + "evidence_sha256": "f176191f243db4cde3281c36d431c051330aaf8bed2317c4bfc271b0ab17c7c3" }, { "scenario": "tools-call-error", @@ -191,7 +191,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "c0a78aef84e5353a71161e750cf1688e0fdaaa937d5592fedb7547ecfd73a64c" + "evidence_sha256": "d8d28432df8232f95c5c7333cca717383761bb2b129649f67e146129d55d9c55" }, { "scenario": "tools-call-image", @@ -200,7 +200,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "d8b841aa282c1ff888eac2d3103399bab5d049971d628dc2225b49603fe3da99" + "evidence_sha256": "c0890e33c4952cdc08d213615d387110ea067aaf22584bae4d2e148a5600be1c" }, { "scenario": "tools-call-mixed-content", @@ -209,7 +209,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "a24ee9f4b00fa5402e4ab02d0fe49836b2292f0f3bf45760aa86050219719890" + "evidence_sha256": "ff8a525bfad910a6a1f4e61cbcf35fbec589c12f2535ae97d5645d6caba447bf" }, { "scenario": "tools-call-sampling", @@ -218,7 +218,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "9a95b1e3dd93a5c2ca1203d2f406135f08b19739f7a3ff29fbc2c6f3ea71d010" + "evidence_sha256": "aeb68fd319a9a0cbe7fbca2bdd0dbb45542d26dabeb0a3e69adb37fc577e958c" }, { "scenario": "tools-call-simple-text", @@ -227,7 +227,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "85a43c02a38ca393c7b6fd7740cdc6a204b62b051066a18f41019b9f23d6327c" + "evidence_sha256": "6f37fcae0bde7d34f8b91e1d59d0bf416c0a019609405abd666754a41be912d7" }, { "scenario": "tools-call-with-progress", @@ -236,16 +236,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "42e6639388cb452a7b6e336bf057634a1a0b2c6c9fd05fdbf566d2d6153f3c6d" - }, - { - "scenario": "tools-list", - "check_id": "wire-schema-valid", - "classification": "genuine_protocol_failure", - "reason": "The 2025-11-25 tools/list wire schema rejects a WebCodex outputSchema that omits the required root type.", - "evidence": "MCP 2025-11-25 Tool outputSchema wire schema; first GitHub Actions conformance baseline raw report", - "expected_status": "FAILURE", - "evidence_sha256": "04ac9b7699a61e4970c6efdd1296dc61971f8097fa21e90873d6992f2e1d40ca" + "evidence_sha256": "13c6bda80dd58f68b3634d402f12edc750b996f695b5e913db59da14a7acfaae" } ], "expected_server_capabilities": { @@ -344,7 +335,7 @@ "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "318016600ed6861ea36888a269d82ca3105673a7d3d7e540c15b2ea493571b2c" + "evidence_sha256": "56f4a8d9b0ea3df7f9ad8f42edb3f6dc249c7f119daa83c9ca4da22bd8f86b16" }, { "scenario": "input-required-result-basic-elicitation", @@ -353,7 +344,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "18bdbb4b49ac7b1a73e056914b6627a1b3ffe9c36cd1c438b72c09a4be638f1b" + "evidence_sha256": "8df152cebc4b7699b9222a681b6cb490cc5bc116f0666b05a01b7f07919a20d1" }, { "scenario": "input-required-result-basic-list-roots", @@ -362,7 +353,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "09733701c42953223f9761ea327f831d5eef286a08e3807e2f2687ffafc45625" + "evidence_sha256": "bcfe48b990672771a1512a70fa61adcc4fc71ce23ffa8d9a633af203db8309e5" }, { "scenario": "input-required-result-basic-sampling", @@ -371,7 +362,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "a220d559b1a7050e97a4634952062d2c2eab238542450d87980f0c0aece9733e" + "evidence_sha256": "469d38b05459eccad7e149170cbbd56e6f9d28c9d46d80f8ced6e2f317adf4fc" }, { "scenario": "input-required-result-capability-check", @@ -380,7 +371,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "253ff0b64d1e6b62721dcd10c4d48f9df0dcd12648f406e7e96a15f19824d3e7" + "evidence_sha256": "dbf49040d276400f78f24ba726743e1cc7628dee7ddd6cd252a12dfcdb4b2632" }, { "scenario": "input-required-result-ignore-extra-params", @@ -389,7 +380,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "WARNING", - "evidence_sha256": "52a17049b7fa7e0e6e9d2654c9d42125aa6b79edde9ee4b37751e60f39bca83f" + "evidence_sha256": "de2c48d4f6018bc152772b53c9fb420f9cb6ba85cd0155c45e4daf348bf21d2e" }, { "scenario": "input-required-result-missing-input-response", @@ -416,7 +407,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "cba19b857b74e5a744f92234cd251678d0cf1ec67feb1563d4b3732a6536e4e3" + "evidence_sha256": "c6775d26e7f2bcf767c6208b388b25716ffff91803b33ef8263aac4472e2ab29" }, { "scenario": "input-required-result-request-state", @@ -425,7 +416,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "bb56f96f2a4741573281dd15c3af52bb3dce1dcf68d966b5eb36cff46aec2ae1" + "evidence_sha256": "db44366f4b2663c8dd18a73b5d243e0201429acc0e80605a95ca278ddd92aa87" }, { "scenario": "input-required-result-result-type", @@ -434,7 +425,7 @@ "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "18bdbb4b49ac7b1a73e056914b6627a1b3ffe9c36cd1c438b72c09a4be638f1b" + "evidence_sha256": "8df152cebc4b7699b9222a681b6cb490cc5bc116f0666b05a01b7f07919a20d1" }, { "scenario": "input-required-result-tampered-state", @@ -461,7 +452,7 @@ "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "7efc3ca5577c0ceb01a87bc6dca89d81cb0dd380440005a6438951c2ed0ecaa2" + "evidence_sha256": "ccebe9670fc81aa853c549c9017baa7d88a371fd5d4625e1726b0a542aefec69" }, { "scenario": "resources-read-text", @@ -470,7 +461,7 @@ "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "7efc3ca5577c0ceb01a87bc6dca89d81cb0dd380440005a6438951c2ed0ecaa2" + "evidence_sha256": "2b815566ba4173981dcb12f46ea1ab73bff3b373796d1d152107023e3efa15f7" }, { "scenario": "resources-templates-read", @@ -479,7 +470,7 @@ "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "7efc3ca5577c0ceb01a87bc6dca89d81cb0dd380440005a6438951c2ed0ecaa2" + "evidence_sha256": "ae122f912511f271c31f53fee8245c1dde3acfcff41c5db301341055cc9917e0" }, { "scenario": "sep-2164-resource-not-found", @@ -497,7 +488,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "e7955d92d110e3187d92d1a199891c9e6a2bda2a880ba499f49e8bc8d1934c32" + "evidence_sha256": "7b694b9546c9ffc28b7018bb47ad1b35eb71982e0e07cf5d238a8b5b07f28f70" }, { "scenario": "server-stateless", @@ -515,7 +506,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "c5d626bc1ba209485e7961f24e1e9a1dbf1fcb30b900565bf45b99fb8af1da28" + "evidence_sha256": "5802de50b88da05f1deb16702e8a9fc63de9536f2ed3b5455691a6d31adf23d1" }, { "scenario": "server-stateless", @@ -524,7 +515,7 @@ "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "9b03051fab076a4266977ed4b87c04f57ac9ea8077e40474f6c4d72971a3ffd9" + "evidence_sha256": "54fb37a17716a6faa4e20eba8bbceb73fa4cf5b7c0ede03c4fc3c950642b6ad0" }, { "scenario": "server-stateless", @@ -596,7 +587,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "6f8af85c68030374f0172b2a48e00df300146d7e14f11a37050da19e52bd661b" + "evidence_sha256": "14101eaa646a4d704365cc1088cffb01fd1e9c022de157cd12ee830ef70e9432" }, { "scenario": "tools-call-embedded-resource", @@ -605,7 +596,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "847e62c211f21d7b531c72225355fd80ff2b99175ee64e59cc7a5cefa3263cbb" + "evidence_sha256": "3381f09b48c05f0a3a98644c1b0851a4aab608cb471d1fde932b44d351c2c283" }, { "scenario": "tools-call-error", @@ -614,7 +605,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "0fa33b84480756ee34f1cb755233ab4fb6de29371e34d623d218da344b0272aa" + "evidence_sha256": "ceda82cecaf90be4000fbb2601518a3f0fa3b6f960b6fe242a9cd050f350eda9" }, { "scenario": "tools-call-image", @@ -623,7 +614,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "eed3a72319705ab76276f68b868a9bfd7c32146fbf5de1fb5e207137f7df4764" + "evidence_sha256": "5da82959f8874a71170b8a9d221b37e0cd7a53ef978f27f2555fb1f6438848f8" }, { "scenario": "tools-call-mixed-content", @@ -632,7 +623,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "e7ba2d298b0f72354f1c2b8e3c01ae103ce371208df94b245c8779e5cd934b0e" + "evidence_sha256": "99afc685f8f53a330164573619be79f552c87dfc138d1aa311187d75569b644a" }, { "scenario": "tools-call-simple-text", @@ -641,7 +632,7 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "7e56ee1105a41b3e15bc7f1231ba28c9b69d8c905d7fa3a224b8f20766d1a5fc" + "evidence_sha256": "d68ec286ab9f5950928708ded11dc1023f1a1e488c318c0ee2b57e9a93f764c5" }, { "scenario": "tools-call-with-progress", @@ -650,10 +641,14 @@ "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", "expected_status": "FAILURE", - "evidence_sha256": "fb6394b7f3c96a61167a0a89ea934533a154f990bfbcdf757be026706d8f20a7" + "evidence_sha256": "21f7c05843f8c02dfaebdb1a8f25b23b9ec413c1cccc100b3267f756d450c994" } ], "expected_server_capabilities": { + "resources": { + "listChanged": false, + "subscribe": false + }, "tools": { "listChanged": false }