diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15b9f3415..907b3ccf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,37 @@ jobs: explicit_resume_advanced_compatibility_schema_and_metadata_are_retained \ http_project_connector_lists_and_dispatches_only_canonical_capabilities + mcp-conformance: + # Dated MCP wire evidence is independent from ordinary unit/integration tests. + # The script pins the external referee and rejects missing or silently skipped + # coverage; known failures require exact check-level classification. + needs: contract + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: linux-ci + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Check conformance report gate + run: python3 scripts/tests/test_mcp_conformance_report.py + - name: Run pinned MCP dated-revision baseline + run: bash scripts/mcp_conformance.sh + - name: Upload MCP conformance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: mcp-conformance-${{ github.sha }} + path: target/mcp-conformance/reports + if-no-files-found: ignore + retention-days: 7 + test-linux-rust: # Complete Linux Rust coverage is mandatory before merge, including owner PRs. # Start it immediately in parallel with contract/frontend work; the stable @@ -760,7 +791,7 @@ jobs: require_lane desktop "$NEEDS_DESKTOP" "$DESKTOP_RESULT" test-native: - needs: [contract, changes, test-docker-server, test-macos, test-windows] + needs: [contract, changes, mcp-conformance, test-docker-server, test-macos, test-windows] # Stable branch-protection context: daily native aggregates and the path-aware # Server image smoke must resolve according to classifier policy. if: always() @@ -772,13 +803,14 @@ jobs: CONTRACT_RESULT: ${{ needs.contract.result }} CHANGES_RESULT: ${{ needs.changes.result }} NEEDS_DOCKER: ${{ needs.changes.outputs.needs_docker }} + MCP_CONFORMANCE_RESULT: ${{ needs.mcp-conformance.result }} DOCKER_RESULT: ${{ needs.test-docker-server.result }} MACOS_RESULT: ${{ needs.test-macos.result }} WINDOWS_RESULT: ${{ needs.test-windows.result }} run: | set -euo pipefail - echo "changes=$CHANGES_RESULT contract=$CONTRACT_RESULT docker=$DOCKER_RESULT macos=$MACOS_RESULT windows=$WINDOWS_RESULT needs_docker=$NEEDS_DOCKER" - if [ "$CHANGES_RESULT" != success ] || [ "$CONTRACT_RESULT" != success ]; then + echo "changes=$CHANGES_RESULT contract=$CONTRACT_RESULT mcp_conformance=$MCP_CONFORMANCE_RESULT docker=$DOCKER_RESULT macos=$MACOS_RESULT windows=$WINDOWS_RESULT needs_docker=$NEEDS_DOCKER" + if [ "$CHANGES_RESULT" != success ] || [ "$CONTRACT_RESULT" != success ] || [ "$MCP_CONFORMANCE_RESULT" != success ]; then echo "CI policy or contract classification did not succeed" >&2 exit 1 fi diff --git a/docs/MCP_CONFORMANCE.md b/docs/MCP_CONFORMANCE.md new file mode 100644 index 000000000..4c5005700 --- /dev/null +++ b/docs/MCP_CONFORMANCE.md @@ -0,0 +1,178 @@ +# MCP conformance baseline + +WebCodex tracks MCP wire compatibility with the upstream +[`modelcontextprotocol/conformance`](https://github.com/modelcontextprotocol/conformance) +referee in addition to its focused Rust tests. The baseline is evidence about the +checks that actually ran; it is not a blanket "MCP compliant" badge. + +## Pinned referee and profiles + +`scripts/mcp_conformance.sh` pins the upstream referee to the immutable commit in +`tests/fixtures/mcp/conformance/baseline.json`. The script rejects an external +checkout at a different commit, archives the pinned Git object into a fresh +script-owned build directory, and runs `npm ci` plus the referee build from that +archive every time. Existing `dist/` output and working-tree modifications are +therefore never trusted as pinned evidence. Updating the pin is a reviewed +test-contract change, not a floating dependency update. + +The ordinary baseline runs two dated server profiles independently: + +- `2026-07-28`, using the stateless per-request protocol and that revision's + frozen requirements; +- `2025-11-25`, using the stateful lifecycle and its reconstructed frozen + requirements. + +Focused repository tests continue to cover `2025-06-18`; the conformance harness +is not used to invent a frozen requirements set that it does not publish. + +Run the same baseline locally with: + +```bash +bash scripts/mcp_conformance.sh +``` + +To reuse an already checked-out referee without changing the pin: + +```bash +WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR=/path/to/conformance \ + bash scripts/mcp_conformance.sh 2026-07-28 +``` + +The supplied checkout must resolve to the exact pinned commit. A checkout supplied +through `WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR` is read-only from this script's +perspective: invalid or mismatched checkouts are rejected rather than deleted or +rewritten, and local working-tree changes are ignored by the pinned `git archive`. +Reports are written to `target/mcp-conformance/reports` by default. An explicitly +supplied report directory must be empty or carry the WebCodex ownership marker +before old profile output is removed. CI uploads the report directory even when +the gate fails so the raw evidence remains inspectable. + +## What is exercised + +The script starts one ignored, test-only Rust fixture on `127.0.0.1:0`. The +fixture composes the same Salvo `AuthMiddleware` and `/mcp` handlers used by the +production HTTP surface, with temporary database/runtime state and authentication +disabled in the isolated test config. The referee therefore reaches real WebCodex parsing, dispatch and +rendering without production credentials or an external deployment. + +No conformance-only tool is added to the production registry. Some upstream +scenarios intentionally expect named reference-fixture tools. When WebCodex does +not expose such a tool, that result is a coverage gap until a test-only adapter is +provided; it must not be relabeled as a product protocol failure merely to make a +summary green. + +Authenticated Runtime/Connector behavior remains covered by focused synthetic- +credential integration tests. The upstream server CLI does not provide a generic +way to inject WebCodex authorization headers, so an authentication-blocked +scenario is inconclusive rather than a pass. + +## Coverage-aware report gate + +The upstream runner saves one `checks.json` file per scenario. After each profile, +`scripts/mcp_conformance_report.py` verifies the raw reports against the exact +required server-scenario list and the reviewed baseline. + +Every `FAILURE`, `WARNING`, or `SKIPPED` check from a **scored required** +scenario must have an exact `, ` classification in +`tests/fixtures/mcp/conformance/baseline.json`. Each classification also pins the +observed status and a SHA-256 fingerprint of the stable failure evidence +(`errorMessage` plus structured details/metadata when present, falling back to +the check's name/description). If the same check ID starts failing for a different reason or +with a different severity, the gate requires review instead of reusing the old +classification. Whole-scenario wildcards are rejected. + +Classifications that rely on an optional capability also bind to the fixture's +actual capability advertisement. Before the referee run, the script probes the +same HTTP endpoint with `server/discover` for `2026-07-28` and `initialize` for +`2025-11-25`, keeps the relevant tools/resources/prompts/logging/completions +capabilities, and compares them with the profile snapshot in the baseline. If the +server starts advertising a previously absent capability, those optional +classifications require review even when the old check ID still fails. + +The referee's `not_scored` server scenarios are still required to run and are +retained as informational protocol evidence. Their protocol verdicts do not +consume expected-failure classifications, but infrastructure failures such as a +scenario timeout or connection failure remain gate-blocking. Supported scored +classifications are: + +| Classification | Meaning | +|---|---| +| `genuine_protocol_failure` | The intended WebCodex behavior was reached and a protocol requirement is known to be violated. | +| `missing_harness_fixture` | The upstream scenario depends on reference-fixture behavior that the product endpoint does not expose. The product behavior remains untested by that check. | +| `optional_capability_not_implemented` | A capability is genuinely optional and is not advertised/implemented. The entry must cite capability/spec evidence. | +| `harness_limitation_pending` | The pinned referee itself marks or demonstrates a limitation/pending check. | +| `inconclusive_infrastructure` | Authentication blocking, timeout, harness exception, or other infrastructure failure. This is recorded but **never satisfies the CI gate**. | + +The gate also fails when: + +- no applicable `SUCCESS`/`FAILURE`/`WARNING` checks were emitted; +- an individual required scenario emits no verdict (for example an empty or + INFO-only report); +- a scored or `not_scored` scenario report expected by the frozen requirements is + missing, duplicated, or an unknown scenario is unexpectedly present; +- a check contains an unknown status; +- a new non-success check in a scored required scenario has no exact classification; +- a classified check changes status/evidence, disappears, or now passes; +- the referee exits abnormally, its exit code disagrees with the scored FAILURE + set, or any scored/not-scored scenario emits `scenario-timeout`, + `wire-schema-harness-error`, one of the pinned referee's generic top-level + scenario-error checks, the outer `Failed to run scenario` shape, or an exact + transport-exception form emitted by the pinned Node runner; +- the server capability advertisement changes while optional-capability baseline + entries depend on the previous advertisement; +- an explicitly classified inconclusive infrastructure result is present. + +For the pinned referee, exit `0` means no scored FAILURE and exit `1` means at +least one scored FAILURE; other exit codes are treated as runner/infrastructure +failures. This is stricter than trusting the process exit code alone because the +raw reports still need complete coverage and reviewed classifications. + +The report gate's own regression suite is dependency-free: + +```bash +python3 scripts/tests/test_mcp_conformance_report.py +``` + +It covers all-skipped output, missing/empty/INFO-only scenarios, unknown statuses, +abnormal referee exits, scored and not-scored infrastructure/transport failures, +changed status/evidence for an existing check ID, capability-advertisement drift, +ambiguous duplicate non-success check IDs, harness-pin mismatch, placeholder evidence, new scored +failures, informational non-success reporting, stale classifications, exact-path +JSON-RPC request-ID normalization, wire-schema payload drift, ordinary lookalike +objects, empty structured evidence, and rejection of broad masks. + +## Raw evidence and review policy + +Each profile retains: + +- the referee's raw per-scenario `checks.json` files; +- the referee stdout/stderr log; +- metadata containing the WebCodex source SHA, referee SHA, profile, upstream + referee exit code, observed server capability projection, exact scored scenario + list, and frozen `not_scored` server scenarios with their upstream reasons; +- the coverage-aware WebCodex summary. + +Baseline updates should be narrow. Add a classification only after reproducing +and understanding the exact check, then record its observed status and evidence +fingerprint. Fingerprints retain semantic diagnostics but normalize two narrowly +scoped referee noise sources: the numeric value (not presence/type) of the two +known SEP-2575 millisecond-epoch response IDs, and the duplicated offending +message only inside the structured `wire-schema-valid` violation list. For a +`ListToolsResult/tools//...` diagnostic, that volatile array ordinal is +resolved through the offending response to the referenced tool name before hashing, +so unrelated tool insertion/reordering does not rename the finding; moving the defect +to a different tool still changes the fingerprint. Wire-schema origin/context/errors/ +specVersion remain hashed, and ordinary objects with similar field names are not +normalized. A changed fingerprint is a review prompt, not something +to refresh mechanically. Remove the entry as soon as the check passes. Do not replace +multiple check IDs with a scenario-wide exception. The SEP-2164 `data.uri` +finding is kept as an advisory SHOULD/WARNING rather than described as a MUST +violation. + +This baseline deliberately records two already-demonstrated 2026 result-shape findings +without fixing production behavior: streamed artifact `resources/read` omits the +cache hints emitted by ordinary resource reads, and a completed Connector Task +embeds its original tool result without a nested `resultType`. Their focused Rust +observations and planned follow-up are recorded in the baseline fixture. Fixes +belong in a separate protocol-behavior change so this baseline remains an +independent measuring instrument. diff --git a/docs/TESTING.md b/docs/TESTING.md index 4d354f171..b625c8406 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -33,6 +33,7 @@ waits, and the cost of each test lane. | real-process job reconciliation harness | Boot a real server plus a WebSocket runner that advertises `job_state_reconciliation`. Scenario A keeps a raw async Job running across a SERVER-only restart and asserts the SAME runner instance, original `job_id`, preserved ownership/project/session, non-regressing sequence/log cursors, `recovered_after_server_restart`, original-process stop, and one side-effect set. Scenario B lets a Job complete while the Server is offline and reconciles the terminal result without duplicate logs or execution. Scenario C forces `run_process` past its synchronous grace window, then proves the handed-off structured Job survives a Server restart and an old Server-epoch observation token refreshes immediately for the same `job_id`. Scenario D uses a delayed Cargo fixture to force a real `cargo_check` validation handoff past its sync window, then proves the same restart/token-refresh/stop contract with the validation command started exactly once. Ordinary Runner-owned Jobs keep the Runner process alive for these scenarios; `run_detached_process` restart survival is a separate supervisor-ownership contract covered by its focused Runner suites and production dogfood. | Local processes, temp dirs/ports/tokens, and a temp project; no production services or QUIC certs. Scenario D intentionally takes roughly the validation sync window plus restart time. | `bash scripts/e2e_job_reconciliation_ws.sh` | | real-process job recovery failure/non-reconciliation harness | Cover the failure and non-reconciliation paths the happy-path reconciliation harness omits, using `WEBCODEX_JOB_RECOVERY_GRACE_SECS=10` (clamped, above the 5s floor) so the deadline is bounded without waiting the 120s default. Scenario C: kill the runner only (server stays up), let the job enter `recovering`, and assert the non-request-triggered recovery-timeout sweep transitions it to `lost` with `runner_recovery_deadline_exceeded`, `ended_at` set once, one list record, stop-on-lost stable, and the command never re-executes. Scenario D: instance B replaces instance A (same client_id, new `agent_instance_id`); A's job becomes `lost` with `runner_instance_replaced`, B starts its own new job, A's late update is rejected, first `ended_at`/reason preserved. Scenario E: a generation-2 Runner registered with `WEBCODEX_RUNNER_DISABLE_JOB_STATE_RECONCILIATION=1` (no capability, no inventory) dispatches a job and, on disconnect, deterministically fences it to `lost` with `runner_disconnected_without_reconciliation` (never `recovering`); after a server restart its public lost receipt remains observable within retention, and a same-client new no-reconciliation instance cannot revive execution. Scenario F: a long job across three server restarts keeps the same `job_id`, runs the command once, keeps `last_update_seq`/log cursors non-regressing and markers non-duplicating, and reaches a terminal `stopped` that survives a third restart with `ended_at` unchanged by terminal inventory replay. | Local processes, temp dirs/ports/tokens, and a temp project; no production services or QUIC certs. | `bash scripts/e2e_job_recovery_failures_ws.sh` | | security auth matrix | Cover OAuth, scope policy, shared-key behavior, token classes, read-only session guards, and denied mutations. | No external identity provider by default; use local fixtures and synthetic tokens. | `cargo test -p webcodex --lib oauth -- --nocapture`; `cargo test -p webcodex --lib scope -- --nocapture`; `cargo test -p webcodex --lib metadata -- --nocapture` | +| MCP conformance baseline | Exercise the real Salvo `/mcp` handlers through a disposable loopback fixture against a pinned upstream referee, then gate on check-level coverage/classification rather than process exit alone. | Loopback product fixture; immutable external referee source/dependencies fetched only by the explicit script/CI lane; no production credentials. | `bash scripts/mcp_conformance.sh`; see [`MCP_CONFORMANCE.md`](MCP_CONFORMANCE.md). | The MCP Apps' DOM/message-order regression tests run without browser or npm dependencies: `node --test src/mcp_tests/*.test.mjs`. They execute the embedded @@ -95,6 +96,14 @@ The lanes above define test semantics; workflows decide when to run them. `test-windows`, and `test-native` aggregates always resolve and verify each child lane is `success` when required or `skipped` when not required, avoiding a skipped required-check context that could leave branch protection pending. +- MCP dated-revision evidence has its own bounded `mcp-conformance` lane. It pins + and freshly builds the upstream referee, runs the `2026-07-28` and `2025-11-25` + server requirements against a test-only loopback WebCodex endpoint, validates + the report gate itself, and uploads raw reports on success or failure. + `test-native` requires this lane to succeed, so missing per-scenario verdicts, + abnormal/infrastructure runs, stale or changed classifications, and unclassified + new failures are merge-blocking. See [`MCP_CONFORMANCE.md`](MCP_CONFORMANCE.md) + for baseline semantics. - Linux Rust execution remains package-sharded: the server package `webcodex`, the Runner/LSP packages, and the remaining workspace crates run in parallel. The Runner/LSP shard compiles with `--features runner-real-process-tests` to prevent diff --git a/scripts/mcp_conformance.sh b/scripts/mcp_conformance.sh new file mode 100644 index 000000000..527a8b176 --- /dev/null +++ b/scripts/mcp_conformance.sh @@ -0,0 +1,472 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + +# The conformance referee is intentionally immutable. Updating it is a reviewed +# baseline change because scenario/check semantics can change between commits. +BASELINE="tests/fixtures/mcp/conformance/baseline.json" +HARNESS_COMMIT="$(python3 - "$BASELINE" <<'PY' +import json +import pathlib +import re +import sys +value = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +commit = value.get("harness_commit", "") +if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise SystemExit("baseline harness_commit must be a full lowercase Git SHA") +print(commit) +PY +)" +WORK_ROOT="${WEBCODEX_MCP_CONFORMANCE_WORK_ROOT:-target/mcp-conformance}" +HARNESS_DIR_EXTERNAL=0 +if [ "${WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR+x}" = x ]; then + HARNESS_DIR="${WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR}" + HARNESS_DIR_EXTERNAL=1 +else + HARNESS_DIR="$WORK_ROOT/harness" +fi +REPORT_ROOT_EXTERNAL=0 +if [ "${WEBCODEX_MCP_CONFORMANCE_REPORT_ROOT+x}" = x ]; then + REPORT_ROOT="${WEBCODEX_MCP_CONFORMANCE_REPORT_ROOT}" + REPORT_ROOT_EXTERNAL=1 +else + REPORT_ROOT="$WORK_ROOT/reports" +fi +PROFILES=("$@") +HARNESS_OWNER_MARKER=".webcodex-mcp-conformance-harness" +REPORT_OWNER_MARKER=".webcodex-mcp-conformance-reports" +harness_build_dir="" +fixture_dir="" +fixture_pid="" +cleanup_started=0 + +if [ "$#" -eq 0 ]; then + PROFILES=("2026-07-28" "2025-11-25") +fi +for profile in "${PROFILES[@]}"; do + case "$profile" in + 2026-07-28|2025-11-25) ;; + *) echo "unsupported MCP conformance profile: $profile" >&2; exit 2 ;; + esac +done + +if [ -z "$WORK_ROOT" ] || [ -z "$HARNESS_DIR" ] || [ -z "$REPORT_ROOT" ]; then + echo "MCP conformance work, harness, and report paths must be non-empty" >&2 + exit 2 +fi +mkdir -p "$WORK_ROOT" + +fixture_alive() { + [ -n "$fixture_pid" ] || return 1 + kill -0 -- "-$fixture_pid" 2>/dev/null || kill -0 "$fixture_pid" 2>/dev/null +} + +signal_fixture() { + local signal="$1" + kill "-$signal" -- "-$fixture_pid" 2>/dev/null || kill "-$signal" "$fixture_pid" 2>/dev/null || true +} + +stop_fixture() { + [ -n "$fixture_pid" ] || return 0 + touch "$fixture_dir/stop" 2>/dev/null || true + for _ in $(seq 1 50); do + if ! fixture_alive; then break; fi + sleep 0.1 + done + if fixture_alive; then + signal_fixture TERM + fi + for _ in $(seq 1 50); do + if ! fixture_alive; then break; fi + sleep 0.1 + done + if fixture_alive; then + signal_fixture KILL + fi + wait "$fixture_pid" 2>/dev/null || true + fixture_pid="" +} + +cleanup() { + [ "$cleanup_started" -eq 0 ] || return 0 + cleanup_started=1 + stop_fixture + if [ -n "$fixture_dir" ]; then + rm -rf "$fixture_dir" + fi + if [ -n "$harness_build_dir" ]; then + rm -rf "$harness_build_dir" + fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +is_git_worktree() { + git -C "$1" rev-parse --is-inside-work-tree >/dev/null 2>&1 +} + +prepare_harness_source() { + if [ "$HARNESS_DIR_EXTERNAL" -eq 1 ]; then + if ! is_git_worktree "$HARNESS_DIR"; then + echo "WEBCODEX_MCP_CONFORMANCE_HARNESS_DIR must name an existing Git worktree; refusing to modify it: $HARNESS_DIR" >&2 + exit 2 + fi + elif [ ! -e "$HARNESS_DIR" ]; then + mkdir -p "$HARNESS_DIR" + touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" + git -C "$HARNESS_DIR" init -q + git -C "$HARNESS_DIR" remote add origin https://github.com/modelcontextprotocol/conformance.git + elif ! is_git_worktree "$HARNESS_DIR"; then + echo "default harness path exists but is not a Git worktree; refusing to remove it: $HARNESS_DIR" >&2 + exit 2 + fi + + if [ "$HARNESS_DIR_EXTERNAL" -eq 0 ]; then + origin_url="$(git -C "$HARNESS_DIR" remote get-url origin 2>/dev/null || true)" + case "$origin_url" in + https://github.com/modelcontextprotocol/conformance.git|https://github.com/modelcontextprotocol/conformance) ;; + *) + echo "default harness checkout has an unexpected origin; refusing to modify it: $HARNESS_DIR" >&2 + exit 2 + ;; + esac + # Older versions of this script created the same default checkout before the + # ownership marker existed. Adopt it only when it is already at the pinned + # commit and has no tracked edits; otherwise ownership is ambiguous and we + # refuse to mutate it. Externally supplied worktrees are always read-only. + if [ ! -f "$HARNESS_DIR/$HARNESS_OWNER_MARKER" ]; then + legacy_actual="$(git -C "$HARNESS_DIR" rev-parse HEAD 2>/dev/null || true)" + if [ "$legacy_actual" != "$HARNESS_COMMIT" ] || \ + ! git -C "$HARNESS_DIR" diff --quiet --no-ext-diff || \ + ! git -C "$HARNESS_DIR" diff --cached --quiet --no-ext-diff; then + echo "default harness checkout is unowned and not an exact clean pinned checkout; refusing to modify it: $HARNESS_DIR" >&2 + exit 2 + fi + touch "$HARNESS_DIR/$HARNESS_OWNER_MARKER" + fi + fi + + actual="$(git -C "$HARNESS_DIR" rev-parse HEAD 2>/dev/null || true)" + if [ "$actual" != "$HARNESS_COMMIT" ]; then + if [ "$HARNESS_DIR_EXTERNAL" -eq 1 ]; then + echo "MCP conformance harness must be exactly $HARNESS_COMMIT, got ${actual:-unresolved}; external checkout is never modified" >&2 + exit 2 + fi + GIT_TERMINAL_PROMPT=0 git -C "$HARNESS_DIR" fetch -q --depth 1 origin "$HARNESS_COMMIT" + git -C "$HARNESS_DIR" checkout -q --detach FETCH_HEAD + actual="$(git -C "$HARNESS_DIR" rev-parse HEAD)" + fi + if [ "$actual" != "$HARNESS_COMMIT" ]; then + echo "MCP conformance harness must be exactly $HARNESS_COMMIT, got $actual" >&2 + exit 2 + fi +} + +prepare_report_root() { + if [ -e "$REPORT_ROOT" ] && [ ! -d "$REPORT_ROOT" ]; then + echo "MCP conformance report root is not a directory: $REPORT_ROOT" >&2 + exit 2 + fi + if [ ! -e "$REPORT_ROOT" ]; then + mkdir -p "$REPORT_ROOT" + touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" + elif [ ! -f "$REPORT_ROOT/$REPORT_OWNER_MARKER" ]; then + if find "$REPORT_ROOT" -mindepth 1 -maxdepth 1 -print -quit | grep -q .; then + echo "MCP report root is non-empty and lacks the WebCodex ownership marker; refusing to delete its contents: $REPORT_ROOT" >&2 + exit 2 + fi + touch "$REPORT_ROOT/$REPORT_OWNER_MARKER" + fi + rm -rf "$REPORT_ROOT/2026-07-28" "$REPORT_ROOT/2025-11-25" + rm -f "$REPORT_ROOT/fixture.log" "$REPORT_ROOT/server-capabilities.json" +} + +prepare_harness_build() { + harness_build_dir="$(mktemp -d "$WORK_ROOT/harness-build.XXXXXX")" + git -C "$HARNESS_DIR" archive "$HARNESS_COMMIT" | tar -x -C "$harness_build_dir" + if [ ! -f "$harness_build_dir/package-lock.json" ]; then + echo "pinned harness has no package-lock.json" >&2 + exit 2 + fi + npm --prefix "$harness_build_dir" ci --ignore-scripts --no-audit --no-fund + npm --prefix "$harness_build_dir" run build + if [ ! -f "$harness_build_dir/dist/index.js" ]; then + echo "pinned harness build did not produce dist/index.js" >&2 + exit 2 + fi +} + +prepare_harness_source +prepare_report_root +prepare_harness_build +python3 scripts/tests/test_mcp_conformance_report.py + +# Compile before starting the bounded readiness clock. A cold WebCodex test build +# can take several minutes on small builders; readiness should measure server +# startup, not Rust compilation time. +cargo test --locked -p webcodex --lib mcp_conformance_fixture_server --no-run + +fixture_dir="$(mktemp -d "${TMPDIR:-/tmp}/webcodex-mcp-conformance.XXXXXX")" +url_file="$fixture_dir/url" +stop_file="$fixture_dir/stop" +fixture_log="$REPORT_ROOT/fixture.log" +# Keep the entire Cargo + test-binary fixture tree in one private process group so +# fallback teardown cannot leave the loopback server alive if Cargo exits first. +WEBCODEX_MCP_CONFORMANCE_URL_FILE="$url_file" \ +WEBCODEX_MCP_CONFORMANCE_STOP_FILE="$stop_file" \ + python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ + cargo test --locked -p webcodex --lib mcp_conformance_fixture_server -- --ignored --nocapture \ + >"$fixture_log" 2>&1 & +fixture_pid=$! + +ready_deadline=$((SECONDS + 60)) +while [ ! -s "$url_file" ]; do + if ! fixture_alive; then + echo "MCP conformance fixture exited before publishing its URL" >&2 + tail -80 "$fixture_log" >&2 || true + exit 1 + fi + if [ "$SECONDS" -ge "$ready_deadline" ]; then + echo "timed out waiting for MCP conformance fixture URL" >&2 + tail -80 "$fixture_log" >&2 || true + exit 1 + fi + sleep 0.1 +done +fixture_url="$(tr -d '\r\n' < "$url_file")" +case "$fixture_url" in + http://127.0.0.1:*'/mcp') ;; + *) echo "fixture published unexpected URL: $fixture_url" >&2; exit 1 ;; +esac + +capabilities_file="$REPORT_ROOT/server-capabilities.json" +python3 - "$fixture_url" "$capabilities_file" <<'PY' +import json +import pathlib +import sys +import urllib.error +import urllib.request + +server_url, output_path = sys.argv[1:] + +def rpc_request(payload, headers=None): + encoded = json.dumps(payload, separators=(",", ":")).encode("utf-8") + request_headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + if headers: + request_headers.update(headers) + request = urllib.request.Request( + server_url, + data=encoded, + headers=request_headers, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + raw = response.read().decode("utf-8") + content_type = response.headers.get("Content-Type", "") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise SystemExit(f"capability probe HTTP {exc.code}: {body[:500]}") from exc + except OSError as exc: + raise SystemExit(f"capability probe transport error: {exc}") from exc + + messages = [] + if "text/event-stream" in content_type: + for line in raw.splitlines(): + if not line.startswith("data:"): + continue + candidate = line[len("data:"):].strip() + if candidate: + messages.append(json.loads(candidate)) + else: + messages.append(json.loads(raw)) + request_id = payload["id"] + message = next((item for item in messages if item.get("id") == request_id), None) + if not isinstance(message, dict): + raise SystemExit("capability probe did not receive the matching JSON-RPC response") + if "error" in message: + raise SystemExit(f"capability probe returned JSON-RPC error: {message['error']}") + result = message.get("result") + if not isinstance(result, dict): + raise SystemExit("capability probe response has no result object") + capabilities = result.get("capabilities") + if not isinstance(capabilities, dict): + raise SystemExit("capability probe result has no capabilities object") + return capabilities + +modern = rpc_request( + { + "jsonrpc": "2.0", + "id": "webcodex-capabilities-2026", + "method": "server/discover", + "params": { + "_meta": { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": {}, + "io.modelcontextprotocol/clientInfo": { + "name": "webcodex-conformance-capability-probe", + "version": "1.0", + }, + } + }, + }, + { + "MCP-Protocol-Version": "2026-07-28", + "Mcp-Method": "server/discover", + }, +) +legacy = rpc_request( + { + "jsonrpc": "2.0", + "id": "webcodex-capabilities-2025", + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { + "name": "webcodex-conformance-capability-probe", + "version": "1.0", + }, + }, + } +) + +relevant_keys = ("completions", "logging", "prompts", "resources", "tools") +def project(capabilities): + return {key: capabilities[key] for key in relevant_keys if key in capabilities} + +pathlib.Path(output_path).write_text( + json.dumps( + { + "2026-07-28": project(modern), + "2025-11-25": project(legacy), + }, + indent=2, + sort_keys=True, + ) + "\n", + encoding="utf-8", +) +PY + +server_sha="$(git rev-parse HEAD)" +overall=0 +for profile in "${PROFILES[@]}"; do + profile_root="$REPORT_ROOT/$profile" + raw="$profile_root/raw" + mkdir -p "$raw" + log="$profile_root/harness.log" + set +e + node "$harness_build_dir/dist/index.js" server \ + --url "$fixture_url" \ + --requirements "$profile" \ + --output-dir "$raw" \ + --timeout 10000 \ + >"$log" 2>&1 + harness_exit=$? + set -e + + metadata="$profile_root/metadata.json" + python3 - "$harness_build_dir/requirements/$profile.yaml" "$metadata" "$profile" \ + "$server_sha" "$HARNESS_COMMIT" "$harness_exit" "$capabilities_file" <<'PY' +import json +import pathlib +import sys + +( + requirements_path, + output_path, + profile, + server_sha, + harness_sha, + harness_exit, + capabilities_path, +) = sys.argv[1:] +capability_map = json.loads(pathlib.Path(capabilities_path).read_text(encoding="utf-8")) +server_capabilities = capability_map.get(profile) +if not isinstance(server_capabilities, dict): + raise SystemExit(f"missing capability probe result for {profile}") +server = [] +not_scored = [] +section = None +current_not_scored = None + +def retain_not_scored(entry): + if not entry or entry.get("leg") != "server": + return + scenario = entry.get("scenario") + reason = entry.get("reason") + if not scenario or not reason: + raise SystemExit(f"malformed server not_scored entry in {requirements_path}: {entry}") + not_scored.append({"scenario": scenario, "reason": reason}) + +for raw in pathlib.Path(requirements_path).read_text(encoding="utf-8").splitlines(): + if raw in {"server:", "client:", "not_scored:"}: + retain_not_scored(current_not_scored) + current_not_scored = None + section = raw[:-1] + continue + if raw and not raw.startswith(" "): + retain_not_scored(current_not_scored) + current_not_scored = None + section = None + continue + if section == "server" and raw.startswith(" - "): + server.append(raw[4:].strip()) + elif section == "not_scored": + if raw.startswith(" - scenario: "): + retain_not_scored(current_not_scored) + current_not_scored = {"scenario": raw[len(" - scenario: "):].strip()} + elif current_not_scored is not None and raw.startswith(" leg: "): + current_not_scored["leg"] = raw[len(" leg: "):].strip() + elif current_not_scored is not None and raw.startswith(" reason: "): + current_not_scored["reason"] = raw[len(" reason: "):].strip() +retain_not_scored(current_not_scored) + +if not server: + raise SystemExit(f"no scored server scenarios found in {requirements_path}") +if len(server) != len(set(server)): + raise SystemExit(f"duplicate scored server scenarios in {requirements_path}") +not_scored_names = [entry["scenario"] for entry in not_scored] +if len(not_scored_names) != len(set(not_scored_names)): + raise SystemExit(f"duplicate server not_scored scenarios in {requirements_path}") +if set(server) & set(not_scored_names): + raise SystemExit(f"server scenario is both scored and not_scored in {requirements_path}") + +pathlib.Path(output_path).write_text( + json.dumps( + { + "schema_version": 1, + "profile": profile, + "server_sha": server_sha, + "harness_sha": harness_sha, + "harness_exit_code": int(harness_exit), + "server_capabilities": server_capabilities, + "required_scenarios": server, + "not_scored_scenarios": not_scored, + }, + indent=2, + sort_keys=True, + ) + "\n", + encoding="utf-8", +) +PY + + summary="$profile_root/summary.json" + if ! python3 scripts/mcp_conformance_report.py \ + --reports "$raw" \ + --metadata "$metadata" \ + --baseline "$BASELINE" \ + --profile "$profile" \ + --summary "$summary"; then + overall=1 + fi +done + +exit "$overall" diff --git a/scripts/mcp_conformance_report.py b/scripts/mcp_conformance_report.py new file mode 100644 index 000000000..d44a073ab --- /dev/null +++ b/scripts/mcp_conformance_report.py @@ -0,0 +1,707 @@ +#!/usr/bin/env python3 +"""Coverage-aware gate for pinned MCP conformance reports. + +The upstream conformance runner intentionally answers a different question from +this gate: it executes scenarios and reports their checks. WebCodex keeps the +raw upstream output, then this script verifies that the expected scenario set was +actually exercised and that every non-success result has a narrow, reviewed +classification. A zero upstream process exit is never treated as proof of +coverage or compliance. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +CLASSIFICATIONS = { + "genuine_protocol_failure", + "missing_harness_fixture", + "optional_capability_not_implemented", + "harness_limitation_pending", + "inconclusive_infrastructure", +} +VALID_STATUSES = {"SUCCESS", "FAILURE", "WARNING", "SKIPPED", "INFO"} +NON_SUCCESS_STATUSES = {"FAILURE", "WARNING", "SKIPPED"} +APPLICABLE_STATUSES = {"SUCCESS", "FAILURE", "WARNING"} +VERDICT_STATUSES = {"SUCCESS", "FAILURE", "WARNING", "SKIPPED"} +SCENARIO_DIR_RE = re.compile(r"^server-(.+)-\d{4}-\d{2}-\d{2}T.*Z$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") + +# The pinned referee uses Date.now() only for these two stateless diagnostic +# probes. Keep this deliberately narrow: a large integer in any other check is +# semantic evidence, not generic timestamp noise. +DYNAMIC_JSONRPC_ID_CHECKS = { + "sep-2575-http-server-no-independent-requests-on-stream", + "sep-2575-server-no-log-without-loglevel", +} +DYNAMIC_JSONRPC_ID_PATH = ("details", "response", "id") +DYNAMIC_JSONRPC_ID_SENTINEL = { + "$webcodexEvidence": "dynamic-millisecond-jsonrpc-id", + "sourceType": "integer", +} +WIRE_SCHEMA_CHECK_ID = "wire-schema-valid" +WIRE_SCHEMA_MESSAGE_SENTINEL = "" +WIRE_SCHEMA_TOOL_INDEX_RE = re.compile(r"ListToolsResult/tools/(?P\d+)") + +# Structural or top-level scenario failures emitted by the immutable referee pin. +# These IDs mean the scenario/harness aborted outside its intended protocol +# assertions. Per-assertion task failures and setup checks are deliberately not +# blanket-listed; transport exceptions from those paths are recognized separately. +PINNED_REFEREE_INFRA_CHECK_IDS = { + "scenario-timeout", + "wire-schema-harness-error", + "json-schema-2020-12-error", + "server-session-lifecycle-error", + "server-sse-multiple-streams-error", + "server-sse-polling-error", +} +PINNED_TRANSPORT_ERROR_RE = re.compile( + r"^(?:(?:Failed(?::| to initialize:| to set up tests:)|Error:)\s*)?" + r"(?:fetch failed|connect E(?:CONNREFUSED|CONNRESET|HOSTUNREACH|NETUNREACH)\b.*)$" +) + + +class GateError(RuntimeError): + """Raised when report or baseline input is malformed.""" + + +@dataclass(frozen=True) +class Classification: + scenario: str + check_id: str + expected_status: str + evidence_sha256: str + classification: str + reason: str + evidence: str + + @property + def key(self) -> tuple[str, str]: + return (self.scenario, self.check_id) + + +def _normalize_evidence_value( + value: Any, + *, + path: tuple[str, ...] = (), + normalize_dynamic_probe_id: bool = False, +) -> Any: + """Normalize only pin-specific noise at an exact known evidence path.""" + + if isinstance(value, list): + return [ + _normalize_evidence_value( + item, + path=path, + normalize_dynamic_probe_id=normalize_dynamic_probe_id, + ) + for item in value + ] + if not isinstance(value, dict): + return value + + normalized: dict[str, Any] = {} + for field, item in value.items(): + item_path = path + (field,) + if ( + normalize_dynamic_probe_id + and item_path == DYNAMIC_JSONRPC_ID_PATH + and value.get("jsonrpc") == "2.0" + and isinstance(item, int) + and not isinstance(item, bool) + and abs(item) >= 1_000_000_000_000 + ): + # Preserve field presence/type semantics while ignoring only the + # Date.now()-style numeric value used by the pinned probe. + normalized[field] = DYNAMIC_JSONRPC_ID_SENTINEL + continue + normalized[field] = _normalize_evidence_value( + item, + path=item_path, + normalize_dynamic_probe_id=normalize_dynamic_probe_id, + ) + return normalized + + +def _wire_schema_errors_with_tool_names(errors: Any, message: Any) -> Any: + """Replace volatile tools/list array ordinals with the referenced tool name.""" + + if not isinstance(errors, list) or not isinstance(message, dict): + return errors + result = message.get("result") + tools = result.get("tools") if isinstance(result, dict) else None + if not isinstance(tools, list): + return errors + + def replace_tool_index(match: re.Match[str]) -> str: + tool_index = int(match.group("index")) + if tool_index >= len(tools): + return match.group(0) + tool = tools[tool_index] + if not isinstance(tool, dict): + return match.group(0) + tool_name = tool.get("name") + if not isinstance(tool_name, str) or not tool_name: + return match.group(0) + return "ListToolsResult/tools[" + json.dumps(tool_name, ensure_ascii=False) + "]" + + return [ + WIRE_SCHEMA_TOOL_INDEX_RE.sub(replace_tool_index, error) + if isinstance(error, str) + else error + for error in errors + ] + + +def _wire_schema_semantic_details(check: dict[str, Any]) -> Any | None: + """Return structured wire-schema evidence with only the duplicated message normalized.""" + + if check.get("id") != WIRE_SCHEMA_CHECK_ID: + return None + details = check.get("details") + if not isinstance(details, dict) or not isinstance(details.get("messagesValidated"), int): + return None + violations = details.get("violations") + if not isinstance(violations, list) or not violations: + return None + + normalized = _normalize_evidence_value(details, path=("details",)) + normalized_violations = normalized.get("violations") + if not isinstance(normalized_violations, list): + return None + + recognized = False + for index, violation in enumerate(violations): + if not isinstance(violation, dict): + continue + if ( + violation.get("origin") not in {"harness", "implementation"} + or not isinstance(violation.get("context"), str) + or not isinstance(violation.get("errors"), list) + or not isinstance(violation.get("specVersion"), str) + or "message" not in violation + ): + continue + normalized_violation = normalized_violations[index] + if not isinstance(normalized_violation, dict): + continue + # Tool insertion/reordering changes the numeric array position in the + # referee diagnostic even when the same named tool has the same schema + # defect. Resolve that ordinal through the offending response before the + # complete message is replaced. If resolution fails, keep the raw index + # so the gate fails closed. + normalized_violation["errors"] = _wire_schema_errors_with_tool_names( + normalized_violation.get("errors"), violation.get("message") + ) + # The same complete JSON-RPC object is also rendered into the referee's + # errorMessage. Retain message *presence* with a sentinel and hash the + # authoritative origin/context/errors/specVersion fields instead. + normalized_violation["message"] = WIRE_SCHEMA_MESSAGE_SENTINEL + recognized = True + return normalized if recognized else None + + +def _has_substantive_evidence(value: Any) -> bool: + if value is None or value == "": + return False + if isinstance(value, dict): + return any(_has_substantive_evidence(item) for item in value.values()) + if isinstance(value, list): + return any(_has_substantive_evidence(item) for item in value) + return True + + +def check_evidence_sha256(check: dict[str, Any]) -> str: + """Hash stable failure evidence while excluding only proven referee noise.""" + + payload: dict[str, Any] = {} + wire_schema_details = _wire_schema_semantic_details(check) + if wire_schema_details is not None: + # The structured violations are the authoritative evidence. The pinned + # referee's errorMessage redundantly serializes the entire offending + # message, so including it would reintroduce unrelated tool/schema drift. + payload["details"] = wire_schema_details + if "metadata" in check: + payload["metadata"] = _normalize_evidence_value( + check.get("metadata"), path=("metadata",) + ) + else: + normalize_dynamic_probe_id = check.get("id") in DYNAMIC_JSONRPC_ID_CHECKS + for field in ("errorMessage", "details", "metadata"): + if field in check: + payload[field] = _normalize_evidence_value( + check.get(field), + path=(field,), + normalize_dynamic_probe_id=normalize_dynamic_probe_id, + ) + + if not payload or not any(_has_substantive_evidence(value) for value in payload.values()): + payload = { + "name": check.get("name"), + "description": check.get("description"), + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _infrastructure_reason(check: dict[str, Any]) -> str | None: + # These are structural/top-level failure shapes emitted by the immutable referee + # pin. Do not scan arbitrary assertion text for words such as "timeout": a + # valid protocol assertion may contain those words. + check_id = check.get("id") + if check_id in PINNED_REFEREE_INFRA_CHECK_IDS: + if check_id == "scenario-timeout": + return "referee scenario timeout" + if check_id == "wire-schema-harness-error": + return "referee wire-schema harness error" + return f"referee top-level scenario failure ({check_id})" + if check.get("description") == "Failed to run scenario": + return "referee failed to run scenario" + error_message = check.get("errorMessage") + if isinstance(error_message, str) and PINNED_TRANSPORT_ERROR_RE.fullmatch(error_message.strip()): + return "referee transport exception" + return None + + +def _load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise GateError(f"cannot read JSON {path}: {exc}") from exc + + +def _scenario_from_result_dir(path: Path) -> str: + match = SCENARIO_DIR_RE.match(path.parent.name) + if not match: + raise GateError( + f"unexpected upstream result directory {path.parent.name!r}; " + "expected server--" + ) + return match.group(1) + + +def load_reports(report_root: Path) -> dict[str, list[dict[str, Any]]]: + reports: dict[str, list[dict[str, Any]]] = {} + for checks_path in sorted(report_root.glob("server-*/checks.json")): + scenario = _scenario_from_result_dir(checks_path) + if scenario in reports: + raise GateError( + f"multiple raw reports found for scenario {scenario!r}; clean the output directory" + ) + value = _load_json(checks_path) + if not isinstance(value, list): + raise GateError(f"{checks_path} must contain a JSON array") + checks: list[dict[str, Any]] = [] + for index, check in enumerate(value): + if not isinstance(check, dict): + raise GateError(f"{checks_path}[{index}] is not an object") + check_id = check.get("id") + status = check.get("status") + if not isinstance(check_id, str) or not check_id: + raise GateError(f"{checks_path}[{index}] has no non-empty check id") + if not isinstance(status, str) or not status: + raise GateError(f"{checks_path}[{index}] has no non-empty status") + if status not in VALID_STATUSES: + raise GateError(f"{checks_path}[{index}] has unknown status {status!r}") + checks.append(check) + reports[scenario] = checks + return reports + + +def load_classifications(baseline_path: Path, profile: str) -> dict[tuple[str, str], Classification]: + baseline = _load_json(baseline_path) + if not isinstance(baseline, dict) or baseline.get("schema_version") != 2: + raise GateError("baseline must be an object with schema_version=2") + profiles = baseline.get("profiles") + if not isinstance(profiles, dict) or profile not in profiles: + raise GateError(f"baseline has no profile {profile!r}") + profile_config = profiles[profile] + if not isinstance(profile_config, dict): + raise GateError(f"baseline profile {profile!r} must be an object") + raw_entries = profile_config.get("classifications", []) + if not isinstance(raw_entries, list): + raise GateError(f"baseline profile {profile!r} classifications must be an array") + + entries: dict[tuple[str, str], Classification] = {} + for index, raw in enumerate(raw_entries): + if not isinstance(raw, dict): + raise GateError(f"classification #{index + 1} must be an object") + scenario = raw.get("scenario") + check_id = raw.get("check_id") + expected_status = raw.get("expected_status") + evidence_sha256 = raw.get("evidence_sha256") + kind = raw.get("classification") + reason = raw.get("reason") + evidence = raw.get("evidence") + if not isinstance(scenario, str) or not scenario: + raise GateError(f"classification #{index + 1} needs a non-empty scenario") + # Whole-scenario masks are deliberately unsupported. They can hide a newly + # failing check when an older, unrelated check is already expected to fail. + if not isinstance(check_id, str) or not check_id or check_id == "*": + raise GateError( + f"classification {scenario!r} must name one exact check_id; broad scenario masks are forbidden" + ) + if expected_status not in NON_SUCCESS_STATUSES: + raise GateError( + f"classification {scenario}:{check_id} needs expected_status in " + f"{sorted(NON_SUCCESS_STATUSES)}, got {expected_status!r}" + ) + if not isinstance(evidence_sha256, str) or not SHA256_RE.fullmatch(evidence_sha256): + raise GateError( + f"classification {scenario}:{check_id} needs a lowercase 64-hex evidence_sha256" + ) + if evidence_sha256 == "0" * 64: + raise GateError( + f"classification {scenario}:{check_id} has placeholder evidence_sha256" + ) + if kind not in CLASSIFICATIONS: + raise GateError( + f"classification {scenario}:{check_id} has unsupported kind {kind!r}" + ) + if not isinstance(reason, str) or not reason.strip(): + raise GateError(f"classification {scenario}:{check_id} needs a reason") + if not isinstance(evidence, str) or not evidence.strip(): + raise GateError(f"classification {scenario}:{check_id} needs evidence") + entry = Classification( + scenario, + check_id, + expected_status, + evidence_sha256, + kind, + reason.strip(), + evidence.strip(), + ) + if entry.key in entries: + raise GateError(f"duplicate classification for {scenario}:{check_id}") + entries[entry.key] = entry + return entries + + +def evaluate_reports( + report_root: Path, + metadata_path: Path, + baseline_path: Path, + profile: str, +) -> dict[str, Any]: + metadata = _load_json(metadata_path) + if not isinstance(metadata, dict): + raise GateError("metadata must be a JSON object") + if metadata.get("schema_version") != 1: + raise GateError("metadata must have schema_version=1") + if metadata.get("profile") != profile: + raise GateError( + f"metadata profile {metadata.get('profile')!r} does not match requested {profile!r}" + ) + server_sha = metadata.get("server_sha") + harness_sha = metadata.get("harness_sha") + if not isinstance(server_sha, str) or not GIT_SHA_RE.fullmatch(server_sha): + raise GateError("metadata server_sha must be a full lowercase Git SHA") + if not isinstance(harness_sha, str) or not GIT_SHA_RE.fullmatch(harness_sha): + raise GateError("metadata harness_sha must be a full lowercase Git SHA") + baseline_metadata = _load_json(baseline_path) + if not isinstance(baseline_metadata, dict) or baseline_metadata.get("schema_version") != 2: + raise GateError("baseline must be an object with schema_version=2") + pinned_harness = baseline_metadata.get("harness_commit") + if not isinstance(pinned_harness, str) or not GIT_SHA_RE.fullmatch(pinned_harness): + raise GateError("baseline harness_commit must be a full lowercase Git SHA") + if harness_sha != pinned_harness: + raise GateError( + f"metadata harness_sha {harness_sha} does not match pinned baseline {pinned_harness}" + ) + harness_exit_code = metadata.get("harness_exit_code") + if isinstance(harness_exit_code, bool) or not isinstance(harness_exit_code, int): + raise GateError("metadata harness_exit_code must be an integer") + required = metadata.get("required_scenarios") + if not isinstance(required, list) or not required or not all(isinstance(item, str) and item for item in required): + raise GateError("metadata required_scenarios must be a non-empty string array") + if len(required) != len(set(required)): + raise GateError("metadata required_scenarios contains duplicates") + + raw_not_scored = metadata.get("not_scored_scenarios", []) + if not isinstance(raw_not_scored, list): + raise GateError("metadata not_scored_scenarios must be an array") + not_scored: dict[str, str] = {} + for index, raw in enumerate(raw_not_scored): + if not isinstance(raw, dict): + raise GateError(f"metadata not_scored_scenarios[{index}] must be an object") + scenario = raw.get("scenario") + reason = raw.get("reason") + if not isinstance(scenario, str) or not scenario or not isinstance(reason, str) or not reason: + raise GateError( + f"metadata not_scored_scenarios[{index}] requires non-empty scenario and reason" + ) + if scenario in not_scored: + raise GateError(f"metadata not_scored_scenarios duplicates {scenario!r}") + not_scored[scenario] = reason + + required_set = set(required) + if required_set & set(not_scored): + raise GateError("metadata scenario cannot be both required and not_scored") + + reports = load_reports(report_root) + classifications = load_classifications(baseline_path, profile) + problems: list[str] = [] + optional_classifications = [ + entry + for entry in classifications.values() + if entry.classification == "optional_capability_not_implemented" + ] + server_capabilities = metadata.get("server_capabilities") + if optional_classifications: + if not isinstance(server_capabilities, dict): + raise GateError( + "metadata server_capabilities must be an object when optional-capability classifications exist" + ) + profiles = baseline_metadata.get("profiles") + profile_config = profiles.get(profile) if isinstance(profiles, dict) else None + expected_capabilities = ( + profile_config.get("expected_server_capabilities") + if isinstance(profile_config, dict) + else None + ) + if not isinstance(expected_capabilities, dict): + raise GateError( + f"baseline profile {profile!r} needs expected_server_capabilities for optional classifications" + ) + if server_capabilities != expected_capabilities: + problems.append( + "server capability advertisement changed; optional-capability classifications " + "require review" + ) + elif server_capabilities is not None and not isinstance(server_capabilities, dict): + raise GateError("metadata server_capabilities must be an object when present") + missing_scenarios = sorted(required_set - set(reports)) + missing_not_scored_scenarios = sorted(set(not_scored) - set(reports)) + expected_scenarios = required_set | set(not_scored) + unexpected_scenarios = sorted(set(reports) - expected_scenarios) + if missing_scenarios: + problems.append("missing required scenario reports: " + ", ".join(missing_scenarios)) + if missing_not_scored_scenarios: + problems.append( + "missing not-scored scenario reports: " + ", ".join(missing_not_scored_scenarios) + ) + if unexpected_scenarios: + problems.append("unexpected scenario reports: " + ", ".join(unexpected_scenarios)) + + invalid_classification_scenarios = sorted( + {entry.scenario for entry in classifications.values() if entry.scenario not in required_set} + ) + if invalid_classification_scenarios: + problems.append( + "classifications may target scored required scenarios only: " + + ", ".join(invalid_classification_scenarios) + ) + + statuses = Counter() + informational_statuses = Counter() + emitted: dict[tuple[str, str], set[str]] = {} + applicable_checks = 0 + unclassified: list[str] = [] + expected: list[dict[str, str]] = [] + informational_non_success: list[dict[str, str]] = [] + inconclusive: list[str] = [] + evidence_mismatches: list[str] = [] + infrastructure_failures: list[str] = [] + baseline_observations: list[dict[str, str]] = [] + + for scenario, checks in sorted(reports.items()): + scored = scenario in required_set + if scored: + checks_by_id: dict[str, list[dict[str, Any]]] = {} + for check in checks: + checks_by_id.setdefault(check["id"], []).append(check) + ambiguous_duplicates = sorted( + check_id + for check_id, repeated in checks_by_id.items() + if len(repeated) > 1 + and any(item["status"] in NON_SUCCESS_STATUSES for item in repeated) + ) + if ambiguous_duplicates: + problems.append( + f"required scenario {scenario!r} emitted duplicate non-success check IDs " + "that cannot be matched to one baseline entry: " + + ", ".join(ambiguous_duplicates) + ) + if scored and not any(check["status"] in VERDICT_STATUSES for check in checks): + problems.append(f"required scenario {scenario!r} emitted no verdict checks") + if not scored and not checks: + problems.append(f"not-scored scenario {scenario!r} emitted an empty report") + for check in checks: + check_id = check["id"] + status = check["status"] + key = (scenario, check_id) + infra_reason = _infrastructure_reason(check) + if infra_reason is not None and status in NON_SUCCESS_STATUSES: + infrastructure_failures.append( + f"{scenario}:{check_id} [{status}] ({infra_reason})" + ) + if not scored: + informational_statuses[status] += 1 + if scenario in not_scored and status in NON_SUCCESS_STATUSES: + informational_non_success.append( + { + "scenario": scenario, + "check_id": check_id, + "status": status, + "reason": not_scored[scenario], + } + ) + continue + + statuses[status] += 1 + emitted.setdefault(key, set()).add(status) + if status in APPLICABLE_STATUSES: + applicable_checks += 1 + if status in NON_SUCCESS_STATUSES: + entry = classifications.get(key) + label = f"{scenario}:{check_id} [{status}]" + observed_evidence = check_evidence_sha256(check) + baseline_observations.append( + { + "scenario": scenario, + "check_id": check_id, + "expected_status": status, + "evidence_sha256": observed_evidence, + } + ) + if entry is None: + unclassified.append(label) + continue + if entry.expected_status != status: + evidence_mismatches.append( + f"{label} expected status {entry.expected_status}" + ) + continue + if entry.evidence_sha256 != observed_evidence: + evidence_mismatches.append( + f"{label} evidence changed: expected {entry.evidence_sha256}, " + f"observed {observed_evidence}" + ) + continue + expected.append( + { + "scenario": scenario, + "check_id": check_id, + "status": status, + "evidence_sha256": observed_evidence, + "classification": entry.classification, + "reason": entry.reason, + "evidence": entry.evidence, + } + ) + if entry.classification == "inconclusive_infrastructure": + inconclusive.append(label) + + if applicable_checks == 0: + problems.append("zero applicable SUCCESS/FAILURE/WARNING checks were emitted") + if harness_exit_code not in {0, 1}: + problems.append( + f"conformance referee exited abnormally with code {harness_exit_code}; only 0/1 are normal" + ) + scored_failures = statuses.get("FAILURE", 0) + if harness_exit_code == 0 and scored_failures > 0: + problems.append( + f"referee exit code 0 disagrees with {scored_failures} scored FAILURE check(s)" + ) + if harness_exit_code == 1 and scored_failures == 0: + problems.append("referee exit code 1 had no scored FAILURE checks") + if infrastructure_failures: + problems.append( + "infrastructure failures cannot satisfy the gate: " + + ", ".join(sorted(infrastructure_failures)) + ) + if unclassified: + problems.append("unclassified non-success checks: " + ", ".join(sorted(unclassified))) + if evidence_mismatches: + problems.append( + "classified check status/evidence changed and requires review: " + + ", ".join(sorted(evidence_mismatches)) + ) + if inconclusive: + problems.append( + "inconclusive infrastructure results cannot satisfy the gate: " + + ", ".join(sorted(inconclusive)) + ) + + stale: list[str] = [] + for key, entry in sorted(classifications.items()): + if entry.scenario not in required_set: + continue + states = emitted.get(key) + label = f"{entry.scenario}:{entry.check_id}" + if not states: + stale.append(label + " (check not emitted)") + elif states <= {"INFO"}: + stale.append(label + " (no verdict emitted)") + elif states <= {"SUCCESS", "INFO"} and "SUCCESS" in states: + stale.append(label + " (now passing)") + if stale: + problems.append("stale or uncovered classifications: " + ", ".join(stale)) + + return { + "schema_version": 2, + "profile": profile, + "server_sha": server_sha, + "harness_sha": harness_sha, + "harness_exit_code": harness_exit_code, + "server_capabilities": server_capabilities, + "required_scenario_count": len(required), + "not_scored_scenario_count": len(not_scored), + "reported_scenario_count": len(reports), + "applicable_check_count": applicable_checks, + "status_counts": dict(sorted(statuses.items())), + "informational_status_counts": dict(sorted(informational_statuses.items())), + "classified_non_success": expected, + "informational_non_success": informational_non_success, + "baseline_observations": baseline_observations, + "infrastructure_failures": infrastructure_failures, + "missing_scenarios": missing_scenarios, + "missing_not_scored_scenarios": missing_not_scored_scenarios, + "unexpected_scenarios": unexpected_scenarios, + "problems": problems, + "gate_passed": not problems, + } + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reports", required=True, type=Path) + parser.add_argument("--metadata", required=True, type=Path) + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--profile", required=True) + parser.add_argument("--summary", type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + try: + summary = evaluate_reports(args.reports, args.metadata, args.baseline, args.profile) + except GateError as exc: + print(f"MCP conformance report gate input error: {exc}", file=sys.stderr) + return 2 + + rendered = json.dumps(summary, indent=2, sort_keys=True) + "\n" + if args.summary: + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if summary["gate_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_mcp_conformance_report.py b/scripts/tests/test_mcp_conformance_report.py new file mode 100644 index 000000000..c28e348e2 --- /dev/null +++ b/scripts/tests/test_mcp_conformance_report.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "mcp_conformance_report.py" +SPEC = importlib.util.spec_from_file_location("mcp_conformance_report", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +report_gate = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = report_gate +SPEC.loader.exec_module(report_gate) + + +class ReportGateTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.raw = self.root / "raw" + self.raw.mkdir() + self.metadata = self.root / "metadata.json" + self.baseline = self.root / "baseline.json" + self.profile = "2026-07-28" + + def tearDown(self) -> None: + self.tmp.cleanup() + + def write_metadata( + self, + scenarios: list[str], + not_scored: list[dict[str, str]] | None = None, + harness_exit_code: int = 0, + server_capabilities: dict[str, object] | None = None, + ) -> None: + self.metadata.write_text( + json.dumps( + { + "profile": self.profile, + "schema_version": 1, + "server_sha": "a" * 40, + "harness_sha": "b" * 40, + "harness_exit_code": harness_exit_code, + "server_capabilities": server_capabilities + if server_capabilities is not None + else {"tools": {"listChanged": False}}, + "required_scenarios": scenarios, + "not_scored_scenarios": not_scored or [], + } + ), + encoding="utf-8", + ) + + def write_baseline( + self, + entries: list[dict[str, str]], + expected_server_capabilities: dict[str, object] | None = None, + ) -> None: + self.baseline.write_text( + json.dumps( + { + "schema_version": 2, + "harness_commit": "b" * 40, + "profiles": { + self.profile: { + "expected_server_capabilities": expected_server_capabilities + if expected_server_capabilities is not None + else {"tools": {"listChanged": False}}, + "classifications": entries, + } + }, + } + ), + encoding="utf-8", + ) + + def write_checks(self, scenario: str, checks: list[dict[str, object]]) -> None: + result = self.raw / f"server-{scenario}-2026-09-14T00-00-00-000Z" + result.mkdir() + (result / "checks.json").write_text(json.dumps(checks), encoding="utf-8") + + @staticmethod + def check( + check_id: str, + status: str, + *, + error_message: str | None = None, + description: str | None = None, + ) -> dict[str, object]: + result: dict[str, object] = { + "id": check_id, + "name": check_id, + "description": description or check_id, + "status": status, + "timestamp": "2026-09-14T00:00:00Z", + } + if error_message is not None: + result["errorMessage"] = error_message + return result + + @staticmethod + def classification( + check: dict[str, object], + kind: str = "genuine_protocol_failure", + *, + scenario: str = "scenario-a", + ) -> dict[str, str]: + return { + "scenario": scenario, + "check_id": str(check["id"]), + "expected_status": str(check["status"]), + "evidence_sha256": report_gate.check_evidence_sha256(check), + "classification": kind, + "reason": "reviewed baseline reason", + "evidence": "MCP 2026-07-28 section/example", + } + + def evaluate(self): + return report_gate.evaluate_reports(self.raw, self.metadata, self.baseline, self.profile) + + def test_successful_nonempty_coverage_passes(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("wire-shape", "SUCCESS")]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + self.assertEqual(summary["applicable_check_count"], 1) + + def test_exact_classified_failure_passes(self) -> None: + failed = self.check("known-gap", "FAILURE", error_message="stable protocol mismatch") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(failed)]) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_all_skipped_is_not_mistaken_for_coverage(self) -> None: + skipped = self.check("not-applicable", "SKIPPED") + self.write_metadata(["scenario-a"]) + self.write_baseline( + [self.classification(skipped, "inconclusive_infrastructure")] + ) + self.write_checks("scenario-a", [skipped]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("zero applicable" in problem for problem in summary["problems"])) + + def test_missing_required_scenario_fails(self) -> None: + self.write_metadata(["scenario-a", "scenario-b"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertEqual(summary["missing_scenarios"], ["scenario-b"]) + + def test_unexpected_scenario_fails(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("scenario-extra", [self.check("extra", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertEqual(summary["unexpected_scenarios"], ["scenario-extra"]) + + def test_duplicate_scenario_reports_are_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + duplicate = self.raw / "server-scenario-a-2026-09-14T00-00-01-000Z" + duplicate.mkdir() + (duplicate / "checks.json").write_text( + json.dumps([self.check("ok-again", "SUCCESS")]), encoding="utf-8" + ) + with self.assertRaisesRegex(report_gate.GateError, "multiple raw reports"): + self.evaluate() + + def test_required_and_not_scored_overlap_is_rejected(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "scenario-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + with self.assertRaisesRegex(report_gate.GateError, "both required and not_scored"): + self.evaluate() + + def test_required_empty_report_fails_even_with_other_coverage(self) -> None: + self.write_metadata(["scenario-a", "scenario-b"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("scenario-b", []) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("scenario-b" in p and "no verdict" in p for p in summary["problems"])) + + def test_required_info_only_report_fails_even_with_other_coverage(self) -> None: + self.write_metadata(["scenario-a", "scenario-b"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("scenario-b", [self.check("info", "INFO")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("scenario-b" in p and "no verdict" in p for p in summary["problems"])) + + def test_unknown_status_is_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("broken", "ERROR")]) + with self.assertRaisesRegex(report_gate.GateError, "unknown status"): + self.evaluate() + + def test_duplicate_success_check_id_is_allowed(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + self.write_checks( + "scenario-a", + [self.check("duplicate", "SUCCESS"), self.check("duplicate", "SUCCESS")], + ) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_duplicate_non_success_check_id_fails_closed(self) -> None: + first = self.check("duplicate", "FAILURE", error_message="first failure") + second = self.check("duplicate", "FAILURE", error_message="second failure") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(first)]) + self.write_checks("scenario-a", [first, second]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("duplicate non-success check IDs" in p for p in summary["problems"])) + + def test_metadata_harness_sha_must_match_baseline_pin(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline([]) + metadata = json.loads(self.metadata.read_text(encoding="utf-8")) + metadata["harness_sha"] = "c" * 40 + self.metadata.write_text(json.dumps(metadata), encoding="utf-8") + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + with self.assertRaisesRegex(report_gate.GateError, "does not match pinned baseline"): + self.evaluate() + + def test_not_scored_failure_is_visible_but_does_not_require_classification(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "extension"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("extension-a", [self.check("pending", "FAILURE")]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + self.assertEqual(summary["informational_status_counts"], {"FAILURE": 1}) + self.assertEqual(summary["informational_non_success"][0]["reason"], "extension") + + def test_not_scored_timeout_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "extension"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "extension-a", + [ + self.check( + "scenario-timeout", + "FAILURE", + error_message="Scenario 'extension-a' did not complete within 10000ms.", + ) + ], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_not_scored_duplicate_check_ids_remain_informational(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "extension-a", + [self.check("repeated", "FAILURE"), self.check("repeated", "FAILURE")], + ) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + self.assertEqual(summary["informational_status_counts"], {"FAILURE": 2}) + + def test_missing_not_scored_scenario_fails_coverage_integrity(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "pending-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertEqual(summary["missing_not_scored_scenarios"], ["pending-a"]) + + def test_empty_not_scored_report_fails_coverage_integrity(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "pending-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks("pending-a", []) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("not-scored scenario" in p and "empty" in p for p in summary["problems"])) + + def test_classified_harness_error_remains_inconclusive(self) -> None: + failed = self.check("scenario-a", "FAILURE", error_message="unreviewable harness result") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline( + [self.classification(failed, "inconclusive_infrastructure")] + ) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("inconclusive infrastructure" in p for p in summary["problems"])) + + def test_abnormal_harness_exit_is_infrastructure_failure(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=137) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("exited abnormally" in p for p in summary["problems"])) + + def test_exit_one_without_scored_failure_is_rejected(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("exit code 1 had no scored FAILURE" in p for p in summary["problems"])) + + def test_exit_zero_with_scored_failure_is_rejected(self) -> None: + failed = self.check("unexpected-failure", "FAILURE", error_message="protocol mismatch") + self.write_metadata(["scenario-a"], harness_exit_code=0) + self.write_baseline([self.classification(failed)]) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("exit code 0" in p and "scored FAILURE" in p for p in summary["problems"])) + + def test_new_failure_requires_exact_classification(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("new-regression", "FAILURE")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("new-regression" in p for p in summary["problems"])) + + def test_changed_failure_evidence_requires_review(self) -> None: + expected = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="Unknown tool: test_simple_text", + ) + actual = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="Tool returned a different protocol error", + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_changed_failure_details_with_same_message_requires_review(self) -> None: + expected = self.check( + "stable-message", + "FAILURE", + error_message="protocol assertion failed", + ) + expected["details"] = {"actual": 200, "expected": 400} + actual = self.check( + "stable-message", + "FAILURE", + error_message="protocol assertion failed", + ) + actual["details"] = {"actual": 500, "expected": 400} + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_timestamp_jsonrpc_id_does_not_change_evidence(self) -> None: + expected = self.check( + "sep-2575-http-server-no-independent-requests-on-stream", + "FAILURE", + error_message="diagnostic tool is unavailable", + ) + expected["details"] = { + "untestable": True, + "response": { + "jsonrpc": "2.0", + "id": 1_789_000_000_001, + "error": {"code": -32602, "message": "tool unavailable"}, + }, + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = 1_789_000_000_999 + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_missing_timestamp_jsonrpc_id_requires_review(self) -> None: + expected = self.check( + "sep-2575-http-server-no-independent-requests-on-stream", + "FAILURE", + error_message="diagnostic tool is unavailable", + ) + expected["details"] = { + "response": { + "jsonrpc": "2.0", + "id": 1_789_000_000_001, + "error": {"code": -32602, "message": "tool unavailable"}, + } + } + actual = json.loads(json.dumps(expected)) + del actual["details"]["response"]["id"] + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_timestamp_jsonrpc_id_type_change_requires_review(self) -> None: + expected = self.check( + "sep-2575-http-server-no-independent-requests-on-stream", + "FAILURE", + error_message="diagnostic tool is unavailable", + ) + expected["details"] = { + "response": {"jsonrpc": "2.0", "id": 1_789_000_000_001, "error": {"code": -32602}} + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = "" + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_fixed_jsonrpc_id_change_still_requires_review(self) -> None: + expected = self.check( + "sep-2575-http-server-no-independent-requests-on-stream", + "FAILURE", + error_message="protocol assertion failed", + ) + expected["details"] = { + "response": {"jsonrpc": "2.0", "id": 1, "error": {"code": -32602}} + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = 2 + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_timestamp_id_on_unrelated_check_is_not_normalized(self) -> None: + expected = self.check("other-check", "FAILURE", error_message="protocol assertion failed") + expected["details"] = { + "response": {"jsonrpc": "2.0", "id": 1_789_000_000_001, "error": {"code": -32602}} + } + actual = json.loads(json.dumps(expected)) + actual["details"]["response"]["id"] = 1_789_000_000_999 + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_wire_schema_full_message_drift_does_not_change_evidence(self) -> None: + expected = self.check( + "wire-schema-valid", + "FAILURE", + error_message='Schema validation failed; offending message: {"tools":[{"name":"old-tool"}]}', + ) + expected["details"] = { + "messagesValidated": 5, + "violations": [ + { + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["outputSchema: must have required property 'type'"], + "message": { + "jsonrpc": "2.0", + "id": 1, + "result": {"tools": [{"name": "old-tool"}]}, + }, + } + ], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["message"]["result"] = { + "tools": [{"name": "new-unrelated-tool", "description": "changed"}] + } + actual["errorMessage"] = ( + 'Schema validation failed; offending message: ' + '{"tools":[{"name":"new-unrelated-tool","description":"changed"}]}' + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertTrue(summary["gate_passed"]) + + def test_wire_schema_tool_index_shift_uses_tool_identity(self) -> None: + expected = self.check("wire-schema-valid", "FAILURE", error_message="Schema validation failed") + expected["details"] = { + "messagesValidated": 5, + "violations": [{ + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["ListToolsResult/tools/1/outputSchema: must have required property 'type' (result of 'tools/list')"], + "message": {"jsonrpc": "2.0", "id": 1, "result": {"tools": [{"name": "other"}, {"name": "ssh_resource"}]}}, + }], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["errors"] = [ + "ListToolsResult/tools/2/outputSchema: must have required property 'type' (result of 'tools/list')" + ] + actual["details"]["violations"][0]["message"]["result"]["tools"] = [ + {"name": "inserted"}, {"name": "other"}, {"name": "ssh_resource"} + ] + self.assertEqual( + report_gate.check_evidence_sha256(expected), + report_gate.check_evidence_sha256(actual), + ) + + def test_wire_schema_tool_identity_change_requires_review(self) -> None: + expected = self.check("wire-schema-valid", "FAILURE", error_message="Schema validation failed") + expected["details"] = { + "messagesValidated": 5, + "violations": [{ + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["ListToolsResult/tools/0/outputSchema: must have required property 'type' (result of 'tools/list')"], + "message": {"jsonrpc": "2.0", "id": 1, "result": {"tools": [{"name": "ssh_resource"}]}}, + }], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["message"]["result"]["tools"][0]["name"] = "different_tool" + self.assertNotEqual( + report_gate.check_evidence_sha256(expected), + report_gate.check_evidence_sha256(actual), + ) + + def test_wire_schema_error_change_still_requires_review(self) -> None: + expected = self.check( + "wire-schema-valid", + "FAILURE", + error_message="Schema validation failed", + ) + expected["details"] = { + "messagesValidated": 5, + "violations": [ + { + "origin": "implementation", + "specVersion": "2025-11-25", + "context": "stateful response to 'tools/list'", + "errors": ["outputSchema: must have required property 'type'"], + "message": {"jsonrpc": "2.0", "id": 1, "result": {}}, + } + ], + } + actual = json.loads(json.dumps(expected)) + actual["details"]["violations"][0]["errors"] = [ + "inputSchema: must have required property 'type'" + ] + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_business_message_change_is_not_wire_schema_noise(self) -> None: + expected = self.check( + "business-error", + "FAILURE", + error_message="tool failed", + ) + expected["details"] = { + "origin": "application", + "context": "tool-call", + "errors": [], + "specVersion": "business-v1", + "message": {"code": "FIXTURE_MISSING"}, + } + actual = json.loads(json.dumps(expected)) + actual["details"]["message"]["code"] = "AUTH_DENIED" + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_empty_structured_evidence_falls_back_to_description(self) -> None: + expected = self.check("empty-details", "FAILURE", description="first diagnostic") + expected["details"] = {} + actual = self.check("empty-details", "FAILURE", description="changed diagnostic") + actual["details"] = {} + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("evidence changed" in p for p in summary["problems"])) + + def test_not_scored_wire_schema_harness_error_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "extension-a", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "extension-a", + [self.check("wire-schema-harness-error", "FAILURE", error_message="harness emitted invalid JSON-RPC")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_not_scored_scenario_catch_error_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "json-schema-2020-12", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "json-schema-2020-12", + [self.check("json-schema-2020-12-error", "FAILURE", error_message="Failed: fetch failed")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_not_scored_top_level_scenario_error_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "server-sse-polling", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "server-sse-polling", + [ + self.check( + "server-sse-polling-error", + "FAILURE", + error_message="Error: unexpected scenario exception", + ) + ], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("top-level scenario failure" in p for p in summary["problems"])) + + def test_not_scored_transport_exception_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "tasks-mrtr-composition", "reason": "extension"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "tasks-mrtr-composition", + [ + self.check( + "sep-2663-mrtr-synchronous-before-task-creation", + "FAILURE", + error_message="fetch failed", + ) + ], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("transport exception" in p for p in summary["problems"])) + + def test_not_scored_setup_failure_is_infrastructure_failure(self) -> None: + self.write_metadata( + ["scenario-a"], + [{"scenario": "http-header-validation", "reason": "pending"}], + ) + self.write_baseline([]) + self.write_checks("scenario-a", [self.check("ok", "SUCCESS")]) + self.write_checks( + "http-header-validation", + [self.check("sep-2243-server-standard-setup", "FAILURE", error_message="Failed to set up tests: fetch failed")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_connection_failure_cannot_hide_behind_fixture_classification(self) -> None: + expected = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="Unknown tool: test_simple_text", + ) + actual = self.check( + "tools-call-simple-text", + "FAILURE", + error_message="connect ECONNREFUSED 127.0.0.1:12345", + description="Failed to run scenario", + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected, "missing_harness_fixture")]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("infrastructure failures" in p for p in summary["problems"])) + + def test_status_change_requires_review(self) -> None: + expected = self.check("subscription-ack", "SKIPPED") + actual = self.check( + "subscription-ack", + "FAILURE", + error_message="subscription protocol assertion failed", + ) + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline( + [self.classification(expected, "optional_capability_not_implemented")] + ) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("expected status SKIPPED" in p for p in summary["problems"])) + + def test_warning_to_failure_requires_review(self) -> None: + expected = self.check("advisory", "WARNING", error_message="SHOULD mismatch") + actual = self.check("advisory", "FAILURE", error_message="SHOULD mismatch") + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [actual]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("expected status WARNING" in p for p in summary["problems"])) + + def test_optional_classification_requires_same_capability_advertisement(self) -> None: + failed = self.check( + "prompts-list", + "FAILURE", + error_message="Failed: method not found", + ) + self.write_metadata( + ["scenario-a"], + harness_exit_code=1, + server_capabilities={"tools": {"listChanged": False}, "prompts": {}}, + ) + self.write_baseline( + [self.classification(failed, "optional_capability_not_implemented")], + expected_server_capabilities={"tools": {"listChanged": False}}, + ) + self.write_checks("scenario-a", [failed]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("capability advertisement changed" in p for p in summary["problems"])) + + def test_passing_expected_failure_is_stale(self) -> None: + expected = self.check("fixed-check", "FAILURE", error_message="known bug") + self.write_metadata(["scenario-a"]) + self.write_baseline([self.classification(expected)]) + self.write_checks("scenario-a", [self.check("fixed-check", "SUCCESS")]) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("now passing" in p for p in summary["problems"])) + + def test_info_only_expected_failure_is_uncovered(self) -> None: + expected = self.check("no-verdict", "FAILURE", error_message="known bug") + self.write_metadata(["scenario-a"]) + self.write_baseline([self.classification(expected)]) + self.write_checks( + "scenario-a", + [self.check("no-verdict", "INFO"), self.check("ok", "SUCCESS")], + ) + summary = self.evaluate() + self.assertFalse(summary["gate_passed"]) + self.assertTrue(any("no verdict emitted" in p for p in summary["problems"])) + + def test_placeholder_evidence_hash_is_rejected(self) -> None: + self.write_metadata(["scenario-a"], harness_exit_code=1) + self.write_baseline( + [ + { + "scenario": "scenario-a", + "check_id": "failure", + "expected_status": "FAILURE", + "evidence_sha256": "0" * 64, + "classification": "genuine_protocol_failure", + "reason": "not reviewed", + "evidence": "placeholder", + } + ] + ) + self.write_checks("scenario-a", [self.check("failure", "FAILURE")]) + with self.assertRaisesRegex(report_gate.GateError, "placeholder evidence_sha256"): + self.evaluate() + + def test_broad_expected_failure_masks_are_rejected(self) -> None: + self.write_metadata(["scenario-a"]) + self.write_baseline( + [ + { + "scenario": "scenario-a", + "check_id": "*", + "expected_status": "FAILURE", + "evidence_sha256": "0" * 64, + "classification": "genuine_protocol_failure", + "reason": "too broad", + "evidence": "none", + } + ] + ) + self.write_checks("scenario-a", [self.check("failure", "FAILURE")]) + with self.assertRaisesRegex(report_gate.GateError, "broad scenario masks"): + self.evaluate() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/mcp_tests.rs b/src/mcp_tests.rs index 666472c30..2920fca2f 100644 --- a/src/mcp_tests.rs +++ b/src/mcp_tests.rs @@ -214,6 +214,8 @@ mod agent_continuation_app; mod artifact_export; #[path = "mcp_tests/computer_app.rs"] mod computer_app; +#[path = "mcp_tests/conformance.rs"] +mod conformance; #[path = "mcp_tests/file_import.rs"] mod file_import; #[path = "mcp_tests/goal_plan_app.rs"] diff --git a/src/mcp_tests/artifact_export.rs b/src/mcp_tests/artifact_export.rs index f9d035c8b..52775f145 100644 --- a/src/mcp_tests/artifact_export.rs +++ b/src/mcp_tests/artifact_export.rs @@ -1052,6 +1052,9 @@ async fn http_mcp_artifact_export_resources_read_streams_valid_json_blob() { assert_eq!(body["jsonrpc"], "2.0"); assert_eq!(body["id"], 3190); assert_eq!(body["result"]["resultType"], "complete"); + // MCP 2026 conformance baseline finding: the optimized streaming path has + // not yet been aligned with the cache hints emitted by ordinary resources/read. + // This baseline records the current wire shape only; a focused follow-up fixes it. assert!(body["result"].get("ttlMs").is_none()); assert!(body["result"].get("cacheScope").is_none()); assert_eq!(body["result"]["contents"][0]["uri"], uri); diff --git a/src/mcp_tests/conformance.rs b/src/mcp_tests/conformance.rs new file mode 100644 index 000000000..e2db650c4 --- /dev/null +++ b/src/mcp_tests/conformance.rs @@ -0,0 +1,61 @@ +use super::*; +use salvo::conn::{Acceptor, Listener, TcpListener}; +use std::path::PathBuf; +use std::time::Duration; +use tokio::time::Instant; + +const CONFORMANCE_URL_FILE_ENV: &str = "WEBCODEX_MCP_CONFORMANCE_URL_FILE"; +const CONFORMANCE_STOP_FILE_ENV: &str = "WEBCODEX_MCP_CONFORMANCE_STOP_FILE"; +const CONFORMANCE_FIXTURE_LIFETIME: Duration = Duration::from_secs(15 * 60); + +/// External MCP conformance uses a real loopback socket, but the endpoint is +/// still the same production Salvo/AuthMiddleware/mcp_post composition used by +/// the in-process HTTP tests. It is ignored because it deliberately waits for an +/// external referee process; `scripts/mcp_conformance.sh` owns its lifecycle. +#[tokio::test] +#[ignore = "started only by scripts/mcp_conformance.sh with the pinned external MCP referee"] +async fn mcp_conformance_fixture_server() { + let url_file = PathBuf::from( + std::env::var_os(CONFORMANCE_URL_FILE_ENV) + .expect("conformance fixture requires WEBCODEX_MCP_CONFORMANCE_URL_FILE"), + ); + let stop_file = PathBuf::from( + std::env::var_os(CONFORMANCE_STOP_FILE_ENV) + .expect("conformance fixture requires WEBCODEX_MCP_CONFORMANCE_STOP_FILE"), + ); + + let config = test_config(None); + let (_db_tmp, db) = test_db(); + let runtime = Arc::new(test_runtime_from_model_surface_env(None)); + let router = build_test_router(config, db, runtime); + let acceptor = TcpListener::new("127.0.0.1:0").bind().await; + let addr = acceptor.holdings()[0] + .local_addr + .clone() + .into_std() + .expect("MCP conformance listener must be TCP"); + let server_task = tokio::spawn(async move { + Server::new(acceptor).serve(router).await; + }); + + let temporary_url_file = url_file.with_extension("tmp"); + std::fs::write(&temporary_url_file, format!("http://{addr}/mcp\n")) + .expect("write MCP conformance fixture URL"); + std::fs::rename(&temporary_url_file, &url_file) + .expect("publish MCP conformance fixture URL atomically"); + + let deadline = Instant::now() + CONFORMANCE_FIXTURE_LIFETIME; + loop { + if stop_file.exists() { + break; + } + assert!( + Instant::now() < deadline, + "external MCP conformance fixture exceeded its bounded lifetime" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + + server_task.abort(); + let _ = server_task.await; +} diff --git a/src/mcp_tests/project_connector.rs b/src/mcp_tests/project_connector.rs index 74df38474..7b14a77de 100644 --- a/src/mcp_tests/project_connector.rs +++ b/src/mcp_tests/project_connector.rs @@ -907,6 +907,12 @@ async fn http_project_connector_2026_tasks_poll_durable_execution_across_reopen( let completed_body: Value = completed.take_json().await.unwrap(); assert_eq!(completed_body["result"]["status"], "completed"); assert_eq!(completed_body["result"]["taskId"], task_id); + // MCP 2026 conformance baseline finding: the completed Task is finalized, + // but the nested original tool result is not yet discriminated as complete. + // Keep the observation explicit until the focused result-shape follow-up. + assert!(completed_body["result"]["result"] + .get("resultType") + .is_none()); assert_eq!( completed_body["result"]["result"]["structuredContent"]["data"]["execution"] ["execution_status"], diff --git a/tests/fixtures/mcp/conformance/baseline.json b/tests/fixtures/mcp/conformance/baseline.json new file mode 100644 index 000000000..921527d4b --- /dev/null +++ b/tests/fixtures/mcp/conformance/baseline.json @@ -0,0 +1,672 @@ +{ + "schema_version": 2, + "harness_commit": "7169291ec0b68eb370fddcd9947313ab0d5e4156", + "profiles": { + "2025-11-25": { + "classifications": [ + { + "scenario": "completion-complete", + "check_id": "completion-complete", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "3fb31b87a162a1318b40f9d1ddee34b892156f5d42e865bb31a78e9cbb2fa5e3" + }, + { + "scenario": "logging-set-level", + "check_id": "logging-set-level", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "2eea842958ca1c21934dbf7f31c526278313834d87d01c77b24d28e2ea1d7f17" + }, + { + "scenario": "prompts-get-embedded-resource", + "check_id": "prompts-get-embedded-resource", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" + }, + { + "scenario": "prompts-get-simple", + "check_id": "prompts-get-simple", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" + }, + { + "scenario": "prompts-get-with-args", + "check_id": "prompts-get-with-args", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" + }, + { + "scenario": "prompts-get-with-image", + "check_id": "prompts-get-with-image", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "69302e542bba424021834142e01875b229b97440c67eedbc4b671b87ed03640d" + }, + { + "scenario": "prompts-list", + "check_id": "prompts-list", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "500284cbe8b04997354d450cc6004d54c245c404c89baa1d45e8f657bda7ff6b" + }, + { + "scenario": "resources-list", + "check_id": "resources-list", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "940374420a5847d2786704f3964d1f6a9b6f93d45d5caca7c867109789721091" + }, + { + "scenario": "resources-subscribe", + "check_id": "resources-subscribe", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "6fa45e84678894951bba5b600f36e0eeb6fa9f4760bc4e54aa7aa4fd03caa465" + }, + { + "scenario": "resources-unsubscribe", + "check_id": "resources-unsubscribe", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "6fa45e84678894951bba5b600f36e0eeb6fa9f4760bc4e54aa7aa4fd03caa465" + }, + { + "scenario": "tools-call-with-logging", + "check_id": "tools-call-with-logging", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "2eea842958ca1c21934dbf7f31c526278313834d87d01c77b24d28e2ea1d7f17" + }, + { + "scenario": "dns-rebinding-protection", + "check_id": "localhost-host-rebinding-rejected", + "classification": "genuine_protocol_failure", + "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", + "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "55fef6d01afbb12e2235724cc78cef53e13ad7b61fb8b387d4b260c19ce74b89" + }, + { + "scenario": "elicitation-sep1034-defaults", + "check_id": "elicitation-sep1034-general", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "5ab028bbd3467c6dc5fd5c8bd1c19af5732391da6e7c6543737d21962039db54" + }, + { + "scenario": "elicitation-sep1330-enums", + "check_id": "elicitation-sep1330-general", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named elicitation reference tool that is absent from the product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "6043343fce6e7d657c351276e50c1092274ab5679572e2cfbac15397db729633" + }, + { + "scenario": "resources-read-binary", + "check_id": "resources-read-binary", + "classification": "optional_capability_not_implemented", + "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "5ec605e03cf3cb3c48dae40d35358da3caffebcba8576ed4050cc206d0a12327" + }, + { + "scenario": "resources-read-text", + "check_id": "resources-read-text", + "classification": "optional_capability_not_implemented", + "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "5ec605e03cf3cb3c48dae40d35358da3caffebcba8576ed4050cc206d0a12327" + }, + { + "scenario": "resources-templates-read", + "check_id": "resources-templates-read", + "classification": "optional_capability_not_implemented", + "reason": "The legacy 2025-11-25 WebCodex surface does not advertise/implement the resources capability exercised by this scenario.", + "evidence": "MCP 2025-11-25 resources capability is optional; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "5ec605e03cf3cb3c48dae40d35358da3caffebcba8576ed4050cc206d0a12327" + }, + { + "scenario": "tools-call-audio", + "check_id": "tools-call-audio", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "f52ab456f88f932ae8fbda66972454a1eb01e97650550de960ce5f78d251eafe" + }, + { + "scenario": "tools-call-elicitation", + "check_id": "tools-call-elicitation", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "2e5b0ff0322968f3f71159940edc711a62a8570ff5b807d17060695f40b921e2" + }, + { + "scenario": "tools-call-embedded-resource", + "check_id": "tools-call-embedded-resource", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "f176191f243db4cde3281c36d431c051330aaf8bed2317c4bfc271b0ab17c7c3" + }, + { + "scenario": "tools-call-error", + "check_id": "tools-call-error", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "d8d28432df8232f95c5c7333cca717383761bb2b129649f67e146129d55d9c55" + }, + { + "scenario": "tools-call-image", + "check_id": "tools-call-image", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "c0890e33c4952cdc08d213615d387110ea067aaf22584bae4d2e148a5600be1c" + }, + { + "scenario": "tools-call-mixed-content", + "check_id": "tools-call-mixed-content", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "ff8a525bfad910a6a1f4e61cbcf35fbec589c12f2535ae97d5645d6caba447bf" + }, + { + "scenario": "tools-call-sampling", + "check_id": "tools-call-sampling", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "aeb68fd319a9a0cbe7fbca2bdd0dbb45542d26dabeb0a3e69adb37fc577e958c" + }, + { + "scenario": "tools-call-simple-text", + "check_id": "tools-call-simple-text", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "6f37fcae0bde7d34f8b91e1d59d0bf416c0a019609405abd666754a41be912d7" + }, + { + "scenario": "tools-call-with-progress", + "check_id": "tools-call-with-progress", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "13c6bda80dd58f68b3634d402f12edc750b996f695b5e913db59da14a7acfaae" + } + ], + "expected_server_capabilities": { + "tools": { + "listChanged": false + } + } + }, + "2026-07-28": { + "classifications": [ + { + "scenario": "caching", + "check_id": "sep-2549-prompts-list-caching-hints", + "classification": "optional_capability_not_implemented", + "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", + "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "7af406127f9e0edb58db00d3c7a974c4a64e9dc2eb966779a6de918741a9f5f8" + }, + { + "scenario": "caching", + "check_id": "sep-2549-resources-templates-list-caching-hints", + "classification": "optional_capability_not_implemented", + "reason": "The cacheable endpoint belongs to an optional capability that the current WebCodex surface does not implement.", + "evidence": "SEP-2549 applies when the corresponding optional endpoint exists; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "e3fb42ce657d615df87789d7f92cc5b379ed29c676d48cd6f60f27515347ec75" + }, + { + "scenario": "caching", + "check_id": "sep-2549-resources-read-caching-hints", + "classification": "missing_harness_fixture", + "reason": "resources/read caching could not be exercised because the fixture exposed no reference resource to read.", + "evidence": "SEP-2549 caching scenario result from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "e8e86cdaa5319e85265044f35b33d006da90dbab5ee709fd55269c6e9892d0d0" + }, + { + "scenario": "completion-complete", + "check_id": "completion-complete", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "662966f5250ef4190d1d1dfa6f5b05348cf61d995e15b8a435240872b3273ae1" + }, + { + "scenario": "prompts-get-embedded-resource", + "check_id": "prompts-get-embedded-resource", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" + }, + { + "scenario": "prompts-get-simple", + "check_id": "prompts-get-simple", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" + }, + { + "scenario": "prompts-get-with-args", + "check_id": "prompts-get-with-args", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" + }, + { + "scenario": "prompts-get-with-image", + "check_id": "prompts-get-with-image", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "9b485366f340ecc7b0a03461e3949fac498d9d362fffbdde82d0fcc5f0a7b224" + }, + { + "scenario": "prompts-list", + "check_id": "prompts-list", + "classification": "optional_capability_not_implemented", + "reason": "This protocol capability is optional and is not advertised/implemented by the current WebCodex MCP surface.", + "evidence": "MCP server capability model plus first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "96a058e400b47135503f462d35b6865092c786aee626904bcfd88ffa4f51aa02" + }, + { + "scenario": "dns-rebinding-protection", + "check_id": "localhost-host-rebinding-rejected", + "classification": "genuine_protocol_failure", + "reason": "WebCodex accepted invalid Host/Origin input instead of rejecting the rebinding probe.", + "evidence": "MCP transport security / DNS rebinding protection; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "56f4a8d9b0ea3df7f9ad8f42edb3f6dc249c7f119daa83c9ca4da22bd8f86b16" + }, + { + "scenario": "input-required-result-basic-elicitation", + "check_id": "sep-2322-elicitation-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "8df152cebc4b7699b9222a681b6cb490cc5bc116f0666b05a01b7f07919a20d1" + }, + { + "scenario": "input-required-result-basic-list-roots", + "check_id": "sep-2322-list-roots-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "bcfe48b990672771a1512a70fa61adcc4fc71ce23ffa8d9a633af203db8309e5" + }, + { + "scenario": "input-required-result-basic-sampling", + "check_id": "sep-2322-sampling-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "469d38b05459eccad7e149170cbbd56e6f9d28c9d46d80f8ced6e2f317adf4fc" + }, + { + "scenario": "input-required-result-capability-check", + "check_id": "sep-2322-respect-client-capabilities", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "dbf49040d276400f78f24ba726743e1cc7628dee7ddd6cd252a12dfcdb4b2632" + }, + { + "scenario": "input-required-result-ignore-extra-params", + "check_id": "sep-2322-ignore-unexpected-params", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "WARNING", + "evidence_sha256": "de2c48d4f6018bc152772b53c9fb420f9cb6ba85cd0155c45e4daf348bf21d2e" + }, + { + "scenario": "input-required-result-missing-input-response", + "check_id": "sep-2322-missing-response-rerequests", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "WARNING", + "evidence_sha256": "e6fd1be79feba098d507c9f3d21453d995503479cbcc91d09c33fbf331721e66" + }, + { + "scenario": "input-required-result-multi-round", + "check_id": "sep-2322-multi-round-r1", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "3afcf29210ae83e39892a28ac859e41baac1ab6838e5ad869bd193f08cfd505b" + }, + { + "scenario": "input-required-result-multiple-input-requests", + "check_id": "sep-2322-multiple-inputs-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "c6775d26e7f2bcf767c6208b388b25716ffff91803b33ef8263aac4472e2ab29" + }, + { + "scenario": "input-required-result-request-state", + "check_id": "sep-2322-request-state-incomplete", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "db44366f4b2663c8dd18a73b5d243e0201429acc0e80605a95ca278ddd92aa87" + }, + { + "scenario": "input-required-result-result-type", + "check_id": "sep-2322-result-type-included", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "8df152cebc4b7699b9222a681b6cb490cc5bc116f0666b05a01b7f07919a20d1" + }, + { + "scenario": "input-required-result-tampered-state", + "check_id": "sep-2322-reject-tampered-state", + "classification": "missing_harness_fixture", + "reason": "The SEP-2322 scenario requires a named diagnostic/reference tool; this baseline fixture does not yet inject conformance-only tools.", + "evidence": "Pinned SEP-2322 conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "ae62de054b2a479f9f52a8d06482d068dac60854ce75e4f0fb84580995a826d1" + }, + { + "scenario": "input-required-result-non-tool-request", + "check_id": "sep-2322-non-tool-incomplete", + "classification": "optional_capability_not_implemented", + "reason": "This MRTR probe uses prompts/get, but the current WebCodex surface does not implement the optional prompts capability.", + "evidence": "SEP-2322 non-tool request scenario plus server capability model; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "19847c48286d6012d963b8274997698a9c1a9cdfb53fc7f9980729303d5c62bc" + }, + { + "scenario": "resources-read-binary", + "check_id": "resources-read-binary", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "ccebe9670fc81aa853c549c9017baa7d88a371fd5d4625e1726b0a542aefec69" + }, + { + "scenario": "resources-read-text", + "check_id": "resources-read-text", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "2b815566ba4173981dcb12f46ea1ab73bff3b373796d1d152107023e3efa15f7" + }, + { + "scenario": "resources-templates-read", + "check_id": "resources-templates-read", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario expects reference resource content/URIs that the product MCP surface does not provide.", + "evidence": "Pinned resource conformance fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "ae122f912511f271c31f53fee8245c1dde3acfcff41c5db301341055cc9917e0" + }, + { + "scenario": "sep-2164-resource-not-found", + "check_id": "sep-2164-data-uri", + "classification": "genuine_protocol_failure", + "reason": "The resource-not-found error omits the recommended data.uri echo.", + "evidence": "SEP-2164 resource-not-found error data; first GitHub Actions conformance baseline raw report", + "expected_status": "WARNING", + "evidence_sha256": "03e182c64ad2607c2749b90609cbf22dc87eda95ae0951f9c7a4c2f1227aabe0" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-http-server-no-independent-requests-on-stream", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "7b694b9546c9ffc28b7018bb47ad1b35eb71982e0e07cf5d238a8b5b07f28f70" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-missing-capability-http-400", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "00318fb6a220abec56b408ee72fa031253a43ec35e93d0247bd7f49f3f0ce1e3" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-no-log-without-loglevel", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "5802de50b88da05f1deb16702e8a9fc63de9536f2ed3b5455691a6d31adf23d1" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-rejects-undeclared-capability", + "classification": "missing_harness_fixture", + "reason": "The check explicitly requires a named diagnostic tool that the product registry does not expose.", + "evidence": "Pinned server-stateless diagnostic fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "54fb37a17716a6faa4e20eba8bbceb73fa4cf5b7c0ede03c4fc3c950642b6ad0" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-request-meta-invalid-missing-meta", + "classification": "genuine_protocol_failure", + "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", + "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "f72e87a4dcd615f71164cc3d3a94bc2e9a8db5fd4da6364c1379115262c62a15" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-request-meta-invalid-missing-protocol-version", + "classification": "genuine_protocol_failure", + "reason": "The request is rejected with WebCodex header-mismatch code -32020 instead of the required -32602 Invalid params.", + "evidence": "SEP-2575 request metadata validation; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "453717c56bda626408c987bb9e6c8dd5c2582e1523481557197a85f3169fc463" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-honors-notification-filter", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "3526c6b7099730a5aa95fcdb13c9a9256c97a831c6adf4a6fcea428948faedc1" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-sends-prompts-list-changed-on-subscription", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "bc64f7ce41051860e4ea6a058fa1fc33182b920635312562fa174807df371b2c" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-sends-subscription-ack", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "3526c6b7099730a5aa95fcdb13c9a9256c97a831c6adf4a6fcea428948faedc1" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-sends-tools-list-changed-on-subscription", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "9b3210383e2285dfff82e29bfce618a0124128dac675e6d498f11495e20af5f6" + }, + { + "scenario": "server-stateless", + "check_id": "sep-2575-server-tags-subscription-id", + "classification": "optional_capability_not_implemented", + "reason": "The referee marked this check not applicable because WebCodex did not advertise the related subscription/listChanged capability.", + "evidence": "SEP-2575 check details from first GitHub Actions conformance baseline raw report", + "expected_status": "SKIPPED", + "evidence_sha256": "3526c6b7099730a5aa95fcdb13c9a9256c97a831c6adf4a6fcea428948faedc1" + }, + { + "scenario": "tools-call-audio", + "check_id": "tools-call-audio", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "14101eaa646a4d704365cc1088cffb01fd1e9c022de157cd12ee830ef70e9432" + }, + { + "scenario": "tools-call-embedded-resource", + "check_id": "tools-call-embedded-resource", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "3381f09b48c05f0a3a98644c1b0851a4aab608cb471d1fde932b44d351c2c283" + }, + { + "scenario": "tools-call-error", + "check_id": "tools-call-error", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "ceda82cecaf90be4000fbb2601518a3f0fa3b6f960b6fe242a9cd050f350eda9" + }, + { + "scenario": "tools-call-image", + "check_id": "tools-call-image", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "5da82959f8874a71170b8a9d221b37e0cd7a53ef978f27f2555fb1f6438848f8" + }, + { + "scenario": "tools-call-mixed-content", + "check_id": "tools-call-mixed-content", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "99afc685f8f53a330164573619be79f552c87dfc138d1aa311187d75569b644a" + }, + { + "scenario": "tools-call-simple-text", + "check_id": "tools-call-simple-text", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "d68ec286ab9f5950928708ded11dc1023f1a1e488c318c0ee2b57e9a93f764c5" + }, + { + "scenario": "tools-call-with-progress", + "check_id": "tools-call-with-progress", + "classification": "missing_harness_fixture", + "reason": "The upstream scenario requires a named reference test tool that WebCodex intentionally does not expose in its product registry.", + "evidence": "Pinned conformance scenario fixture contract; first GitHub Actions conformance baseline raw report", + "expected_status": "FAILURE", + "evidence_sha256": "21f7c05843f8c02dfaebdb1a8f25b23b9ec413c1cccc100b3267f756d450c994" + } + ], + "expected_server_capabilities": { + "resources": { + "listChanged": false, + "subscribe": false + }, + "tools": { + "listChanged": false + } + } + } + }, + "supplemental_findings": [ + { + "id": "2026-streamed-resource-cache-hints", + "classification": "genuine_protocol_failure", + "evidence": "src/mcp_tests/artifact_export.rs::http_mcp_artifact_export_resources_read_streams_valid_json_blob", + "summary": "The streamed artifact resources/read result currently omits ttlMs and cacheScope. This baseline records the finding but does not change production framing." + }, + { + "id": "2026-completed-task-nested-result-type", + "classification": "genuine_protocol_failure", + "evidence": "src/mcp_tests/project_connector.rs::http_project_connector_2026_tasks_poll_durable_execution_across_reopen", + "summary": "A completed Connector Task currently embeds its original tool result without resultType. This baseline records the finding but does not change production framing." + } + ] +}